Use Protocol Buffers for data serialization

This commit is contained in:
Carles Fernandez 2019-04-20 11:52:00 +02:00
parent 35b5ec1112
commit 811cdc9693
No known key found for this signature in database
GPG Key ID: 4C583C52B0C3877D
11 changed files with 435 additions and 7 deletions

View File

@ -416,6 +416,7 @@ set(GNSSSDR_GNSS_SIM_LOCAL_VERSION "master")
set(GNSSSDR_GPSTK_LOCAL_VERSION "2.10.6")
set(GNSSSDR_MATIO_LOCAL_VERSION "1.5.14")
set(GNSSSDR_PUGIXML_LOCAL_VERSION "1.9")
set(GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION "3.7.1")
if(CMAKE_VERSION VERSION_LESS "3.0.2") # Fix for CentOS 7
set(GNSSSDR_GFLAGS_LOCAL_VERSION "2.2.1")
@ -1784,6 +1785,77 @@ endif()
################################################################################
# Protocol Buffers https://github.com/protocolbuffers/protobuf
################################################################################
option(ENABLE_PROTOBUF "Enable Protocol Buffers" ON)
if(ENABLE_PROTOBUF)
find_package(Protobuf)
set_package_properties(Protobuf PROPERTIES
URL "https://developers.google.com/protocol-buffers/"
DESCRIPTION "A language-neutral, platform-neutral extensible mechanism for serializing structured data"
PURPOSE "Used to serialize output data."
TYPE OPTIONAL
)
if(NOT Protobuf_FOUND)
if(CMAKE_VERSION VERSION_LESS 3.2)
ExternalProject_Add(protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
GIT_REPOSITORY https://github.com/protocolbuffers/protobuf
GIT_TAG v${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/protobuf/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
UPDATE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/protobuf/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/autogen.sh
CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/protobuf/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/configure --prefix=<INSTALL_DIR>
BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
)
else()
ExternalProject_Add(protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
PREFIX ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
GIT_REPOSITORY https://github.com/protocolbuffers/protobuf
GIT_TAG v${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/protobuf/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}
UPDATE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/protobuf/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/autogen.sh
CONFIGURE_COMMAND CXXFLAGS=-fvisibility=hidden ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/protobuf/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/configure --prefix=<INSTALL_DIR>
BUILD_COMMAND ${CMAKE_MAKE_PROGRAM}
INSTALL_COMMAND ${CMAKE_MAKE_PROGRAM} install
BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}
${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/bin/protoc
)
endif()
if(NOT TARGET protobuf::libprotobuf)
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/include)
add_library(protobuf::libprotobuf STATIC IMPORTED)
add_dependencies(protobuf::libprotobuf protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION})
set_target_properties(protobuf::libprotobuf PROPERTIES
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}"
INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/include"
INTERFACE_LINK_LIBRARIES "${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}protobuf${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
endif()
if(NOT TARGET protobuf::protoc)
add_executable(protobuf::protoc IMPORTED)
add_dependencies(protobuf::protoc protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION})
set_target_properties(protobuf::protoc PROPERTIES
IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/bin/protoc"
INTERFACE_LINK_LIBRARIES "${CMAKE_CURRENT_BINARY_DIR}/protobuf-${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}protoc${CMAKE_STATIC_LIBRARY_SUFFIX}"
)
endif()
set_package_properties(Protobuf PROPERTIES
PURPOSE "Protocol Buffers v${GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION} will be downloaded and built when doing '${CMAKE_MAKE_PROGRAM_PRETTY_NAME}'."
)
endif()
endif()
################################################################################
# Doxygen - http://www.doxygen.nl (OPTIONAL, used if found)
################################################################################

View File

@ -0,0 +1,40 @@
syntax = "proto3";
package gnss_sdr;
message GnssSynchro {
string system = 1;
string signal = 2;
uint32 prn = 3;
int32 channel_id = 4;
double acq_delay_samples = 5;
double acq_doppler_hz = 6;
uint64 acq_samplestamp_samples = 7;
uint32 acq_doppler_step = 8;
bool flag_valid_acquisition = 9;
int64 fs = 10;
double prompt_i = 11;
double prompt_q = 12;
double cn0_db_hz = 13;
double carrier_doppler_hz = 14;
double carrier_phase_rads = 15;
double code_phase_samples = 16;
uint64 tracking_sample_counter = 17;
bool flag_valid_symbol_output = 18;
int32 correlation_length_ms = 19;
bool flag_valid_word = 20;
uint32 tow_at_current_symbol_ms = 21;
double pseudorange_m = 22;
double rx_time = 23;
bool flag_valid_pseudorange = 24;
double interp_tow_ms = 25;
}
message Observables {
repeated GnssSynchro observable = 1;
}

View File

@ -16,6 +16,7 @@
# along with GNSS-SDR. If not, see <https://www.gnu.org/licenses/>.
#
add_subdirectory(protobuf)
add_subdirectory(system_parameters)
add_subdirectory(libs)
add_subdirectory(receiver)

View File

@ -43,6 +43,7 @@ target_link_libraries(core_monitor
Boost::system
Gnuradio::runtime
core_system_parameters
core_protobuf
PRIVATE
Gnuradio::pmt
)

View File

@ -37,6 +37,7 @@
Gnss_Synchro_Udp_Sink::Gnss_Synchro_Udp_Sink(std::vector<std::string> addresses, const uint16_t& port) : socket{io_service}
{
serdes = Serdes_Gnss_Synchro();
for (const auto& address : addresses)
{
boost::asio::ip::udp::endpoint endpoint(boost::asio::ip::address::from_string(address, error), port);
@ -45,13 +46,20 @@ Gnss_Synchro_Udp_Sink::Gnss_Synchro_Udp_Sink(std::vector<std::string> addresses,
}
bool Gnss_Synchro_Udp_Sink::write_gnss_synchro(const std::vector<Gnss_Synchro>& stocks)
bool Gnss_Synchro_Udp_Sink::write_gnss_synchro(const std::vector<Gnss_Synchro>& stocks, bool protocolbuffers)
{
std::ostringstream archive_stream;
boost::archive::binary_oarchive oa{archive_stream};
oa << stocks;
std::string outbound_data = archive_stream.str();
std::string outbound_data;
if (protocolbuffers == false)
{
std::ostringstream archive_stream;
boost::archive::binary_oarchive oa{archive_stream};
oa << stocks;
outbound_data = archive_stream.str();
}
else
{
outbound_data = serdes.createProtobuffer(stocks);
}
for (const auto& endpoint : endpoints)
{
socket.open(endpoint.protocol(), error);

View File

@ -33,6 +33,7 @@
#define GNSS_SDR_GNSS_SYNCHRO_UDP_SINK_H_
#include "gnss_synchro.h"
#include "serdes_gnss_synchro.h"
#include <boost/asio.hpp>
#include <boost/system/error_code.hpp>
#include <cstdint>
@ -48,7 +49,7 @@ class Gnss_Synchro_Udp_Sink
{
public:
Gnss_Synchro_Udp_Sink(std::vector<std::string> addresses, const uint16_t& port);
bool write_gnss_synchro(const std::vector<Gnss_Synchro>& stocks);
bool write_gnss_synchro(const std::vector<Gnss_Synchro>& stocks, bool protocolbuffers = false);
private:
boost::asio::io_service io_service;
@ -56,6 +57,7 @@ private:
boost::system::error_code error;
std::vector<boost::asio::ip::udp::endpoint> endpoints;
std::vector<Gnss_Synchro> stocks;
Serdes_Gnss_Synchro serdes;
};

View File

@ -0,0 +1,37 @@
# Copyright (C) 2012-2019 (see AUTHORS file for a list of contributors)
#
# This file is part of GNSS-SDR.
#
# GNSS-SDR is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# GNSS-SDR is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with GNSS-SDR. If not, see <https://www.gnu.org/licenses/>.
#
protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${CMAKE_SOURCE_DIR}/docs/protobuf/gnss_synchro.proto)
add_library(core_protobuf serdes_gnss_synchro.h ${PROTO_SRCS} ${PROTO_HDRS})
target_link_libraries(core_protobuf
PUBLIC
protobuf::libprotobuf
)
target_include_directories(core_protobuf
PUBLIC
${CMAKE_SOURCE_DIR}/src/core/system_parameters
)
get_filename_component(PROTO_INCLUDE_HEADERS ${PROTO_HDRS} DIRECTORY)
set_property(TARGET core_protobuf
APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES
$<BUILD_INTERFACE:${PROTO_INCLUDE_HEADERS}>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>
)

View File

@ -0,0 +1,158 @@
/*!
* \file serdes_gnss_synchro.h
* \brief Serialization / Deserialization of Gnss_Synchro objects using
* Protocol Buffers
* \author Carles Fernandez-Prades, 2019. cfernandez(at)cttc.es
*
* -------------------------------------------------------------------------
*
* Copyright (C) 2010-2019 (see AUTHORS file for a list of contributors)
*
* GNSS-SDR is a software defined Global Navigation
* Satellite Systems receiver
*
* This file is part of GNSS-SDR.
*
* GNSS-SDR is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* GNSS-SDR is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with GNSS-SDR. If not, see <https://www.gnu.org/licenses/>.
*
* -------------------------------------------------------------------------
*/
#ifndef GNSS_SDR_SERDES_GNSS_SYNCHRO_H_
#define GNSS_SDR_SERDES_GNSS_SYNCHRO_H_
#include "gnss_synchro.h"
#include "gnss_synchro.pb.h" // file created by Protocol Buffers at compile time
#include <cstring> // for memcpy
#include <vector>
/*!
* \brief This class implements serialization and deserialization of
* Gnss_Synchro objects using Protocol Buffers.
*/
class Serdes_Gnss_Synchro
{
public:
Serdes_Gnss_Synchro()
{
// Verify that the version of the library that we linked against is
// compatible with the version of the headers we compiled against.
GOOGLE_PROTOBUF_VERIFY_VERSION;
}
~Serdes_Gnss_Synchro()
{
google::protobuf::ShutdownProtobufLibrary();
}
inline std::string createProtobuffer(const std::vector<Gnss_Synchro>& vgs) //!< Serialization into a string
{
gnss_sdr::Observables observables;
std::string data;
for (int i = 0; i < vgs.size(); ++i)
{
gnss_sdr::GnssSynchro* obs = observables.add_observable();
Gnss_Synchro gs = vgs[i];
char c[2];
c[0] = gs.System;
c[1] = '\0';
const std::string sys(c);
char cc[3];
cc[0] = gs.Signal[0];
cc[1] = gs.Signal[1];
cc[2] = '\0';
const std::string sig(cc);
obs->set_system(sys);
obs->set_signal(sig);
obs->set_prn(gs.PRN);
obs->set_channel_id(gs.Channel_ID);
obs->set_acq_delay_samples(gs.Acq_delay_samples);
obs->set_acq_doppler_hz(gs.Acq_doppler_hz);
obs->set_acq_samplestamp_samples(gs.Acq_samplestamp_samples);
obs->set_acq_doppler_step(gs.Acq_doppler_step);
obs->set_flag_valid_acquisition(gs.Flag_valid_acquisition);
obs->set_fs(gs.fs);
obs->set_prompt_i(gs.Prompt_I);
obs->set_prompt_q(gs.Prompt_Q);
obs->set_cn0_db_hz(gs.CN0_dB_hz);
obs->set_carrier_doppler_hz(gs.Carrier_Doppler_hz);
obs->set_code_phase_samples(gs.Code_phase_samples);
obs->set_tracking_sample_counter(gs.Tracking_sample_counter);
obs->set_flag_valid_symbol_output(gs.Flag_valid_symbol_output);
obs->set_correlation_length_ms(gs.correlation_length_ms);
obs->set_fs(gs.Flag_valid_word);
obs->set_fs(gs.TOW_at_current_symbol_ms);
obs->set_pseudorange_m(gs.Pseudorange_m);
obs->set_rx_time(gs.RX_time);
obs->set_flag_valid_pseudorange(gs.Flag_valid_pseudorange);
obs->set_interp_tow_ms(gs.interp_TOW_ms);
}
observables.SerializeToString(&data);
return data;
}
inline std::vector<Gnss_Synchro> readProtobuffer(const gnss_sdr::Observables& obs) //!< Deserialization
{
std::vector<Gnss_Synchro> vgs;
vgs.reserve(obs.observable_size());
for (int i = 0; i < obs.observable_size(); ++i)
{
const gnss_sdr::GnssSynchro& gs_read = obs.observable(i);
Gnss_Synchro gs = Gnss_Synchro();
std::string sys = gs_read.system();
gs.System = *sys.c_str();
std::string sig = gs_read.signal();
std::memcpy(static_cast<void*>(gs.Signal), sig.c_str(), 3);
gs.PRN = gs_read.prn();
gs.Channel_ID = gs_read.channel_id();
gs.Acq_delay_samples = gs_read.acq_delay_samples();
gs.Acq_doppler_hz = gs_read.acq_doppler_hz();
gs.Acq_samplestamp_samples = gs_read.acq_samplestamp_samples();
gs.Acq_doppler_step = gs_read.acq_doppler_step();
gs.Flag_valid_acquisition = gs_read.flag_valid_acquisition();
gs.fs = gs_read.fs();
gs.Prompt_I = gs_read.prompt_i();
gs.Prompt_Q = gs_read.prompt_q();
gs.CN0_dB_hz = gs_read.cn0_db_hz();
gs.Carrier_Doppler_hz = gs_read.carrier_doppler_hz();
gs.Tracking_sample_counter = gs_read.tracking_sample_counter();
gs.Flag_valid_symbol_output = gs_read.flag_valid_symbol_output();
gs.correlation_length_ms = gs_read.correlation_length_ms();
gs.Flag_valid_word = gs_read.flag_valid_word();
gs.TOW_at_current_symbol_ms = gs_read.tow_at_current_symbol_ms();
gs.Pseudorange_m = gs_read.pseudorange_m();
gs.RX_time = gs_read.rx_time();
gs.Flag_valid_pseudorange = gs_read.flag_valid_pseudorange();
gs.interp_TOW_ms = gs_read.interp_tow_ms();
vgs.push_back(gs);
}
return vgs;
}
};
#endif // GNSS_SDR_SERDES_GNSS_SYNCHRO_H_

View File

@ -393,6 +393,7 @@ if(ENABLE_UNIT_TESTING)
system_testing_lib
core_receiver
core_system_parameters
core_protobuf
)
if(ENABLE_UNIT_TESTING_EXTRA)
target_link_libraries(run_tests PUBLIC Gpstk::gpstk)

View File

@ -66,6 +66,7 @@ DECLARE_string(log_dir);
#include "unit-tests/control-plane/gnss_block_factory_test.cc"
#include "unit-tests/control-plane/gnss_flowgraph_test.cc"
#include "unit-tests/control-plane/in_memory_configuration_test.cc"
#include "unit-tests/control-plane/protobuf_test.cc"
#include "unit-tests/control-plane/string_converter_test.cc"
#include "unit-tests/signal-processing-blocks/acquisition/galileo_e1_pcps_8ms_ambiguous_acquisition_gsoc2013_test.cc"
#include "unit-tests/signal-processing-blocks/acquisition/galileo_e1_pcps_ambiguous_acquisition_gsoc2013_test.cc"

View File

@ -0,0 +1,107 @@
/*!
* \file protobuf_test.cc
* \brief This file implements tests for Serdes_Gnss_Synchro
* \author Carles Fernandez-Prades, 2019. cfernandez(at)cttc.es
*
* -------------------------------------------------------------------------
*
* Copyright (C) 2010-2019 (see AUTHORS file for a list of contributors)
*
* GNSS-SDR is a software defined Global Navigation
* Satellite Systems receiver
*
* This file is part of GNSS-SDR.
*
* GNSS-SDR is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* GNSS-SDR is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with GNSS-SDR. If not, see <https://www.gnu.org/licenses/>.
*
* -------------------------------------------------------------------------
*/
#include "serdes_gnss_synchro.h"
TEST(Protobuf, Works)
{
uint32_t prn_true = 17;
uint32_t prn_true2 = 23;
std::string sys = "G";
std::string sig = "1C";
Gnss_Synchro gs = Gnss_Synchro();
gs.System = *sys.c_str();
std::memcpy(static_cast<void*>(gs.Signal), sig.c_str(), 3);
gs.PRN = prn_true;
gs.Channel_ID = 3;
gs.Acq_delay_samples = 431;
gs.Acq_doppler_hz = 1234;
gs.Acq_samplestamp_samples = 10000;
gs.Acq_doppler_step = 125;
gs.Flag_valid_acquisition = true;
gs.fs = 10000000;
gs.Prompt_I = 10000.0;
gs.Prompt_Q = 0.01;
gs.CN0_dB_hz = 39;
gs.Carrier_Doppler_hz = 321;
gs.Tracking_sample_counter = 11000;
gs.Flag_valid_symbol_output = false;
gs.correlation_length_ms = 1;
gs.Flag_valid_word = false;
gs.TOW_at_current_symbol_ms = 12345;
gs.Pseudorange_m = 22000002.1;
gs.RX_time = 4321;
gs.Flag_valid_pseudorange = false;
gs.interp_TOW_ms = 20;
gs.Pseudorange_m = 22000002.1;
gs.Carrier_phase_rads = 45.4;
gs.Carrier_Doppler_hz = 321;
gs.CN0_dB_hz = 39;
Serdes_Gnss_Synchro serdes = Serdes_Gnss_Synchro();
// Create a vector of Gnss_Synchro objects
std::vector<Gnss_Synchro> vgs;
vgs.push_back(gs);
gs.PRN = prn_true2;
vgs.push_back(gs);
// Serialize data
std::string serialized_data = serdes.createProtobuffer(vgs);
// Recover data from serialization
gnss_sdr::Observables obs;
obs.ParseFromString(serialized_data);
// Check that recovered data is ok
// We can access like this:
std::vector<Gnss_Synchro> vgs_read = serdes.readProtobuffer(obs);
Gnss_Synchro gs_read = vgs_read[0];
uint32_t prn_read = gs_read.PRN;
uint32_t prn_read2 = vgs_read[1].PRN;
// or without the need of gnss_synchro:
int obs_size = obs.observable_size();
uint32_t prn_read3 = obs.observable(0).prn();
EXPECT_EQ(prn_true, prn_read);
EXPECT_EQ(prn_true2, prn_read2);
EXPECT_EQ(prn_true, prn_read3);
EXPECT_EQ(2, obs_size);
}