Improve error handling when the flow graph fails to start

Avoid segmentation faults due to some common inconsistencies in the configuration file
E.g.: non-existing names for blocks implementation, some mismatched input/output item sizes

Provide hints to the user on how to fix the configuration in case of failure when starting the flow graph
This commit is contained in:
Carles Fernandez 2021-01-24 01:49:16 +01:00
parent 268fc1215c
commit a21c60ecb2
No known key found for this signature in database
GPG Key ID: 4C583C52B0C3877D
6 changed files with 283 additions and 127 deletions

View File

@ -28,6 +28,15 @@ SPDX-FileCopyrightText: 2011-2021 Carles Fernandez-Prades <carles.fernandez@cttc
- Improved handling of change in GNU Radio 3.9 FFT API.
- Improved handling of the filesystem library.
- Do not apply clang-tidy fixes to protobuf-generated headers.
- Refactored private implementation of flow graph connection and disconnection
for improved source code readability.
### Improvements in Usability:
- Avoid segmentation faults in the flow graph connection and/or starting due to
some common inconsistencies in the configuration file.
- Provide hints to the user in case of failed flow graph connection due to
inconsistencies in the configuration file.
&nbsp;

View File

@ -17,6 +17,7 @@
#include "signal_conditioner.h"
#include <glog/logging.h>
#include <stdexcept>
#include <utility>
@ -40,6 +41,18 @@ void SignalConditioner::connect(gr::top_block_sptr top_block)
LOG(WARNING) << "Signal conditioner already connected internally";
return;
}
if (data_type_adapt_ == nullptr)
{
throw std::invalid_argument("DataTypeAdapter implementation not defined");
}
if (in_filt_ == nullptr)
{
throw std::invalid_argument("InputFilter implementation not defined");
}
if (res_ == nullptr)
{
throw std::invalid_argument("Resampler implementation not defined");
}
data_type_adapt_->connect(top_block);
in_filt_->connect(top_block);
res_->connect(top_block);

View File

@ -300,7 +300,6 @@ int ControlThread::run()
}
else
{
LOG(ERROR) << "Unable to connect flowgraph";
return 0;
}
// Start the flowgraph
@ -311,7 +310,6 @@ int ControlThread::run()
}
else
{
LOG(ERROR) << "Unable to start flowgraph";
return 0;
}

View File

@ -374,6 +374,7 @@ std::unique_ptr<GNSSBlockInterface> GNSSBlockFactory::GetChannel(
if (acq_item_type != trk_item_type)
{
LOG(ERROR) << "Acquisition and Tracking blocks must have the same input data type!";
return nullptr;
}
LOG(INFO) << "Instantiating Channel " << channel
@ -388,6 +389,11 @@ std::unique_ptr<GNSSBlockInterface> GNSSBlockFactory::GetChannel(
std::unique_ptr<TrackingInterface> trk_ = GetTrkBlock(configuration, "Tracking_" + signal + appendix2, 1, 1);
std::unique_ptr<TelemetryDecoderInterface> tlm_ = GetTlmBlock(configuration, "TelemetryDecoder_" + signal + appendix3, 1, 1);
if (acq_ == nullptr or trk_ == nullptr or tlm_ == nullptr)
{
return nullptr;
}
std::unique_ptr<GNSSBlockInterface> channel_ = std::make_unique<Channel>(configuration, channel,
std::move(acq_),
std::move(trk_),
@ -1320,8 +1326,7 @@ std::unique_ptr<GNSSBlockInterface> GNSSBlockFactory::GetBlock(
else
{
// Log fatal. This causes execution to stop.
LOG(ERROR) << role << "." << implementation << ": Undefined implementation for block";
LOG(ERROR) << role << " block: Undefined implementation " << implementation;
}
return block;
}
@ -1515,8 +1520,7 @@ std::unique_ptr<AcquisitionInterface> GNSSBlockFactory::GetAcqBlock(
else
{
// Log fatal. This causes execution to stop.
LOG(ERROR) << role << "." << implementation << ": Undefined implementation for block";
LOG(ERROR) << role << " block: Undefined implementation " << implementation;
}
return block;
}
@ -1671,8 +1675,7 @@ std::unique_ptr<TrackingInterface> GNSSBlockFactory::GetTrkBlock(
#endif
else
{
// Log fatal. This causes execution to stop.
LOG(ERROR) << role << "." << implementation << ": Undefined implementation for block";
LOG(ERROR) << role << " block: Undefined implementation " << implementation;
}
return block;
}
@ -1764,8 +1767,7 @@ std::unique_ptr<TelemetryDecoderInterface> GNSSBlockFactory::GetTlmBlock(
else
{
// Log fatal. This causes execution to stop.
LOG(ERROR) << role << "." << implementation << ": Undefined implementation for block";
LOG(ERROR) << role << " block: Undefined implementation " << implementation;
}
return block;
}

View File

@ -128,7 +128,10 @@ void GNSSFlowgraph::init()
// TODO: Create a class interface for SignalSources, derived from GNSSBlockInterface.
// Include GetRFChannels in the interface to avoid read config parameters here
// read the number of RF channels for each front-end
RF_Channels = configuration_->property(sig_source_.at(0)->role() + ".RF_channels", 0);
if (sig_source_.at(0) != nullptr)
{
RF_Channels = configuration_->property(sig_source_.at(0)->role() + ".RF_channels", 0);
}
if (RF_Channels != 0)
{
for (int j = 0; j < RF_Channels; j++)
@ -265,6 +268,7 @@ void GNSSFlowgraph::start()
catch (const std::exception& e)
{
LOG(ERROR) << "Unable to start flowgraph: " << e.what();
print_help();
return;
}
@ -321,11 +325,15 @@ void GNSSFlowgraph::connect()
#if ENABLE_FPGA
if (connect_fpga_flowgraph() != 0)
{
LOG(ERROR) << "Unable to connect flowgraph with FPFA off-loading";
print_help();
return;
}
#else
if (connect_desktop_flowgraph() != 0)
{
LOG(ERROR) << "Unable to connect flowgraph";
print_help();
return;
}
#endif
@ -342,7 +350,7 @@ void GNSSFlowgraph::disconnect()
if (!connected_)
{
LOG(INFO) << "flowgraph was not connected";
LOG(INFO) << "Flowgraph was not connected";
return;
}
connected_ = false;
@ -539,19 +547,28 @@ int GNSSFlowgraph::connect_fpga_flowgraph()
DLOG(INFO) << "Blocks connected internally to the top_block";
// Connect the counter
if (configuration_->property(sig_source_.at(0)->role() + ".enable_FPGA", false) == false)
if (sig_source_.at(0) != nullptr)
{
if (connect_sample_counter() != 0)
if (configuration_->property(sig_source_.at(0)->role() + ".enable_FPGA", false) == false)
{
return 1;
if (connect_sample_counter() != 0)
{
return 1;
}
}
else
{
if (connect_fpga_sample_counter() != 0)
{
return 1;
}
}
}
else
{
if (connect_fpga_sample_counter() != 0)
{
return 1;
}
help_hint_ += " * Check implementation name for SignalSource block\n";
help_hint_ += " Signal Source block implementation for FPGA off-loading should be 'Ad9361_Fpga_Signal_Source'\n";
return 1;
}
if (connect_channels_to_observables() != 0)
@ -575,6 +592,9 @@ int GNSSFlowgraph::connect_fpga_flowgraph()
{
return 1;
}
check_desktop_conf_in_fpga_env();
LOG(INFO) << "The GNU Radio flowgraph for the current GNSS-SDR configuration with FPGA off-loading has been successfully connected";
return 0;
}
@ -654,13 +674,23 @@ int GNSSFlowgraph::connect_signal_sources()
{
for (int i = 0; i < sources_count_; i++)
{
try
if (sig_source_.at(i) != nullptr)
{
sig_source_.at(i)->connect(top_block_);
try
{
sig_source_.at(i)->connect(top_block_);
}
catch (const std::exception& e)
{
LOG(ERROR) << "Can't connect signal source block " << i << " internally: " << e.what();
top_block_->disconnect_all();
return 1;
}
}
catch (const std::exception& e)
else
{
LOG(ERROR) << "Can't connect signal source block " << i << " internally: " << e.what();
help_hint_ += " * Check implementation name for SignalSource" + (i == 0 ? " " : (std::to_string(i) + " ")) + "block\n";
help_hint_ += " Signal Source blocks documentation at https://gnss-sdr.org/docs/sp-blocks/signal-source/\n";
top_block_->disconnect_all();
return 1;
}
@ -689,17 +719,95 @@ int GNSSFlowgraph::disconnect_signal_sources()
}
int GNSSFlowgraph::connect_signal_conditioners()
{
for (auto& sig : sig_conditioner_)
{
try
{
sig->connect(top_block_);
}
catch (const std::exception& e)
{
LOG(ERROR) << "Can't connect signal conditioner block internally: " << e.what();
top_block_->disconnect_all();
std::string reported_error(e.what());
if (std::string::npos != reported_error.find(std::string("itemsize mismatch")))
{
std::string replace_me("copy");
size_t pos = reported_error.find(replace_me);
size_t len = replace_me.length();
reported_error.replace(pos, len, "PassThrough");
help_hint_ += " * Blocks within the Signal Conditioner are connected with mismatched item size\n";
help_hint_ += " Reported error: " + reported_error + '\n';
help_hint_ += " Check the Signal Conditioner documentation at https://gnss-sdr.org/docs/sp-blocks/signal-conditioner/\n";
}
if (std::string::npos != reported_error.find(std::string("DataTypeAdapter")))
{
help_hint_ += " * The DataTypeAdapter implementation set in the configuration file does not exist\n";
help_hint_ += " Check the DataTypeAdapter documentation at https://gnss-sdr.org/docs/sp-blocks/data-type-adapter/\n";
}
if (std::string::npos != reported_error.find(std::string("InputFilter")))
{
help_hint_ += " * The InputFilter implementation set in the configuration file does not exist\n";
help_hint_ += " Check the InputFilter documentation at https://gnss-sdr.org/docs/sp-blocks/input-filter/\n";
}
if (std::string::npos != reported_error.find(std::string("Resampler")))
{
help_hint_ += " * The Resampler implementation set in the configuration file does not exist\n";
help_hint_ += " Check the Resampler documentation at https://gnss-sdr.org/docs/sp-blocks/resampler/\n";
}
return 1;
}
}
DLOG(INFO) << "Signal Conditioner blocks successfully connected to the top_block";
return 0;
}
int GNSSFlowgraph::disconnect_signal_conditioners()
{
for (auto& sig : sig_conditioner_)
{
try
{
sig->disconnect(top_block_);
}
catch (const std::exception& e)
{
LOG(INFO) << "Can't disconnect signal conditioner block internally: " << e.what();
top_block_->disconnect_all();
return 1;
}
}
return 0;
}
int GNSSFlowgraph::connect_channels()
{
for (int i = 0; i < channels_count_; i++)
{
try
if (channels_.at(i) != nullptr)
{
channels_.at(i)->connect(top_block_);
try
{
channels_.at(i)->connect(top_block_);
}
catch (const std::exception& e)
{
LOG(ERROR) << "Can't connect channel " << i << " internally: " << e.what();
top_block_->disconnect_all();
return 1;
}
}
catch (const std::exception& e)
else
{
LOG(ERROR) << "Can't connect channel " << i << " internally: " << e.what();
LOG(ERROR) << "Can't connect channel " << i << " internally";
help_hint_ += " * Check implementation names for Channel" + std::to_string(i) + " inner blocks.\n";
help_hint_ += " Acquisition blocks documentation at https://gnss-sdr.org/docs/sp-blocks/acquisition/\n";
help_hint_ += " Tracking blocks documentation at https://gnss-sdr.org/docs/sp-blocks/tracking/\n";
help_hint_ += " Telemetry Decoder blocks documentation at https://gnss-sdr.org/docs/sp-blocks/telemetry-decoder/\n";
top_block_->disconnect_all();
return 1;
}
@ -730,6 +838,13 @@ int GNSSFlowgraph::disconnect_channels()
int GNSSFlowgraph::connect_observables()
{
if (observables_ == nullptr)
{
help_hint_ += " * Check implementation name for the Observables block\n";
help_hint_ += " Observables block documentation at https://gnss-sdr.org/docs/sp-blocks/observables/\n";
top_block_->disconnect_all();
return 1;
}
try
{
observables_->connect(top_block_);
@ -763,6 +878,13 @@ int GNSSFlowgraph::disconnect_observables()
int GNSSFlowgraph::connect_pvt()
{
if (pvt_ == nullptr)
{
help_hint_ += " * Check implementation name for the PVT block\n";
help_hint_ += " PVT block documentation at https://gnss-sdr.org/docs/sp-blocks/pvt/\n";
top_block_->disconnect_all();
return 1;
}
try
{
pvt_->connect(top_block_);
@ -840,6 +962,7 @@ int GNSSFlowgraph::disconnect_sample_counter()
return 0;
}
#if ENABLE_FPGA
int GNSSFlowgraph::connect_fpga_sample_counter()
{
@ -918,6 +1041,8 @@ int GNSSFlowgraph::connect_signal_sources_to_signal_conditioners()
// GNURADIO max_streams=-1 means infinite ports!
DLOG(INFO) << "sig_source_.at(i)->get_right_block()->output_signature()->max_streams()=" << sig_source_.at(i)->get_right_block()->output_signature()->max_streams();
DLOG(INFO) << "sig_conditioner_.at(signal_conditioner_ID)->get_left_block()->input_signature()=" << sig_conditioner_.at(signal_conditioner_ID)->get_left_block()->input_signature()->max_streams();
size_t output_size = sig_source_.at(i)->item_size();
size_t input_size = sig_conditioner_.at(signal_conditioner_ID)->get_left_block()->input_signature()->sizeof_stream_item(0);
if (sig_source_.at(i)->get_right_block()->output_signature()->max_streams() > 1 or sig_source_.at(i)->get_right_block()->output_signature()->max_streams() == -1)
{
@ -943,12 +1068,28 @@ int GNSSFlowgraph::connect_signal_sources_to_signal_conditioners()
}
}
signal_conditioner_ID++;
// Check configuration inconsistencies
if (output_size != input_size)
{
help_hint_ += " * The Signal Source implementation " + sig_source_.at(i)->implementation() + " has an output with an item size of " + std::to_string(output_size) + " bytes, but it is connected to the Signal Conditioner implementation " + sig_conditioner_.at(signal_conditioner_ID)->implementation() + " with input item size of " + std::to_string(input_size) + "bytes.\n";
help_hint_ += " Output ports must be connected to input ports with the same item size.\n";
}
}
}
}
catch (const std::exception& e)
{
LOG(ERROR) << "Can't connect signal source " << i << " to signal conditioner " << i << ": " << e.what();
LOG(ERROR) << "Can't connect SignalSource" << (i == 0 ? " " : (std::to_string(i) + " ")) << "to SignalConditioner" << (i == 0 ? " " : (std::to_string(i) + " ")) << ": " << e.what();
std::string reported_error(e.what());
if (std::string::npos != reported_error.find(std::string("itemsize mismatch")))
{
std::string replace_me("copy");
size_t pos = reported_error.find(replace_me);
size_t len = replace_me.length();
reported_error.replace(pos, len, "PassThrough");
help_hint_ += " * The SignalSource output item size and the SignalConditioner input item size are mismatched\n";
help_hint_ += " Reported error: " + reported_error + '\n';
}
top_block_->disconnect_all();
return 1;
}
@ -959,6 +1100,65 @@ int GNSSFlowgraph::connect_signal_sources_to_signal_conditioners()
}
int GNSSFlowgraph::disconnect_signal_sources_from_signal_conditioners()
{
int RF_Channels = 0;
int signal_conditioner_ID = 0;
for (int i = 0; i < sources_count_; i++)
{
try
{
// TODO: Remove this array implementation and create generic multistream connector
// (if a signal source has more than 1 stream, then connect it to the multistream signal conditioner)
if (sig_source_.at(i)->implementation() == "Raw_Array_Signal_Source")
{
// Multichannel Array
for (int j = 0; j < GNSS_SDR_ARRAY_SIGNAL_CONDITIONER_CHANNELS; j++)
{
top_block_->disconnect(sig_source_.at(i)->get_right_block(), j, sig_conditioner_.at(i)->get_left_block(), j);
}
}
else
{
// TODO: Create a class interface for SignalSources, derived from GNSSBlockInterface.
// Include GetRFChannels in the interface to avoid read config parameters here
// read the number of RF channels for each front-end
RF_Channels = configuration_->property(sig_source_.at(i)->role() + ".RF_channels", 1);
for (int j = 0; j < RF_Channels; j++)
{
if (sig_source_.at(i)->get_right_block()->output_signature()->max_streams() > 1 or sig_source_.at(i)->get_right_block()->output_signature()->max_streams() == -1)
{
top_block_->disconnect(sig_source_.at(i)->get_right_block(), j, sig_conditioner_.at(signal_conditioner_ID)->get_left_block(), 0);
}
else
{
if (j == 0)
{
// RF_channel 0 backward compatibility with single channel sources
top_block_->disconnect(sig_source_.at(i)->get_right_block(), 0, sig_conditioner_.at(signal_conditioner_ID)->get_left_block(), 0);
}
else
{
// Multiple channel sources using multiple output blocks of single channel (requires RF_channel selector in call)
top_block_->disconnect(sig_source_.at(i)->get_right_block(j), 0, sig_conditioner_.at(signal_conditioner_ID)->get_left_block(), 0);
}
}
signal_conditioner_ID++;
}
}
}
catch (const std::exception& e)
{
LOG(INFO) << "Can't disconnect signal source " << i << " to signal conditioner " << i << ": " << e.what();
top_block_->disconnect_all();
return 1;
}
}
return 0;
}
int GNSSFlowgraph::connect_signal_conditioners_to_channels()
{
for (int i = 0; i < channels_count_; i++)
@ -1096,7 +1296,7 @@ int GNSSFlowgraph::connect_signal_conditioners_to_channels()
}
signal_conditioner_connected_.at(selected_signal_conditioner_ID) = true; // annotate that this signal conditioner is connected
DLOG(INFO) << "signal conditioner " << selected_signal_conditioner_ID << " successfully connected to channel " << i;
DLOG(INFO) << "Signal conditioner " << selected_signal_conditioner_ID << " successfully connected to channel " << i;
}
return 0;
}
@ -1349,104 +1549,6 @@ int GNSSFlowgraph::disconnect_monitors()
}
int GNSSFlowgraph::connect_signal_conditioners()
{
for (auto& sig : sig_conditioner_)
{
try
{
sig->connect(top_block_);
}
catch (const std::exception& e)
{
LOG(ERROR) << "Can't connect signal conditioner block internally: " << e.what();
top_block_->disconnect_all();
return 1;
}
}
DLOG(INFO) << "Signal Conditioner blocks successfully connected to the top_block";
return 0;
}
int GNSSFlowgraph::disconnect_signal_conditioners()
{
for (auto& sig : sig_conditioner_)
{
try
{
sig->disconnect(top_block_);
}
catch (const std::exception& e)
{
LOG(INFO) << "Can't disconnect signal conditioner block internally: " << e.what();
top_block_->disconnect_all();
return 1;
}
}
return 0;
}
int GNSSFlowgraph::disconnect_signal_sources_from_signal_conditioners()
{
int RF_Channels = 0;
int signal_conditioner_ID = 0;
for (int i = 0; i < sources_count_; i++)
{
try
{
// TODO: Remove this array implementation and create generic multistream connector
// (if a signal source has more than 1 stream, then connect it to the multistream signal conditioner)
if (sig_source_.at(i)->implementation() == "Raw_Array_Signal_Source")
{
// Multichannel Array
for (int j = 0; j < GNSS_SDR_ARRAY_SIGNAL_CONDITIONER_CHANNELS; j++)
{
top_block_->disconnect(sig_source_.at(i)->get_right_block(), j, sig_conditioner_.at(i)->get_left_block(), j);
}
}
else
{
// TODO: Create a class interface for SignalSources, derived from GNSSBlockInterface.
// Include GetRFChannels in the interface to avoid read config parameters here
// read the number of RF channels for each front-end
RF_Channels = configuration_->property(sig_source_.at(i)->role() + ".RF_channels", 1);
for (int j = 0; j < RF_Channels; j++)
{
if (sig_source_.at(i)->get_right_block()->output_signature()->max_streams() > 1 or sig_source_.at(i)->get_right_block()->output_signature()->max_streams() == -1)
{
top_block_->disconnect(sig_source_.at(i)->get_right_block(), j, sig_conditioner_.at(signal_conditioner_ID)->get_left_block(), 0);
}
else
{
if (j == 0)
{
// RF_channel 0 backward compatibility with single channel sources
top_block_->disconnect(sig_source_.at(i)->get_right_block(), 0, sig_conditioner_.at(signal_conditioner_ID)->get_left_block(), 0);
}
else
{
// Multiple channel sources using multiple output blocks of single channel (requires RF_channel selector in call)
top_block_->disconnect(sig_source_.at(i)->get_right_block(j), 0, sig_conditioner_.at(signal_conditioner_ID)->get_left_block(), 0);
}
}
signal_conditioner_ID++;
}
}
}
catch (const std::exception& e)
{
LOG(INFO) << "Can't disconnect signal source " << i << " to signal conditioner " << i << ": " << e.what();
top_block_->disconnect_all();
return 1;
}
}
return 0;
}
void GNSSFlowgraph::check_signal_conditioners()
{
// check for unconnected signal conditioners and connect null_sinks
@ -1597,6 +1699,35 @@ void GNSSFlowgraph::assign_channels()
}
void GNSSFlowgraph::print_help()
{
if (!help_hint_.empty())
{
std::cerr << "It seems that your configuration file is not well defined.\n";
std::cerr << "A hint to fix your configuration file:\n";
std::cerr << help_hint_;
}
}
void GNSSFlowgraph::check_desktop_conf_in_fpga_env()
{
int number_of_fpga_acq_channels = 0;
for (int i = 0; i < channels_count_; i++)
{
if (channels_.at(i)->get_left_block_acq() == nullptr)
{
number_of_fpga_acq_channels++;
}
}
if (number_of_fpga_acq_channels != channels_count_)
{
help_hint_ += " * The Acquisition block implementation is not suitable for GNSS-SDR flowgraph with FPGA off-loading\n";
help_hint_ += " If you want to use this configuration in an environment without FPGA, please rebuild GNSS-SDR with CMake option '-DENABLE_FPGA=OFF'\n";
}
}
bool GNSSFlowgraph::send_telemetry_msg(const pmt::pmt_t& msg)
{
// Push ephemeris to PVT telemetry msg in port using a channel out port

View File

@ -216,6 +216,8 @@ private:
void push_back_signal(const Gnss_Signal& gs);
void remove_signal(const Gnss_Signal& gs);
void print_help();
void check_desktop_conf_in_fpga_env();
double project_doppler(const std::string& searched_signal, double primary_freq_doppler_hz);
bool is_multiband() const;
@ -279,6 +281,7 @@ private:
std::map<std::string, StringValue> mapStringValues_;
std::string config_file_;
std::string help_hint_;
std::mutex signal_list_mutex_;