diff --git a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc index 53f63bd07..7eb1f0d3f 100644 --- a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc +++ b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #if USE_GLOG_AND_GFLAGS @@ -69,30 +71,83 @@ IONGSMSSignalSource::IONGSMSSignalSource(const ConfigurationInterface* configura throw std::runtime_error("ION_GSMS_Signal_Source no configured streams were found in the metadata"); } - for (const auto& source : sources_) + for (const auto& source_data : sources_) { + const auto& source = source_data.source; for (std::size_t i = 0; i < source->output_stream_count(); ++i) { copy_blocks_.emplace_back(gr::blocks::copy::make(source->output_stream_item_size(i))); - valves_.emplace_back(gnss_sdr_make_valve(source->output_stream_item_size(i), valve_sample_count(source->output_stream_total_sample_count(i)), queue)); + valves_.emplace_back(gnss_sdr_make_valve(source->output_stream_item_size(i), valve_sample_count(source->output_stream_total_sample_count(i), source_data.sampling_frequency), queue)); } } } +const char* IONGSMSSignalSource::file_uri_scheme() +{ + return "file://"; +} + + +bool IONGSMSSignalSource::starts_with(const std::string& value, const std::string& prefix) +{ + return value.compare(0, prefix.size(), prefix) == 0; +} + + +fs::path IONGSMSSignalSource::absolute_path_key(const fs::path& path) +{ + try + { + return fs::canonical(path); + } + catch (const fs::filesystem_error&) + { + return fs::absolute(path); + } +} + + +fs::path IONGSMSSignalSource::resolve_local_metadata_uri(const fs::path& metadata_path, const std::string& uri) +{ + if (uri.empty()) + { + throw std::runtime_error("ION_GSMS metadata include URI is empty"); + } + + fs::path include_path; + const std::string scheme(file_uri_scheme()); + if (starts_with(uri, scheme)) + { + include_path = fs::path(uri.substr(scheme.size())); + } + else if (uri.find("://") != std::string::npos) + { + throw std::runtime_error( + "ION_GSMS metadata include URI '" + uri + "' is not a local file URI"); + } + else + { + include_path = fs::path(uri); + } + + if (include_path.is_absolute()) + { + return include_path; + } + + return metadata_path.parent_path() / include_path; +} + + void IONGSMSSignalSource::load_metadata() { metadata_ = std::make_shared(); + metadata_files_.clear(); try { - GnssMetadata::XmlProcessor xml_proc; - if (!xml_proc.Load(metadata_filepath_.c_str(), false, *metadata_)) - { - LOG(WARNING) << "Could not load XML metadata file " << metadata_filepath_; - std::cerr << "Could not load XML metadata file " << metadata_filepath_ << std::endl; - std::cout << "GNSS-SDR program ended.\n"; - exit(1); - } + std::vector include_stack; + load_metadata_file(fs::path(metadata_filepath_), *metadata_, include_stack); } catch (GnssMetadata::ApiException& e) { @@ -111,6 +166,54 @@ void IONGSMSSignalSource::load_metadata() } +void IONGSMSSignalSource::load_metadata_file( + const fs::path& metadata_path, + GnssMetadata::Metadata& metadata, + std::vector& include_stack) +{ + const auto metadata_key = absolute_path_key(metadata_path).string(); + if (std::find(include_stack.begin(), include_stack.end(), metadata_key) != include_stack.end()) + { + throw std::runtime_error("ION_GSMS metadata include cycle detected at " + metadata_path.string()); + } + + include_stack.push_back(metadata_key); + try + { + GnssMetadata::XmlProcessor xml_proc; + const auto metadata_path_string = metadata_path.string(); + if (!xml_proc.Load(metadata_path_string.c_str(), false, metadata)) + { + throw std::runtime_error("Could not load XML metadata file " + metadata_path_string); + } + + const auto metadata_directory = metadata_path.parent_path(); + for (const auto& file : metadata.Files()) + { + metadata_files_.push_back({&file, metadata_directory}); + } + + const std::vector includes(metadata.Includes().begin(), metadata.Includes().end()); + metadata.Includes().clear(); + for (const auto& include : includes) + { + GnssMetadata::Metadata included_metadata; + load_metadata_file( + resolve_local_metadata_uri(metadata_path, include.Value()), + included_metadata, + include_stack); + metadata.Splice(included_metadata); + } + } + catch (...) + { + include_stack.pop_back(); + throw; + } + include_stack.pop_back(); +} + + std::vector IONGSMSSignalSource::parse_comma_list(const std::string& str) { std::vector list{}; @@ -175,13 +278,24 @@ std::size_t IONGSMSSignalSource::chunk_cycle_bytes(const GnssMetadata::Block& bl std::size_t IONGSMSSignalSource::infer_block_cycles( const fs::path& data_filepath, const GnssMetadata::Block& block, - const std::size_t block_start_offset) + const std::size_t block_start_offset, + const bool block_extends_to_eof) { if (block.Cycles() != 0) { return block.Cycles(); } + if (!block_extends_to_eof) + { + throw std::runtime_error( + "ION_GSMS_Signal_Source block has cycles=0 before the final lane block; " + "refusing EOF-based cycle inference because later blocks would be unreachable"); + } + LOG(WARNING) << "ION_GSMS_Signal_Source block at offset " << block_start_offset + << " in " << data_filepath.string() + << " has cycles=0; inferring cycle count from EOF. This is a non-standard metadata extension supported only for the final block in a lane."; + const std::size_t cycle_bytes = chunk_cycle_bytes(block); if (cycle_bytes == 0) { @@ -208,56 +322,256 @@ std::size_t IONGSMSSignalSource::infer_block_cycles( std::size_t IONGSMSSignalSource::block_storage_bytes( const fs::path& data_filepath, const GnssMetadata::Block& block, - const std::size_t block_start_offset) + const std::size_t block_start_offset, + const bool block_extends_to_eof) { - const std::size_t cycle_count = infer_block_cycles(data_filepath, block, block_start_offset); + const std::size_t cycle_count = infer_block_cycles(data_filepath, block, block_start_offset, block_extends_to_eof); return block.SizeHeader() + cycle_count * chunk_cycle_bytes(block) + block.SizeFooter(); } -std::vector IONGSMSSignalSource::make_stream_sources(const std::vector& stream_ids) const +std::vector IONGSMSSignalSource::make_stream_sources(const std::vector& stream_ids) const { - std::vector sources{}; - for (const auto& file : metadata_->Files()) + std::vector sources{}; + const auto files = ordered_metadata_files(); + for (const auto& stream_id : stream_ids) { - const fs::path data_filepath = fs::path(metadata_filepath_).parent_path() / file.Url().Value(); - for (const auto& lane : metadata_->Lanes()) + std::vector segments{}; + std::int64_t stream_sampling_frequency = 0; + for (const auto& metadata_file : files) { - if (lane.Id() == file.Lane().Id()) + const auto* file = metadata_file.file; + const fs::path data_filepath = metadata_file.metadata_directory / file->Url().Value(); + for (const auto& lane : metadata_->Lanes()) { - std::size_t block_start_offset = file.Offset(); - for (const auto& block : lane.Blocks()) + if (lane.Id() == file->Lane().Id()) { - if (block_contains_stream(block, stream_ids)) + std::size_t block_start_offset = file->Offset(); + const auto& blocks = lane.Blocks(); + for (auto block_iter = blocks.begin(); block_iter != blocks.end(); ++block_iter) { - auto source = gnss_make_shared( - metadata_filepath_, - file, - block, - block_start_offset, - stream_ids); - - sources.push_back(source); + const auto& block = *block_iter; + const bool block_extends_to_eof = std::next(block_iter) == blocks.end(); + if (block_contains_stream(block, std::vector{stream_id})) + { + segments.push_back({data_filepath, &block, block_start_offset, block_extends_to_eof}); + stream_sampling_frequency = reconcile_sampling_frequency( + stream_sampling_frequency, + stream_sampling_frequency_hz(lane, block, stream_id), + stream_id); + } + if (!block_extends_to_eof) + { + block_start_offset += block_storage_bytes(data_filepath, block, block_start_offset, block_extends_to_eof); + } } - block_start_offset += block_storage_bytes(data_filepath, block, block_start_offset); + break; } - break; } } + if (segments.empty()) + { + throw std::runtime_error("ION_GSMS_Signal_Source requested stream '" + stream_id + "' was not found in the metadata"); + } + if (stream_sampling_frequency == 0) + { + stream_sampling_frequency = sampling_frequency_; + } + sources.push_back({gnss_make_shared(segments, std::vector{stream_id}), stream_sampling_frequency}); } return sources; } -std::uint64_t IONGSMSSignalSource::valve_sample_count(std::uint64_t total_sample_count) const +std::vector IONGSMSSignalSource::ordered_metadata_files() const { - if (total_sample_count == 0 || sampling_frequency_ <= 0) + std::vector files = metadata_files_; + if (files.empty()) + { + const auto metadata_directory = fs::path(metadata_filepath_).parent_path(); + for (const auto& file : metadata_->Files()) + { + files.push_back({&file, metadata_directory}); + } + } + + std::vector ordered_files{}; + std::vector remaining_files = files; + auto find_by_url = [](const std::vector& candidates, const std::string& url) { + return std::find_if(candidates.begin(), candidates.end(), [&url](const auto& file) { + return file.file != nullptr && file.file->Url().Value() == url; + }); + }; + + auto append_chain = [&ordered_files, &remaining_files, &find_by_url](MetadataFileData file_data) { + while (file_data.file != nullptr) + { + ordered_files.push_back(file_data); + remaining_files.erase(std::remove_if(remaining_files.begin(), remaining_files.end(), [&file_data](const auto& candidate) { + return candidate.file == file_data.file; + }), + remaining_files.end()); + const auto next = file_data.file->Next().Value(); + if (next.empty()) + { + file_data = {}; + } + else + { + const auto next_file = find_by_url(remaining_files, next); + file_data = next_file == remaining_files.end() ? MetadataFileData{} : *next_file; + } + } + }; + + for (const auto& fileset : metadata_->FileSets()) + { + for (const auto& file_url : fileset.FileUrls()) + { + const auto first_file = find_by_url(remaining_files, file_url.Value()); + if (first_file != remaining_files.end()) + { + append_chain(*first_file); + } + } + } + + while (!remaining_files.empty()) + { + auto first_file = std::find_if(remaining_files.begin(), remaining_files.end(), [&remaining_files, &find_by_url](const auto& file) { + const auto previous = file.file->Previous().Value(); + return previous.empty() || find_by_url(remaining_files, previous) == remaining_files.end(); + }); + if (first_file == remaining_files.end()) + { + first_file = remaining_files.begin(); + } + + append_chain(*first_file); + } + + return ordered_files; +} + + +const GnssMetadata::System* IONGSMSSignalSource::resolve_system(const GnssMetadata::System& system) const +{ + if (!system.IsReference() && system.BaseFrequency().toHertz() > 0.0) + { + return &system; + } + + for (const auto& metadata_system : metadata_->Systems()) + { + if (metadata_system.Id() == system.Id()) + { + return &metadata_system; + } + } + + return system.IsReference() ? nullptr : &system; +} + + +double IONGSMSSignalSource::lane_base_frequency_hz(const GnssMetadata::Lane& lane) const +{ + double base_frequency = 0.0; + for (const auto& lane_system : lane.Systems()) + { + const auto* system = resolve_system(lane_system); + if (system == nullptr) + { + continue; + } + + const double system_base_frequency = system->BaseFrequency().toHertz(); + if (system_base_frequency <= 0.0) + { + continue; + } + if (base_frequency > 0.0 && std::llround(base_frequency) != std::llround(system_base_frequency)) + { + throw std::runtime_error("ION_GSMS_Signal_Source lane references systems with inconsistent freqbase values"); + } + base_frequency = system_base_frequency; + } + + return base_frequency; +} + + +std::int64_t IONGSMSSignalSource::stream_sampling_frequency_hz( + const GnssMetadata::Lane& lane, + const GnssMetadata::Block& block, + const std::string& stream_id) const +{ + const double base_frequency = lane_base_frequency_hz(lane); + if (base_frequency <= 0.0) + { + return 0; + } + + std::size_t rate_factor = 0; + for (const auto& chunk : block.Chunks()) + { + for (const auto& lump : chunk.Lumps()) + { + for (const auto& stream : lump.Streams()) + { + if (stream.Id() != stream_id) + { + continue; + } + if (rate_factor != 0 && rate_factor != stream.RateFactor()) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream '" + stream_id + "' appears with inconsistent ratefactor values"); + } + rate_factor = stream.RateFactor(); + } + } + } + + if (rate_factor == 0) + { + return 0; + } + + const double sampling_frequency = base_frequency * static_cast(rate_factor); + if (sampling_frequency <= 0.0 || sampling_frequency > static_cast(std::numeric_limits::max())) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream '" + stream_id + "' has an unsupported sampling frequency"); + } + + return static_cast(std::llround(sampling_frequency)); +} + + +std::int64_t IONGSMSSignalSource::reconcile_sampling_frequency( + const std::int64_t current_frequency, + const std::int64_t candidate_frequency, + const std::string& stream_id) +{ + if (candidate_frequency <= 0) + { + return current_frequency; + } + if (current_frequency > 0 && current_frequency != candidate_frequency) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream '" + stream_id + "' appears with inconsistent sampling frequencies"); + } + return candidate_frequency; +} + + +std::uint64_t IONGSMSSignalSource::valve_sample_count(std::uint64_t total_sample_count, const std::int64_t sampling_frequency) const +{ + if (total_sample_count == 0 || sampling_frequency <= 0) { return total_sample_count; } - const auto tail_samples = static_cast(std::ceil(minimum_tail_s_ * static_cast(sampling_frequency_))); + const auto tail_samples = static_cast(std::ceil(minimum_tail_s_ * static_cast(sampling_frequency))); if (total_sample_count <= tail_samples) { std::cout << "Warning: ION_GSMS_Signal_Source stream has " << total_sample_count @@ -273,8 +587,9 @@ std::uint64_t IONGSMSSignalSource::valve_sample_count(std::uint64_t total_sample void IONGSMSSignalSource::connect(gr::top_block_sptr top_block) { std::size_t cumulative_index = 0; - for (const auto& source : sources_) + for (const auto& source_data : sources_) { + const auto& source = source_data.source; for (std::size_t i = 0; i < source->output_stream_count(); ++i, ++cumulative_index) { top_block->connect(source, i, copy_blocks_[cumulative_index], 0); @@ -287,8 +602,9 @@ void IONGSMSSignalSource::connect(gr::top_block_sptr top_block) void IONGSMSSignalSource::disconnect(gr::top_block_sptr top_block) { std::size_t cumulative_index = 0; - for (const auto& source : sources_) + for (const auto& source_data : sources_) { + const auto& source = source_data.source; for (std::size_t i = 0; i < source->output_stream_count(); ++i, ++cumulative_index) { top_block->disconnect(source, i, copy_blocks_[cumulative_index], 0); @@ -314,14 +630,16 @@ gr::basic_block_sptr IONGSMSSignalSource::get_right_block() gr::basic_block_sptr IONGSMSSignalSource::get_right_block(int RF_channel) { - if (RF_channel < 0 || RF_channel >= static_cast(copy_blocks_.size())) + if (RF_channel < 0 || RF_channel >= static_cast(valves_.size())) { LOG(WARNING) << "'RF_channel' out of bounds while trying to get signal source right block."; - if (valves_.empty()) - { - return {}; - } - return valves_[0]; + return {}; } return valves_[RF_channel]; } + + +size_t IONGSMSSignalSource::getRfChannels() const +{ + return valves_.size(); +} diff --git a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h index 98032a95f..d5c842bef 100644 --- a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h +++ b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h @@ -21,6 +21,7 @@ #include "configuration_interface.h" #include "file_source_base.h" +#include "gnss_sdr_filesystem.h" #include "gnss_sdr_timestamp.h" #include "ion_gsms.h" #include @@ -54,6 +55,7 @@ protected: gr::basic_block_sptr get_left_block() override; gr::basic_block_sptr get_right_block() override; gr::basic_block_sptr get_right_block(int RF_channel) override; + size_t getRfChannels() const override; inline size_t item_size() override { @@ -61,36 +63,70 @@ protected: { return 0; } - return (*sources_.begin())->output_stream_item_size(0); + return sources_.begin()->source->output_stream_item_size(0); } private: static constexpr double kMinimumTailSeconds = 0.2; + struct StreamSourceData + { + IONGSMSFileSource::sptr source; + std::int64_t sampling_frequency = 0; + }; + + struct MetadataFileData + { + const GnssMetadata::File* file = nullptr; + fs::path metadata_directory; + }; + static std::vector parse_comma_list(const std::string& str); static bool block_contains_stream(const GnssMetadata::Block& block, const std::vector& stream_ids); static std::size_t chunk_cycle_bytes(const GnssMetadata::Block& block); static std::size_t infer_block_cycles( const fs::path& data_filepath, const GnssMetadata::Block& block, - std::size_t block_start_offset); + std::size_t block_start_offset, + bool block_extends_to_eof); static std::size_t block_storage_bytes( const fs::path& data_filepath, const GnssMetadata::Block& block, - std::size_t block_start_offset); + std::size_t block_start_offset, + bool block_extends_to_eof); + static const char* file_uri_scheme(); + static bool starts_with(const std::string& value, const std::string& prefix); + static fs::path absolute_path_key(const fs::path& path); + static fs::path resolve_local_metadata_uri(const fs::path& metadata_path, const std::string& uri); - std::vector make_stream_sources(const std::vector& stream_ids) const; + std::vector make_stream_sources(const std::vector& stream_ids) const; + std::vector ordered_metadata_files() const; + const GnssMetadata::System* resolve_system(const GnssMetadata::System& system) const; + double lane_base_frequency_hz(const GnssMetadata::Lane& lane) const; + std::int64_t stream_sampling_frequency_hz( + const GnssMetadata::Lane& lane, + const GnssMetadata::Block& block, + const std::string& stream_id) const; + static std::int64_t reconcile_sampling_frequency( + std::int64_t current_frequency, + std::int64_t candidate_frequency, + const std::string& stream_id); void load_metadata(); - std::uint64_t valve_sample_count(std::uint64_t total_sample_count) const; + void load_metadata_file( + const fs::path& metadata_path, + GnssMetadata::Metadata& metadata, + std::vector& include_stack); + std::uint64_t valve_sample_count(std::uint64_t total_sample_count, std::int64_t sampling_frequency) const; std::vector stream_ids_; - std::vector sources_; + std::vector sources_; std::vector> copy_blocks_; std::vector> valves_; std::string metadata_filepath_; std::shared_ptr metadata_; + std::vector metadata_files_; gnss_shared_ptr timestamp_block_; std::string timestamp_file_; diff --git a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc index 21f8d59ee..c970fbc1a 100644 --- a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc +++ b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc @@ -35,70 +35,120 @@ IONGSMSFileSource::IONGSMSFileSource( const GnssMetadata::Block& block, const std::size_t block_start_offset, const std::vector& stream_ids) + : IONGSMSFileSource( + std::vector{make_segment_descriptor(metadata_filepath, file, block, block_start_offset)}, + stream_ids) +{ +} + + +IONGSMSFileSource::IONGSMSFileSource( + const std::vector& segments, + const std::vector& stream_ids) : gr::sync_block( "ion_gsms_file_source", gr::io_signature::make(0, 0, 0), - make_output_signature(block, stream_ids)), - file_stream_(metadata_filepath.parent_path() / file.Url().Value(), std::ios::in | std::ios::binary), + make_output_signature(segments, stream_ids)), io_buffer_offset_(0), maximum_item_rate_(0), chunk_cycle_length_(0), - cycles_remaining_(0) + cycles_remaining_(0), + current_segment_index_(0), + output_stream_ids_(segment_output_stream_ids(segments, stream_ids)) { - fs::path data_filepath = metadata_filepath.parent_path() / file.Url().Value(); - const auto output_stream_ids = block_output_stream_ids(block, stream_ids); - - if (!file_stream_.is_open()) - { - LOG(WARNING) << "ION_GSMS_Signal_Source - Unable to open the samples file: " << (data_filepath).c_str(); - std::cerr << "ION_GSMS_Signal_Source - Unable to open the samples file: " << (data_filepath).c_str() << std::endl; - std::cout << "GNSS-SDR program ended.\n"; - exit(1); - } - - // Skip to this block's sample payload, after the lane offset and block header. - file_stream_.seekg(static_cast(block_start_offset + block.SizeHeader()), std::ios::beg); - - output_stream_count_ = output_stream_ids.size(); + output_stream_count_ = output_stream_ids_.size(); output_stream_item_sizes_.assign(output_stream_count_, 0); output_stream_item_rates_.assign(output_stream_count_, 0); + output_stream_total_sample_counts_.assign(output_stream_count_, 0); - for (const auto& chunk : block.Chunks()) + if (segments.empty()) { - chunk_data_.emplace_back(std::make_shared(chunk, output_stream_ids, 0)); - chunk_cycle_length_ += chunk.CountWords() * chunk.SizeWord(); + throw std::runtime_error("ION_GSMS_Signal_Source requires at least one source segment"); + } + + for (const auto& descriptor : segments) + { + if (descriptor.block == nullptr) + { + throw std::runtime_error("ION_GSMS_Signal_Source source segment has no block metadata"); + } + + SegmentData segment; + segment.data_filepath = descriptor.data_filepath; + segment.block = descriptor.block; + segment.block_start_offset = descriptor.block_start_offset; + segment.output_stream_item_rates.assign(output_stream_count_, 0); + + for (const auto& chunk : descriptor.block->Chunks()) + { + segment.chunk_data.emplace_back(std::make_shared(chunk, output_stream_ids_, 0)); + segment.chunk_cycle_length += chunk.CountWords() * chunk.SizeWord(); + for (std::size_t i = 0; i < output_stream_count_; ++i) + { + const auto chunk_item_rate = segment.chunk_data.back()->output_stream_item_rate(i); + if (chunk_item_rate == 0) + { + continue; + } + + const auto chunk_item_size = segment.chunk_data.back()->output_stream_item_size(i); + if (output_stream_item_sizes_[i] != 0 && output_stream_item_sizes_[i] != chunk_item_size) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream appears with inconsistent output item sizes"); + } + output_stream_item_sizes_[i] = chunk_item_size; + segment.output_stream_item_rates[i] += chunk_item_rate; + segment.maximum_item_rate = std::max(segment.output_stream_item_rates[i], segment.maximum_item_rate); + } + } + + std::size_t cycle_count = descriptor.block->Cycles(); + if (cycle_count == 0) + { + if (!descriptor.block_extends_to_eof) + { + throw std::runtime_error( + "ION_GSMS_Signal_Source block has cycles=0 before the final lane block; " + "refusing EOF-based cycle inference because later blocks would be unreachable"); + } + const std::string warning = "ION_GSMS_Signal_Source block at offset " + std::to_string(descriptor.block_start_offset) + + " in " + segment.data_filepath.string() + + " has cycles=0; inferring cycle count from EOF. This is a non-standard metadata extension supported only for the final block in a lane."; + LOG(WARNING) << warning; + std::cerr << "Warning: " << warning << std::endl; + cycle_count = infer_cycle_count_from_file(segment.data_filepath, *descriptor.block, descriptor.block_start_offset, segment.chunk_cycle_length); + } + segment.cycle_count = cycle_count; + for (std::size_t i = 0; i < output_stream_count_; ++i) { - const auto chunk_item_rate = chunk_data_.back()->output_stream_item_rate(i); - if (chunk_item_rate == 0) - { - continue; - } - - const auto chunk_item_size = chunk_data_.back()->output_stream_item_size(i); - if (output_stream_item_sizes_[i] != 0 && output_stream_item_sizes_[i] != chunk_item_size) - { - throw std::runtime_error("ION_GSMS_Signal_Source stream appears with inconsistent output item sizes"); - } - output_stream_item_sizes_[i] = chunk_item_size; - output_stream_item_rates_[i] += chunk_item_rate; - maximum_item_rate_ = std::max(output_stream_item_rates_[i], maximum_item_rate_); + output_stream_item_rates_[i] = std::max(output_stream_item_rates_[i], segment.output_stream_item_rates[i]); + output_stream_total_sample_counts_[i] += cycle_count * segment.output_stream_item_rates[i]; } + maximum_item_rate_ = std::max(maximum_item_rate_, segment.maximum_item_rate); + segments_.push_back(std::move(segment)); } - output_stream_total_sample_counts_.resize(output_stream_count_); - - std::size_t cycle_count = block.Cycles(); - if (cycle_count == 0) - { - cycle_count = infer_cycle_count_from_file(data_filepath, block, block_start_offset, chunk_cycle_length_); - } - cycles_remaining_ = cycle_count; - for (std::size_t i = 0; i < output_stream_count_; ++i) { - output_stream_total_sample_counts_[i] = cycle_count * output_stream_item_rates_[i]; + if (output_stream_item_sizes_[i] == 0) + { + throw std::runtime_error("ION_GSMS_Signal_Source requested stream is not present in source segments"); + } } + + current_segment_index_ = 0; + advance_to_next_segment(); +} + + +IONGSMSFileSource::SegmentDescriptor IONGSMSFileSource::make_segment_descriptor( + const fs::path& metadata_filepath, + const GnssMetadata::File& file, + const GnssMetadata::Block& block, + const std::size_t block_start_offset) +{ + return {metadata_filepath.parent_path() / file.Url().Value(), &block, block_start_offset, true}; } @@ -166,6 +216,94 @@ int IONGSMSFileSource::output_item_size_for_stream_id(const GnssMetadata::Block& } +std::vector IONGSMSFileSource::segment_output_stream_ids(const std::vector& segments, const std::vector& stream_ids) +{ + std::vector output_stream_ids; + for (const auto& stream_id : stream_ids) + { + for (const auto& segment : segments) + { + if (segment.block != nullptr && block_contains_stream_id(*segment.block, stream_id)) + { + output_stream_ids.push_back(stream_id); + break; + } + } + } + + return output_stream_ids; +} + + +int IONGSMSFileSource::output_item_size_for_stream_id(const std::vector& segments, const std::string& stream_id) +{ + int item_size = 0; + for (const auto& segment : segments) + { + if (segment.block == nullptr) + { + continue; + } + const auto current_item_size = output_item_size_for_stream_id(*segment.block, stream_id); + if (current_item_size == 0) + { + continue; + } + if (item_size != 0 && item_size != current_item_size) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream appears with inconsistent output item sizes"); + } + item_size = current_item_size; + } + + return item_size; +} + + +std::size_t IONGSMSFileSource::output_stream_count() const +{ + return output_stream_count_; +} + + +std::size_t IONGSMSFileSource::output_stream_item_size(std::size_t stream_index) const +{ + return output_stream_item_sizes_[stream_index]; +} + + +std::size_t IONGSMSFileSource::output_stream_total_sample_count(std::size_t stream_index) const +{ + return output_stream_total_sample_counts_[stream_index]; +} + + +gr::io_signature::sptr IONGSMSFileSource::make_output_signature(const std::vector& segments, const std::vector& stream_ids) +{ + const auto output_stream_ids = segment_output_stream_ids(segments, stream_ids); + if (output_stream_ids.empty()) + { + throw std::runtime_error("ION_GSMS_Signal_Source requested streams are not present in source segments"); + } + std::vector item_sizes{}; + + for (const auto& stream_id : output_stream_ids) + { + const auto item_size = output_item_size_for_stream_id(segments, stream_id); + if (item_size == 0) + { + throw std::runtime_error("ION_GSMS_Signal_Source requested stream is not present in source segments"); + } + item_sizes.push_back(item_size); + } + + return gr::io_signature::makev( + static_cast(item_sizes.size()), + static_cast(item_sizes.size()), + item_sizes); +} + + std::size_t IONGSMSFileSource::infer_cycle_count_from_file( const fs::path& data_filepath, const GnssMetadata::Block& block, @@ -194,43 +332,38 @@ std::size_t IONGSMSFileSource::infer_cycle_count_from_file( } -std::size_t IONGSMSFileSource::output_stream_count() const +bool IONGSMSFileSource::advance_to_next_segment() { - return output_stream_count_; -} - - -std::size_t IONGSMSFileSource::output_stream_item_size(std::size_t stream_index) const -{ - return output_stream_item_sizes_[stream_index]; -} - - -std::size_t IONGSMSFileSource::output_stream_total_sample_count(std::size_t stream_index) const -{ - return output_stream_total_sample_counts_[stream_index]; -} - - -gr::io_signature::sptr IONGSMSFileSource::make_output_signature(const GnssMetadata::Block& block, const std::vector& stream_ids) -{ - const auto output_stream_ids = block_output_stream_ids(block, stream_ids); - std::vector item_sizes{}; - - for (const auto& stream_id : output_stream_ids) + file_stream_.close(); + while (current_segment_index_ < segments_.size()) { - const auto item_size = output_item_size_for_stream_id(block, stream_id); - if (item_size == 0) + auto& segment = segments_[current_segment_index_]; + if (segment.cycle_count == 0) { - throw std::runtime_error("ION_GSMS_Signal_Source requested stream is not present in block"); + ++current_segment_index_; + continue; } - item_sizes.push_back(item_size); + + file_stream_.open(segment.data_filepath, std::ios::in | std::ios::binary); + if (!file_stream_.is_open()) + { + LOG(WARNING) << "ION_GSMS_Signal_Source - Unable to open the samples file: " << segment.data_filepath.c_str(); + std::cerr << "ION_GSMS_Signal_Source - Unable to open the samples file: " << segment.data_filepath.c_str() << std::endl; + std::cout << "GNSS-SDR program ended.\n"; + exit(1); + } + + file_stream_.seekg(static_cast(segment.block_start_offset + segment.block->SizeHeader()), std::ios::beg); + cycles_remaining_ = segment.cycle_count; + chunk_cycle_length_ = segment.chunk_cycle_length; + maximum_item_rate_ = segment.maximum_item_rate; + return true; } - return gr::io_signature::makev( - static_cast(item_sizes.size()), - static_cast(item_sizes.size()), - item_sizes); + cycles_remaining_ = 0; + chunk_cycle_length_ = 0; + maximum_item_rate_ = 0; + return false; } @@ -241,7 +374,10 @@ int IONGSMSFileSource::work( { if (cycles_remaining_ == 0) { - return WORK_DONE; + if (!advance_to_next_segment()) + { + return WORK_DONE; + } } if (noutput_items <= 0 || maximum_item_rate_ == 0 || chunk_cycle_length_ == 0) { @@ -256,6 +392,7 @@ int IONGSMSFileSource::work( return 0; } const std::size_t cycles_to_read = std::min(max_sample_output, cycles_remaining_); + auto& segment = segments_[current_segment_index_]; // Resize the IO buffer to fit exactly the maximum amount of samples that will be outputted. io_buffer_.resize(cycles_to_read * chunk_cycle_length_); @@ -272,7 +409,8 @@ int IONGSMSFileSource::work( if (cycles_read == 0) { cycles_remaining_ = 0; - return WORK_DONE; + ++current_segment_index_; + return 0; } cycles_remaining_ = cycles_read >= cycles_remaining_ ? 0 : cycles_remaining_ - cycles_read; if (bytes_read < bytes_to_read) @@ -288,7 +426,7 @@ int IONGSMSFileSource::work( while (io_buffer_offset_ < bytes_to_decode) { // Iterate chunks within a chunk cycle - for (auto& chunk : chunk_data_) + for (auto& chunk : segment.chunk_data) { // Copy chunk into a separate buffer where the samples will be shifted from. const std::size_t bytes_copied = chunk->read_from_buffer(reinterpret_cast(io_buffer_.data()), io_buffer_offset_); @@ -301,6 +439,11 @@ int IONGSMSFileSource::work( } } + if (cycles_remaining_ == 0) + { + ++current_segment_index_; + } + // Call `produce(int, int)` with the appropriate item count for each output stream. for (std::size_t i = 0; i < items_produced_.size(); ++i) { diff --git a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h index d96e01373..873ba8070 100644 --- a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h +++ b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h @@ -39,6 +39,14 @@ class IONGSMSFileSource : public gr::sync_block public: using sptr = gnss_shared_ptr; + struct SegmentDescriptor + { + fs::path data_filepath; + const GnssMetadata::Block* block = nullptr; + std::size_t block_start_offset = 0; + bool block_extends_to_eof = false; + }; + IONGSMSFileSource( const fs::path& metadata_filepath, const GnssMetadata::File& file, @@ -46,6 +54,10 @@ public: std::size_t block_start_offset, const std::vector& stream_ids); + IONGSMSFileSource( + const std::vector& segments, + const std::vector& stream_ids); + int work( int noutput_items, gr_vector_const_void_star& input_items, @@ -56,15 +68,35 @@ public: std::size_t output_stream_total_sample_count(std::size_t stream_index) const; private: - static gr::io_signature::sptr make_output_signature(const GnssMetadata::Block& block, const std::vector& stream_ids); + struct SegmentData + { + fs::path data_filepath; + const GnssMetadata::Block* block = nullptr; + std::size_t block_start_offset = 0; + std::size_t cycle_count = 0; + std::size_t chunk_cycle_length = 0; + std::size_t maximum_item_rate = 0; + std::vector output_stream_item_rates; + std::vector> chunk_data; + }; + + static SegmentDescriptor make_segment_descriptor( + const fs::path& metadata_filepath, + const GnssMetadata::File& file, + const GnssMetadata::Block& block, + std::size_t block_start_offset); + static gr::io_signature::sptr make_output_signature(const std::vector& segments, const std::vector& stream_ids); static bool block_contains_stream_id(const GnssMetadata::Block& block, const std::string& stream_id); static std::vector block_output_stream_ids(const GnssMetadata::Block& block, const std::vector& stream_ids); + static std::vector segment_output_stream_ids(const std::vector& segments, const std::vector& stream_ids); static int output_item_size_for_stream_id(const GnssMetadata::Block& block, const std::string& stream_id); + static int output_item_size_for_stream_id(const std::vector& segments, const std::string& stream_id); static std::size_t infer_cycle_count_from_file( const fs::path& data_filepath, const GnssMetadata::Block& block, std::size_t block_start_offset, std::size_t chunk_cycle_length); + bool advance_to_next_segment(); std::ifstream file_stream_; std::vector io_buffer_; @@ -75,9 +107,11 @@ private: std::vector output_stream_item_rates_; std::vector output_stream_total_sample_counts_; std::size_t maximum_item_rate_; - std::vector> chunk_data_; std::size_t chunk_cycle_length_; std::size_t cycles_remaining_; + std::vector segments_; + std::size_t current_segment_index_; + std::vector output_stream_ids_; }; /** \} */ diff --git a/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc b/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc index 76706f898..8dd66421f 100644 --- a/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc +++ b/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc @@ -66,13 +66,10 @@ IONGSMSChunkData::IONGSMSChunkData(const GnssMetadata::Chunk& chunk, const std:: throw std::runtime_error("ION_GSMS_Signal_Source metadata describes a lump pattern larger than its chunk"); } - const std::size_t pattern_repeat_count = total_bitsize / pattern_bitsize; output_stream_count_ = stream_ids.size(); output_stream_item_size_.assign(output_stream_count_, 0); output_stream_item_rate_.assign(output_stream_count_, 0); std::vector output_stream_seen(output_stream_count_, false); - std::vector pattern_output_item_rates(output_stream_count_, 0); - std::vector pattern_streams; for (const auto& lump : chunk.Lumps()) { for (const auto& stream : lump.Streams()) @@ -90,8 +87,8 @@ IONGSMSChunkData::IONGSMSChunkData(const GnssMetadata::Chunk& chunk, const std:: output_index = static_cast(relative_output_index + output_stream_offset); output_item_rate = stream_output_item_rate(stream); output_item_size = stream_output_item_size(stream); - output_item_offset = pattern_output_item_rates[relative_output_index]; - pattern_output_item_rates[relative_output_index] += output_item_rate; + output_item_offset = output_stream_item_rate_[relative_output_index]; + output_stream_item_rate_[relative_output_index] += output_item_rate; if (output_stream_item_size_[relative_output_index] != 0 && output_stream_item_size_[relative_output_index] != output_item_size) @@ -107,40 +104,11 @@ IONGSMSChunkData::IONGSMSChunkData(const GnssMetadata::Chunk& chunk, const std:: } } - pattern_streams.emplace_back(lump, stream, stream_encoding, output_index, output_item_offset, output_item_size); + streams_.emplace_back(lump, stream, stream_encoding, output_index, output_item_offset, output_item_size); } } - for (std::size_t i = 0; i < output_stream_item_rate_.size(); ++i) - { - output_stream_item_rate_[i] = pattern_output_item_rates[i] * pattern_repeat_count; - } - - for (std::size_t repeat = 0; repeat < pattern_repeat_count; ++repeat) - { - for (const auto& stream_metadata : pattern_streams) - { - std::size_t output_item_offset = 0; - if (stream_metadata.output_index != -1) - { - const auto relative_output_index = static_cast(stream_metadata.output_index) - output_stream_offset_; - const std::size_t chronological_repeat = stream_metadata.lump.Shift() == GnssMetadata::Lump::shiftRight - ? (pattern_repeat_count - repeat - 1) - : repeat; - output_item_offset = chronological_repeat * pattern_output_item_rates[relative_output_index] + - stream_metadata.output_item_offset; - } - streams_.emplace_back( - stream_metadata.lump, - stream_metadata.stream, - stream_metadata.stream_encoding, - stream_metadata.output_index, - output_item_offset, - stream_metadata.output_item_size); - } - } - - padding_bitsize_ = total_bitsize - pattern_bitsize * pattern_repeat_count; + padding_bitsize_ = total_bitsize - pattern_bitsize; } @@ -258,12 +226,7 @@ void IONGSMSChunkData::unpack_words(gr_vector_void_star& outputs, std::vector ctx{data, countwords_}; + IONGSMSChunkUnpackingCtx ctx{data, countwords_, chunk_.Shift() == GnssMetadata::Chunk::Right}; // Head padding if (padding_bitsize_ > 0 && chunk_.Padding() == GnssMetadata::Chunk::Head) diff --git a/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h b/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h index 9fec8ea8e..c4bd0c890 100644 --- a/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h +++ b/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h @@ -36,11 +36,14 @@ struct IONGSMSChunkUnpackingCtx const WT* data_ = nullptr; // Not owned by this class, MUST NOT destroy. std::size_t word_count_ = 0; std::size_t bit_offset_ = 0; + bool read_lsb_first_ = false; IONGSMSChunkUnpackingCtx( WT* data_buffer, - std::size_t data_buffer_word_count) : data_(data_buffer), - word_count_(data_buffer_word_count) + std::size_t data_buffer_word_count, + bool read_lsb_first) : data_(data_buffer), + word_count_(data_buffer_word_count), + read_lsb_first_(read_lsb_first) { } @@ -61,9 +64,17 @@ struct IONGSMSChunkUnpackingCtx throw std::runtime_error("ION_GSMS_Signal_Source tried to read past the chunk boundary"); } const std::size_t bit_index = absolute_bit % word_bitsize_; - const std::size_t word_bit = word_bitsize_ - 1 - bit_index; - value <<= 1; - value |= (static_cast(data_[word_index]) >> word_bit) & 0x01U; + const std::size_t word_bit = read_lsb_first_ ? bit_index : word_bitsize_ - 1 - bit_index; + const uint64_t bit_value = (static_cast(data_[word_index]) >> word_bit) & 0x01U; + if (read_lsb_first_) + { + value |= bit_value << bit; + } + else + { + value <<= 1; + value |= bit_value; + } } bit_offset_ += bit_count; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index efcbf7145..288a8837c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1166,6 +1166,7 @@ if(NOT ENABLE_PACKAGING AND NOT ENABLE_FPGA) GTest::GTest GTest::Main Volkgnsssdr::volkgnsssdr + signal_source_adapters signal_source_gr_blocks signal_source_libs core_receiver diff --git a/tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc b/tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc index 321a1aaec..a333ac129 100644 --- a/tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc +++ b/tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc @@ -15,12 +15,16 @@ #include "gnss_sdr_filesystem.h" #include "gnss_sdr_flags.h" +#include "gnss_sdr_make_unique.h" +#include "in_memory_configuration.h" #include "ion_gsms.h" #include "ion_gsms_chunk_data.h" +#include "ion_gsms_signal_source.h" #include #include #include #include +#include #include #include #include @@ -196,7 +200,7 @@ GnssMetadata::Block make_file_block( const std::size_t size_header = 0, const std::size_t size_footer = 0) { - const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 8, 4); auto chunk = make_chunk(stream, 1, 1); GnssMetadata::Block block; @@ -218,6 +222,13 @@ void write_binary_file(const fs::path& path, const std::vector& bytes) } +void write_text_file(const fs::path& path, const std::string& text) +{ + std::ofstream file(path.c_str(), std::ios::out | std::ios::trunc); + file << text; +} + + GnssMetadata::File make_data_file(const fs::path& data_path, const std::size_t offset) { GnssMetadata::File file; @@ -250,6 +261,26 @@ std::vector run_file_source( } +std::vector run_file_source( + const std::vector& segments) +{ + auto source = gnss_make_shared(segments, std::vector{"L1"}); + auto sink = gr::blocks::vector_sink_b::make(); + auto top_block = gr::make_top_block("IONGSMSFileSourceTest"); + top_block->connect(source, 0, sink, 0); + top_block->run(); + + const auto sink_data = sink->data(); + std::vector output; + output.reserve(sink_data.size()); + for (const auto item : sink_data) + { + output.push_back(static_cast(item)); + } + return output; +} + + std::vector run_complex_file_source( const fs::path& metadata_path, const GnssMetadata::File& file, @@ -311,6 +342,17 @@ TEST(IONGSMSChunkDataTest, HonorsStreamShiftRightWithoutOverrun) } +TEST(IONGSMSChunkDataTest, HonorsRightShiftedChunkBitOrder) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 8, 4); + + const auto output = decode_int8_stream(stream, {0b00111001}, 1, 1, GnssMetadata::Chunk::Undefined, GnssMetadata::Chunk::Tail, GnssMetadata::Chunk::Right); + + const std::vector expected{1, -2, -1, 0}; + EXPECT_EQ(expected, output); +} + + TEST(IONGSMSChunkDataTest, DecodesBigEndianWordsAsUnsigned) { const auto stream = make_stream(GnssMetadata::IonStream::IF, "OB", 4, 16, 1, GnssMetadata::IonStream::Left); @@ -322,13 +364,13 @@ TEST(IONGSMSChunkDataTest, DecodesBigEndianWordsAsUnsigned) } -TEST(IONGSMSChunkDataTest, RepeatsSingleLumpPayloadToFillChunk) +TEST(IONGSMSChunkDataTest, DecodesOneDeclaredLumpAndLeavesTailPadding) { const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); const auto output = decode_int8_stream(stream, {0b01101100}); - const std::vector expected{1, -2, -1, 0}; + const std::vector expected{1}; EXPECT_EQ(expected, output); } @@ -362,7 +404,7 @@ TEST(IONGSMSChunkDataTest, DecodesMultipleLumpsInDeclaredOrder) } -TEST(IONGSMSChunkDataTest, RepeatsOrderedMultiLumpPatternToFillChunk) +TEST(IONGSMSChunkDataTest, DecodesOrderedMultiLumpPatternOnce) { const auto first_stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1, GnssMetadata::IonStream::Undefined, GnssMetadata::IonStream::shiftUndefined, "L1"); const auto second_stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1, GnssMetadata::IonStream::Undefined, GnssMetadata::IonStream::shiftUndefined, "L2"); @@ -384,8 +426,8 @@ TEST(IONGSMSChunkDataTest, RepeatsOrderedMultiLumpPatternToFillChunk) const auto output = decode_int8_chunk(chunk, {"L1", "L2"}, {0b01101100}); ASSERT_EQ(2U, output.size()); - const std::vector expected_first{1, -1}; - const std::vector expected_second{-2, 0}; + const std::vector expected_first{1}; + const std::vector expected_second{-2}; EXPECT_EQ(expected_first, output[0]); EXPECT_EQ(expected_second, output[1]); } @@ -412,7 +454,7 @@ TEST(IONGSMSChunkDataTest, CollapsesRepeatedStreamIdIntoOneOutput) const auto output = decode_int8_chunk(chunk, {"L1"}, {0b01101100}); ASSERT_EQ(1U, output.size()); - const std::vector expected{1, -2, -1, 0}; + const std::vector expected{1, -2}; EXPECT_EQ(expected, output[0]); } @@ -506,6 +548,38 @@ TEST(IONGSMSFileSourceTest, DecodesOnlyCompleteCyclesReadFromFile) } +TEST(IONGSMSFileSourceTest, InfersZeroCyclesFromEofForStandaloneBlock) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_file_source_zero_cycles_standalone.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_file_source_zero_cycles_standalone.sdrx"; + write_binary_file(data_path, {static_cast(0b01101100U), static_cast(0b00011011U)}); + + const auto file = make_data_file(data_path, 0); + const auto block = make_file_block(0); + + const auto output = run_file_source(metadata_path, file, block, 0); + + const std::vector expected{1, -2, -1, 0, 0, 1, -2, -1}; + EXPECT_EQ(expected, output); + + fs::remove(data_path); +} + + +TEST(IONGSMSFileSourceTest, RejectsZeroCycleSegmentUnlessItExtendsToEof) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_file_source_zero_cycles_nonfinal.bin"; + + const auto block = make_file_block(0); + const std::vector segments{ + {data_path, &block, 0}}; + + EXPECT_THROW(gnss_make_shared(segments, std::vector{"L1"}), std::runtime_error); +} + + TEST(IONGSMSFileSourceTest, CollapsesRepeatedStreamIdAcrossChunksIntoOneOutput) { const fs::path temp_dir(GetTempDir()); @@ -513,7 +587,7 @@ TEST(IONGSMSFileSourceTest, CollapsesRepeatedStreamIdAcrossChunksIntoOneOutput) const fs::path metadata_path = temp_dir / "ion_gsms_file_source_repeated_stream.sdrx"; write_binary_file(data_path, {static_cast(0b01101100U), static_cast(0b00011011U)}); - const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 8, 4); auto first_chunk = make_chunk(stream, 1, 1); auto second_chunk = make_chunk(stream, 1, 1); @@ -532,6 +606,29 @@ TEST(IONGSMSFileSourceTest, CollapsesRepeatedStreamIdAcrossChunksIntoOneOutput) } +TEST(IONGSMSFileSourceTest, ConcatenatesSegmentsForSameStream) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path first_data_path = temp_dir / "ion_gsms_file_source_segment_1.bin"; + const fs::path second_data_path = temp_dir / "ion_gsms_file_source_segment_2.bin"; + write_binary_file(first_data_path, {static_cast(0b01101100U)}); + write_binary_file(second_data_path, {static_cast(0b00011011U)}); + + const auto block = make_file_block(1); + const std::vector segments{ + {first_data_path, &block, 0}, + {second_data_path, &block, 0}}; + + const auto output = run_file_source(segments); + + const std::vector expected{1, -2, -1, 0, 0, 1, -2, -1}; + EXPECT_EQ(expected, output); + + fs::remove(first_data_path); + fs::remove(second_data_path); +} + + TEST(IONGSMSFileSourceTest, StartsReadingAtProvidedBlockOffset) { const fs::path temp_dir(GetTempDir()); @@ -575,3 +672,334 @@ TEST(IONGSMSFileSourceTest, EmitsFp32IqAsComplexItems) fs::remove(data_path); } + + +TEST(IONGSMSSignalSourceTest, UsesMetadataSamplingFrequencyPerRequestedStream) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_signal_source_multirate.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_signal_source_multirate.sdrx"; + write_binary_file(data_path, std::vector(10, 0)); + write_text_file(metadata_path, + "\n" + "\n" + " \n" + " 10.0\n" + " \n" + " \n" + " 1.0\n" + " 0.0\n" + " 1.0\n" + " \n" + " \n" + " \n" + " \n" + " 10\n" + " \n" + " 1\n" + " 1\n" + " Left\n" + " \n" + " \n" + " 2\n" + " 1\n" + " 2\n" + " Left\n" + " IF\n" + " SIGN\n" + " \n" + " \n" + " \n" + " 5\n" + " 1\n" + " 5\n" + " Left\n" + " IF\n" + " SIGN\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " ion_gsms_signal_source_multirate.bin\n" + " 2026-06-21T00:00:00Z\n" + " \n" + " \n" + "\n"); + + auto queue = std::make_shared>(); + auto config = std::make_shared(); + config->set_property("Test.metadata_filename", metadata_path.string()); + config->set_property("Test.streams", "L1,L2"); + config->set_property("Test.sampling_frequency", "100"); + + std::unique_ptr source = + std::make_unique(config.get(), "Test", 0, 1, queue.get()); + EXPECT_EQ(2U, source->getRfChannels()); + EXPECT_EQ(nullptr, source->get_right_block(2)); + + auto first_sink = gr::blocks::vector_sink_b::make(); + auto second_sink = gr::blocks::vector_sink_b::make(); + auto top_block = gr::make_top_block("IONGSMSSignalSourceTest"); + source->connect(top_block); + top_block->connect(source->get_right_block(0), 0, first_sink, 0); + top_block->connect(source->get_right_block(1), 0, second_sink, 0); + top_block->run(); + + EXPECT_EQ(16U, first_sink->data().size()); + EXPECT_EQ(40U, second_sink->data().size()); + + fs::remove(data_path); + fs::remove(metadata_path); +} + + +TEST(IONGSMSSignalSourceTest, LoadsStreamsFromRelativeIncludedMetadata) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path include_dir = temp_dir / "ion_gsms_signal_source_include_dir"; + const fs::path data_path = include_dir / "ion_gsms_signal_source_included.bin"; + const fs::path root_metadata_path = temp_dir / "ion_gsms_signal_source_include_root.sdrx"; + const fs::path included_metadata_path = include_dir / "child.sdrx"; + fs::create_directories(include_dir); + write_binary_file(data_path, std::vector(10, 0)); + write_text_file(root_metadata_path, + "\n" + "\n" + " ion_gsms_signal_source_include_dir/child.sdrx\n" + "\n"); + write_text_file(included_metadata_path, + "\n" + "\n" + " \n" + " 10.0\n" + " \n" + " \n" + " 1.0\n" + " 0.0\n" + " 1.0\n" + " \n" + " \n" + " \n" + " \n" + " 10\n" + " \n" + " 1\n" + " 1\n" + " Left\n" + " \n" + " \n" + " 4\n" + " 2\n" + " 8\n" + " Left\n" + " IF\n" + " TC\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " ion_gsms_signal_source_included.bin\n" + " 2026-06-21T00:00:00Z\n" + " \n" + " \n" + "\n"); + + auto queue = std::make_shared>(); + auto config = std::make_shared(); + config->set_property("Test.metadata_filename", root_metadata_path.string()); + config->set_property("Test.streams", "L1"); + config->set_property("Test.sampling_frequency", "100"); + + std::unique_ptr source = + std::make_unique(config.get(), "Test", 0, 1, queue.get()); + EXPECT_EQ(1U, source->getRfChannels()); + + auto sink = gr::blocks::vector_sink_b::make(); + auto top_block = gr::make_top_block("IONGSMSSignalSourceIncludeTest"); + source->connect(top_block); + top_block->connect(source->get_right_block(0), 0, sink, 0); + top_block->run(); + + EXPECT_EQ(32U, sink->data().size()); + + fs::remove(data_path); + fs::remove(included_metadata_path); + fs::remove(root_metadata_path); + fs::remove(include_dir); +} + + +TEST(IONGSMSSignalSourceTest, UsesFileSetEntriesAsPreferredFileStarts) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path first_data_path = temp_dir / "ion_gsms_signal_source_fileset_first.bin"; + const fs::path second_data_path = temp_dir / "ion_gsms_signal_source_fileset_second.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_signal_source_fileset.sdrx"; + write_binary_file(first_data_path, {static_cast(0b01101100U), static_cast(0b01101100U)}); + write_binary_file(second_data_path, {static_cast(0b00011011U), static_cast(0b00011011U)}); + write_text_file(metadata_path, + "\n" + "\n" + " \n" + " 10.0\n" + " \n" + " \n" + " 1.0\n" + " 0.0\n" + " 1.0\n" + " \n" + " \n" + " ion_gsms_signal_source_fileset_first.bin\n" + " \n" + " \n" + " \n" + " \n" + " 2\n" + " \n" + " 1\n" + " 1\n" + " Left\n" + " \n" + " \n" + " 4\n" + " 2\n" + " 8\n" + " Left\n" + " IF\n" + " TC\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " ion_gsms_signal_source_fileset_second.bin\n" + " 2026-06-21T00:00:01Z\n" + " \n" + " \n" + " \n" + " ion_gsms_signal_source_fileset_first.bin\n" + " 2026-06-21T00:00:00Z\n" + " \n" + " \n" + "\n"); + + auto queue = std::make_shared>(); + auto config = std::make_shared(); + config->set_property("Test.metadata_filename", metadata_path.string()); + config->set_property("Test.streams", "L1"); + config->set_property("Test.sampling_frequency", "100"); + + std::unique_ptr source = + std::make_unique(config.get(), "Test", 0, 1, queue.get()); + + auto sink = gr::blocks::vector_sink_b::make(); + auto top_block = gr::make_top_block("IONGSMSSignalSourceFileSetTest"); + source->connect(top_block); + top_block->connect(source->get_right_block(0), 0, sink, 0); + top_block->run(); + + std::vector output; + output.reserve(sink->data().size()); + for (const auto item : sink->data()) + { + output.push_back(static_cast(item)); + } + const std::vector expected{1, -2, -1, 0, 1, -2, -1, 0}; + EXPECT_EQ(expected, output); + + fs::remove(first_data_path); + fs::remove(second_data_path); + fs::remove(metadata_path); +} + + +TEST(IONGSMSSignalSourceTest, RejectsZeroCycleBlockBeforeLaterLaneBlocks) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_signal_source_zero_cycles_nonfinal.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_signal_source_zero_cycles_nonfinal.sdrx"; + write_binary_file(data_path, {static_cast(0b01101100U), static_cast(0b00011011U)}); + write_text_file(metadata_path, + "\n" + "\n" + " \n" + " 10.0\n" + " \n" + " \n" + " 1.0\n" + " 0.0\n" + " 1.0\n" + " \n" + " \n" + " \n" + " \n" + " 0\n" + " \n" + " 1\n" + " 1\n" + " Left\n" + " \n" + " \n" + " 4\n" + " 2\n" + " 8\n" + " Left\n" + " IF\n" + " TC\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " 1\n" + " \n" + " 1\n" + " 1\n" + " Left\n" + " \n" + " \n" + " 4\n" + " 2\n" + " 8\n" + " Left\n" + " IF\n" + " TC\n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " \n" + " ion_gsms_signal_source_zero_cycles_nonfinal.bin\n" + " 2026-06-21T00:00:00Z\n" + " \n" + " \n" + "\n"); + + auto queue = std::make_shared>(); + auto config = std::make_shared(); + config->set_property("Test.metadata_filename", metadata_path.string()); + config->set_property("Test.streams", "L1"); + config->set_property("Test.sampling_frequency", "100"); + + EXPECT_THROW( + { + std::unique_ptr source = + std::make_unique(config.get(), "Test", 0, 1, queue.get()); + (void)source; + }, + std::runtime_error); + + fs::remove(data_path); + fs::remove(metadata_path); +}