From 6b77e477f158a29413e650a42c8329870a13d003 Mon Sep 17 00:00:00 2001 From: Carles Fernandez Date: Sun, 21 Jun 2026 00:03:24 +0200 Subject: [PATCH] Improve ION_GSMS_Signal_Source implementation --- docs/CHANGELOG.md | 8 + .../adapters/ion_gsms_signal_source.cc | 222 ++++-- .../adapters/ion_gsms_signal_source.h | 24 +- .../signal_source/gnuradio_blocks/ion_gsms.cc | 213 ++++-- .../signal_source/gnuradio_blocks/ion_gsms.h | 13 +- .../signal_source/libs/ion_gsms_chunk_data.cc | 667 +++++++++++++++--- .../signal_source/libs/ion_gsms_chunk_data.h | 84 ++- .../libs/ion_gsms_chunk_unpacking_ctx.h | 152 +--- .../libs/ion_gsms_stream_encodings.h | 3 +- tests/CMakeLists.txt | 5 + .../sources/ion_gsms_chunk_data_test.cc | 577 +++++++++++++++ 11 files changed, 1593 insertions(+), 375 deletions(-) create mode 100644 tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 272c0f48f..922b70f77 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -35,6 +35,14 @@ All notable changes to GNSS-SDR will be documented in this file. alias. It can auto-detect `.gns` file layout information, unpack 2-, 4-, 8-, and 16-bit samples, and expose multi-channel recordings as independent RF output streams. +- Reworked + [`ION_GSMS_Signal_Source`](https://gnss-sdr.org/docs/sp-blocks/signal-source/#implementation-ion_gsms_signal_source) + support for ION GNSS SDR metadata files. The source now validates and + deduplicates requested streams, honors file offsets, block headers/footers, + and omitted cycle counts, and stops finite captures cleanly with a guarded + valve tail. Chunk unpacking now handles word endianness, padding, shifts, + repeated lump patterns, repeated stream IDs, standard integer encodings, and + FP32 streams as `float` or `gr_complex` outputs. - Improved Galileo HAS robustness and ICD compliance, including stricter MT1 validation, correct cache/Do-Not-Use handling, TOW fallback for E6 HAS pages, preserved mask/IOD correction context, and corrected HAS application in diff --git a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc index c92c35433..53f63bd07 100644 --- a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc +++ b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.cc @@ -2,26 +2,28 @@ * \file ion_gsms_signal_source.h * \brief GNSS-SDR Signal Source that reads sample streams following ION's GNSS-SDR metadata standard * \author Víctor Castillo Agüero, 2024. victorcastilloaguero(at)gmail.com + * \author Carles Fernandez, 2026 carles.fernandez(at)cttc.es * * ----------------------------------------------------------------------------- * * GNSS-SDR is a Global Navigation Satellite System software-defined receiver. * This file is part of GNSS-SDR. * - * Copyright (C) 2010-2024 (see AUTHORS file for a list of contributors) + * Copyright (C) 2010-2026 (see AUTHORS file for a list of contributors) * SPDX-License-Identifier: GPL-3.0-or-later * * ----------------------------------------------------------------------------- */ #include "ion_gsms_signal_source.h" -#include "gnss_sdr_flags.h" #include "gnss_sdr_string_literals.h" #include "gnss_sdr_valve.h" #include +#include +#include #include #include -#include +#include #if USE_GLOG_AND_GFLAGS #include @@ -31,29 +33,6 @@ using namespace std::string_literals; -namespace -{ -std::vector parse_comma_list(const std::string& str) -{ - std::vector list{}; - std::size_t prev_comma_at{0}; - - while (prev_comma_at < str.size()) - { - std::size_t comma_at = str.find_first_of(',', prev_comma_at); - if (comma_at == std::string::npos) - { - comma_at = str.size(); - } - list.emplace_back(str.substr(prev_comma_at, (comma_at - prev_comma_at))); - prev_comma_at = comma_at + 1; - } - - return list; -} -} // anonymous namespace - - IONGSMSSignalSource::IONGSMSSignalSource(const ConfigurationInterface* configuration, const std::string& role, unsigned int in_streams, @@ -62,6 +41,8 @@ IONGSMSSignalSource::IONGSMSSignalSource(const ConfigurationInterface* configura : SignalSourceBase(configuration, role, "ION_GSMS_Signal_Source"s), stream_ids_(parse_comma_list(configuration->property(role + ".streams"s, ""s))), metadata_filepath_(configuration->property(role + ".metadata_filename"s, "./example_capture_metadata.sdrx"s)), + minimum_tail_s_(kMinimumTailSeconds), + sampling_frequency_(configuration->property(role + ".sampling_frequency"s, configuration->property("GNSS-SDR.internal_fs_sps"s, int64_t(0)))), in_streams_(in_streams), out_streams_(out_streams) { @@ -73,19 +54,27 @@ IONGSMSSignalSource::IONGSMSSignalSource(const ConfigurationInterface* configura { LOG(ERROR) << "A signal source does not have an output stream"; } + if (stream_ids_.empty()) + { + throw std::runtime_error("ION_GSMS_Signal_Source requires at least one stream id in the .streams property"); + } // Parse XML metadata file load_metadata(); // Make source vector sources_ = make_stream_sources(stream_ids_); + if (sources_.empty()) + { + throw std::runtime_error("ION_GSMS_Signal_Source no configured streams were found in the metadata"); + } for (const auto& source : sources_) { for (std::size_t i = 0; i < source->output_stream_count(); ++i) { copy_blocks_.emplace_back(gr::blocks::copy::make(source->output_stream_item_size(i))); - valves_.emplace_back(gnss_sdr_make_valve(source->output_stream_item_size(i), source->output_stream_total_sample_count(i), queue)); + valves_.emplace_back(gnss_sdr_make_valve(source->output_stream_item_size(i), valve_sample_count(source->output_stream_total_sample_count(i)), queue)); } } } @@ -122,60 +111,135 @@ void IONGSMSSignalSource::load_metadata() } +std::vector IONGSMSSignalSource::parse_comma_list(const std::string& str) +{ + std::vector list{}; + std::size_t prev_comma_at{0}; + + while (prev_comma_at < str.size()) + { + std::size_t comma_at = str.find_first_of(',', prev_comma_at); + if (comma_at == std::string::npos) + { + comma_at = str.size(); + } + auto value = str.substr(prev_comma_at, (comma_at - prev_comma_at)); + const auto first_non_space = value.find_first_not_of(" \t\n\r"); + if (first_non_space != std::string::npos) + { + const auto last_non_space = value.find_last_not_of(" \t\n\r"); + value = value.substr(first_non_space, last_non_space - first_non_space + 1); + if (std::find(list.begin(), list.end(), value) == list.end()) + { + list.push_back(value); + } + } + prev_comma_at = comma_at + 1; + } + + return list; +} + + +bool IONGSMSSignalSource::block_contains_stream(const GnssMetadata::Block& block, const std::vector& stream_ids) +{ + for (const auto& chunk : block.Chunks()) + { + for (const auto& lump : chunk.Lumps()) + { + for (const auto& stream : lump.Streams()) + { + if (std::find(stream_ids.begin(), stream_ids.end(), stream.Id()) != stream_ids.end()) + { + return true; + } + } + } + } + + return false; +} + + +std::size_t IONGSMSSignalSource::chunk_cycle_bytes(const GnssMetadata::Block& block) +{ + std::size_t bytes = 0; + for (const auto& chunk : block.Chunks()) + { + bytes += chunk.CountWords() * chunk.SizeWord(); + } + return bytes; +} + + +std::size_t IONGSMSSignalSource::infer_block_cycles( + const fs::path& data_filepath, + const GnssMetadata::Block& block, + const std::size_t block_start_offset) +{ + if (block.Cycles() != 0) + { + return block.Cycles(); + } + + const std::size_t cycle_bytes = chunk_cycle_bytes(block); + if (cycle_bytes == 0) + { + return 0; + } + + const std::size_t file_size = fs::file_size(data_filepath); + if (file_size <= block_start_offset) + { + return 0; + } + + const std::size_t remaining_bytes = file_size - block_start_offset; + const std::size_t block_overhead = block.SizeHeader() + block.SizeFooter(); + if (remaining_bytes <= block_overhead) + { + return 0; + } + + return (remaining_bytes - block_overhead) / cycle_bytes; +} + + +std::size_t IONGSMSSignalSource::block_storage_bytes( + const fs::path& data_filepath, + const GnssMetadata::Block& block, + const std::size_t block_start_offset) +{ + const std::size_t cycle_count = infer_block_cycles(data_filepath, block, block_start_offset); + return block.SizeHeader() + cycle_count * chunk_cycle_bytes(block) + block.SizeFooter(); +} + + std::vector IONGSMSSignalSource::make_stream_sources(const std::vector& stream_ids) const { std::vector sources{}; for (const auto& file : metadata_->Files()) { + const fs::path data_filepath = fs::path(metadata_filepath_).parent_path() / file.Url().Value(); for (const auto& lane : metadata_->Lanes()) { if (lane.Id() == file.Lane().Id()) { + std::size_t block_start_offset = file.Offset(); for (const auto& block : lane.Blocks()) { - bool block_done = false; - for (const auto& chunk : block.Chunks()) + if (block_contains_stream(block, stream_ids)) { - for (const auto& lump : chunk.Lumps()) - { - for (const auto& stream : lump.Streams()) - { - bool found = false; - for (const auto& stream_id : stream_ids) - { - if (stream_id == stream.Id()) - { - found = true; - break; - } - } - if (found) - { - auto source = gnss_make_shared( - metadata_filepath_, - file, - block, - stream_ids); + auto source = gnss_make_shared( + metadata_filepath_, + file, + block, + block_start_offset, + stream_ids); - sources.push_back(source); - - // This file source will take care of any other matching streams in this block - // We can skip the rest of this block - block_done = true; - break; - } - } - - if (block_done) - { - break; - } - } - if (block_done) - { - break; - } + sources.push_back(source); } + block_start_offset += block_storage_bytes(data_filepath, block, block_start_offset); } break; } @@ -186,6 +250,26 @@ std::vector IONGSMSSignalSource::make_stream_sources(co } +std::uint64_t IONGSMSSignalSource::valve_sample_count(std::uint64_t total_sample_count) const +{ + if (total_sample_count == 0 || sampling_frequency_ <= 0) + { + return total_sample_count; + } + + const auto tail_samples = static_cast(std::ceil(minimum_tail_s_ * static_cast(sampling_frequency_))); + if (total_sample_count <= tail_samples) + { + std::cout << "Warning: ION_GSMS_Signal_Source stream has " << total_sample_count + << " samples, which is shorter than the configured " << tail_samples + << " sample receiver tail. Setting the valve to process 1 sample.\n"; + return 1; + } + + return total_sample_count - tail_samples; +} + + void IONGSMSSignalSource::connect(gr::top_block_sptr top_block) { std::size_t cumulative_index = 0; @@ -233,6 +317,10 @@ gr::basic_block_sptr IONGSMSSignalSource::get_right_block(int RF_channel) if (RF_channel < 0 || RF_channel >= static_cast(copy_blocks_.size())) { LOG(WARNING) << "'RF_channel' out of bounds while trying to get signal source right block."; + if (valves_.empty()) + { + return {}; + } return valves_[0]; } return valves_[RF_channel]; diff --git a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h index bcdedbc3d..98032a95f 100644 --- a/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h +++ b/src/algorithms/signal_source/adapters/ion_gsms_signal_source.h @@ -2,13 +2,14 @@ * \file ion_gsms_signal_source.h * \brief GNSS-SDR Signal Source that reads sample streams following ION's GNSS-SDR metadata standard * \author Víctor Castillo Agüero, 2024. victorcastilloaguero(at)gmail.com + * \author Carles Fernandez, 2026 carles.fernandez(at)cttc.es * * ----------------------------------------------------------------------------- * * GNSS-SDR is a Global Navigation Satellite System software-defined receiver. * This file is part of GNSS-SDR. * - * Copyright (C) 2010-2024 (see AUTHORS file for a list of contributors) + * Copyright (C) 2010-2026 (see AUTHORS file for a list of contributors) * SPDX-License-Identifier: GPL-3.0-or-later * * ----------------------------------------------------------------------------- @@ -56,13 +57,32 @@ protected: inline size_t item_size() override { + if (sources_.empty()) + { + return 0; + } return (*sources_.begin())->output_stream_item_size(0); } private: + static constexpr double kMinimumTailSeconds = 0.2; + + static std::vector parse_comma_list(const std::string& str); + static bool block_contains_stream(const GnssMetadata::Block& block, const std::vector& stream_ids); + static std::size_t chunk_cycle_bytes(const GnssMetadata::Block& block); + static std::size_t infer_block_cycles( + const fs::path& data_filepath, + const GnssMetadata::Block& block, + std::size_t block_start_offset); + static std::size_t block_storage_bytes( + const fs::path& data_filepath, + const GnssMetadata::Block& block, + std::size_t block_start_offset); + std::vector make_stream_sources(const std::vector& stream_ids) const; void load_metadata(); + std::uint64_t valve_sample_count(std::uint64_t total_sample_count) const; std::vector stream_ids_; std::vector sources_; @@ -74,6 +94,8 @@ private: gnss_shared_ptr timestamp_block_; std::string timestamp_file_; + double minimum_tail_s_; + int64_t sampling_frequency_; uint32_t in_streams_; uint32_t out_streams_; diff --git a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc index 22a25868c..21f8d59ee 100644 --- a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc +++ b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.cc @@ -2,13 +2,14 @@ * \file ion_gsms.cc * \brief GNU Radio block that reads a Block from a file following ION's GNSS-SDR metadata standard * \author Víctor Castillo Agüero, 2024. victorcastilloaguero(at)gmail.com + * \author Carles Fernandez, 2026 carles.fernandez(at)cttc.es * * ----------------------------------------------------------------------------- * * GNSS-SDR is a Global Navigation Satellite System software-defined receiver. * This file is part of GNSS-SDR. * - * Copyright (C) 2010-2024 (see AUTHORS file for a list of contributors) + * Copyright (C) 2010-2026 (see AUTHORS file for a list of contributors) * SPDX-License-Identifier: GPL-3.0-or-later * * ----------------------------------------------------------------------------- @@ -20,6 +21,7 @@ #include #include #include +#include #if USE_GLOG_AND_GFLAGS #include @@ -31,6 +33,7 @@ IONGSMSFileSource::IONGSMSFileSource( const fs::path& metadata_filepath, const GnssMetadata::File& file, const GnssMetadata::Block& block, + const std::size_t block_start_offset, const std::vector& stream_ids) : gr::sync_block( "ion_gsms_file_source", @@ -39,10 +42,11 @@ IONGSMSFileSource::IONGSMSFileSource( file_stream_(metadata_filepath.parent_path() / file.Url().Value(), std::ios::in | std::ios::binary), io_buffer_offset_(0), maximum_item_rate_(0), - chunk_cycle_length_(0) + chunk_cycle_length_(0), + cycles_remaining_(0) { fs::path data_filepath = metadata_filepath.parent_path() / file.Url().Value(); - std::size_t block_offset = file.Offset(); + const auto output_stream_ids = block_output_stream_ids(block, stream_ids); if (!file_stream_.is_open()) { @@ -52,34 +56,44 @@ IONGSMSFileSource::IONGSMSFileSource( exit(1); } - // Skip offset and block header - file_stream_.seekg(file.Offset() + block_offset + block.SizeHeader()); + // Skip to this block's sample payload, after the lane offset and block header. + file_stream_.seekg(static_cast(block_start_offset + block.SizeHeader()), std::ios::beg); + + output_stream_count_ = output_stream_ids.size(); + output_stream_item_sizes_.assign(output_stream_count_, 0); + output_stream_item_rates_.assign(output_stream_count_, 0); - std::size_t output_stream_offset = 0; for (const auto& chunk : block.Chunks()) { - chunk_data_.emplace_back(std::make_shared(chunk, stream_ids, output_stream_offset)); + chunk_data_.emplace_back(std::make_shared(chunk, output_stream_ids, 0)); chunk_cycle_length_ += chunk.CountWords() * chunk.SizeWord(); - const std::size_t out_count = chunk_data_.back()->output_stream_count(); - output_stream_offset += out_count; - for (std::size_t i = 0; i < out_count; ++i) + for (std::size_t i = 0; i < output_stream_count_; ++i) { - output_stream_item_sizes_.push_back(chunk_data_.back()->output_stream_item_size(i)); - output_stream_item_rates_.push_back(chunk_data_.back()->output_stream_item_rate(i)); - maximum_item_rate_ = std::max(chunk_data_.back()->output_stream_item_rate(i), maximum_item_rate_); + const auto chunk_item_rate = chunk_data_.back()->output_stream_item_rate(i); + if (chunk_item_rate == 0) + { + continue; + } + + const auto chunk_item_size = chunk_data_.back()->output_stream_item_size(i); + if (output_stream_item_sizes_[i] != 0 && output_stream_item_sizes_[i] != chunk_item_size) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream appears with inconsistent output item sizes"); + } + output_stream_item_sizes_[i] = chunk_item_size; + output_stream_item_rates_[i] += chunk_item_rate; + maximum_item_rate_ = std::max(output_stream_item_rates_[i], maximum_item_rate_); } } - output_stream_count_ = output_stream_offset; output_stream_total_sample_counts_.resize(output_stream_count_); std::size_t cycle_count = block.Cycles(); if (cycle_count == 0) { - // Read the whole file - const std::size_t file_size = fs::file_size(data_filepath); - cycle_count = std::floor((file_size - block_offset - block.SizeHeader()) / chunk_cycle_length_); + cycle_count = infer_cycle_count_from_file(data_filepath, block, block_start_offset, chunk_cycle_length_); } + cycles_remaining_ = cycle_count; for (std::size_t i = 0; i < output_stream_count_; ++i) { @@ -88,6 +102,98 @@ IONGSMSFileSource::IONGSMSFileSource( } +bool IONGSMSFileSource::block_contains_stream_id(const GnssMetadata::Block& block, const std::string& stream_id) +{ + for (const auto& chunk : block.Chunks()) + { + for (const auto& lump : chunk.Lumps()) + { + for (const auto& stream : lump.Streams()) + { + if (stream.Id() == stream_id) + { + return true; + } + } + } + } + + return false; +} + + +std::vector IONGSMSFileSource::block_output_stream_ids(const GnssMetadata::Block& block, const std::vector& stream_ids) +{ + std::vector output_stream_ids; + for (const auto& stream_id : stream_ids) + { + if (std::find(output_stream_ids.begin(), output_stream_ids.end(), stream_id) == output_stream_ids.end() && + block_contains_stream_id(block, stream_id)) + { + output_stream_ids.push_back(stream_id); + } + } + + return output_stream_ids; +} + + +int IONGSMSFileSource::output_item_size_for_stream_id(const GnssMetadata::Block& block, const std::string& stream_id) +{ + int item_size = 0; + for (const auto& chunk : block.Chunks()) + { + for (const auto& lump : chunk.Lumps()) + { + for (const auto& stream : lump.Streams()) + { + if (stream.Id() != stream_id) + { + continue; + } + + const auto current_item_size = static_cast(IONGSMSChunkData::stream_output_item_size(stream)); + if (item_size != 0 && item_size != current_item_size) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream appears with inconsistent output item sizes"); + } + item_size = current_item_size; + } + } + } + + return item_size; +} + + +std::size_t IONGSMSFileSource::infer_cycle_count_from_file( + const fs::path& data_filepath, + const GnssMetadata::Block& block, + const std::size_t block_start_offset, + const std::size_t chunk_cycle_length) +{ + if (chunk_cycle_length == 0) + { + throw std::runtime_error("ION_GSMS_Signal_Source block has zero-length chunk cycle"); + } + + const std::size_t file_size = fs::file_size(data_filepath); + const std::size_t payload_start = block_start_offset + block.SizeHeader(); + if (file_size <= payload_start) + { + return 0; + } + + std::size_t payload_bytes = file_size - payload_start; + if (payload_bytes <= block.SizeFooter()) + { + return 0; + } + payload_bytes -= block.SizeFooter(); + return payload_bytes / chunk_cycle_length; +} + + std::size_t IONGSMSFileSource::output_stream_count() const { return output_stream_count_; @@ -108,42 +214,22 @@ std::size_t IONGSMSFileSource::output_stream_total_sample_count(std::size_t stre gr::io_signature::sptr IONGSMSFileSource::make_output_signature(const GnssMetadata::Block& block, const std::vector& stream_ids) { - int nstreams = 0; + const auto output_stream_ids = block_output_stream_ids(block, stream_ids); std::vector item_sizes{}; - for (const auto& chunk : block.Chunks()) + for (const auto& stream_id : output_stream_ids) { - for (const auto& lump : chunk.Lumps()) + const auto item_size = output_item_size_for_stream_id(block, stream_id); + if (item_size == 0) { - for (const auto& stream : lump.Streams()) - { - bool found = false; - for (const auto& stream_id : stream_ids) - { - if (stream_id == stream.Id()) - { - found = true; - break; - } - } - if (found) - { - ++nstreams; - std::size_t sample_bitsize = stream.Packedbits() / stream.RateFactor(); - if (stream.Packedbits() >= 2 * stream.RateFactor() * stream.Quantization()) - { - // Samples have 'Complex' format - sample_bitsize /= 2; - } - item_sizes.push_back(bits_to_item_size(sample_bitsize)); - } - } + throw std::runtime_error("ION_GSMS_Signal_Source requested stream is not present in block"); } + item_sizes.push_back(item_size); } return gr::io_signature::makev( - nstreams, - nstreams, + static_cast(item_sizes.size()), + static_cast(item_sizes.size()), item_sizes); } @@ -153,16 +239,26 @@ int IONGSMSFileSource::work( gr_vector_const_void_star& input_items __attribute__((unused)), gr_vector_void_star& output_items) { + if (cycles_remaining_ == 0) + { + return WORK_DONE; + } + if (noutput_items <= 0 || maximum_item_rate_ == 0 || chunk_cycle_length_ == 0) + { + return 0; + } + // Compute the maximum number of samples that will be copied across all output buffer. // If there are more than one output buffer (multichannel set up), the one with the most samples will be used as the maximum. - // - // Complex samples produce 2 items each (I and Q). In order to account for them, we subtract 1 from `noutput_items` and - // then floor the division. During testing, not doing this caused `max_sample_output` to oscillate between two values, thus - // resizing the `io_buffer_` on each call to `work()`. - const std::size_t max_sample_output = std::floor((noutput_items - 1.0) / maximum_item_rate_); + const std::size_t max_sample_output = static_cast(noutput_items) / maximum_item_rate_; + if (max_sample_output == 0) + { + return 0; + } + const std::size_t cycles_to_read = std::min(max_sample_output, cycles_remaining_); // Resize the IO buffer to fit exactly the maximum amount of samples that will be outputted. - io_buffer_.resize(max_sample_output * chunk_cycle_length_); + io_buffer_.resize(cycles_to_read * chunk_cycle_length_); // We will be walking the IO buffer with this variable. io_buffer_offset_ = 0; @@ -170,13 +266,26 @@ int IONGSMSFileSource::work( // Read samples from file into IO buffer const std::size_t bytes_to_read = io_buffer_.size(); file_stream_.read(io_buffer_.data(), bytes_to_read); + const auto bytes_read = static_cast(file_stream_.gcount()); + const std::size_t cycles_read = bytes_read / chunk_cycle_length_; + const std::size_t bytes_to_decode = cycles_read * chunk_cycle_length_; + if (cycles_read == 0) + { + cycles_remaining_ = 0; + return WORK_DONE; + } + cycles_remaining_ = cycles_read >= cycles_remaining_ ? 0 : cycles_remaining_ - cycles_read; + if (bytes_read < bytes_to_read) + { + cycles_remaining_ = 0; + } // Reset `items_produced_` vector. This vector will accumulate the amount of items produced for each output stream. items_produced_.clear(); items_produced_.resize(output_items.size()); // Walk the IO buffer one chunk cycle at a time. See ION documentation for a definition of chunk and chunk cycle. - while (io_buffer_offset_ < bytes_to_read) + while (io_buffer_offset_ < bytes_to_decode) { // Iterate chunks within a chunk cycle for (auto& chunk : chunk_data_) diff --git a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h index 4a6baa7e9..d96e01373 100644 --- a/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h +++ b/src/algorithms/signal_source/gnuradio_blocks/ion_gsms.h @@ -2,13 +2,14 @@ * \file ion_gsms.h * \brief GNU Radio block that reads a Block from a file following ION's GNSS-SDR metadata standard * \author Víctor Castillo Agüero, 2024. victorcastilloaguero(at)gmail.com + * \author Carles Fernandez, 2026 carles.fernandez(at)cttc.es * * ----------------------------------------------------------------------------- * * GNSS-SDR is a Global Navigation Satellite System software-defined receiver. * This file is part of GNSS-SDR. * - * Copyright (C) 2010-2024 (see AUTHORS file for a list of contributors) + * Copyright (C) 2010-2026 (see AUTHORS file for a list of contributors) * SPDX-License-Identifier: GPL-3.0-or-later * * ----------------------------------------------------------------------------- @@ -42,6 +43,7 @@ public: const fs::path& metadata_filepath, const GnssMetadata::File& file, const GnssMetadata::Block& block, + std::size_t block_start_offset, const std::vector& stream_ids); int work( @@ -55,6 +57,14 @@ public: private: static gr::io_signature::sptr make_output_signature(const GnssMetadata::Block& block, const std::vector& stream_ids); + static bool block_contains_stream_id(const GnssMetadata::Block& block, const std::string& stream_id); + static std::vector block_output_stream_ids(const GnssMetadata::Block& block, const std::vector& stream_ids); + static int output_item_size_for_stream_id(const GnssMetadata::Block& block, const std::string& stream_id); + static std::size_t infer_cycle_count_from_file( + const fs::path& data_filepath, + const GnssMetadata::Block& block, + std::size_t block_start_offset, + std::size_t chunk_cycle_length); std::ifstream file_stream_; std::vector io_buffer_; @@ -67,6 +77,7 @@ private: std::size_t maximum_item_rate_; std::vector> chunk_data_; std::size_t chunk_cycle_length_; + std::size_t cycles_remaining_; }; /** \} */ diff --git a/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc b/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc index 66ffa1e62..76706f898 100644 --- a/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc +++ b/src/algorithms/signal_source/libs/ion_gsms_chunk_data.cc @@ -2,20 +2,26 @@ * \file ion_gsms_chunk_data.cc * \brief Holds logic for reading and decoding samples from a chunk * \author Víctor Castillo Agüero, 2024. victorcastilloaguero(at)gmail.com + * \author Carles Fernandez, 2026 carles.fernandez(at)cttc.es * * ----------------------------------------------------------------------------- * * GNSS-SDR is a Global Navigation Satellite System software-defined receiver. * This file is part of GNSS-SDR. * - * Copyright (C) 2010-2024 (see AUTHORS file for a list of contributors) + * Copyright (C) 2010-2026 (see AUTHORS file for a list of contributors) * SPDX-License-Identifier: GPL-3.0-or-later * * ----------------------------------------------------------------------------- */ #include "ion_gsms_chunk_data.h" +#include +#include #include +#include +#include +#include #if USE_GLOG_AND_GFLAGS #include #else @@ -26,7 +32,8 @@ IONGSMSChunkData::IONGSMSChunkData(const GnssMetadata::Chunk& chunk, const std::vector& stream_ids, std::size_t output_stream_offset) : chunk_(chunk), sizeword_(chunk_.SizeWord()), - countwords_(chunk_.CountWords()) + countwords_(chunk_.CountWords()), + output_stream_offset_(output_stream_offset) { // Instantiate the Allocator functor Allocator allocator(countwords_, buffer_); @@ -34,47 +41,106 @@ IONGSMSChunkData::IONGSMSChunkData(const GnssMetadata::Chunk& chunk, const std:: with_word_type(sizeword_, allocator); const std::size_t total_bitsize = sizeword_ * countwords_ * 8; - std::size_t used_bitsize = 0; - std::size_t output_streams = 0; + std::size_t pattern_bitsize = 0; + for (const auto& lump : chunk.Lumps()) + { + std::size_t lump_bitsize = 0; + for (const auto& stream : lump.Streams()) + { + lump_bitsize += stream.Packedbits(); + } + + if (lump_bitsize == 0) + { + throw std::runtime_error("ION_GSMS_Signal_Source lump must occupy at least one bit"); + } + pattern_bitsize += lump_bitsize; + } + + if (pattern_bitsize == 0) + { + throw std::runtime_error("ION_GSMS_Signal_Source chunk must occupy at least one bit"); + } + if (pattern_bitsize > total_bitsize) + { + throw std::runtime_error("ION_GSMS_Signal_Source metadata describes a lump pattern larger than its chunk"); + } + + const std::size_t pattern_repeat_count = total_bitsize / pattern_bitsize; + output_stream_count_ = stream_ids.size(); + output_stream_item_size_.assign(output_stream_count_, 0); + output_stream_item_rate_.assign(output_stream_count_, 0); + std::vector output_stream_seen(output_stream_count_, false); + std::vector pattern_output_item_rates(output_stream_count_, 0); + std::vector pattern_streams; for (const auto& lump : chunk.Lumps()) { for (const auto& stream : lump.Streams()) { - used_bitsize += stream.Packedbits(); - - bool found = false; - for (const auto& stream_id : stream_ids) - { - if (stream_id == stream.Id()) - { - found = true; - break; - } - } + const auto stream_id = std::find(stream_ids.begin(), stream_ids.end(), stream.Id()); + const bool found = stream_id != stream_ids.end(); + const auto stream_encoding = GnssMetadata::encoding_from_string(stream.Encoding()); + int output_index = -1; + std::size_t output_item_offset = 0; + std::size_t output_item_rate = 0; + std::size_t output_item_size = 0; if (found) { - streams_.emplace_back(lump, stream, GnssMetadata::encoding_from_string(stream.Encoding()), output_streams + output_stream_offset); - ++output_streams; - std::size_t sample_bitsize = stream.Packedbits() / stream.RateFactor(); - std::size_t sample_rate = stream.RateFactor(); - if (stream.Packedbits() >= 2 * stream.RateFactor() * stream.Quantization()) + const auto relative_output_index = static_cast(std::distance(stream_ids.begin(), stream_id)); + output_index = static_cast(relative_output_index + output_stream_offset); + output_item_rate = stream_output_item_rate(stream); + output_item_size = stream_output_item_size(stream); + output_item_offset = pattern_output_item_rates[relative_output_index]; + pattern_output_item_rates[relative_output_index] += output_item_rate; + + if (output_stream_item_size_[relative_output_index] != 0 && + output_stream_item_size_[relative_output_index] != output_item_size) { - // Samples have 'Complex' format - sample_bitsize /= 2; - sample_rate *= 2; + throw std::runtime_error("ION_GSMS_Signal_Source stream appears with inconsistent output item sizes"); + } + output_stream_item_size_[relative_output_index] = output_item_size; + + if (!output_stream_seen[relative_output_index]) + { + output_stream_indices_.push_back(output_index); + output_stream_seen[relative_output_index] = true; } - output_stream_item_size_.push_back(bits_to_item_size(sample_bitsize)); - output_stream_item_rate_.push_back(sample_rate); - } - else - { - streams_.emplace_back(lump, stream, GnssMetadata::encoding_from_string(stream.Encoding()), -1); } + + pattern_streams.emplace_back(lump, stream, stream_encoding, output_index, output_item_offset, output_item_size); } } - output_stream_count_ = output_streams; - padding_bitsize_ = total_bitsize - used_bitsize; + for (std::size_t i = 0; i < output_stream_item_rate_.size(); ++i) + { + output_stream_item_rate_[i] = pattern_output_item_rates[i] * pattern_repeat_count; + } + + for (std::size_t repeat = 0; repeat < pattern_repeat_count; ++repeat) + { + for (const auto& stream_metadata : pattern_streams) + { + std::size_t output_item_offset = 0; + if (stream_metadata.output_index != -1) + { + const auto relative_output_index = static_cast(stream_metadata.output_index) - output_stream_offset_; + const std::size_t chronological_repeat = stream_metadata.lump.Shift() == GnssMetadata::Lump::shiftRight + ? (pattern_repeat_count - repeat - 1) + : repeat; + output_item_offset = chronological_repeat * pattern_output_item_rates[relative_output_index] + + stream_metadata.output_item_offset; + } + streams_.emplace_back( + stream_metadata.lump, + stream_metadata.stream, + stream_metadata.stream_encoding, + stream_metadata.output_index, + output_item_offset, + stream_metadata.output_item_size); + } + } + + padding_bitsize_ = total_bitsize - pattern_bitsize * pattern_repeat_count; } @@ -85,6 +151,52 @@ IONGSMSChunkData::~IONGSMSChunkData() } +uint64_t IONGSMSChunkData::low_bits_mask(const std::size_t bits) +{ + if (bits >= 64) + { + return std::numeric_limits::max(); + } + return (uint64_t{1} << bits) - 1U; +} + + +template +Word IONGSMSChunkData::byte_swap_word(Word value) +{ + if constexpr (sizeof(Word) == 1) + { + return value; + } + + Word swapped = 0; + for (std::size_t byte = 0; byte < sizeof(Word); ++byte) + { + swapped <<= 8U; + swapped |= (value >> (byte * 8U)) & static_cast(0xFFU); + } + return swapped; +} + + +bool IONGSMSChunkData::host_is_little_endian() +{ + const uint16_t one = 1; + return *reinterpret_cast(&one) == 1U; +} + + +bool IONGSMSChunkData::source_endianness_is_different(const GnssMetadata::Chunk::WordEndian endian) +{ + if (endian == GnssMetadata::Chunk::Undefined) + { + return false; + } + + return (endian == GnssMetadata::Chunk::Little) != host_is_little_endian(); +} + + std::size_t IONGSMSChunkData::read_from_buffer(uint8_t* buffer, std::size_t offset) { memset(buffer_, 0, sizeword_ * countwords_); @@ -98,16 +210,16 @@ void IONGSMSChunkData::write_to_output(gr_vector_void_star& outputs, std::vector switch (sizeword_) { case 1: - unpack_words(outputs, output_items); + unpack_words(outputs, output_items); break; case 2: - unpack_words(outputs, output_items); + unpack_words(outputs, output_items); break; case 4: - unpack_words(outputs, output_items); + unpack_words(outputs, output_items); break; case 8: - unpack_words(outputs, output_items); + unpack_words(outputs, output_items); break; default: LOG(ERROR) << "Unknown word size (" << std::to_string(sizeword_) << "), unpacking nothing."; @@ -138,13 +250,20 @@ template void IONGSMSChunkData::unpack_words(gr_vector_void_star& outputs, std::vector& output_items) { WT* data = static_cast(buffer_); - // TODO - Swap endianness if needed + if (source_endianness_is_different(chunk_.Endian())) + { + for (std::size_t i = 0; i < countwords_; ++i) + { + data[i] = byte_swap_word(data[i]); + } + } - IONGSMSChunkUnpackingCtx ctx{ - chunk_.Shift(), - data, - countwords_, - }; + if (chunk_.Shift() == GnssMetadata::Chunk::Right) + { + std::reverse(data, data + countwords_); + } + + IONGSMSChunkUnpackingCtx ctx{data, countwords_}; // Head padding if (padding_bitsize_ > 0 && chunk_.Padding() == GnssMetadata::Chunk::Head) @@ -152,8 +271,10 @@ void IONGSMSChunkData::unpack_words(gr_vector_void_star& outputs, std::vector(outputs[output_index]) + + output_item_offset * output_item_size; + output_items[output_index] += write_stream_samples(ctx, lump, stream, encoding, &output); } } + + for (int output_index : output_stream_indices_) + { + const auto relative_output_index = static_cast(output_index) - output_stream_offset_; + const auto items_written = output_items[output_index] - initial_output_items[output_index]; + outputs[output_index] = static_cast(outputs[output_index]) + items_written * output_stream_item_size_[relative_output_index]; + } } @@ -176,96 +306,437 @@ std::size_t IONGSMSChunkData::write_stream_samples( const GnssMetadata::StreamEncoding stream_encoding, void** out) { - std::size_t sample_bitsize = stream.Packedbits() / stream.RateFactor(); - std::size_t sample_count = stream.RateFactor(); - - if (stream.Packedbits() >= 2 * stream.RateFactor() * stream.Quantization()) + const std::size_t padding_bits = stream_padding_bits(stream); + if (padding_bits > 0 && stream.Alignment() == GnssMetadata::IonStream::Undefined) { - // Samples have 'Complex' format - sample_bitsize /= 2; - sample_count *= 2; + throw std::runtime_error("ION_GSMS_Signal_Source stream has packed padding bits but no alignment metadata"); } - if (sample_bitsize <= 8) + if (padding_bits > 0 && stream.Alignment() == GnssMetadata::IonStream::Right) { - write_n_samples(ctx, lump.Shift(), sample_bitsize, sample_count, stream_encoding, reinterpret_cast(out)); - } - else if (sample_bitsize <= 16) - { - write_n_samples(ctx, lump.Shift(), sample_bitsize, sample_count, stream_encoding, reinterpret_cast(out)); - } - else if (sample_bitsize <= 32) - { - write_n_samples(ctx, lump.Shift(), sample_bitsize, sample_count, stream_encoding, reinterpret_cast(out)); - } - else if (sample_bitsize <= 64) - { - write_n_samples(ctx, lump.Shift(), sample_bitsize, sample_count, stream_encoding, reinterpret_cast(out)); + ctx.shift_padding(padding_bits); } - return sample_count; + std::size_t items_written = 0; + if (stream_encoding == GnssMetadata::StreamEncodings::FP) + { + items_written = write_fp32_samples(ctx, lump.Shift(), stream, out); + } + else + { + const auto sample_bitsize = stream_output_item_bits(stream); + if (sample_bitsize <= 8) + { + items_written = write_n_samples(ctx, lump.Shift(), stream, stream_encoding, reinterpret_cast(out)); + } + else if (sample_bitsize <= 16) + { + items_written = write_n_samples(ctx, lump.Shift(), stream, stream_encoding, reinterpret_cast(out)); + } + else if (sample_bitsize <= 32) + { + items_written = write_n_samples(ctx, lump.Shift(), stream, stream_encoding, reinterpret_cast(out)); + } + else if (sample_bitsize <= 64) + { + items_written = write_n_samples(ctx, lump.Shift(), stream, stream_encoding, reinterpret_cast(out)); + } + } + + if (padding_bits > 0 && stream.Alignment() == GnssMetadata::IonStream::Left) + { + ctx.shift_padding(padding_bits); + } + + return items_written; } template -void IONGSMSChunkData::write_n_samples( +std::size_t IONGSMSChunkData::write_n_samples( IONGSMSChunkUnpackingCtx& ctx, GnssMetadata::Lump::LumpShift lump_shift, - uint8_t sample_bitsize, - std::size_t sample_count, + const GnssMetadata::IonStream& stream, GnssMetadata::StreamEncoding stream_encoding, OT** out) { - if (lump_shift == GnssMetadata::Lump::shiftRight) + const auto sample_bitsize = static_cast(stream.Quantization()); + const auto items_per_sample = stream_is_complex(stream.Format()) ? 2U : 1U; + std::vector> samples; + samples.reserve(stream.RateFactor()); + + for (std::size_t i = 0; i < stream.RateFactor(); ++i) { - auto* sample = static_cast(*out); - sample += sample_count; - for (std::size_t i = 0; i < sample_count; ++i) + std::array sample{}; + const auto first = decode_sample(sample_bitsize, ctx.read_bits(sample_bitsize), stream_encoding); + if (!stream_is_complex(stream.Format())) { - *sample = 0; - ctx.shift_sample(sample_bitsize, sample); - decode_sample(sample_bitsize, sample, stream_encoding); - --sample; + sample[0] = (stream.Format() == GnssMetadata::IonStream::IFn) ? static_cast(-first) : first; + samples.push_back(sample); + continue; + } + + const auto second = decode_sample(sample_bitsize, ctx.read_bits(sample_bitsize), stream_encoding); + switch (stream.Format()) + { + case GnssMetadata::IonStream::IQ: + case GnssMetadata::IonStream::Int8IQ: + case GnssMetadata::IonStream::Int16IQ: + sample[0] = first; + sample[1] = second; + break; + case GnssMetadata::IonStream::IQn: + sample[0] = first; + sample[1] = static_cast(-second); + break; + case GnssMetadata::IonStream::InQ: + sample[0] = static_cast(-first); + sample[1] = second; + break; + case GnssMetadata::IonStream::InQn: + sample[0] = static_cast(-first); + sample[1] = static_cast(-second); + break; + case GnssMetadata::IonStream::QI: + sample[0] = second; + sample[1] = first; + break; + case GnssMetadata::IonStream::QIn: + sample[0] = static_cast(-second); + sample[1] = first; + break; + case GnssMetadata::IonStream::QnI: + sample[0] = second; + sample[1] = static_cast(-first); + break; + case GnssMetadata::IonStream::QnIn: + sample[0] = static_cast(-second); + sample[1] = static_cast(-first); + break; + default: + throw std::runtime_error("ION_GSMS_Signal_Source unsupported complex stream format"); + } + samples.push_back(sample); + } + + const bool reverse_samples = samples_are_reversed(stream.Shift(), lump_shift); + auto* sample_out = static_cast(*out); + if (reverse_samples) + { + for (auto it = samples.rbegin(); it != samples.rend(); ++it) + { + for (std::size_t i = 0; i < items_per_sample; ++i) + { + *sample_out++ = (*it)[i]; + } } } - else // if (lump_shift == GnssMetadata::Lump::shiftLeft || lump_shift == GnssMetadata::Lump::shiftUndefined) + else { - auto* sample = static_cast(*out); - for (std::size_t i = 0; i < sample_count; ++i) + for (const auto& sample : samples) { - *sample = 0; - ctx.shift_sample(sample_bitsize, sample); - decode_sample(sample_bitsize, sample, stream_encoding); - ++sample; + for (std::size_t i = 0; i < items_per_sample; ++i) + { + *sample_out++ = sample[i]; + } } } - (*out) += sample_count; + *out = sample_out; + return samples.size() * items_per_sample; +} + + +template +std::size_t IONGSMSChunkData::write_fp32_samples( + IONGSMSChunkUnpackingCtx& ctx, + const GnssMetadata::Lump::LumpShift lump_shift, + const GnssMetadata::IonStream& stream, + void** out) +{ + if (stream.Quantization() != 32) + { + throw std::runtime_error("ION_GSMS_Signal_Source only supports FP32 stream encoding"); + } + + const bool reverse_samples = samples_are_reversed(stream.Shift(), lump_shift); + if (!stream_is_complex(stream.Format())) + { + std::vector samples; + samples.reserve(stream.RateFactor()); + for (std::size_t i = 0; i < stream.RateFactor(); ++i) + { + auto sample = read_fp32_sample(ctx); + if (stream.Format() == GnssMetadata::IonStream::IFn) + { + sample = -sample; + } + samples.push_back(sample); + } + + auto* sample_out = static_cast(*out); + if (reverse_samples) + { + for (auto it = samples.rbegin(); it != samples.rend(); ++it) + { + *sample_out++ = *it; + } + } + else + { + for (const auto sample : samples) + { + *sample_out++ = sample; + } + } + + *out = sample_out; + return samples.size(); + } + + std::vector samples; + samples.reserve(stream.RateFactor()); + for (std::size_t i = 0; i < stream.RateFactor(); ++i) + { + const auto first = read_fp32_sample(ctx); + const auto second = read_fp32_sample(ctx); + samples.push_back(complex_sample_from_format(stream.Format(), first, second)); + } + + auto* sample_out = static_cast(*out); + if (reverse_samples) + { + for (auto it = samples.rbegin(); it != samples.rend(); ++it) + { + *sample_out++ = *it; + } + } + else + { + for (const auto& sample : samples) + { + *sample_out++ = sample; + } + } + + *out = sample_out; + return samples.size(); } // Static utilities template -void IONGSMSChunkData::decode_sample(const uint8_t sample_bitsize, Sample* sample, const GnssMetadata::StreamEncoding encoding) +Sample IONGSMSChunkData::decode_sample(const uint8_t sample_bitsize, const uint64_t raw_sample, const GnssMetadata::StreamEncoding encoding) { - // using SampleType = std::remove_pointer_t; - switch (sample_bitsize) + if (sample_bitsize == 0 || sample_bitsize > 64) { - case 2: - *sample = GnssMetadata::two_bit_look_up[encoding][*sample]; + throw std::runtime_error("ION_GSMS_Signal_Source unsupported sample quantization"); + } + + const auto raw = raw_sample & low_bits_mask(sample_bitsize); + const auto magnitude_mask = low_bits_mask(sample_bitsize - 1U); + int64_t decoded = 0; + + switch (encoding) + { + case GnssMetadata::StreamEncodings::SIGN: + decoded = (raw & 0x01U) ? -1 : 1; break; - case 3: - *sample = GnssMetadata::three_bit_look_up[encoding][*sample]; + case GnssMetadata::StreamEncodings::OB: + decoded = static_cast(raw) - static_cast(uint64_t{1} << (sample_bitsize - 1U)); break; - case 4: - *sample = GnssMetadata::four_bit_look_up[encoding][*sample]; + case GnssMetadata::StreamEncodings::SM: + decoded = (raw & (uint64_t{1} << (sample_bitsize - 1U))) ? -static_cast(raw & magnitude_mask) : static_cast(raw & magnitude_mask); break; - case 5: - *sample = GnssMetadata::five_bit_look_up[encoding][*sample]; + case GnssMetadata::StreamEncodings::MS: + decoded = (raw & 0x01U) ? -static_cast((raw >> 1U) & magnitude_mask) : static_cast((raw >> 1U) & magnitude_mask); break; + case GnssMetadata::StreamEncodings::TC: + if ((raw & (uint64_t{1} << (sample_bitsize - 1U))) != 0) + { + decoded = static_cast(raw | ~low_bits_mask(sample_bitsize)); + } + else + { + decoded = static_cast(raw); + } + break; + case GnssMetadata::StreamEncodings::OG: + { + uint64_t binary = raw; + for (uint64_t mask = binary >> 1U; mask != 0; mask >>= 1U) + { + binary ^= mask; + } + decoded = static_cast(binary) - static_cast(uint64_t{1} << (sample_bitsize - 1U)); + break; + } + case GnssMetadata::StreamEncodings::OBA: + decoded = static_cast(raw << 1U) - static_cast((uint64_t{1} << sample_bitsize) - 1U); + break; + case GnssMetadata::StreamEncodings::SMA: + decoded = (raw & (uint64_t{1} << (sample_bitsize - 1U))) ? -static_cast(((raw & magnitude_mask) << 1U) | 0x01U) : static_cast(((raw & magnitude_mask) << 1U) | 0x01U); + break; + case GnssMetadata::StreamEncodings::MSA: + decoded = (raw & 0x01U) ? -static_cast((((raw >> 1U) & magnitude_mask) << 1U) | 0x01U) : static_cast((((raw >> 1U) & magnitude_mask) << 1U) | 0x01U); + break; + case GnssMetadata::StreamEncodings::TCA: + { + const auto shifted = raw << 1U; + if ((raw & (uint64_t{1} << (sample_bitsize - 1U))) != 0) + { + decoded = static_cast(shifted | ~low_bits_mask(sample_bitsize + 1U)) + 1; + } + else + { + decoded = static_cast(shifted) + 1; + } + break; + } + case GnssMetadata::StreamEncodings::OGA: + { + uint64_t binary = raw; + for (uint64_t mask = binary >> 1U; mask != 0; mask >>= 1U) + { + binary ^= mask; + } + decoded = static_cast(binary << 1U) - static_cast((uint64_t{1} << sample_bitsize) - 1U); + break; + } default: - // TODO - Is this an error that can happen? - // for now we'll just do nothing, if the sample is this wide it may need no decoding - break; + throw std::runtime_error("ION_GSMS_Signal_Source unsupported stream encoding"); + } + + return static_cast(decoded); +} + + +template +float IONGSMSChunkData::read_fp32_sample(IONGSMSChunkUnpackingCtx& ctx) +{ + const auto raw_sample = static_cast(ctx.read_bits(32)); + float sample = 0.0F; + static_assert(sizeof(sample) == sizeof(raw_sample), "FP32 sample storage must be 32 bits"); + std::memcpy(&sample, &raw_sample, sizeof(sample)); + return sample; +} + + +gr_complex IONGSMSChunkData::complex_sample_from_format(const GnssMetadata::IonStream::SampleFormat format, const float first, const float second) +{ + switch (format) + { + case GnssMetadata::IonStream::IQ: + case GnssMetadata::IonStream::Int8IQ: + case GnssMetadata::IonStream::Int16IQ: + return {first, second}; + case GnssMetadata::IonStream::IQn: + return {first, -second}; + case GnssMetadata::IonStream::InQ: + return {-first, second}; + case GnssMetadata::IonStream::InQn: + return {-first, -second}; + case GnssMetadata::IonStream::QI: + return {second, first}; + case GnssMetadata::IonStream::QIn: + return {-second, first}; + case GnssMetadata::IonStream::QnI: + return {second, -first}; + case GnssMetadata::IonStream::QnIn: + return {-second, -first}; + default: + throw std::runtime_error("ION_GSMS_Signal_Source unsupported complex stream format"); } } + + +bool IONGSMSChunkData::stream_is_complex(const GnssMetadata::IonStream::SampleFormat format) +{ + return format != GnssMetadata::IonStream::IF && format != GnssMetadata::IonStream::IFn; +} + + +uint8_t IONGSMSChunkData::stream_output_item_bits(const GnssMetadata::IonStream& stream) +{ + std::size_t output_bits = stream.Quantization(); + const auto encoding = GnssMetadata::encoding_from_string(stream.Encoding()); + if (encoding == GnssMetadata::StreamEncodings::UNKNOWN) + { + throw std::runtime_error("ION_GSMS_Signal_Source unknown stream encoding: " + stream.Encoding()); + } + if (encoding == GnssMetadata::StreamEncodings::FP) + { + if (stream.Quantization() != 32) + { + throw std::runtime_error("ION_GSMS_Signal_Source only supports FP32 stream encoding"); + } + return stream_is_complex(stream.Format()) ? static_cast(sizeof(gr_complex) * 8U) : 32U; + } + if ((encoding == GnssMetadata::StreamEncodings::OBA || + encoding == GnssMetadata::StreamEncodings::SMA || + encoding == GnssMetadata::StreamEncodings::MSA || + encoding == GnssMetadata::StreamEncodings::TCA || + encoding == GnssMetadata::StreamEncodings::OGA) && + output_bits >= 8) + { + ++output_bits; + } + + if (output_bits == 0 || output_bits > 64) + { + throw std::runtime_error("ION_GSMS_Signal_Source unsupported stream quantization"); + } + + return static_cast(output_bits); +} + + +std::size_t IONGSMSChunkData::stream_output_item_size(const GnssMetadata::IonStream& stream) +{ + if (stream_outputs_gr_complex(stream)) + { + stream_output_item_bits(stream); + return sizeof(gr_complex); + } + + return bits_to_item_size(stream_output_item_bits(stream)); +} + + +std::size_t IONGSMSChunkData::stream_output_item_rate(const GnssMetadata::IonStream& stream) +{ + const std::size_t output_items_per_sample = stream_outputs_gr_complex(stream) ? 1U : (stream_is_complex(stream.Format()) ? 2U : 1U); + return stream.RateFactor() * output_items_per_sample; +} + + +bool IONGSMSChunkData::stream_outputs_gr_complex(const GnssMetadata::IonStream& stream) +{ + return GnssMetadata::encoding_from_string(stream.Encoding()) == GnssMetadata::StreamEncodings::FP && + stream_is_complex(stream.Format()); +} + + +bool IONGSMSChunkData::samples_are_reversed(const GnssMetadata::IonStream::StreamShift stream_shift, const GnssMetadata::Lump::LumpShift lump_shift) +{ + return stream_shift == GnssMetadata::IonStream::shiftRight || + (stream_shift == GnssMetadata::IonStream::shiftUndefined && lump_shift == GnssMetadata::Lump::shiftRight); +} + + +std::size_t IONGSMSChunkData::stream_padding_bits(const GnssMetadata::IonStream& stream) +{ + if (stream.RateFactor() == 0) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream ratefactor must be greater than zero"); + } + + const std::size_t components_per_sample = stream_is_complex(stream.Format()) ? 2U : 1U; + const std::size_t used_bits = stream.RateFactor() * components_per_sample * stream.Quantization(); + if (stream.Packedbits() < used_bits) + { + throw std::runtime_error("ION_GSMS_Signal_Source stream packedbits is smaller than its sample payload"); + } + + return stream.Packedbits() - used_bits; +} diff --git a/src/algorithms/signal_source/libs/ion_gsms_chunk_data.h b/src/algorithms/signal_source/libs/ion_gsms_chunk_data.h index 5170af848..695bac9a9 100644 --- a/src/algorithms/signal_source/libs/ion_gsms_chunk_data.h +++ b/src/algorithms/signal_source/libs/ion_gsms_chunk_data.h @@ -2,13 +2,14 @@ * \file ion_gsms_chunk_data.h * \brief Holds logic for reading and decoding samples from a chunk * \author Víctor Castillo Agüero, 2024. victorcastilloaguero(at)gmail.com + * \author Carles Fernandez, 2026 carles.fernandez(at)cttc.es * * ----------------------------------------------------------------------------- * * GNSS-SDR is a Global Navigation Satellite System software-defined receiver. * This file is part of GNSS-SDR. * - * Copyright (C) 2010-2024 (see AUTHORS file for a list of contributors) + * Copyright (C) 2010-2026 (see AUTHORS file for a list of contributors) * SPDX-License-Identifier: GPL-3.0-or-later * * ----------------------------------------------------------------------------- @@ -20,10 +21,12 @@ #include "ion_gsms_chunk_unpacking_ctx.h" #include "ion_gsms_stream_encodings.h" #include +#include #include #include #include #include +#include #include #include @@ -47,19 +50,17 @@ inline std::size_t bits_to_item_size(std::size_t bit_count) return 8; } - // You are asking too much of this humble processor - std::cerr << "Item size too large (" << std::to_string(bit_count) << "), returning nonsense.\n"; - return 1; + throw std::runtime_error("ION_GSMS_Signal_Source item size is larger than 64 bits"); } // Define a functor that has a templated operator() struct Allocator { - size_t countwords_; + std::size_t countwords_; void*& buffer_; // Using void* to hold any type of pointer - Allocator(size_t countwords, void*& buffer) + Allocator(std::size_t countwords, void*& buffer) : countwords_(countwords), buffer_(buffer) {} template @@ -87,25 +88,24 @@ struct Deleter template -void with_word_type(uint8_t word_size, Callback callback) +void with_word_type(std::size_t word_size, Callback callback) { switch (word_size) { case 1: - callback.template operator()(); + callback.template operator()(); break; case 2: - callback.template operator()(); + callback.template operator()(); break; case 4: - callback.template operator()(); + callback.template operator()(); break; case 8: - callback.template operator()(); + callback.template operator()(); break; default: - std::cerr << "Unknown word size (" << std::to_string(word_size) << "), returning nonsense.\n"; - break; + throw std::runtime_error("ION_GSMS_Signal_Source unsupported chunk word size: " + std::to_string(word_size)); } } @@ -130,6 +130,11 @@ public: std::size_t output_stream_item_size(std::size_t stream_index) const; std::size_t output_stream_item_rate(std::size_t stream_index) const; + static bool stream_is_complex(GnssMetadata::IonStream::SampleFormat format); + static uint8_t stream_output_item_bits(const GnssMetadata::IonStream& stream); + static std::size_t stream_output_item_size(const GnssMetadata::IonStream& stream); + static std::size_t stream_output_item_rate(const GnssMetadata::IonStream& stream); + private: template void unpack_words(gr_vector_void_star& outputs, std::vector& output_items); @@ -143,21 +148,43 @@ private: void** out); template - void write_n_samples( + std::size_t write_n_samples( IONGSMSChunkUnpackingCtx& ctx, GnssMetadata::Lump::LumpShift lump_shift, - uint8_t sample_bitsize, - std::size_t sample_count, + const GnssMetadata::IonStream& stream, GnssMetadata::StreamEncoding stream_encoding, OT** out); + template + std::size_t write_fp32_samples( + IONGSMSChunkUnpackingCtx& ctx, + GnssMetadata::Lump::LumpShift lump_shift, + const GnssMetadata::IonStream& stream, + void** out); + template - static void decode_sample(uint8_t sample_bitsize, Sample* sample, GnssMetadata::StreamEncoding encoding); + static Sample decode_sample(uint8_t sample_bitsize, uint64_t raw_sample, GnssMetadata::StreamEncoding encoding); + + template + static float read_fp32_sample(IONGSMSChunkUnpackingCtx& ctx); + + static gr_complex complex_sample_from_format(GnssMetadata::IonStream::SampleFormat format, float first, float second); + static bool stream_outputs_gr_complex(const GnssMetadata::IonStream& stream); + static bool samples_are_reversed(GnssMetadata::IonStream::StreamShift stream_shift, GnssMetadata::Lump::LumpShift lump_shift); + static std::size_t stream_padding_bits(const GnssMetadata::IonStream& stream); + static uint64_t low_bits_mask(std::size_t bits); + + template + static Word byte_swap_word(Word value); + + static bool host_is_little_endian(); + static bool source_endianness_is_different(GnssMetadata::Chunk::WordEndian endian); const GnssMetadata::Chunk& chunk_; - uint8_t sizeword_; - uint8_t countwords_; - uint8_t padding_bitsize_; + std::size_t sizeword_; + std::size_t countwords_; + std::size_t padding_bitsize_; + std::size_t output_stream_offset_; std::size_t output_stream_count_; std::vector output_stream_item_size_; std::vector output_stream_item_rate_; @@ -168,21 +195,28 @@ private: const GnssMetadata::IonStream& stream; GnssMetadata::StreamEncoding stream_encoding; int output_index = -1; + std::size_t output_item_offset = 0; + std::size_t output_item_size = 0; stream_metadata_t( const GnssMetadata::Lump& lump_, const GnssMetadata::IonStream& stream_, GnssMetadata::StreamEncoding stream_encoding_, - int output_index_ = -1) : lump(lump_), - stream(stream_), - stream_encoding(stream_encoding_), - output_index(output_index_) + int output_index_ = -1, + std::size_t output_item_offset_ = 0, + std::size_t output_item_size_ = 0) : lump(lump_), + stream(stream_), + stream_encoding(stream_encoding_), + output_index(output_index_), + output_item_offset(output_item_offset_), + output_item_size(output_item_size_) { } }; std::vector streams_; + std::vector output_stream_indices_; - void* buffer_; + void* buffer_ = nullptr; }; #endif // GNSS_SDR_ION_GSMS_CHUNK_DATA_H diff --git a/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h b/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h index 6799f1080..9fec8ea8e 100644 --- a/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h +++ b/src/algorithms/signal_source/libs/ion_gsms_chunk_unpacking_ctx.h @@ -19,9 +19,9 @@ #ifndef GNSS_SDR_ION_GSMS_CHUNK_UNPACKING_CTX_H #define GNSS_SDR_ION_GSMS_CHUNK_UNPACKING_CTX_H -#include -#include +#include #include +#include /** \addtogroup Signal_Source * \{ */ @@ -33,149 +33,41 @@ struct IONGSMSChunkUnpackingCtx { static constexpr uint8_t word_bitsize_ = sizeof(WT) * 8; - const GnssMetadata::Chunk::WordShift word_shift_direction_; - WT* iterator_ = nullptr; // Not owned by this class, MUST NOT destroy - WT current_word_{}; - uint8_t bitshift_ = 0; + const WT* data_ = nullptr; // Not owned by this class, MUST NOT destroy. + std::size_t word_count_ = 0; + std::size_t bit_offset_ = 0; IONGSMSChunkUnpackingCtx( - const GnssMetadata::Chunk::WordShift word_shift, WT* data_buffer, - uint8_t data_buffer_word_count) : word_shift_direction_(word_shift) + std::size_t data_buffer_word_count) : data_(data_buffer), + word_count_(data_buffer_word_count) { - if (word_shift_direction_ == GnssMetadata::Chunk::Left) - { - iterator_ = data_buffer; - } - else if (word_shift_direction_ == GnssMetadata::Chunk::Right) - { - iterator_ = &data_buffer[data_buffer_word_count]; - } - if (iterator_) - { - advance_word(); // Initializes current_word_ - } } - void advance_word() + void shift_padding(std::size_t n_bits) { - WT word = *iterator_; - if (word_shift_direction_ == GnssMetadata::Chunk::Left) - { - ++iterator_; - } - else if (word_shift_direction_ == GnssMetadata::Chunk::Right) - { - --iterator_; - } - - current_word_ = word; + bit_offset_ += n_bits; } - void shift_current_word(uint8_t n) + uint64_t read_bits(std::size_t bit_count) { - if ((n % word_bitsize_) == 0) + uint64_t value = 0; + for (std::size_t bit = 0; bit < bit_count; ++bit) { - for (uint8_t i = 0; i < (n / word_bitsize_); ++i) + const std::size_t absolute_bit = bit_offset_ + bit; + const std::size_t word_index = absolute_bit / word_bitsize_; + if (word_index >= word_count_) { - advance_word(); + throw std::runtime_error("ION_GSMS_Signal_Source tried to read past the chunk boundary"); } - return; + const std::size_t bit_index = absolute_bit % word_bitsize_; + const std::size_t word_bit = word_bitsize_ - 1 - bit_index; + value <<= 1; + value |= (static_cast(data_[word_index]) >> word_bit) & 0x01U; } - if (word_shift_direction_ == GnssMetadata::Chunk::Left) - { - current_word_ <<= n; - } - else if (word_shift_direction_ == GnssMetadata::Chunk::Right) - { - current_word_ >>= n; - } - - bitshift_ += n; - if (bitshift_ >= word_bitsize_) - { - advance_word(); - bitshift_ -= word_bitsize_; - } - } - - void shift_padding(uint8_t n_bits) - { - if (n_bits == 0) - { - return; - } - - if ((n_bits + (bitshift_ % word_bitsize_)) >= word_bitsize_) - { - const uint8_t bits_shifted = word_bitsize_ - (bitshift_ % word_bitsize_); - - shift_current_word(bits_shifted); - shift_padding(n_bits - bits_shifted); - } - else - { - shift_current_word(n_bits); - } - } - - template - void shift_sample(uint8_t sample_bitsize, OT* output, uint8_t output_bit_offset = 0) - { - if (sample_bitsize % word_bitsize_ == 0) - { - const uint8_t words_per_sample = sample_bitsize / word_bitsize_; - for (uint8_t i = 0; i < words_per_sample; ++i) - { - if (word_shift_direction_ == GnssMetadata::Chunk::Left) - { - *output |= (current_word_ << ((words_per_sample - 1 - i) * word_bitsize_)); - } - else if (word_shift_direction_ == GnssMetadata::Chunk::Right) - { - *output |= (current_word_ << (i * word_bitsize_)); - // TODO - reverse bit order of sample? maybe? - } - advance_word(); - } - } - else if ((sample_bitsize + (bitshift_ % word_bitsize_)) > word_bitsize_) - { - const uint8_t bits_shifted = word_bitsize_ - (bitshift_ % word_bitsize_); - - if (word_shift_direction_ == GnssMetadata::Chunk::Left) - { - WT mask = ~((1 << (word_bitsize_ - bits_shifted)) - 1); - *output |= ((current_word_ & mask) >> output_bit_offset); - } - else if (word_shift_direction_ == GnssMetadata::Chunk::Right) - { - WT mask = ((1 << (bits_shifted)) - 1); - *output |= (current_word_ & mask) << output_bit_offset; - // TODO - reverse bit order of sample? maybe? - } - - shift_current_word(bits_shifted); - shift_sample(sample_bitsize - bits_shifted, output, bits_shifted); - } - else - { - if (word_shift_direction_ == GnssMetadata::Chunk::Left) - { - WT mask = ~((1 << (word_bitsize_ - sample_bitsize)) - 1); - OT sample = (current_word_ & mask) >> (word_bitsize_ - sample_bitsize); - *output |= (sample) >> output_bit_offset; - } - else if (word_shift_direction_ == GnssMetadata::Chunk::Right) - { - WT mask = ((1 << (sample_bitsize)) - 1); - *output |= (current_word_ & mask) << output_bit_offset; - // TODO - reverse bit order of sample? maybe? - } - - shift_current_word(sample_bitsize); - } + bit_offset_ += bit_count; + return value; } }; diff --git a/src/algorithms/signal_source/libs/ion_gsms_stream_encodings.h b/src/algorithms/signal_source/libs/ion_gsms_stream_encodings.h index 5b39e6497..955485881 100644 --- a/src/algorithms/signal_source/libs/ion_gsms_stream_encodings.h +++ b/src/algorithms/signal_source/libs/ion_gsms_stream_encodings.h @@ -46,6 +46,7 @@ constexpr unsigned char MSA = 8; constexpr unsigned char TCA = 9; constexpr unsigned char OGA = 10; constexpr unsigned char FP = 11; +constexpr unsigned char UNKNOWN = 255; } // namespace StreamEncodings @@ -99,7 +100,7 @@ inline StreamEncoding encoding_from_string(const std::string& str) { return StreamEncodings::FP; } - return 0; + return StreamEncodings::UNKNOWN; } template diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0ce5f7f95..efcbf7145 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1144,6 +1144,11 @@ if(NOT ENABLE_PACKAGING AND NOT ENABLE_FPGA) ${CMAKE_CURRENT_SOURCE_DIR}/single_test_main.cc ${CMAKE_CURRENT_SOURCE_DIR}/unit-tests/signal-processing-blocks/sources/unpack_2bit_samples_test.cc ) + if(ENABLE_ION) + list(APPEND GNURADIO_BLOCK_TEST_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc + ) + endif() if(USE_CMAKE_TARGET_SOURCES) add_executable(gnuradio_block_test) target_sources(gnuradio_block_test PRIVATE ${GNURADIO_BLOCK_TEST_SOURCES}) diff --git a/tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc b/tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc new file mode 100644 index 000000000..a5088a848 --- /dev/null +++ b/tests/unit-tests/signal-processing-blocks/sources/ion_gsms_chunk_data_test.cc @@ -0,0 +1,577 @@ +/*! + * \file ion_gsms_chunk_data_test.cc + * \brief Unit tests for ION GNSS Metadata Standard chunk unpacking + * \author Carles Fernandez-Prades, 2026. cfernandez(at)cttc.es + * ----------------------------------------------------------------------------- + * + * GNSS-SDR is a Global Navigation Satellite System software-defined receiver. + * This file is part of GNSS-SDR. + * + * Copyright (C) 2010-2026 (see AUTHORS file for a list of contributors) + * SPDX-License-Identifier: GPL-3.0-or-later + * + * ----------------------------------------------------------------------------- + */ + +#include "ion_gsms_chunk_data.h" +#include "ion_gsms.h" +#include "gnss_sdr_filesystem.h" +#include "gnss_sdr_flags.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +GnssMetadata::IonStream make_stream( + const GnssMetadata::IonStream::SampleFormat format, + const std::string& encoding, + const std::size_t quantization, + const std::size_t packed_bits, + const std::size_t rate_factor, + const GnssMetadata::IonStream::StreamAlignment alignment = GnssMetadata::IonStream::Undefined, + const GnssMetadata::IonStream::StreamShift shift = GnssMetadata::IonStream::shiftUndefined, + const std::string& id = "L1") +{ + GnssMetadata::IonStream stream(id); + stream.Format(format); + stream.Encoding(encoding); + stream.Quantization(quantization); + stream.Packedbits(packed_bits); + stream.RateFactor(rate_factor); + stream.Alignment(alignment); + stream.Shift(shift); + return stream; +} + + +GnssMetadata::Chunk make_chunk( + const GnssMetadata::IonStream& stream, + const std::size_t size_word, + const std::size_t count_words, + const GnssMetadata::Chunk::WordEndian endian = GnssMetadata::Chunk::Undefined, + const GnssMetadata::Chunk::WordPadding padding = GnssMetadata::Chunk::Tail, + const GnssMetadata::Chunk::WordShift shift = GnssMetadata::Chunk::Left, + const GnssMetadata::Lump::LumpShift lump_shift = GnssMetadata::Lump::shiftUndefined) +{ + GnssMetadata::Lump lump; + lump.Shift(lump_shift); + lump.Streams().push_back(stream); + + GnssMetadata::Chunk chunk; + chunk.SizeWord(size_word); + chunk.CountWords(count_words); + chunk.Endian(endian); + chunk.Padding(padding); + chunk.Shift(shift); + chunk.Lumps().push_back(lump); + return chunk; +} + + +std::vector> decode_int8_chunk( + const GnssMetadata::Chunk& chunk, + const std::vector& stream_ids, + const std::vector& bytes) +{ + IONGSMSChunkData chunk_data(chunk, stream_ids, 0); + + auto input = bytes; + chunk_data.read_from_buffer(input.data(), 0); + + std::vector> output(chunk_data.output_stream_count()); + gr_vector_void_star outputs(chunk_data.output_stream_count()); + for (std::size_t i = 0; i < output.size(); ++i) + { + output[i].resize(chunk_data.output_stream_item_rate(i), 0); + outputs[i] = output[i].data(); + } + std::vector output_items(chunk_data.output_stream_count(), 0); + + chunk_data.write_to_output(outputs, output_items); + for (std::size_t i = 0; i < output.size(); ++i) + { + output[i].resize(static_cast(output_items[i])); + } + return output; +} + + +std::vector decode_int8_stream( + const GnssMetadata::IonStream& stream, + const std::vector& bytes, + const std::size_t size_word = 1, + const std::size_t count_words = 1, + const GnssMetadata::Chunk::WordEndian endian = GnssMetadata::Chunk::Undefined, + const GnssMetadata::Chunk::WordPadding padding = GnssMetadata::Chunk::Tail, + const GnssMetadata::Chunk::WordShift chunk_shift = GnssMetadata::Chunk::Left, + const GnssMetadata::Lump::LumpShift lump_shift = GnssMetadata::Lump::shiftUndefined) +{ + auto chunk = make_chunk(stream, size_word, count_words, endian, padding, chunk_shift, lump_shift); + return decode_int8_chunk(chunk, {stream.Id()}, bytes).front(); +} + + +std::vector fp32_little_endian_bytes(const std::vector& samples) +{ + std::vector bytes; + bytes.reserve(samples.size() * sizeof(uint32_t)); + for (const auto sample : samples) + { + uint32_t raw = 0; + static_assert(sizeof(sample) == sizeof(raw), "FP32 sample storage must be 32 bits"); + std::memcpy(&raw, &sample, sizeof(raw)); + bytes.push_back(static_cast(raw & 0xFFU)); + bytes.push_back(static_cast((raw >> 8U) & 0xFFU)); + bytes.push_back(static_cast((raw >> 16U) & 0xFFU)); + bytes.push_back(static_cast((raw >> 24U) & 0xFFU)); + } + return bytes; +} + + +std::vector decode_float_stream( + const GnssMetadata::IonStream& stream, + const std::vector& bytes, + const std::size_t size_word = 4, + const std::size_t count_words = 1, + const GnssMetadata::Chunk::WordEndian endian = GnssMetadata::Chunk::Little, + const GnssMetadata::Chunk::WordPadding padding = GnssMetadata::Chunk::Tail, + const GnssMetadata::Chunk::WordShift chunk_shift = GnssMetadata::Chunk::Left, + const GnssMetadata::Lump::LumpShift lump_shift = GnssMetadata::Lump::shiftUndefined) +{ + auto chunk = make_chunk(stream, size_word, count_words, endian, padding, chunk_shift, lump_shift); + IONGSMSChunkData chunk_data(chunk, {stream.Id()}, 0); + + auto input = bytes; + chunk_data.read_from_buffer(input.data(), 0); + + std::vector output(chunk_data.output_stream_item_rate(0), 0.0F); + gr_vector_void_star outputs(chunk_data.output_stream_count()); + outputs[0] = output.data(); + std::vector output_items(chunk_data.output_stream_count(), 0); + + chunk_data.write_to_output(outputs, output_items); + output.resize(static_cast(output_items[0])); + return output; +} + + +std::vector decode_complex_stream( + const GnssMetadata::IonStream& stream, + const std::vector& bytes, + const std::size_t size_word = 4, + const std::size_t count_words = 2, + const GnssMetadata::Chunk::WordEndian endian = GnssMetadata::Chunk::Little, + const GnssMetadata::Chunk::WordPadding padding = GnssMetadata::Chunk::Tail, + const GnssMetadata::Chunk::WordShift chunk_shift = GnssMetadata::Chunk::Left, + const GnssMetadata::Lump::LumpShift lump_shift = GnssMetadata::Lump::shiftUndefined) +{ + auto chunk = make_chunk(stream, size_word, count_words, endian, padding, chunk_shift, lump_shift); + IONGSMSChunkData chunk_data(chunk, {stream.Id()}, 0); + + auto input = bytes; + chunk_data.read_from_buffer(input.data(), 0); + + std::vector output(chunk_data.output_stream_item_rate(0)); + gr_vector_void_star outputs(chunk_data.output_stream_count()); + outputs[0] = output.data(); + std::vector output_items(chunk_data.output_stream_count(), 0); + + chunk_data.write_to_output(outputs, output_items); + output.resize(static_cast(output_items[0])); + return output; +} + + +GnssMetadata::Block make_file_block( + const std::size_t cycles, + const std::size_t size_header = 0, + const std::size_t size_footer = 0) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); + auto chunk = make_chunk(stream, 1, 1); + + GnssMetadata::Block block; + block.Cycles(cycles); + block.SizeHeader(size_header); + block.SizeFooter(size_footer); + block.Chunks().push_back(chunk); + return block; +} + + +void write_binary_file(const fs::path& path, const std::vector& bytes) +{ + std::ofstream file(path.c_str(), std::ios::out | std::ios::binary | std::ios::trunc); + for (const auto byte : bytes) + { + file.put(static_cast(byte)); + } +} + + +GnssMetadata::File make_data_file(const fs::path& data_path, const std::size_t offset) +{ + GnssMetadata::File file; + file.Url(GnssMetadata::AnyUri(data_path.filename().string())); + file.Offset(offset); + return file; +} + + +std::vector run_file_source( + const fs::path& metadata_path, + const GnssMetadata::File& file, + const GnssMetadata::Block& block, + const std::size_t block_start_offset) +{ + auto source = gnss_make_shared(metadata_path, file, block, block_start_offset, std::vector{"L1"}); + auto sink = gr::blocks::vector_sink_b::make(); + auto top_block = gr::make_top_block("IONGSMSFileSourceTest"); + top_block->connect(source, 0, sink, 0); + top_block->run(); + + const auto sink_data = sink->data(); + std::vector output; + output.reserve(sink_data.size()); + for (const auto item : sink_data) + { + output.push_back(static_cast(item)); + } + return output; +} + + +std::vector run_complex_file_source( + const fs::path& metadata_path, + const GnssMetadata::File& file, + const GnssMetadata::Block& block, + const std::size_t block_start_offset) +{ + auto source = gnss_make_shared(metadata_path, file, block, block_start_offset, std::vector{"L1"}); + auto sink = gr::blocks::vector_sink_c::make(); + auto top_block = gr::make_top_block("IONGSMSFileSourceTest"); + top_block->connect(source, 0, sink, 0); + top_block->run(); + + return sink->data(); +} +} // namespace + + +TEST(IONGSMSChunkDataTest, DecodesSignIqNFromDeclaredFormat) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IQn, "SIGN", 1, 8, 1, GnssMetadata::IonStream::Left); + + const auto output = decode_int8_stream(stream, {0b01000000}); + + const std::vector expected{1, 1}; + EXPECT_EQ(expected, output); +} + + +TEST(IONGSMSChunkDataTest, DecodesInQnFromDeclaredFormat) +{ + const auto stream = make_stream(GnssMetadata::IonStream::InQn, "SIGN", 1, 8, 1, GnssMetadata::IonStream::Left); + + const auto output = decode_int8_stream(stream, {0b01000000}); + + const std::vector expected{-1, 1}; + EXPECT_EQ(expected, output); +} + + +TEST(IONGSMSChunkDataTest, HonorsRightAlignedStreamPadding) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 8, 1, GnssMetadata::IonStream::Right); + + const auto output = decode_int8_stream(stream, {0b00000011}); + + const std::vector expected{-1}; + EXPECT_EQ(expected, output); +} + + +TEST(IONGSMSChunkDataTest, HonorsStreamShiftRightWithoutOverrun) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 8, 2, GnssMetadata::IonStream::Left, GnssMetadata::IonStream::shiftRight); + + const auto output = decode_int8_stream(stream, {0b01100000}); + + const std::vector expected{-2, 1}; + EXPECT_EQ(expected, output); +} + + +TEST(IONGSMSChunkDataTest, DecodesBigEndianWordsAsUnsigned) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "OB", 4, 16, 1, GnssMetadata::IonStream::Left); + + const auto output = decode_int8_stream(stream, {0x12, 0x34}, 2, 1, GnssMetadata::Chunk::Big); + + const std::vector expected{-7}; + EXPECT_EQ(expected, output); +} + + +TEST(IONGSMSChunkDataTest, RepeatsSingleLumpPayloadToFillChunk) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); + + const auto output = decode_int8_stream(stream, {0b01101100}); + + const std::vector expected{1, -2, -1, 0}; + EXPECT_EQ(expected, output); +} + + +TEST(IONGSMSChunkDataTest, DecodesMultipleLumpsInDeclaredOrder) +{ + const auto first_stream = make_stream(GnssMetadata::IonStream::IF, "TC", 4, 4, 1, GnssMetadata::IonStream::Undefined, GnssMetadata::IonStream::shiftUndefined, "L1"); + const auto second_stream = make_stream(GnssMetadata::IonStream::IF, "TC", 4, 4, 1, GnssMetadata::IonStream::Undefined, GnssMetadata::IonStream::shiftUndefined, "L2"); + + GnssMetadata::Lump first_lump; + first_lump.Streams().push_back(first_stream); + GnssMetadata::Lump second_lump; + second_lump.Streams().push_back(second_stream); + + GnssMetadata::Chunk chunk; + chunk.SizeWord(1); + chunk.CountWords(1); + chunk.Endian(GnssMetadata::Chunk::Undefined); + chunk.Padding(GnssMetadata::Chunk::Tail); + chunk.Shift(GnssMetadata::Chunk::Left); + chunk.Lumps().push_back(first_lump); + chunk.Lumps().push_back(second_lump); + + const auto output = decode_int8_chunk(chunk, {"L1", "L2"}, {0b00011111}); + + ASSERT_EQ(2U, output.size()); + const std::vector expected_first{1}; + const std::vector expected_second{-1}; + EXPECT_EQ(expected_first, output[0]); + EXPECT_EQ(expected_second, output[1]); +} + + +TEST(IONGSMSChunkDataTest, RepeatsOrderedMultiLumpPatternToFillChunk) +{ + const auto first_stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1, GnssMetadata::IonStream::Undefined, GnssMetadata::IonStream::shiftUndefined, "L1"); + const auto second_stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1, GnssMetadata::IonStream::Undefined, GnssMetadata::IonStream::shiftUndefined, "L2"); + + GnssMetadata::Lump first_lump; + first_lump.Streams().push_back(first_stream); + GnssMetadata::Lump second_lump; + second_lump.Streams().push_back(second_stream); + + GnssMetadata::Chunk chunk; + chunk.SizeWord(1); + chunk.CountWords(1); + chunk.Endian(GnssMetadata::Chunk::Undefined); + chunk.Padding(GnssMetadata::Chunk::Tail); + chunk.Shift(GnssMetadata::Chunk::Left); + chunk.Lumps().push_back(first_lump); + chunk.Lumps().push_back(second_lump); + + const auto output = decode_int8_chunk(chunk, {"L1", "L2"}, {0b01101100}); + + ASSERT_EQ(2U, output.size()); + const std::vector expected_first{1, -1}; + const std::vector expected_second{-2, 0}; + EXPECT_EQ(expected_first, output[0]); + EXPECT_EQ(expected_second, output[1]); +} + + +TEST(IONGSMSChunkDataTest, CollapsesRepeatedStreamIdIntoOneOutput) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); + + GnssMetadata::Lump first_lump; + first_lump.Streams().push_back(stream); + GnssMetadata::Lump second_lump; + second_lump.Streams().push_back(stream); + + GnssMetadata::Chunk chunk; + chunk.SizeWord(1); + chunk.CountWords(1); + chunk.Endian(GnssMetadata::Chunk::Undefined); + chunk.Padding(GnssMetadata::Chunk::Tail); + chunk.Shift(GnssMetadata::Chunk::Left); + chunk.Lumps().push_back(first_lump); + chunk.Lumps().push_back(second_lump); + + const auto output = decode_int8_chunk(chunk, {"L1"}, {0b01101100}); + + ASSERT_EQ(1U, output.size()); + const std::vector expected{1, -2, -1, 0}; + EXPECT_EQ(expected, output[0]); +} + + +TEST(IONGSMSChunkDataTest, DecodesFp32IfSamples) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "FP", 32, 64, 2); + + const auto output = decode_float_stream(stream, fp32_little_endian_bytes({1.25F, -0.5F}), 4, 2); + + ASSERT_EQ(2U, output.size()); + EXPECT_FLOAT_EQ(1.25F, output[0]); + EXPECT_FLOAT_EQ(-0.5F, output[1]); +} + + +TEST(IONGSMSChunkDataTest, NegatesFp32IfnSamples) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IFn, "FP", 32, 32, 1); + + const auto output = decode_float_stream(stream, fp32_little_endian_bytes({2.0F})); + + ASSERT_EQ(1U, output.size()); + EXPECT_FLOAT_EQ(-2.0F, output[0]); +} + + +TEST(IONGSMSChunkDataTest, DecodesFp32IqAsGrComplex) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IQ, "FP", 32, 64, 1); + const auto chunk = make_chunk(stream, 4, 2, GnssMetadata::Chunk::Little); + IONGSMSChunkData chunk_data(chunk, {stream.Id()}, 0); + + EXPECT_EQ(sizeof(gr_complex), chunk_data.output_stream_item_size(0)); + EXPECT_EQ(1U, chunk_data.output_stream_item_rate(0)); + + const auto output = decode_complex_stream(stream, fp32_little_endian_bytes({1.5F, -2.25F})); + + ASSERT_EQ(1U, output.size()); + EXPECT_FLOAT_EQ(1.5F, output[0].real()); + EXPECT_FLOAT_EQ(-2.25F, output[0].imag()); +} + + +TEST(IONGSMSChunkDataTest, AppliesFp32ComplexFormatSignAndOrder) +{ + const auto stream = make_stream(GnssMetadata::IonStream::QIn, "FP", 32, 64, 1); + + const auto output = decode_complex_stream(stream, fp32_little_endian_bytes({3.0F, 4.0F})); + + ASSERT_EQ(1U, output.size()); + EXPECT_FLOAT_EQ(-4.0F, output[0].real()); + EXPECT_FLOAT_EQ(3.0F, output[0].imag()); +} + + +TEST(IONGSMSChunkDataTest, RejectsUnsupportedFloatingPointQuantization) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "FP", 64, 64, 1); + const auto chunk = make_chunk(stream, 8, 1); + + EXPECT_THROW(IONGSMSChunkData(chunk, {stream.Id()}, 0), std::runtime_error); +} + + +TEST(IONGSMSChunkDataTest, RejectsUnsupportedChunkWordSizeAtConstruction) +{ + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); + const auto chunk = make_chunk(stream, 3, 1); + + EXPECT_THROW(IONGSMSChunkData(chunk, {stream.Id()}, 0), std::runtime_error); +} + + +TEST(IONGSMSFileSourceTest, DecodesOnlyCompleteCyclesReadFromFile) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_file_source_short_read.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_file_source_short_read.sdrx"; + write_binary_file(data_path, {static_cast('H'), static_cast(0b01101100U), static_cast(0b00011011U)}); + + const auto file = make_data_file(data_path, 0); + const auto block = make_file_block(3, 1); + + const auto output = run_file_source(metadata_path, file, block, 0); + + const std::vector expected{1, -2, -1, 0, 0, 1, -2, -1}; + EXPECT_EQ(expected, output); + + fs::remove(data_path); +} + + +TEST(IONGSMSFileSourceTest, CollapsesRepeatedStreamIdAcrossChunksIntoOneOutput) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_file_source_repeated_stream.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_file_source_repeated_stream.sdrx"; + write_binary_file(data_path, {static_cast(0b01101100U), static_cast(0b00011011U)}); + + const auto stream = make_stream(GnssMetadata::IonStream::IF, "TC", 2, 2, 1); + auto first_chunk = make_chunk(stream, 1, 1); + auto second_chunk = make_chunk(stream, 1, 1); + + GnssMetadata::Block block; + block.Cycles(1); + block.Chunks().push_back(first_chunk); + block.Chunks().push_back(second_chunk); + + const auto file = make_data_file(data_path, 0); + const auto output = run_file_source(metadata_path, file, block, 0); + + const std::vector expected{1, -2, -1, 0, 0, 1, -2, -1}; + EXPECT_EQ(expected, output); + + fs::remove(data_path); +} + + +TEST(IONGSMSFileSourceTest, StartsReadingAtProvidedBlockOffset) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_file_source_block_offset.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_file_source_block_offset.sdrx"; + write_binary_file(data_path, {static_cast('P'), static_cast('P'), static_cast('A'), static_cast(0b11111111U), static_cast('F'), static_cast('B'), static_cast(0b01101100U), static_cast('G')}); + + const auto file = make_data_file(data_path, 2); + const auto block = make_file_block(1, 1, 1); + const std::size_t second_block_offset = 2 + 1 + 1 + 1; + + const auto output = run_file_source(metadata_path, file, block, second_block_offset); + + const std::vector expected{1, -2, -1, 0}; + EXPECT_EQ(expected, output); + + fs::remove(data_path); +} + + +TEST(IONGSMSFileSourceTest, EmitsFp32IqAsComplexItems) +{ + const fs::path temp_dir(GetTempDir()); + const fs::path data_path = temp_dir / "ion_gsms_file_source_fp32_iq.bin"; + const fs::path metadata_path = temp_dir / "ion_gsms_file_source_fp32_iq.sdrx"; + write_binary_file(data_path, fp32_little_endian_bytes({1.5F, -2.25F})); + + const auto stream = make_stream(GnssMetadata::IonStream::IQ, "FP", 32, 64, 1); + auto chunk = make_chunk(stream, 4, 2, GnssMetadata::Chunk::Little); + + GnssMetadata::Block block; + block.Cycles(1); + block.Chunks().push_back(chunk); + + const auto file = make_data_file(data_path, 0); + const auto output = run_complex_file_source(metadata_path, file, block, 0); + + ASSERT_EQ(1U, output.size()); + EXPECT_FLOAT_EQ(1.5F, output[0].real()); + EXPECT_FLOAT_EQ(-2.25F, output[0].imag()); + + fs::remove(data_path); +}