From a1b06f747b2a03c3b939269dbdf28be419996c80 Mon Sep 17 00:00:00 2001 From: Carles Fernandez Date: Sun, 21 Jun 2020 21:10:22 +0200 Subject: [PATCH] Add benchmarks of small code snippets --- .clang-format | 2 +- CMakeLists.txt | 6 ++ docs/changelog.md | 2 + src/tests/CMakeLists.txt | 4 + src/tests/benchmarks/CMakeLists.txt | 112 +++++++++++++++++++++ src/tests/benchmarks/README.md | 66 ++++++++++++ src/tests/benchmarks/benchmark_copy.cc | 83 +++++++++++++++ src/tests/benchmarks/benchmark_detector.cc | 87 ++++++++++++++++ src/tests/benchmarks/benchmark_preamble.cc | 62 ++++++++++++ 9 files changed, 423 insertions(+), 1 deletion(-) create mode 100644 src/tests/benchmarks/CMakeLists.txt create mode 100644 src/tests/benchmarks/README.md create mode 100644 src/tests/benchmarks/benchmark_copy.cc create mode 100644 src/tests/benchmarks/benchmark_detector.cc create mode 100644 src/tests/benchmarks/benchmark_preamble.cc diff --git a/.clang-format b/.clang-format index 7679d254e..c8cd017d2 100644 --- a/.clang-format +++ b/.clang-format @@ -53,7 +53,7 @@ IncludeBlocks: Merge IncludeCategories: - Regex: '^.*.h"' Priority: 1 - - Regex: '^.*(boost|gflags|glog|gnsssdr|gnuradio|gpstk|gsl|gtest|pmt|uhd|volk)/' + - Regex: '^.*(benchmark|boost|gflags|glog|gnsssdr|gnuradio|gpstk|gsl|gtest|pmt|uhd|volk)/' Priority: 2 - Regex: '^.*(armadillo|iio|matio|pugixml)' Priority: 2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 207b1ec1a..0bb760e64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,10 @@ if(ENABLE_FPGA) set(ENABLE_INSTALL_TESTS ON) endif() +option(ENABLE_BENCHMARKS "Build code snippets benchmarks" OFF) +if(CMAKE_VERSION VERSION_LESS 3.5.1) + set(ENABLE_BENCHMARKS OFF) +endif() ################################################################################ @@ -320,6 +324,7 @@ set(GNSSSDR_GPSTK_LOCAL_VERSION "3.0.0") set(GNSSSDR_MATIO_LOCAL_VERSION "1.5.17") set(GNSSSDR_PUGIXML_LOCAL_VERSION "1.10") set(GNSSSDR_PROTOCOLBUFFERS_LOCAL_VERSION "3.12.3") +set(GNSSSDR_BENCHMARK_LOCAL_VERSION "1.5.1") if(CMAKE_VERSION VERSION_LESS "3.0.2") set(GNSSSDR_GFLAGS_LOCAL_VERSION "2.2.1") # Fix for CentOS 7 @@ -3045,6 +3050,7 @@ add_feature_info(ENABLE_SYSTEM_TESTING_EXTRA ENABLE_SYSTEM_TESTING_EXTRA "Enable add_feature_info(ENABLE_OWN_GPSTK ENABLE_OWN_GPSTK "Forces the downloading and building of GPSTk for system tests.") add_feature_info(ENABLE_GNSS_SIM_INSTALL ENABLE_GNSS_SIM_INSTALL "Enables downloading and building of gnss-sim.") add_feature_info(ENABLE_INSTALL_TESTS ENABLE_INSTALL_TESTS "Install test binaries when doing '${CMAKE_MAKE_PROGRAM_PRETTY_NAME} install'.") +add_feature_info(ENABLE_BENCHMARKS ENABLE_BENCHMARKS "Enables building of code snippet benchmarks.") message(STATUS "") message(STATUS "***************************************") diff --git a/docs/changelog.md b/docs/changelog.md index 21c028a1b..237fc68b7 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -37,6 +37,8 @@ SPDX-FileCopyrightText: 2011-2020 Carles Fernandez-Prades :Debug>$<$:Release>$<$:RelWithDebInfo>$<$:MinSizeRel>$<$:Debug>$<$:Debug>$<$:RelWithDebInfo>$<$:RelWithDebInfo>$<$:Debug> + ) + if(CMAKE_GENERATOR STREQUAL Xcode) + set(BENCHMARK_BUILD_COMMAND "xcodebuild" "-configuration" $<$:Debug>$<$:Release>$<$:RelWithDebInfo>$<$:MinSizeRel> "-target" "benchmark_main") + endif() + + if((CMAKE_VERSION VERSION_GREATER 3.12.0) AND NOT (CMAKE_GENERATOR STREQUAL Xcode)) + set(BENCHMARK_PARALLEL_BUILD "--parallel 2") + endif() + + ExternalProject_Add(google-benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION} + GIT_REPOSITORY https://github.com/google/benchmark + GIT_TAG v${GNSSSDR_BENCHMARK_LOCAL_VERSION} + SOURCE_DIR ${CMAKE_BINARY_DIR}/thirdparty/benchmark + BINARY_DIR ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION} + CMAKE_ARGS ${BENCHMARK_COMPILER} + -DBENCHMARK_ENABLE_GTEST_TESTS=OFF + ${BENCHMARK_TOOLCHAIN_ARG} + -DCMAKE_BUILD_TYPE=$<$:Debug>$<$:Release>$<$:RelWithDebInfo>$<$:MinSizeRel>$<$:Debug>$<$:Debug>$<$:RelWithDebInfo>$<$:RelWithDebInfo>$<$:Debug> + BUILD_COMMAND ${BENCHMARK_BUILD_COMMAND} ${BENCHMARK_PARALLEL_BUILD} + UPDATE_COMMAND "" + PATCH_COMMAND "" + BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + INSTALL_COMMAND "" + ) + + add_library(benchmark::benchmark STATIC IMPORTED) + add_dependencies(benchmark::benchmark google-benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}) + file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/thirdparty/benchmark/include) + set_target_properties(benchmark::benchmark PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" + IMPORTED_CONFIGURATIONS "None;Debug;Release;RelWithDebInfo;MinSizeRel" + MAP_IMPORTED_CONFIG_NOOPTWITHASM Debug + MAP_IMPORTED_CONFIG_COVERAGE Debug + MAP_IMPORTED_CONFIG_O2WITHASM RelWithDebInfo + MAP_IMPORTED_CONFIG_O3WITHASM RelWithDebInfo + MAP_IMPORTED_CONFIG_ASAN Debug + IMPORTED_LOCATION_NONE ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + IMPORTED_LOCATION_DEBUG ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + IMPORTED_LOCATION_RELEASE ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + IMPORTED_LOCATION_RELWITHDEBINFO ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + IMPORTED_LOCATION_MINSIZEREL ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_BINARY_DIR}/thirdparty/benchmark/include + INTERFACE_LINK_LIBRARIES "${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX};${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark_main${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + if((CMAKE_GENERATOR STREQUAL Xcode) OR MSVC) + set_target_properties(benchmark::benchmark PROPERTIES + IMPORTED_LOCATION_DEBUG ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/Debug/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + IMPORTED_LOCATION_RELEASE ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/Release/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + IMPORTED_LOCATION_RELWITHDEBINFO ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/RelWithDebInfo/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + IMPORTED_LOCATION_MINSIZEREL ${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/MinSizeRel/${CMAKE_FIND_LIBRARY_PREFIXES}benchmark${CMAKE_STATIC_LIBRARY_SUFFIX} + INTERFACE_LINK_LIBRARIES "${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/$<$:Debug/>$<$:Release/>$<$:RelWithDebInfo/>$<$:MinSizeRel/>benchmark${CMAKE_STATIC_LIBRARY_SUFFIX};${CMAKE_BINARY_DIR}/benchmark-${GNSSSDR_BENCHMARK_LOCAL_VERSION}/src/$<$:Debug/>$<$:Release/>$<$:RelWithDebInfo/>$<$:MinSizeRel/>benchmark_main${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + endif() + set_target_properties(benchmark::benchmark PROPERTIES + INTERFACE_LINK_LIBRARIES Threads::Threads + ) + +endif() + +add_executable(benchmark_copy benchmark_copy.cc) +target_link_libraries(benchmark_copy PRIVATE benchmark::benchmark) + +add_custom_command(TARGET benchmark_copy POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ + ${LOCAL_INSTALL_BASE_DIR}/install/$ +) + + +add_executable(benchmark_detector benchmark_detector.cc) +target_link_libraries(benchmark_detector PRIVATE core_system_parameters benchmark::benchmark) + +add_custom_command(TARGET benchmark_detector POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ + ${LOCAL_INSTALL_BASE_DIR}/install/$ +) + + +add_executable(benchmark_preamble benchmark_preamble.cc) +target_link_libraries(benchmark_preamble PRIVATE core_system_parameters benchmark::benchmark) + +add_custom_command(TARGET benchmark_preamble POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy $ + ${LOCAL_INSTALL_BASE_DIR}/install/$ +) diff --git a/src/tests/benchmarks/README.md b/src/tests/benchmarks/README.md new file mode 100644 index 000000000..d67a57bf2 --- /dev/null +++ b/src/tests/benchmarks/README.md @@ -0,0 +1,66 @@ + +[comment]: # ( +SPDX-License-Identifier: GPL-3.0-or-later +) + +[comment]: # ( +SPDX-FileCopyrightText: 2020 Carles Fernandez-Prades +) + + +## Benchmarks + +This is a collection of implementation benchmarks based on +[Benchmark](https://github.com/google/benchmark). It is useful to developers for +assessing which is the fastest implementation of a given snippet of code in +their machine, and to keep a track about them over different machines and +compilers. The results may vary over different system architectures, compiler +versions, building modes, C++ standard version used by the compiler, etc. + +If Benchmark is not found in your system, CMake will download, build and +statically link against that library for you at building time. + +This collection is only built if the option `ENABLE_BENCHMARKS` is enabled when +configuring `gnss-sdr` project's building: + +``` +$ cmake -DENABLE_BENCHMARKS=ON .. +``` + +## Basic usage + +Just execute the binaries generated in your `install` folder. + +Example: + +``` +$ cd ../install +$ ./benchmark_copy +``` + +### Output formats + +The benchmarks support multiple output formats. Use the +`--benchmark_format=` flag (or set the +`BENCHMARK_FORMAT=` environment variable) to set the format +type. `console` is the default format. + +Write benchmark results to a file with the `--benchmark_out=` option. + +Example: + +``` +$ ./benchmark_copy --benchmark_format=json --benchmark_out=benchmark_copy.json +``` + +### Statistics + +The number of runs of each benchmark is specified globally by the +`--benchmark_repetitions` flag. When a benchmark is run more than once the mean, +median and standard deviation of the runs will be reported. + +Example: + +``` +$ ./benchmark_copy --benchmark_repetitions=10 +``` diff --git a/src/tests/benchmarks/benchmark_copy.cc b/src/tests/benchmarks/benchmark_copy.cc new file mode 100644 index 000000000..f4c0e4a03 --- /dev/null +++ b/src/tests/benchmarks/benchmark_copy.cc @@ -0,0 +1,83 @@ +/*! + * \file benchmark_copy.cc + * \brief Benchmark for memory copy implementations + * \author Carles Fernandez-Prades, 2020. cfernandez(at)cttc.es + * + * Based on https://stackoverflow.com/a/40109182 + * + * + * ------------------------------------------------------------------------- + * + * Copyright (C) 2020 (see AUTHORS file for a list of contributors) + * + * GNSS-SDR is a software defined Global Navigation + * Satellite Systems receiver + * + * This file is part of GNSS-SDR. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + * ------------------------------------------------------------------------- + */ + +#include +#include +#include +#include +#include + +constexpr int N = 8184; + +void bm_memcpy(benchmark::State& state) +{ + std::vector> orig(N); + std::vector> dest(N); + + while (state.KeepRunning()) + { + memcpy(dest.data(), orig.data(), N * sizeof(std::complex)); + } +} + + +void bm_stdmemcpy(benchmark::State& state) +{ + std::vector> orig(N); + std::vector> dest(N); + + while (state.KeepRunning()) + { + std::memcpy(dest.data(), orig.data(), N * sizeof(std::complex)); + } +} + + +void bm_stdcopy(benchmark::State& state) +{ + std::vector> orig(N); + std::vector> dest(N); + + while (state.KeepRunning()) + { + std::copy(orig.begin(), orig.end(), dest.begin()); + } +} + + +void bm_stdcopy_n(benchmark::State& state) +{ + std::vector> orig(N); + std::vector> dest(N); + + while (state.KeepRunning()) + { + std::copy_n(orig.begin(), N, dest.begin()); + } +} + +BENCHMARK(bm_memcpy); +BENCHMARK(bm_stdmemcpy); +BENCHMARK(bm_stdcopy); +BENCHMARK(bm_stdcopy_n); + +BENCHMARK_MAIN(); diff --git a/src/tests/benchmarks/benchmark_detector.cc b/src/tests/benchmarks/benchmark_detector.cc new file mode 100644 index 000000000..c80b7f321 --- /dev/null +++ b/src/tests/benchmarks/benchmark_detector.cc @@ -0,0 +1,87 @@ +/*! + * \file benchmark_detector.cc + * \brief Benchmark for preamble detection implementations + * \author Carles Fernandez-Prades, 2020. cfernandez(at)cttc.es + * + * + * ------------------------------------------------------------------------- + * + * Copyright (C) 2020 (see AUTHORS file for a list of contributors) + * + * GNSS-SDR is a software defined Global Navigation + * Satellite Systems receiver + * + * This file is part of GNSS-SDR. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + * ------------------------------------------------------------------------- + */ + +#include "GPS_L1_CA.h" +#include +#include +#include +#include +#include +#include +#include + + +void bm_forloop(benchmark::State& state) +{ + int32_t corr_value = 0; + std::vector d_symbol_history(GPS_CA_PREAMBLE_LENGTH_SYMBOLS, 0.0); + std::array d_preamble_samples{}; + + // fill the inputs + std::random_device rd; + std::default_random_engine e2(rd()); + std::uniform_real_distribution<> dist(-1.0, 1.0); + std::generate(d_symbol_history.begin(), d_symbol_history.end(), [&dist, &e2]() { return dist(e2); }); + + std::generate(d_preamble_samples.begin(), d_preamble_samples.end(), [n = 0]() mutable { return (GPS_CA_PREAMBLE[n++] == '1' ? 1 : -1); }); + + while (state.KeepRunning()) + { + for (int32_t i = 0; i < GPS_CA_PREAMBLE_LENGTH_BITS; i++) + { + if (d_symbol_history[i] < 0.0) + { + corr_value -= d_preamble_samples[i]; + } + else + { + corr_value += d_preamble_samples[i]; + } + } + } +} + + +void bm_lambda(benchmark::State& state) +{ + int32_t corr_value = 0; + std::vector d_symbol_history(GPS_CA_PREAMBLE_LENGTH_SYMBOLS, 0.0); + std::array d_preamble_samples{}; + + // fill the inputs + std::random_device rd; + std::default_random_engine e2(rd()); + std::uniform_real_distribution<> dist(-1.0, 1.0); + std::generate(d_symbol_history.begin(), d_symbol_history.end(), [&dist, &e2]() { return dist(e2); }); + + std::generate(d_preamble_samples.begin(), d_preamble_samples.end(), [n = 0]() mutable { return (GPS_CA_PREAMBLE[n++] == '1' ? 1 : -1); }); + + while (state.KeepRunning()) + { + corr_value += std::accumulate(d_symbol_history.begin(), + d_symbol_history.begin() + GPS_CA_PREAMBLE_LENGTH_BITS, + 0, + [&d_preamble_samples, n = 0](float a, float b) mutable { return (b > 0.0 ? a + d_preamble_samples[n++] : a - d_preamble_samples[n++]); }); + } +} + +BENCHMARK(bm_forloop); +BENCHMARK(bm_lambda); +BENCHMARK_MAIN(); diff --git a/src/tests/benchmarks/benchmark_preamble.cc b/src/tests/benchmarks/benchmark_preamble.cc new file mode 100644 index 000000000..1bd634cf7 --- /dev/null +++ b/src/tests/benchmarks/benchmark_preamble.cc @@ -0,0 +1,62 @@ +/*! + * \file benchmark_preamble.cc + * \brief Benchmark for preamble conversion implementations + * \author Carles Fernandez-Prades, 2020. cfernandez(at)cttc.es + * + * + * ------------------------------------------------------------------------- + * + * Copyright (C) 2020 (see AUTHORS file for a list of contributors) + * + * GNSS-SDR is a software defined Global Navigation + * Satellite Systems receiver + * + * This file is part of GNSS-SDR. + * + * SPDX-License-Identifier: GPL-3.0-or-later + * + * ------------------------------------------------------------------------- + */ + +#include "GPS_L1_CA.h" +#include +#include +#include + +void bm_forloop(benchmark::State& state) +{ + std::array d_preamble_samples{}; + int32_t n = 0; + while (state.KeepRunning()) + { + n = 0; + for (int32_t i = 0; i < GPS_CA_PREAMBLE_LENGTH_BITS; i++) + { + if (GPS_CA_PREAMBLE[i] == '1') + { + d_preamble_samples[n] = 1; + n++; + } + else + { + d_preamble_samples[n] = -1; + n++; + } + } + } +} + + +void bm_lambda(benchmark::State& state) +{ + std::array d_preamble_samples{}; + while (state.KeepRunning()) + { + std::generate(d_preamble_samples.begin(), d_preamble_samples.end(), [n = 0]() mutable { return (GPS_CA_PREAMBLE[n++] == '1' ? 1 : -1); }); + } +} + + +BENCHMARK(bm_forloop); +BENCHMARK(bm_lambda); +BENCHMARK_MAIN();