mirror of https://github.com/gnss-sdr/gnss-sdr
tests: split testcases in ctest for granular failure reports
This commit is contained in:
Carles Fernandez
parent
48f65b067b
commit
c4f4f80b45
|
|
@ -22,24 +22,40 @@ if(DEFINED __INCLUDED_VOLK_ADD_TEST)
|
|||
endif()
|
||||
set(__INCLUDED_VOLK_ADD_TEST TRUE)
|
||||
|
||||
########################################################################
|
||||
# Generate a test executable which can be used in ADD_TEST to call
|
||||
# various subtests.
|
||||
#
|
||||
# SOURCES - sources for the test
|
||||
# TARGET_DEPS - build target dependencies (e.g., libraries)
|
||||
########################################################################
|
||||
|
||||
function(VOLK_GEN_TEST executable_name)
|
||||
include(CMakeParseArgumentsCopy)
|
||||
CMAKE_PARSE_ARGUMENTS(VOLK_TEST "" "" "SOURCES;TARGET_DEPS;EXTRA_LIB_DIRS;ENVIRONS;ARGS" ${ARGN})
|
||||
add_executable(${executable_name} ${VOLK_TEST_SOURCES})
|
||||
target_link_libraries(${executable_name} ${VOLK_TEST_TARGET_DEPS})
|
||||
endfunction()
|
||||
|
||||
########################################################################
|
||||
# Add a unit test and setup the environment for it.
|
||||
# Encloses ADD_TEST, with additional functionality to create a shell
|
||||
# script that sets the environment to gain access to in-build binaries
|
||||
# properly. The following variables are used to pass in settings:
|
||||
# A test executable has to be generated with VOLK_GEN_TEST beforehand.
|
||||
# The executable name has to be passed as argument.
|
||||
#
|
||||
# NAME - the test name
|
||||
# SOURCES - sources for the test
|
||||
# TARGET_DEPS - build target dependencies (e.g., libraries)
|
||||
# EXTRA_LIB_DIRS - other directories for the library path
|
||||
# ENVIRONS - other environment key/value pairs
|
||||
# ARGS - arguments for the test
|
||||
########################################################################
|
||||
function(VOLK_ADD_TEST test_name)
|
||||
function(VOLK_ADD_TEST test_name executable_name)
|
||||
|
||||
#parse the arguments for component names
|
||||
include(CMakeParseArgumentsCopy)
|
||||
CMAKE_PARSE_ARGUMENTS(VOLK_TEST "" "" "SOURCES;TARGET_DEPS;EXTRA_LIB_DIRS;ENVIRONS;ARGS" ${ARGN})
|
||||
CMAKE_PARSE_ARGUMENTS(VOLK_TEST "" "" "TARGET_DEPS;EXTRA_LIB_DIRS;ENVIRONS;ARGS" ${ARGN})
|
||||
|
||||
#set the initial environs to use
|
||||
set(environs ${VOLK_TEST_ENVIRONS})
|
||||
|
|
@ -65,7 +81,7 @@ function(VOLK_ADD_TEST test_name)
|
|||
#"add_test" command, via the $<FOO:BAR> operator; make sure the
|
||||
#test's directory is first, since it ($1) is prepended to PATH.
|
||||
unset(TARGET_DIR_LIST)
|
||||
foreach(target ${test_name} ${VOLK_TEST_TARGET_DEPS})
|
||||
foreach(target ${executable_name} ${VOLK_TEST_TARGET_DEPS})
|
||||
list(APPEND TARGET_DIR_LIST "\$<TARGET_FILE_DIR:${target}>")
|
||||
endforeach()
|
||||
|
||||
|
|
@ -134,18 +150,17 @@ function(VOLK_ADD_TEST test_name)
|
|||
file(APPEND ${sh_file} "export ${environ}\n")
|
||||
endforeach(environ)
|
||||
|
||||
set(VOLK_TEST_ARGS "${test_name}")
|
||||
|
||||
#redo the test args to have a space between each
|
||||
string(REPLACE ";" " " VOLK_TEST_ARGS "${VOLK_TEST_ARGS}")
|
||||
|
||||
#finally: append the test name to execute
|
||||
file(APPEND ${sh_file} ${test_name} " " ${VOLK_TEST_ARGS} "\n")
|
||||
file(APPEND ${sh_file} "${CMAKE_CROSSCOMPILING_EMULATOR} ${executable_name} ${VOLK_TEST_ARGS}\n")
|
||||
|
||||
#make the shell file executable
|
||||
execute_process(COMMAND chmod +x ${sh_file})
|
||||
|
||||
add_executable(${test_name} ${VOLK_TEST_SOURCES})
|
||||
target_link_libraries(${test_name} ${VOLK_TEST_TARGET_DEPS})
|
||||
|
||||
#add the shell file as the test to execute;
|
||||
#use the form that allows for $<FOO:BAR> substitutions,
|
||||
#then combine the script arguments inside the script.
|
||||
|
|
@ -196,10 +211,8 @@ function(VOLK_ADD_TEST test_name)
|
|||
file(APPEND ${bat_file} ${test_name} " " ${VOLK_TEST_ARGS} "\n")
|
||||
file(APPEND ${bat_file} "\n")
|
||||
|
||||
add_executable(${test_name} ${VOLK_TEST_SOURCES})
|
||||
target_link_libraries(${test_name} ${VOLK_TEST_TARGET_DEPS})
|
||||
|
||||
add_test(${test_name} ${bat_file})
|
||||
endif(WIN32)
|
||||
|
||||
endfunction(VOLK_ADD_TEST)
|
||||
|
||||
|
|
|
|||
|
|
@ -345,7 +345,7 @@ macro(gen_template tmpl output)
|
|||
)
|
||||
endmacro(gen_template)
|
||||
|
||||
make_directory(${PROJECT_BINARY_DIR}/include/volk_gnsssdr)
|
||||
file(MAKE_DIRECTORY ${PROJECT_BINARY_DIR}/include/volk_gnsssdr)
|
||||
|
||||
gen_template(${PROJECT_SOURCE_DIR}/tmpl/volk_gnsssdr.tmpl.h ${PROJECT_BINARY_DIR}/include/volk_gnsssdr/volk_gnsssdr.h)
|
||||
gen_template(${PROJECT_SOURCE_DIR}/tmpl/volk_gnsssdr.tmpl.c ${PROJECT_BINARY_DIR}/lib/volk_gnsssdr.c)
|
||||
|
|
@ -604,18 +604,24 @@ if(ENABLE_TESTING)
|
|||
|
||||
#include Boost headers
|
||||
include_directories(${Boost_INCLUDE_DIRS})
|
||||
link_directories(${Boost_LIBRARY_DIRS})
|
||||
make_directory(${CMAKE_CURRENT_BINARY_DIR}/.unittest)
|
||||
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/.unittest)
|
||||
set_source_files_properties(
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/testqa.cc PROPERTIES
|
||||
COMPILE_DEFINITIONS "BOOST_TEST_DYN_LINK;BOOST_TEST_MAIN"
|
||||
)
|
||||
|
||||
include(VolkAddTest)
|
||||
VOLK_ADD_TEST(test_all
|
||||
VOLK_GEN_TEST("volk_gnsssdr_test_all"
|
||||
SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/testqa.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/qa_utils.cc
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/qa_utils.cc
|
||||
TARGET_DEPS volk_gnsssdr
|
||||
)
|
||||
)
|
||||
foreach(kernel ${h_files})
|
||||
get_filename_component(kernel ${kernel} NAME)
|
||||
string(REPLACE ".h" "" kernel ${kernel})
|
||||
if(NOT ${kernel} MATCHES puppet*)
|
||||
VOLK_ADD_TEST(${kernel} "volk_gnsssdr_test_all")
|
||||
endif(NOT ${kernel} MATCHES puppet*)
|
||||
endforeach()
|
||||
|
||||
endif(ENABLE_TESTING)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@
|
|||
|
||||
void print_qa_xml(std::vector<volk_gnsssdr_test_results_t> results, unsigned int nfails);
|
||||
|
||||
int main()
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
bool qa_ret_val = 0;
|
||||
|
||||
|
|
@ -45,47 +45,72 @@ int main()
|
|||
volk_gnsssdr_test_params_t test_params(def_tol, def_scalar, def_vlen, def_iter,
|
||||
def_benchmark_mode, def_kernel_regex);
|
||||
std::vector<volk_gnsssdr_test_case_t> test_cases = init_test_list(test_params);
|
||||
|
||||
std::vector<std::string> qa_failures;
|
||||
std::vector<volk_gnsssdr_test_results_t> results;
|
||||
// Test every kernel reporting failures when they occur
|
||||
for (unsigned int ii = 0; ii < test_cases.size(); ++ii)
|
||||
if (argc > 1)
|
||||
{
|
||||
bool qa_result = false;
|
||||
volk_gnsssdr_test_case_t test_case = test_cases[ii];
|
||||
try
|
||||
for (unsigned int ii = 0; ii < test_cases.size(); ++ii)
|
||||
{
|
||||
qa_result = run_volk_gnsssdr_tests(test_case.desc(), test_case.kernel_ptr(), test_case.name(),
|
||||
test_case.test_parameters(), &results, test_case.puppet_master_name());
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// TODO: what exceptions might we need to catch and how do we handle them?
|
||||
std::cerr << "Exception found on kernel: " << test_case.name() << std::endl;
|
||||
qa_result = false;
|
||||
}
|
||||
|
||||
if (qa_result)
|
||||
{
|
||||
std::cerr << "Failure on " << test_case.name() << std::endl;
|
||||
qa_failures.push_back(test_case.name());
|
||||
if (std::string(argv[1]) == test_cases[ii].name())
|
||||
{
|
||||
volk_gnsssdr_test_case_t test_case = test_cases[ii];
|
||||
if (run_volk_gnsssdr_tests(test_case.desc(), test_case.kernel_ptr(),
|
||||
test_case.name(),
|
||||
test_case.test_parameters(), &results,
|
||||
test_case.puppet_master_name()))
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
std::cerr << "Did not run a test for kernel: " << std::string(argv[1]) << " !" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Generate XML results
|
||||
print_qa_xml(results, qa_failures.size());
|
||||
|
||||
// Summarize QA results
|
||||
std::cerr << "Kernel QA finished: " << qa_failures.size() << " failures out of "
|
||||
<< test_cases.size() << " tests." << std::endl;
|
||||
if (qa_failures.size() > 0)
|
||||
else
|
||||
{
|
||||
std::cerr << "The following kernels failed QA:" << std::endl;
|
||||
for (unsigned int ii = 0; ii < qa_failures.size(); ++ii)
|
||||
std::vector<std::string> qa_failures;
|
||||
// Test every kernel reporting failures when they occur
|
||||
for (unsigned int ii = 0; ii < test_cases.size(); ++ii)
|
||||
{
|
||||
std::cerr << " " << qa_failures[ii] << std::endl;
|
||||
bool qa_result = false;
|
||||
volk_gnsssdr_test_case_t test_case = test_cases[ii];
|
||||
try
|
||||
{
|
||||
qa_result = run_volk_gnsssdr_tests(test_case.desc(), test_case.kernel_ptr(), test_case.name(),
|
||||
test_case.test_parameters(), &results, test_case.puppet_master_name());
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
// TODO: what exceptions might we need to catch and how do we handle them?
|
||||
std::cerr << "Exception found on kernel: " << test_case.name() << std::endl;
|
||||
qa_result = false;
|
||||
}
|
||||
|
||||
if (qa_result)
|
||||
{
|
||||
std::cerr << "Failure on " << test_case.name() << std::endl;
|
||||
qa_failures.push_back(test_case.name());
|
||||
}
|
||||
}
|
||||
|
||||
// Generate XML results
|
||||
print_qa_xml(results, qa_failures.size());
|
||||
|
||||
// Summarize QA results
|
||||
std::cerr << "Kernel QA finished: " << qa_failures.size() << " failures out of "
|
||||
<< test_cases.size() << " tests." << std::endl;
|
||||
if (qa_failures.size() > 0)
|
||||
{
|
||||
std::cerr << "The following kernels failed QA:" << std::endl;
|
||||
for (unsigned int ii = 0; ii < qa_failures.size(); ++ii)
|
||||
{
|
||||
std::cerr << " " << qa_failures[ii] << std::endl;
|
||||
}
|
||||
qa_ret_val = 1;
|
||||
}
|
||||
qa_ret_val = 1;
|
||||
}
|
||||
|
||||
return qa_ret_val;
|
||||
|
|
@ -128,7 +153,6 @@ void print_qa_xml(std::vector<volk_gnsssdr_test_results_t> results, unsigned int
|
|||
qa_file << " </testsuite>" << std::endl;
|
||||
}
|
||||
|
||||
|
||||
qa_file << "</testsuites>" << std::endl;
|
||||
qa_file.close();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ typedef struct volk_gnsssdr_func_desc
|
|||
const char **impl_names;
|
||||
const int *impl_deps;
|
||||
const bool *impl_alignment;
|
||||
const size_t n_impls;
|
||||
size_t n_impls;
|
||||
} volk_gnsssdr_func_desc_t;
|
||||
|
||||
//! Prints a list of machines available
|
||||
|
|
@ -68,12 +68,12 @@ VOLK_API size_t volk_gnsssdr_get_alignment(void);
|
|||
*/
|
||||
VOLK_API bool volk_gnsssdr_is_aligned(const void *ptr);
|
||||
|
||||
// clang-format off
|
||||
%for kern in kernels:
|
||||
|
||||
//! A function pointer to the dispatcher implementation
|
||||
extern VOLK_API ${kern.pname} ${kern.name};
|
||||
|
||||
// clang-format off
|
||||
//! A function pointer to the fastest aligned implementation
|
||||
extern VOLK_API ${kern.pname} ${kern.name}_a;
|
||||
|
||||
|
|
@ -86,9 +86,7 @@ extern VOLK_API void ${kern.name}_manual(${kern.arglist_full}, const char* impl_
|
|||
//! Get description parameters for this kernel
|
||||
extern VOLK_API volk_gnsssdr_func_desc_t ${kern.name}_get_func_desc(void);
|
||||
%endfor
|
||||
// clang-format off
|
||||
|
||||
__VOLK_DECL_END
|
||||
|
||||
// clang-format on
|
||||
|
||||
#endif /*INCLUDED_VOLK_GNSSSDR_RUNTIME*/
|
||||
|
|
|
|||
Loading…
Reference in New Issue