From ddce5c6dc9ae5c75e3481696cd7f282cc1a4a1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Mon, 5 Jun 2023 15:48:00 -0400 Subject: [PATCH 01/10] refactor(test): update results as fixture cleanups This is less wasteful as it makes use of the output of existing tests. It should simplify adding support for running the tests. --- README.md | 2 +- regression-tests/CMakeLists.txt | 46 ++++++++++++------- .../cmake/ExecuteWithRedirection.cmake | 14 ++++++ .../cmake/UpdateTestResults.cmake.in | 16 ------- 4 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 regression-tests/cmake/ExecuteWithRedirection.cmake delete mode 100644 regression-tests/cmake/UpdateTestResults.cmake.in diff --git a/README.md b/README.md index 9824ae3..342a665 100644 --- a/README.md +++ b/README.md @@ -139,4 +139,4 @@ processing the directory. The CMake project `regression-tests/CMakeLists.txt` runs the test suite of cppfront. See "Regression tests" at [`./.github/workflows/ci.yml`](./.github/workflows/ci.yml) for how to set it up. -To update the test results, build with `--target cppfront_update_test_results`. +To have the test results updated, configure with `-DCPPFRONT_DEVELOPING=TRUE`. diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index e8b1916..516f090 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 3.23) project(cppfront-regression-tests LANGUAGES NONE) +option(CPPFRONT_DEVELOPING "Test results are updated.") + enable_testing() set(CPPFRONT_NO_MAGIC 1) @@ -8,8 +10,7 @@ find_package(cppfront REQUIRED) set(REGRESSION_TESTS_DIR "${CMAKE_CURRENT_LIST_DIR}/../cppfront/regression-tests") -configure_file("cmake/UpdateTestResults.cmake.in" "UpdateTestResults.cmake" @ONLY) -add_custom_target(cppfront_update_test_results) +configure_file("cmake/ExecuteWithRedirection.cmake" "ExecuteWithRedirection.cmake" COPYONLY) function(cppfront_command_tests) cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE;EXPECTED_FILE" "EXTRA_FLAGS") @@ -18,23 +19,35 @@ function(cppfront_command_tests) cmake_path(GET ARG_SOURCE STEM test_name) - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${test_name}") - add_custom_target( - "${test_name}" - COMMAND - "${CMAKE_COMMAND}" - -D "TEST_NAME=${test_name}" - -D "EXTRA_FLAGS=${ARG_EXTRA_FLAGS}" - -P "UpdateTestResults.cmake" - ) - add_dependencies(cppfront_update_test_results "${test_name}") - + if (NOT "${test_name}" MATCHES [[.*-error$]]) + set(gen_cpp_src "${test_name}.cpp") + set(COMMAND_ERROR_IS_FATAL "COMMAND_ERROR_IS_FATAL" "ANY") + endif () add_test( NAME "codegen/${test_name}" - COMMAND cppfront::cppfront "${ARG_SOURCE}" ${ARG_EXTRA_FLAGS} + COMMAND + "${CMAKE_COMMAND}" + -D "OUTPUT_FILE=${ARG_SOURCE}.output" + -P "ExecuteWithRedirection.cmake" + -- + "${CPPFRONT_EXECUTABLE}" "${ARG_SOURCE}" ${ARG_EXTRA_FLAGS} + ${COMMAND_ERROR_IS_FATAL} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) + if (CPPFRONT_DEVELOPING) + add_test( + NAME "codegen/update/${test_name}" + COMMAND + "${CMAKE_COMMAND}" + -E copy_if_different + "${gen_cpp_src}" + "${ARG_SOURCE}.output" + "${REGRESSION_TESTS_DIR}/test-results/" + ) + set_tests_properties("codegen/update/${test_name}" PROPERTIES FIXTURES_CLEANUP "codegen/${test_name}") + endif () + set(expected_output_file "${REGRESSION_TESTS_DIR}/test-results/${ARG_SOURCE}.output") if (EXISTS "${expected_output_file}") file(READ "${expected_output_file}" expected_output) @@ -57,11 +70,10 @@ function(cppfront_command_tests) endif () if (ARG_EXPECTED_FILE) - cmake_path(REPLACE_EXTENSION ARG_SOURCE "cpp" OUTPUT_VARIABLE gen_cpp_src) - + configure_file("${ARG_EXPECTED_FILE}" "${gen_cpp_src}.original" COPYONLY) add_test( NAME "codegen/check/${test_name}" - COMMAND "${CMAKE_COMMAND}" -E compare_files "${gen_cpp_src}" "${ARG_EXPECTED_FILE}" + COMMAND "${CMAKE_COMMAND}" -E compare_files "${gen_cpp_src}" "${gen_cpp_src}.original" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) diff --git a/regression-tests/cmake/ExecuteWithRedirection.cmake b/regression-tests/cmake/ExecuteWithRedirection.cmake new file mode 100644 index 0000000..9f98aa6 --- /dev/null +++ b/regression-tests/cmake/ExecuteWithRedirection.cmake @@ -0,0 +1,14 @@ +if (NOT "${CMAKE_ARGV5}" STREQUAL "--") + message(FATAL_ERROR "Unexpected argument.") +endif () +foreach (i RANGE 6 ${CMAKE_ARGC}) + list(APPEND command_args "${CMAKE_ARGV${i}}") # Probably doesn't handle nested `;`. +endforeach () +execute_process( + COMMAND ${command_args} + OUTPUT_VARIABLE OUTPUT + ERROR_VARIABLE OUTPUT + ECHO_OUTPUT_VARIABLE + ECHO_ERROR_VARIABLE +) +file(WRITE "${OUTPUT_FILE}" "${OUTPUT}") diff --git a/regression-tests/cmake/UpdateTestResults.cmake.in b/regression-tests/cmake/UpdateTestResults.cmake.in deleted file mode 100644 index 0694ebb..0000000 --- a/regression-tests/cmake/UpdateTestResults.cmake.in +++ /dev/null @@ -1,16 +0,0 @@ -set(SOURCE "${TEST_NAME}.cpp2") -set(EXPECTED_FILE "test-results/${TEST_NAME}.cpp") -set(EXPECTED_FILE_OUTPUT "test-results/${SOURCE}.output") -set(FILE_OUTPUT "@CMAKE_CURRENT_BINARY_DIR@/${TEST_NAME}/output") -if (NOT "${TEST_NAME}" MATCHES [[.*-error$]]) - set(COMMAND_ERROR_IS_FATAL "COMMAND_ERROR_IS_FATAL" "ANY") -endif () -execute_process( - COMMAND "@CPPFRONT_EXECUTABLE@" "${SOURCE}" ${EXTRA_FLAGS} -o "${EXPECTED_FILE}" - WORKING_DIRECTORY "@REGRESSION_TESTS_DIR@" - OUTPUT_FILE "${FILE_OUTPUT}" - ERROR_FILE "${FILE_OUTPUT}" - ${COMMAND_ERROR_IS_FATAL} -) -execute_process( - COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${FILE_OUTPUT}" "@REGRESSION_TESTS_DIR@/${EXPECTED_FILE_OUTPUT}") From 12672519919190c52c90ae083badb485bdc1e989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Mon, 5 Jun 2023 17:28:53 -0400 Subject: [PATCH 02/10] refactor(test): order build after codegen Ordering by relative cost reduces turnarounds. Previously, codegen update happened last. --- regression-tests/CMakeLists.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index 516f090..dda81cc 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -46,6 +46,7 @@ function(cppfront_command_tests) "${REGRESSION_TESTS_DIR}/test-results/" ) set_tests_properties("codegen/update/${test_name}" PROPERTIES FIXTURES_CLEANUP "codegen/${test_name}") + set(build_test_depends "codegen/update/${test_name}") endif () set(expected_output_file "${REGRESSION_TESTS_DIR}/test-results/${ARG_SOURCE}.output") @@ -96,19 +97,18 @@ function(cppfront_build_tests) mixed-postfix-expression-custom-formatting ) - cmake_path(GET ARG_SOURCE STEM stem) - if (stem IN_LIST codegen_only_tests) + cmake_path(GET ARG_SOURCE STEM test_name) + if (test_name IN_LIST codegen_only_tests) return() endif () - set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/${stem}") - set(test_name "build/${stem}") + set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/${test_name}") configure_file("${REGRESSION_TESTS_DIR}/${ARG_SOURCE}" "${test_dir}/${ARG_SOURCE}" COPYONLY) configure_file("cmake/CMakeLists.txt.in" "${test_dir}/CMakeLists.txt" @ONLY) add_test( - NAME "${test_name}" + NAME "build/${test_name}" COMMAND "${CMAKE_CTEST_COMMAND}" --build-and-test "${test_dir}" "${test_dir}/build" @@ -120,6 +120,7 @@ function(cppfront_build_tests) "-Dcppfront-exe_DIR=${cppfront-exe_DIR}" "-DCPPFRONT_FLAGS=${ARG_EXTRA_FLAGS}" ) + set_tests_properties("build/${test_name}" PROPERTIES DEPENDS "${build_test_depends};codegen/check/${test_name}") endfunction() function(cppfront_tests) From ed9d6969c6a09e1f16f50d34b6301c1a5b9c396c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Mon, 5 Jun 2023 22:37:35 -0400 Subject: [PATCH 03/10] feat(test): run tests and update build result Although the project's language is now `CXX`, the setup of the C++ environment is as was before. I.e., - The environment of the build test can only be set globally (through CMake defaults, the environment, etc.). - The environment of this project should match that of the build test. It doesn't attempt to propagate any aspect explicitly. --- regression-tests/CMakeLists.txt | 69 ++++++++++++++++++- regression-tests/cmake/CMakeLists.txt.in | 27 ++++++++ .../cmake/ExecuteTestCase.cmake.in | 23 +++++++ .../cmake/test-case-config.cmake.in | 7 ++ 4 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 regression-tests/cmake/ExecuteTestCase.cmake.in create mode 100644 regression-tests/cmake/test-case-config.cmake.in diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index dda81cc..62c3afd 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.23) -project(cppfront-regression-tests LANGUAGES NONE) +project(cppfront-regression-tests LANGUAGES CXX) option(CPPFRONT_DEVELOPING "Test results are updated.") @@ -10,7 +10,31 @@ find_package(cppfront REQUIRED) set(REGRESSION_TESTS_DIR "${CMAKE_CURRENT_LIST_DIR}/../cppfront/regression-tests") +if (CPPFRONT_DEVELOPING) + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(compiler_id "gcc") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + set(compiler_id "apple-clang") + else () + string(TOLOWER "${CMAKE_CXX_COMPILER_ID}" compiler_id) + endif () + string(REGEX MATCH "[0-9]+" compiler_major_version "${CMAKE_CXX_COMPILER_VERSION}") + + set(TEST_RESULTS_BUILD_DIR "${REGRESSION_TESTS_DIR}/test-results/${compiler_id}-${compiler_major_version}") + file(MAKE_DIRECTORY "${TEST_RESULTS_BUILD_DIR}") + + set(compiler_version_command "${CMAKE_CXX_COMPILER}" "--version") + if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") + set(compiler_id "clang") + endif () + execute_process( + COMMAND ${compiler_version_command} + OUTPUT_FILE "${TEST_RESULTS_BUILD_DIR}/${compiler_id}-version.output" + ) +endif () + configure_file("cmake/ExecuteWithRedirection.cmake" "ExecuteWithRedirection.cmake" COPYONLY) +configure_file("cmake/ExecuteTestCase.cmake.in" "ExecuteTestCase.cmake" @ONLY) function(cppfront_command_tests) cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE;EXPECTED_FILE" "EXTRA_FLAGS") @@ -106,6 +130,7 @@ function(cppfront_build_tests) configure_file("${REGRESSION_TESTS_DIR}/${ARG_SOURCE}" "${test_dir}/${ARG_SOURCE}" COPYONLY) configure_file("cmake/CMakeLists.txt.in" "${test_dir}/CMakeLists.txt" @ONLY) + configure_file("cmake/test-case-config.cmake.in" "${test_dir}/test-case-config.cmake.in" COPYONLY) add_test( NAME "build/${test_name}" @@ -119,8 +144,48 @@ function(cppfront_build_tests) "-Dcppfront_DIR=${cppfront_DIR}" "-Dcppfront-exe_DIR=${cppfront-exe_DIR}" "-DCPPFRONT_FLAGS=${ARG_EXTRA_FLAGS}" + # There's `CMAKE_CXX_LINKER_LAUNCHER`, too. So far, it's not needed. + "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_COMMAND};-D;OUTPUT_FILE=${gen_cpp_src}.output;-P;../../ExecuteWithRedirection.cmake;--" + ) + set_tests_properties( + "build/${test_name}" + PROPERTIES + DEPENDS "${build_test_depends};codegen/check/${test_name}" + FIXTURES_SETUP "build/${test_name}" ) - set_tests_properties("build/${test_name}" PROPERTIES DEPENDS "${build_test_depends};codegen/check/${test_name}") + + if (CPPFRONT_DEVELOPING) + add_test( + NAME "build/execute/${test_name}" + COMMAND + "${CMAKE_COMMAND}" + -D "OUTPUT_FILE=${test_dir}/${gen_cpp_src}.execution" + -P "../ExecuteTestCase.cmake" + WORKING_DIRECTORY "${test_dir}" + ) + set_tests_properties( + "build/execute/${test_name}" + PROPERTIES + FIXTURES_REQUIRED "build/${test_name}" + FIXTURES_SETUP "build/execute/${test_name}" + RESOURCE_LOCK "test.exe") + + add_test( + NAME "build/update/${test_name}" + COMMAND + "${CMAKE_COMMAND}" + -E copy_if_different + "build/${gen_cpp_src}.output" + "${gen_cpp_src}.execution" + "${TEST_RESULTS_BUILD_DIR}" + WORKING_DIRECTORY "${test_dir}" + ) + set_tests_properties( + "build/update/${test_name}" + PROPERTIES + FIXTURES_CLEANUP "build/${test_name};build/execute/${test_name}" + ) + endif () endfunction() function(cppfront_tests) diff --git a/regression-tests/cmake/CMakeLists.txt.in b/regression-tests/cmake/CMakeLists.txt.in index 7d26306..b03dafc 100644 --- a/regression-tests/cmake/CMakeLists.txt.in +++ b/regression-tests/cmake/CMakeLists.txt.in @@ -13,3 +13,30 @@ target_compile_options( PRIVATE "$<$:/experimental:module>" ) + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +set(TEST_CASE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_DATADIR}/cmake/test-case") + +install( + TARGETS test-case + EXPORT test-case-targets +) + +install( + EXPORT test-case-targets + DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}" +) + +configure_package_config_file( + "test-case-config.cmake.in" + "test-case-config.cmake" + INSTALL_DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}" + PATH_VARS CMAKE_INSTALL_BINDIR +) + +install( + FILES "${CMAKE_CURRENT_BINARY_DIR}/test-case-config.cmake" + DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}" +) diff --git a/regression-tests/cmake/ExecuteTestCase.cmake.in b/regression-tests/cmake/ExecuteTestCase.cmake.in new file mode 100644 index 0000000..001545f --- /dev/null +++ b/regression-tests/cmake/ExecuteTestCase.cmake.in @@ -0,0 +1,23 @@ +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "build" --prefix "_local" + COMMAND_ERROR_IS_FATAL ANY +) +find_package(test-case REQUIRED PATHS "${CMAKE_CURRENT_BINARY_DIR}/_local" NO_DEFAULT_PATH) +execute_process( + COMMAND + "${CMAKE_COMMAND}" + -E copy + "${TEST_CASE_EXECUTABLE}" + "@TEST_RESULTS_BUILD_DIR@/test.exe" +) +execute_process( + COMMAND + "${CMAKE_COMMAND}" + -D "OUTPUT_FILE=${OUTPUT_FILE}" + -P "@CMAKE_CURRENT_BINARY_DIR@/ExecuteWithRedirection.cmake" + -- + "./test.exe" + COMMAND_ERROR_IS_FATAL ANY + WORKING_DIRECTORY "@TEST_RESULTS_BUILD_DIR@" +) +execute_process(COMMAND "${CMAKE_COMMAND}" -E rm "@TEST_RESULTS_BUILD_DIR@/test.exe") diff --git a/regression-tests/cmake/test-case-config.cmake.in b/regression-tests/cmake/test-case-config.cmake.in new file mode 100644 index 0000000..4f31576 --- /dev/null +++ b/regression-tests/cmake/test-case-config.cmake.in @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.23) +@PACKAGE_INIT@ + +set_and_check( + TEST_CASE_EXECUTABLE + "@PACKAGE_CMAKE_INSTALL_BINDIR@/test-case@CMAKE_EXECUTABLE_SUFFIX@" +) From 65a99b94c2765b80f66779ff543d95bab79640e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Tue, 6 Jun 2023 10:07:02 -0400 Subject: [PATCH 04/10] fix(test): remove outdated condition See . --- regression-tests/CMakeLists.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index 62c3afd..dcc2c56 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -114,17 +114,7 @@ endfunction() function(cppfront_build_tests) cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE" "EXTRA_FLAGS") - # The following tests aren't expected to be buildable C++, even if - # cppfront succeeds. - set( - codegen_only_tests - mixed-postfix-expression-custom-formatting - ) - cmake_path(GET ARG_SOURCE STEM test_name) - if (test_name IN_LIST codegen_only_tests) - return() - endif () set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/${test_name}") From cb398b04fa5945fc2f4da6c8055faaaf1fcc285d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Wed, 7 Jun 2023 15:43:53 -0400 Subject: [PATCH 05/10] refactor: execute tests without developer mode --- regression-tests/CMakeLists.txt | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index dcc2c56..ec60b76 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -144,22 +144,22 @@ function(cppfront_build_tests) FIXTURES_SETUP "build/${test_name}" ) - if (CPPFRONT_DEVELOPING) - add_test( - NAME "build/execute/${test_name}" - COMMAND - "${CMAKE_COMMAND}" - -D "OUTPUT_FILE=${test_dir}/${gen_cpp_src}.execution" - -P "../ExecuteTestCase.cmake" - WORKING_DIRECTORY "${test_dir}" - ) - set_tests_properties( - "build/execute/${test_name}" - PROPERTIES - FIXTURES_REQUIRED "build/${test_name}" - FIXTURES_SETUP "build/execute/${test_name}" - RESOURCE_LOCK "test.exe") + add_test( + NAME "build/execute/${test_name}" + COMMAND + "${CMAKE_COMMAND}" + -D "OUTPUT_FILE=${test_dir}/${gen_cpp_src}.execution" + -P "../ExecuteTestCase.cmake" + WORKING_DIRECTORY "${test_dir}" + ) + set_tests_properties( + "build/execute/${test_name}" + PROPERTIES + FIXTURES_REQUIRED "build/${test_name}" + FIXTURES_SETUP "build/execute/${test_name}" + RESOURCE_LOCK "test.exe") + if (CPPFRONT_DEVELOPING) add_test( NAME "build/update/${test_name}" COMMAND From 857240f8c0cede75d70e14759282bb581d877bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Thu, 8 Jun 2023 11:32:41 -0400 Subject: [PATCH 06/10] feat(test): copy new or changed build results only --- regression-tests/CMakeLists.txt | 37 ++++++++++++++----- .../cmake/ExecuteTestCase.cmake.in | 6 +-- .../cmake/UpdateBuildOutput.cmake.in | 29 +++++++++++++++ 3 files changed, 59 insertions(+), 13 deletions(-) create mode 100644 regression-tests/cmake/UpdateBuildOutput.cmake.in diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index ec60b76..f1e10cb 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -9,8 +9,10 @@ set(CPPFRONT_NO_MAGIC 1) find_package(cppfront REQUIRED) set(REGRESSION_TESTS_DIR "${CMAKE_CURRENT_LIST_DIR}/../cppfront/regression-tests") +set(TEST_RESULTS_DIR "${REGRESSION_TESTS_DIR}/test-results") if (CPPFRONT_DEVELOPING) + # Set `compiler_id` and `compiler_major_version`. if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set(compiler_id "gcc") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") @@ -20,21 +22,36 @@ if (CPPFRONT_DEVELOPING) endif () string(REGEX MATCH "[0-9]+" compiler_major_version "${CMAKE_CXX_COMPILER_VERSION}") - set(TEST_RESULTS_BUILD_DIR "${REGRESSION_TESTS_DIR}/test-results/${compiler_id}-${compiler_major_version}") - file(MAKE_DIRECTORY "${TEST_RESULTS_BUILD_DIR}") + # Setup `BUILD_RESULTS_DIR`. + set(compiler_item_name "${compiler_id}-${compiler_major_version}") + set(BUILD_RESULTS_DIR "${TEST_RESULTS_DIR}/${compiler_item_name}") + file(MAKE_DIRECTORY "${BUILD_RESULTS_DIR}") + # Write compiler version output. set(compiler_version_command "${CMAKE_CXX_COMPILER}" "--version") if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") set(compiler_id "clang") endif () execute_process( COMMAND ${compiler_version_command} - OUTPUT_FILE "${TEST_RESULTS_BUILD_DIR}/${compiler_id}-version.output" + OUTPUT_FILE "${BUILD_RESULTS_DIR}/${compiler_id}-version.output" ) + + # Set `OLD_BUILD_RESULTS_DIRS`. + file( + GLOB build_result_dirs + RELATIVE "${TEST_RESULTS_DIR}" + "${TEST_RESULTS_DIR}/${compiler_id}-*" + ) + list(SORT build_result_dirs) + list(FIND build_result_dirs "${compiler_item_name}" i) + list(SUBLIST build_result_dirs 0 ${i} OLD_BUILD_RESULTS_DIRS) + list(REVERSE OLD_BUILD_RESULTS_DIRS) endif () configure_file("cmake/ExecuteWithRedirection.cmake" "ExecuteWithRedirection.cmake" COPYONLY) configure_file("cmake/ExecuteTestCase.cmake.in" "ExecuteTestCase.cmake" @ONLY) +configure_file("cmake/UpdateBuildOutput.cmake.in" "UpdateBuildOutput.cmake" @ONLY) function(cppfront_command_tests) cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE;EXPECTED_FILE" "EXTRA_FLAGS") @@ -67,13 +84,13 @@ function(cppfront_command_tests) -E copy_if_different "${gen_cpp_src}" "${ARG_SOURCE}.output" - "${REGRESSION_TESTS_DIR}/test-results/" + "${TEST_RESULTS_DIR}" ) set_tests_properties("codegen/update/${test_name}" PROPERTIES FIXTURES_CLEANUP "codegen/${test_name}") set(build_test_depends "codegen/update/${test_name}") endif () - set(expected_output_file "${REGRESSION_TESTS_DIR}/test-results/${ARG_SOURCE}.output") + set(expected_output_file "${TEST_RESULTS_DIR}/${ARG_SOURCE}.output") if (EXISTS "${expected_output_file}") file(READ "${expected_output_file}" expected_output) string(REPLACE "\\" "\\\\" expected_output "${expected_output}") @@ -164,10 +181,10 @@ function(cppfront_build_tests) NAME "build/update/${test_name}" COMMAND "${CMAKE_COMMAND}" - -E copy_if_different - "build/${gen_cpp_src}.output" - "${gen_cpp_src}.execution" - "${TEST_RESULTS_BUILD_DIR}" + -D "GEN_CPP_SRC=${gen_cpp_src}" + -D "OUTPUT_FILE=build/${gen_cpp_src}.output" + -D "EXECUTION_FILE=${gen_cpp_src}.execution" + -P "../UpdateBuildOutput.cmake" WORKING_DIRECTORY "${test_dir}" ) set_tests_properties( @@ -190,7 +207,7 @@ function(cppfront_tests) foreach (src IN LISTS sources) cmake_path(REPLACE_EXTENSION src "cpp" OUTPUT_VARIABLE expected_file) - set(expected_file "${REGRESSION_TESTS_DIR}/test-results/${expected_file}") + set(expected_file "${TEST_RESULTS_DIR}/${expected_file}") if (NOT EXISTS "${expected_file}") set(expected_file "") diff --git a/regression-tests/cmake/ExecuteTestCase.cmake.in b/regression-tests/cmake/ExecuteTestCase.cmake.in index 001545f..9840075 100644 --- a/regression-tests/cmake/ExecuteTestCase.cmake.in +++ b/regression-tests/cmake/ExecuteTestCase.cmake.in @@ -8,7 +8,7 @@ execute_process( "${CMAKE_COMMAND}" -E copy "${TEST_CASE_EXECUTABLE}" - "@TEST_RESULTS_BUILD_DIR@/test.exe" + "@BUILD_RESULTS_DIR@/test.exe" ) execute_process( COMMAND @@ -18,6 +18,6 @@ execute_process( -- "./test.exe" COMMAND_ERROR_IS_FATAL ANY - WORKING_DIRECTORY "@TEST_RESULTS_BUILD_DIR@" + WORKING_DIRECTORY "@BUILD_RESULTS_DIR@" ) -execute_process(COMMAND "${CMAKE_COMMAND}" -E rm "@TEST_RESULTS_BUILD_DIR@/test.exe") +execute_process(COMMAND "${CMAKE_COMMAND}" -E rm "@BUILD_RESULTS_DIR@/test.exe") diff --git a/regression-tests/cmake/UpdateBuildOutput.cmake.in b/regression-tests/cmake/UpdateBuildOutput.cmake.in new file mode 100644 index 0000000..43ac5be --- /dev/null +++ b/regression-tests/cmake/UpdateBuildOutput.cmake.in @@ -0,0 +1,29 @@ +# Updates an output for the current compiler-version +# when it differs from its previous version or +# when no previous version has the output file. +# Objectives: +# - For "absence of output" to mean that "it's the same as the previous version". +# - To commit only new and changed outputs in PRs for easier reviewing. + +file(READ "${OUTPUT_FILE}" new_output) +if (EXISTS "${EXECUTION_FILE}") + file(READ "${EXECUTION_FILE}" new_execution) +endif () +function(write path_var ext) + foreach (build_results_dir IN ITEMS @OLD_BUILD_RESULTS_DIRS@) + set(result_file "@TEST_RESULTS_DIR@/${build_results_dir}/${GEN_CPP_SRC}.${ext}") + if (EXISTS "${result_file}") + file(READ "${result_file}" "old_${ext}") + if (new_${ext} STREQUAL old_${ext}) + return() + else () + break () + endif () + endif () + endforeach () + file(COPY "${${path_var}}" DESTINATION "@BUILD_RESULTS_DIR@") +endfunction() +write(OUTPUT_FILE "output") +if (DEFINED new_execution) + write(EXECUTION_FILE "execution") +endif () From 369aa235126ed91880bf26e2074e93b621a5e188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Thu, 8 Jun 2023 12:41:10 -0400 Subject: [PATCH 07/10] feat(test): check expected build results --- regression-tests/CMakeLists.txt | 86 ++++++++++++------- .../cmake/FindBuildResultFile.cmake.in | 28 ++++++ .../cmake/UpdateBuildOutput.cmake.in | 20 ++--- 3 files changed, 95 insertions(+), 39 deletions(-) create mode 100644 regression-tests/cmake/FindBuildResultFile.cmake.in diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index f1e10cb..15e8d61 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -11,20 +11,21 @@ find_package(cppfront REQUIRED) set(REGRESSION_TESTS_DIR "${CMAKE_CURRENT_LIST_DIR}/../cppfront/regression-tests") set(TEST_RESULTS_DIR "${REGRESSION_TESTS_DIR}/test-results") -if (CPPFRONT_DEVELOPING) - # Set `compiler_id` and `compiler_major_version`. - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set(compiler_id "gcc") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - set(compiler_id "apple-clang") - else () - string(TOLOWER "${CMAKE_CXX_COMPILER_ID}" compiler_id) - endif () - string(REGEX MATCH "[0-9]+" compiler_major_version "${CMAKE_CXX_COMPILER_VERSION}") +# Set `COMPILER_ITEM_NAME`. +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(compiler_id "gcc") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + set(compiler_id "apple-clang") +else () + string(TOLOWER "${CMAKE_CXX_COMPILER_ID}" compiler_id) +endif () +string(REGEX MATCH "[0-9]+" compiler_major_version "${CMAKE_CXX_COMPILER_VERSION}") +set(COMPILER_ITEM_NAME "${compiler_id}-${compiler_major_version}") - # Setup `BUILD_RESULTS_DIR`. - set(compiler_item_name "${compiler_id}-${compiler_major_version}") - set(BUILD_RESULTS_DIR "${TEST_RESULTS_DIR}/${compiler_item_name}") +# Setup `BUILD_RESULTS_DIR`. +set(BUILD_RESULTS_DIR "${TEST_RESULTS_DIR}/${COMPILER_ITEM_NAME}") + +if (CPPFRONT_DEVELOPING) file(MAKE_DIRECTORY "${BUILD_RESULTS_DIR}") # Write compiler version output. @@ -36,23 +37,26 @@ if (CPPFRONT_DEVELOPING) COMMAND ${compiler_version_command} OUTPUT_FILE "${BUILD_RESULTS_DIR}/${compiler_id}-version.output" ) - - # Set `OLD_BUILD_RESULTS_DIRS`. - file( - GLOB build_result_dirs - RELATIVE "${TEST_RESULTS_DIR}" - "${TEST_RESULTS_DIR}/${compiler_id}-*" - ) - list(SORT build_result_dirs) - list(FIND build_result_dirs "${compiler_item_name}" i) - list(SUBLIST build_result_dirs 0 ${i} OLD_BUILD_RESULTS_DIRS) - list(REVERSE OLD_BUILD_RESULTS_DIRS) endif () configure_file("cmake/ExecuteWithRedirection.cmake" "ExecuteWithRedirection.cmake" COPYONLY) configure_file("cmake/ExecuteTestCase.cmake.in" "ExecuteTestCase.cmake" @ONLY) +configure_file("cmake/FindBuildResultFile.cmake.in" "FindBuildResultFile.cmake" @ONLY) configure_file("cmake/UpdateBuildOutput.cmake.in" "UpdateBuildOutput.cmake" @ONLY) +include("${CMAKE_CURRENT_BINARY_DIR}/FindBuildResultFile.cmake") + +function(cppfront_add_check_test) + cmake_parse_arguments(PARSE_ARGV 0 ARG "" "NAME;NEW_FILE;OLD_FILE;FIXTURES_REQUIRED" "") + + add_test( + NAME "${ARG_NAME}" + COMMAND "${CMAKE_COMMAND}" -E compare_files "${ARG_NEW_FILE}" "${ARG_OLD_FILE}" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + ) + set_tests_properties("${ARG_NAME}" PROPERTIES FIXTURES_REQUIRED "${ARG_FIXTURES_REQUIRED}") +endfunction() + function(cppfront_command_tests) cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE;EXPECTED_FILE" "EXTRA_FLAGS") @@ -113,14 +117,13 @@ function(cppfront_command_tests) if (ARG_EXPECTED_FILE) configure_file("${ARG_EXPECTED_FILE}" "${gen_cpp_src}.original" COPYONLY) - add_test( + cppfront_add_check_test( NAME "codegen/check/${test_name}" - COMMAND "${CMAKE_COMMAND}" -E compare_files "${gen_cpp_src}" "${gen_cpp_src}.original" - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + NEW_FILE "${gen_cpp_src}" + OLD_FILE "${gen_cpp_src}.original" + FIXTURES_REQUIRED "codegen/${test_name}" ) - set_tests_properties("codegen/check/${test_name}" PROPERTIES FIXTURES_REQUIRED "codegen/${test_name}") - cppfront_build_tests( SOURCE ${ARG_SOURCE} EXTRA_FLAGS ${ARG_EXTRA_FLAGS} @@ -161,6 +164,16 @@ function(cppfront_build_tests) FIXTURES_SETUP "build/${test_name}" ) + cppfront_find_build_result_file(expected_output_file RESULT_FILE "${gen_cpp_src}.output") + if (expected_output_file) + cppfront_add_check_test( + NAME "build/check/${test_name}" + NEW_FILE "${test_dir}/build/${gen_cpp_src}.output" + OLD_FILE "${expected_output_file}" + FIXTURES_REQUIRED "build/${test_name}" + ) + endif() + add_test( NAME "build/execute/${test_name}" COMMAND @@ -176,7 +189,22 @@ function(cppfront_build_tests) FIXTURES_SETUP "build/execute/${test_name}" RESOURCE_LOCK "test.exe") + cppfront_find_build_result_file(expected_execution_file RESULT_FILE "${gen_cpp_src}.execution") + if (expected_execution_file) + cppfront_add_check_test( + NAME "build/execute/check/${test_name}" + NEW_FILE "${test_dir}/${gen_cpp_src}.execution" + OLD_FILE "${expected_execution_file}" + FIXTURES_REQUIRED "build/execute/${test_name}" + ) + endif() + if (CPPFRONT_DEVELOPING) + # Trigger regeneration to add the `check` tests for newly added results. + if (NOT DEFINED expected_output_file OR NOT DEFINED expected_execution_file) + file(GLOB unused CONFIGURE_DEPENDS "${BUILD_RESULTS_DIR}/${gen_cpp_src}.*") + endif() + add_test( NAME "build/update/${test_name}" COMMAND diff --git a/regression-tests/cmake/FindBuildResultFile.cmake.in b/regression-tests/cmake/FindBuildResultFile.cmake.in new file mode 100644 index 0000000..30eee3f --- /dev/null +++ b/regression-tests/cmake/FindBuildResultFile.cmake.in @@ -0,0 +1,28 @@ +# Set `OLD_BUILD_RESULTS_DIRS`. +file( + GLOB build_result_dirs + RELATIVE "@TEST_RESULTS_DIR@" + "@TEST_RESULTS_DIR@/@compiler_id@-*" +) +list(SORT build_result_dirs) +list(FIND build_result_dirs "@COMPILER_ITEM_NAME@" i) +list(SUBLIST build_result_dirs 0 ${i} OLD_BUILD_RESULTS_DIRS) +list(REVERSE OLD_BUILD_RESULTS_DIRS) + +function(cppfront_find_build_result_file out_var) + cmake_parse_arguments(PARSE_ARGV 0 ARG "OLD_ONLY" "RESULT_FILE" "") + + if (NOT ARG_OLD_ONLY) + set(extra_item "@COMPILER_ITEM_NAME@") + endif () + + foreach (build_results_dir IN ITEMS "${extra_item}" LISTS OLD_BUILD_RESULTS_DIRS) + set(result_file "@TEST_RESULTS_DIR@/${build_results_dir}/${ARG_RESULT_FILE}") + if (EXISTS "${result_file}") + set("${out_var}" "${result_file}" PARENT_SCOPE) + return() + endif () + endforeach () + + unset("${out_var}" PARENT_SCOPE) +endfunction() diff --git a/regression-tests/cmake/UpdateBuildOutput.cmake.in b/regression-tests/cmake/UpdateBuildOutput.cmake.in index 43ac5be..40a23ad 100644 --- a/regression-tests/cmake/UpdateBuildOutput.cmake.in +++ b/regression-tests/cmake/UpdateBuildOutput.cmake.in @@ -5,24 +5,24 @@ # - For "absence of output" to mean that "it's the same as the previous version". # - To commit only new and changed outputs in PRs for easier reviewing. +include("@CMAKE_CURRENT_BINARY_DIR@/FindBuildResultFile.cmake") + file(READ "${OUTPUT_FILE}" new_output) if (EXISTS "${EXECUTION_FILE}") file(READ "${EXECUTION_FILE}" new_execution) endif () + function(write path_var ext) - foreach (build_results_dir IN ITEMS @OLD_BUILD_RESULTS_DIRS@) - set(result_file "@TEST_RESULTS_DIR@/${build_results_dir}/${GEN_CPP_SRC}.${ext}") - if (EXISTS "${result_file}") - file(READ "${result_file}" "old_${ext}") - if (new_${ext} STREQUAL old_${ext}) - return() - else () - break () - endif () + cppfront_find_build_result_file(result_file OLD_ONLY RESULT_FILE "${GEN_CPP_SRC}.${ext}") + if (DEFINED result_file) + file(READ "${result_file}" "old_${ext}") + if (new_${ext} STREQUAL old_${ext}) + return() endif () - endforeach () + endif () file(COPY "${${path_var}}" DESTINATION "@BUILD_RESULTS_DIR@") endfunction() + write(OUTPUT_FILE "output") if (DEFINED new_execution) write(EXECUTION_FILE "execution") From cb0a14228dcc23d820f0c0f59e3520d72fbb7173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Fri, 9 Jun 2023 17:52:47 -0400 Subject: [PATCH 08/10] fix(test): add non-`check` tests when developing Test results will be updated, so make the tests use those results without a reconfiguration in-between. Also, trigger reconfiguration to add the `check` tests. --- regression-tests/CMakeLists.txt | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index 15e8d61..a8ae07a 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -79,7 +79,9 @@ function(cppfront_command_tests) ${COMMAND_ERROR_IS_FATAL} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) + set_tests_properties("codegen/${test_name}" PROPERTIES FIXTURES_SETUP "codegen/${test_name}") + unset(build_test_depends) if (CPPFRONT_DEVELOPING) add_test( NAME "codegen/update/${test_name}" @@ -107,15 +109,10 @@ function(cppfront_command_tests) string(REPLACE "*" "\\*" expected_output "${expected_output}") string(REPLACE "?" "\\?" expected_output "${expected_output}") - set_tests_properties( - "codegen/${test_name}" - PROPERTIES - FIXTURES_SETUP "codegen/${test_name}" - PASS_REGULAR_EXPRESSION "^${expected_output}$" - ) + set_tests_properties("codegen/${test_name}" PROPERTIES PASS_REGULAR_EXPRESSION "^${expected_output}$") endif () - if (ARG_EXPECTED_FILE) + if (EXISTS "${ARG_EXPECTED_FILE}") configure_file("${ARG_EXPECTED_FILE}" "${gen_cpp_src}.original" COPYONLY) cppfront_add_check_test( NAME "codegen/check/${test_name}" @@ -123,7 +120,13 @@ function(cppfront_command_tests) OLD_FILE "${gen_cpp_src}.original" FIXTURES_REQUIRED "codegen/${test_name}" ) + list(APPEND build_test_depends "codegen/check/${test_name}") + elseif (CPPFRONT_DEVELOPING) + # Trigger regeneration to add the `check` tests for newly added results. + file(GLOB unused CONFIGURE_DEPENDS "${ARG_EXPECTED_FILE}") + endif () + if (DEFINED build_test_depends AND DEFINED gen_cpp_src) cppfront_build_tests( SOURCE ${ARG_SOURCE} EXTRA_FLAGS ${ARG_EXTRA_FLAGS} @@ -160,7 +163,7 @@ function(cppfront_build_tests) set_tests_properties( "build/${test_name}" PROPERTIES - DEPENDS "${build_test_depends};codegen/check/${test_name}" + DEPENDS "${build_test_depends}" FIXTURES_SETUP "build/${test_name}" ) @@ -237,10 +240,6 @@ function(cppfront_tests) cmake_path(REPLACE_EXTENSION src "cpp" OUTPUT_VARIABLE expected_file) set(expected_file "${TEST_RESULTS_DIR}/${expected_file}") - if (NOT EXISTS "${expected_file}") - set(expected_file "") - endif () - cppfront_command_tests( SOURCE ${src} EXPECTED_FILE ${expected_file} From 963462c8cb7b7797516b76f17907b97742e5dabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Sun, 18 Jun 2023 13:22:57 -0400 Subject: [PATCH 09/10] fix(test): add glob for new error tests --- regression-tests/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index a8ae07a..b2c3d36 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -235,6 +235,11 @@ function(cppfront_tests) RELATIVE "${REGRESSION_TESTS_DIR}" CONFIGURE_DEPENDS "${REGRESSION_TESTS_DIR}/${ARG_GROUP}-*.cpp2" ) + # Trigger regeneration to recognize as a test to fail codegen. + file( + GLOB unused + CONFIGURE_DEPENDS "${REGRESSION_TESTS_DIR}/${ARG_GROUP}-*-error.cpp2" + ) foreach (src IN LISTS sources) cmake_path(REPLACE_EXTENSION src "cpp" OUTPUT_VARIABLE expected_file) From 08de0a6742e8aae51bc0e65204ea30b8f3b130b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johel=20Ernesto=20Guerrero=20Pe=C3=B1a?= Date: Sun, 23 Jul 2023 15:44:19 -0400 Subject: [PATCH 10/10] fix: account for C++23 tests --- regression-tests/CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index b2c3d36..4416f0a 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -145,6 +145,15 @@ function(cppfront_build_tests) configure_file("cmake/CMakeLists.txt.in" "${test_dir}/CMakeLists.txt" @ONLY) configure_file("cmake/test-case-config.cmake.in" "${test_dir}/test-case-config.cmake.in" COPYONLY) + set( + cxx_23_tests + pure2-bugfix-for-empty-index + ) + set(extra_flags) + if (test_name IN_LIST cxx_23_tests) + list(APPEND extra_flags "-DCMAKE_CXX_STANDARD=23") + endif () + add_test( NAME "build/${test_name}" COMMAND @@ -157,6 +166,7 @@ function(cppfront_build_tests) "-Dcppfront_DIR=${cppfront_DIR}" "-Dcppfront-exe_DIR=${cppfront-exe_DIR}" "-DCPPFRONT_FLAGS=${ARG_EXTRA_FLAGS}" + ${extra_flags} # There's `CMAKE_CXX_LINKER_LAUNCHER`, too. So far, it's not needed. "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_COMMAND};-D;OUTPUT_FILE=${gen_cpp_src}.output;-P;../../ExecuteWithRedirection.cmake;--" )