diff --git a/README.md b/README.md index 9824ae3..342a665 100644 --- a/README.md +++ b/README.md @@ -139,4 +139,4 @@ processing the directory. The CMake project `regression-tests/CMakeLists.txt` runs the test suite of cppfront. See "Regression tests" at [`./.github/workflows/ci.yml`](./.github/workflows/ci.yml) for how to set it up. -To update the test results, build with `--target cppfront_update_test_results`. +To have the test results updated, configure with `-DCPPFRONT_DEVELOPING=TRUE`. diff --git a/regression-tests/CMakeLists.txt b/regression-tests/CMakeLists.txt index e8b1916..4416f0a 100644 --- a/regression-tests/CMakeLists.txt +++ b/regression-tests/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 3.23) -project(cppfront-regression-tests LANGUAGES NONE) +project(cppfront-regression-tests LANGUAGES CXX) + +option(CPPFRONT_DEVELOPING "Test results are updated.") enable_testing() @@ -7,9 +9,53 @@ set(CPPFRONT_NO_MAGIC 1) find_package(cppfront REQUIRED) set(REGRESSION_TESTS_DIR "${CMAKE_CURRENT_LIST_DIR}/../cppfront/regression-tests") +set(TEST_RESULTS_DIR "${REGRESSION_TESTS_DIR}/test-results") + +# Set `COMPILER_ITEM_NAME`. +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(compiler_id "gcc") +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + set(compiler_id "apple-clang") +else () + string(TOLOWER "${CMAKE_CXX_COMPILER_ID}" compiler_id) +endif () +string(REGEX MATCH "[0-9]+" compiler_major_version "${CMAKE_CXX_COMPILER_VERSION}") +set(COMPILER_ITEM_NAME "${compiler_id}-${compiler_major_version}") + +# Setup `BUILD_RESULTS_DIR`. +set(BUILD_RESULTS_DIR "${TEST_RESULTS_DIR}/${COMPILER_ITEM_NAME}") + +if (CPPFRONT_DEVELOPING) + file(MAKE_DIRECTORY "${BUILD_RESULTS_DIR}") + + # Write compiler version output. + set(compiler_version_command "${CMAKE_CXX_COMPILER}" "--version") + if (CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") + set(compiler_id "clang") + endif () + execute_process( + COMMAND ${compiler_version_command} + OUTPUT_FILE "${BUILD_RESULTS_DIR}/${compiler_id}-version.output" + ) +endif () + +configure_file("cmake/ExecuteWithRedirection.cmake" "ExecuteWithRedirection.cmake" COPYONLY) +configure_file("cmake/ExecuteTestCase.cmake.in" "ExecuteTestCase.cmake" @ONLY) +configure_file("cmake/FindBuildResultFile.cmake.in" "FindBuildResultFile.cmake" @ONLY) +configure_file("cmake/UpdateBuildOutput.cmake.in" "UpdateBuildOutput.cmake" @ONLY) -configure_file("cmake/UpdateTestResults.cmake.in" "UpdateTestResults.cmake" @ONLY) -add_custom_target(cppfront_update_test_results) +include("${CMAKE_CURRENT_BINARY_DIR}/FindBuildResultFile.cmake") + +function(cppfront_add_check_test) + cmake_parse_arguments(PARSE_ARGV 0 ARG "" "NAME;NEW_FILE;OLD_FILE;FIXTURES_REQUIRED" "") + + add_test( + NAME "${ARG_NAME}" + COMMAND "${CMAKE_COMMAND}" -E compare_files "${ARG_NEW_FILE}" "${ARG_OLD_FILE}" + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + ) + set_tests_properties("${ARG_NAME}" PROPERTIES FIXTURES_REQUIRED "${ARG_FIXTURES_REQUIRED}") +endfunction() function(cppfront_command_tests) cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE;EXPECTED_FILE" "EXTRA_FLAGS") @@ -18,24 +64,39 @@ function(cppfront_command_tests) cmake_path(GET ARG_SOURCE STEM test_name) - file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/${test_name}") - add_custom_target( - "${test_name}" - COMMAND - "${CMAKE_COMMAND}" - -D "TEST_NAME=${test_name}" - -D "EXTRA_FLAGS=${ARG_EXTRA_FLAGS}" - -P "UpdateTestResults.cmake" - ) - add_dependencies(cppfront_update_test_results "${test_name}") - + if (NOT "${test_name}" MATCHES [[.*-error$]]) + set(gen_cpp_src "${test_name}.cpp") + set(COMMAND_ERROR_IS_FATAL "COMMAND_ERROR_IS_FATAL" "ANY") + endif () add_test( NAME "codegen/${test_name}" - COMMAND cppfront::cppfront "${ARG_SOURCE}" ${ARG_EXTRA_FLAGS} + COMMAND + "${CMAKE_COMMAND}" + -D "OUTPUT_FILE=${ARG_SOURCE}.output" + -P "ExecuteWithRedirection.cmake" + -- + "${CPPFRONT_EXECUTABLE}" "${ARG_SOURCE}" ${ARG_EXTRA_FLAGS} + ${COMMAND_ERROR_IS_FATAL} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" ) + set_tests_properties("codegen/${test_name}" PROPERTIES FIXTURES_SETUP "codegen/${test_name}") - set(expected_output_file "${REGRESSION_TESTS_DIR}/test-results/${ARG_SOURCE}.output") + unset(build_test_depends) + if (CPPFRONT_DEVELOPING) + add_test( + NAME "codegen/update/${test_name}" + COMMAND + "${CMAKE_COMMAND}" + -E copy_if_different + "${gen_cpp_src}" + "${ARG_SOURCE}.output" + "${TEST_RESULTS_DIR}" + ) + set_tests_properties("codegen/update/${test_name}" PROPERTIES FIXTURES_CLEANUP "codegen/${test_name}") + set(build_test_depends "codegen/update/${test_name}") + endif () + + set(expected_output_file "${TEST_RESULTS_DIR}/${ARG_SOURCE}.output") if (EXISTS "${expected_output_file}") file(READ "${expected_output_file}" expected_output) string(REPLACE "\\" "\\\\" expected_output "${expected_output}") @@ -48,25 +109,24 @@ function(cppfront_command_tests) string(REPLACE "*" "\\*" expected_output "${expected_output}") string(REPLACE "?" "\\?" expected_output "${expected_output}") - set_tests_properties( - "codegen/${test_name}" - PROPERTIES - FIXTURES_SETUP "codegen/${test_name}" - PASS_REGULAR_EXPRESSION "^${expected_output}$" - ) + set_tests_properties("codegen/${test_name}" PROPERTIES PASS_REGULAR_EXPRESSION "^${expected_output}$") endif () - if (ARG_EXPECTED_FILE) - cmake_path(REPLACE_EXTENSION ARG_SOURCE "cpp" OUTPUT_VARIABLE gen_cpp_src) - - add_test( + if (EXISTS "${ARG_EXPECTED_FILE}") + configure_file("${ARG_EXPECTED_FILE}" "${gen_cpp_src}.original" COPYONLY) + cppfront_add_check_test( NAME "codegen/check/${test_name}" - COMMAND "${CMAKE_COMMAND}" -E compare_files "${gen_cpp_src}" "${ARG_EXPECTED_FILE}" - WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + NEW_FILE "${gen_cpp_src}" + OLD_FILE "${gen_cpp_src}.original" + FIXTURES_REQUIRED "codegen/${test_name}" ) + list(APPEND build_test_depends "codegen/check/${test_name}") + elseif (CPPFRONT_DEVELOPING) + # Trigger regeneration to add the `check` tests for newly added results. + file(GLOB unused CONFIGURE_DEPENDS "${ARG_EXPECTED_FILE}") + endif () - set_tests_properties("codegen/check/${test_name}" PROPERTIES FIXTURES_REQUIRED "codegen/${test_name}") - + if (DEFINED build_test_depends AND DEFINED gen_cpp_src) cppfront_build_tests( SOURCE ${ARG_SOURCE} EXTRA_FLAGS ${ARG_EXTRA_FLAGS} @@ -77,26 +137,25 @@ endfunction() function(cppfront_build_tests) cmake_parse_arguments(PARSE_ARGV 0 ARG "" "SOURCE" "EXTRA_FLAGS") - # The following tests aren't expected to be buildable C++, even if - # cppfront succeeds. - set( - codegen_only_tests - mixed-postfix-expression-custom-formatting - ) - - cmake_path(GET ARG_SOURCE STEM stem) - if (stem IN_LIST codegen_only_tests) - return() - endif () + cmake_path(GET ARG_SOURCE STEM test_name) - set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/${stem}") - set(test_name "build/${stem}") + set(test_dir "${CMAKE_CURRENT_BINARY_DIR}/${test_name}") configure_file("${REGRESSION_TESTS_DIR}/${ARG_SOURCE}" "${test_dir}/${ARG_SOURCE}" COPYONLY) configure_file("cmake/CMakeLists.txt.in" "${test_dir}/CMakeLists.txt" @ONLY) + configure_file("cmake/test-case-config.cmake.in" "${test_dir}/test-case-config.cmake.in" COPYONLY) + + set( + cxx_23_tests + pure2-bugfix-for-empty-index + ) + set(extra_flags) + if (test_name IN_LIST cxx_23_tests) + list(APPEND extra_flags "-DCMAKE_CXX_STANDARD=23") + endif () add_test( - NAME "${test_name}" + NAME "build/${test_name}" COMMAND "${CMAKE_CTEST_COMMAND}" --build-and-test "${test_dir}" "${test_dir}/build" @@ -107,7 +166,74 @@ function(cppfront_build_tests) "-Dcppfront_DIR=${cppfront_DIR}" "-Dcppfront-exe_DIR=${cppfront-exe_DIR}" "-DCPPFRONT_FLAGS=${ARG_EXTRA_FLAGS}" + ${extra_flags} + # There's `CMAKE_CXX_LINKER_LAUNCHER`, too. So far, it's not needed. + "-DCMAKE_CXX_COMPILER_LAUNCHER=${CMAKE_COMMAND};-D;OUTPUT_FILE=${gen_cpp_src}.output;-P;../../ExecuteWithRedirection.cmake;--" ) + set_tests_properties( + "build/${test_name}" + PROPERTIES + DEPENDS "${build_test_depends}" + FIXTURES_SETUP "build/${test_name}" + ) + + cppfront_find_build_result_file(expected_output_file RESULT_FILE "${gen_cpp_src}.output") + if (expected_output_file) + cppfront_add_check_test( + NAME "build/check/${test_name}" + NEW_FILE "${test_dir}/build/${gen_cpp_src}.output" + OLD_FILE "${expected_output_file}" + FIXTURES_REQUIRED "build/${test_name}" + ) + endif() + + add_test( + NAME "build/execute/${test_name}" + COMMAND + "${CMAKE_COMMAND}" + -D "OUTPUT_FILE=${test_dir}/${gen_cpp_src}.execution" + -P "../ExecuteTestCase.cmake" + WORKING_DIRECTORY "${test_dir}" + ) + set_tests_properties( + "build/execute/${test_name}" + PROPERTIES + FIXTURES_REQUIRED "build/${test_name}" + FIXTURES_SETUP "build/execute/${test_name}" + RESOURCE_LOCK "test.exe") + + cppfront_find_build_result_file(expected_execution_file RESULT_FILE "${gen_cpp_src}.execution") + if (expected_execution_file) + cppfront_add_check_test( + NAME "build/execute/check/${test_name}" + NEW_FILE "${test_dir}/${gen_cpp_src}.execution" + OLD_FILE "${expected_execution_file}" + FIXTURES_REQUIRED "build/execute/${test_name}" + ) + endif() + + if (CPPFRONT_DEVELOPING) + # Trigger regeneration to add the `check` tests for newly added results. + if (NOT DEFINED expected_output_file OR NOT DEFINED expected_execution_file) + file(GLOB unused CONFIGURE_DEPENDS "${BUILD_RESULTS_DIR}/${gen_cpp_src}.*") + endif() + + add_test( + NAME "build/update/${test_name}" + COMMAND + "${CMAKE_COMMAND}" + -D "GEN_CPP_SRC=${gen_cpp_src}" + -D "OUTPUT_FILE=build/${gen_cpp_src}.output" + -D "EXECUTION_FILE=${gen_cpp_src}.execution" + -P "../UpdateBuildOutput.cmake" + WORKING_DIRECTORY "${test_dir}" + ) + set_tests_properties( + "build/update/${test_name}" + PROPERTIES + FIXTURES_CLEANUP "build/${test_name};build/execute/${test_name}" + ) + endif () endfunction() function(cppfront_tests) @@ -119,14 +245,15 @@ function(cppfront_tests) RELATIVE "${REGRESSION_TESTS_DIR}" CONFIGURE_DEPENDS "${REGRESSION_TESTS_DIR}/${ARG_GROUP}-*.cpp2" ) + # Trigger regeneration to recognize as a test to fail codegen. + file( + GLOB unused + CONFIGURE_DEPENDS "${REGRESSION_TESTS_DIR}/${ARG_GROUP}-*-error.cpp2" + ) foreach (src IN LISTS sources) cmake_path(REPLACE_EXTENSION src "cpp" OUTPUT_VARIABLE expected_file) - set(expected_file "${REGRESSION_TESTS_DIR}/test-results/${expected_file}") - - if (NOT EXISTS "${expected_file}") - set(expected_file "") - endif () + set(expected_file "${TEST_RESULTS_DIR}/${expected_file}") cppfront_command_tests( SOURCE ${src} diff --git a/regression-tests/cmake/CMakeLists.txt.in b/regression-tests/cmake/CMakeLists.txt.in index 7d26306..b03dafc 100644 --- a/regression-tests/cmake/CMakeLists.txt.in +++ b/regression-tests/cmake/CMakeLists.txt.in @@ -13,3 +13,30 @@ target_compile_options( PRIVATE "$<$:/experimental:module>" ) + +include(GNUInstallDirs) +include(CMakePackageConfigHelpers) + +set(TEST_CASE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_DATADIR}/cmake/test-case") + +install( + TARGETS test-case + EXPORT test-case-targets +) + +install( + EXPORT test-case-targets + DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}" +) + +configure_package_config_file( + "test-case-config.cmake.in" + "test-case-config.cmake" + INSTALL_DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}" + PATH_VARS CMAKE_INSTALL_BINDIR +) + +install( + FILES "${CMAKE_CURRENT_BINARY_DIR}/test-case-config.cmake" + DESTINATION "${TEST_CASE_INSTALL_CMAKEDIR}" +) diff --git a/regression-tests/cmake/ExecuteTestCase.cmake.in b/regression-tests/cmake/ExecuteTestCase.cmake.in new file mode 100644 index 0000000..9840075 --- /dev/null +++ b/regression-tests/cmake/ExecuteTestCase.cmake.in @@ -0,0 +1,23 @@ +execute_process( + COMMAND "${CMAKE_COMMAND}" --install "build" --prefix "_local" + COMMAND_ERROR_IS_FATAL ANY +) +find_package(test-case REQUIRED PATHS "${CMAKE_CURRENT_BINARY_DIR}/_local" NO_DEFAULT_PATH) +execute_process( + COMMAND + "${CMAKE_COMMAND}" + -E copy + "${TEST_CASE_EXECUTABLE}" + "@BUILD_RESULTS_DIR@/test.exe" +) +execute_process( + COMMAND + "${CMAKE_COMMAND}" + -D "OUTPUT_FILE=${OUTPUT_FILE}" + -P "@CMAKE_CURRENT_BINARY_DIR@/ExecuteWithRedirection.cmake" + -- + "./test.exe" + COMMAND_ERROR_IS_FATAL ANY + WORKING_DIRECTORY "@BUILD_RESULTS_DIR@" +) +execute_process(COMMAND "${CMAKE_COMMAND}" -E rm "@BUILD_RESULTS_DIR@/test.exe") diff --git a/regression-tests/cmake/ExecuteWithRedirection.cmake b/regression-tests/cmake/ExecuteWithRedirection.cmake new file mode 100644 index 0000000..9f98aa6 --- /dev/null +++ b/regression-tests/cmake/ExecuteWithRedirection.cmake @@ -0,0 +1,14 @@ +if (NOT "${CMAKE_ARGV5}" STREQUAL "--") + message(FATAL_ERROR "Unexpected argument.") +endif () +foreach (i RANGE 6 ${CMAKE_ARGC}) + list(APPEND command_args "${CMAKE_ARGV${i}}") # Probably doesn't handle nested `;`. +endforeach () +execute_process( + COMMAND ${command_args} + OUTPUT_VARIABLE OUTPUT + ERROR_VARIABLE OUTPUT + ECHO_OUTPUT_VARIABLE + ECHO_ERROR_VARIABLE +) +file(WRITE "${OUTPUT_FILE}" "${OUTPUT}") diff --git a/regression-tests/cmake/FindBuildResultFile.cmake.in b/regression-tests/cmake/FindBuildResultFile.cmake.in new file mode 100644 index 0000000..30eee3f --- /dev/null +++ b/regression-tests/cmake/FindBuildResultFile.cmake.in @@ -0,0 +1,28 @@ +# Set `OLD_BUILD_RESULTS_DIRS`. +file( + GLOB build_result_dirs + RELATIVE "@TEST_RESULTS_DIR@" + "@TEST_RESULTS_DIR@/@compiler_id@-*" +) +list(SORT build_result_dirs) +list(FIND build_result_dirs "@COMPILER_ITEM_NAME@" i) +list(SUBLIST build_result_dirs 0 ${i} OLD_BUILD_RESULTS_DIRS) +list(REVERSE OLD_BUILD_RESULTS_DIRS) + +function(cppfront_find_build_result_file out_var) + cmake_parse_arguments(PARSE_ARGV 0 ARG "OLD_ONLY" "RESULT_FILE" "") + + if (NOT ARG_OLD_ONLY) + set(extra_item "@COMPILER_ITEM_NAME@") + endif () + + foreach (build_results_dir IN ITEMS "${extra_item}" LISTS OLD_BUILD_RESULTS_DIRS) + set(result_file "@TEST_RESULTS_DIR@/${build_results_dir}/${ARG_RESULT_FILE}") + if (EXISTS "${result_file}") + set("${out_var}" "${result_file}" PARENT_SCOPE) + return() + endif () + endforeach () + + unset("${out_var}" PARENT_SCOPE) +endfunction() diff --git a/regression-tests/cmake/UpdateBuildOutput.cmake.in b/regression-tests/cmake/UpdateBuildOutput.cmake.in new file mode 100644 index 0000000..40a23ad --- /dev/null +++ b/regression-tests/cmake/UpdateBuildOutput.cmake.in @@ -0,0 +1,29 @@ +# Updates an output for the current compiler-version +# when it differs from its previous version or +# when no previous version has the output file. +# Objectives: +# - For "absence of output" to mean that "it's the same as the previous version". +# - To commit only new and changed outputs in PRs for easier reviewing. + +include("@CMAKE_CURRENT_BINARY_DIR@/FindBuildResultFile.cmake") + +file(READ "${OUTPUT_FILE}" new_output) +if (EXISTS "${EXECUTION_FILE}") + file(READ "${EXECUTION_FILE}" new_execution) +endif () + +function(write path_var ext) + cppfront_find_build_result_file(result_file OLD_ONLY RESULT_FILE "${GEN_CPP_SRC}.${ext}") + if (DEFINED result_file) + file(READ "${result_file}" "old_${ext}") + if (new_${ext} STREQUAL old_${ext}) + return() + endif () + endif () + file(COPY "${${path_var}}" DESTINATION "@BUILD_RESULTS_DIR@") +endfunction() + +write(OUTPUT_FILE "output") +if (DEFINED new_execution) + write(EXECUTION_FILE "execution") +endif () diff --git a/regression-tests/cmake/UpdateTestResults.cmake.in b/regression-tests/cmake/UpdateTestResults.cmake.in deleted file mode 100644 index 0694ebb..0000000 --- a/regression-tests/cmake/UpdateTestResults.cmake.in +++ /dev/null @@ -1,16 +0,0 @@ -set(SOURCE "${TEST_NAME}.cpp2") -set(EXPECTED_FILE "test-results/${TEST_NAME}.cpp") -set(EXPECTED_FILE_OUTPUT "test-results/${SOURCE}.output") -set(FILE_OUTPUT "@CMAKE_CURRENT_BINARY_DIR@/${TEST_NAME}/output") -if (NOT "${TEST_NAME}" MATCHES [[.*-error$]]) - set(COMMAND_ERROR_IS_FATAL "COMMAND_ERROR_IS_FATAL" "ANY") -endif () -execute_process( - COMMAND "@CPPFRONT_EXECUTABLE@" "${SOURCE}" ${EXTRA_FLAGS} -o "${EXPECTED_FILE}" - WORKING_DIRECTORY "@REGRESSION_TESTS_DIR@" - OUTPUT_FILE "${FILE_OUTPUT}" - ERROR_FILE "${FILE_OUTPUT}" - ${COMMAND_ERROR_IS_FATAL} -) -execute_process( - COMMAND "${CMAKE_COMMAND}" -E copy_if_different "${FILE_OUTPUT}" "@REGRESSION_TESTS_DIR@/${EXPECTED_FILE_OUTPUT}") diff --git a/regression-tests/cmake/test-case-config.cmake.in b/regression-tests/cmake/test-case-config.cmake.in new file mode 100644 index 0000000..4f31576 --- /dev/null +++ b/regression-tests/cmake/test-case-config.cmake.in @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.23) +@PACKAGE_INIT@ + +set_and_check( + TEST_CASE_EXECUTABLE + "@PACKAGE_CMAKE_INSTALL_BINDIR@/test-case@CMAKE_EXECUTABLE_SUFFIX@" +)