CI: Update and fix (#957)

* CI: Unify diffing in run-tests.shi and update workflow

* CI: Update MacOs tests

Author: jarzec

.github/workflows/regression-tests.yml

@@ -27,27 +27,27 @@ jobs:
             compiler: cl.exe
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Run regression tests - Linux and macOS version
       if: startsWith(matrix.os, 'ubuntu') || matrix.os == 'macos-13'
       run: |
         cd regression-tests
-        bash run-tests.sh -c ${{ matrix.compiler }}
+        bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
 
     - name: Run regression tests - Windows version
       if: matrix.os == 'windows-latest'
       run: |
         "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" && ^
         git config --local core.autocrlf false && ^
         cd regression-tests && ^
-        bash run-tests.sh -c ${{ matrix.compiler }}
+        bash run-tests.sh -c ${{ matrix.compiler }} -l ${{ matrix.os }}
       shell: cmd
 
     - name: Upload patch
       if: ${{ !cancelled() }}
       uses: actions/upload-artifact@v4
       with:
-        name: ${{ matrix.compiler }}-patch.diff
-        path: regression-tests/${{ matrix.compiler }}-patch.diff
+        name: ${{ matrix.os }}-${{ matrix.compiler }}.patch
+        path: regression-tests/${{ matrix.os }}-${{ matrix.compiler }}.patch
         if-no-files-found: ignore

regression-tests/run-tests.sh

@@ -2,13 +2,33 @@
 
 ################
 usage() {
-    echo "Usage: $0 -c <compiler> [-t <tests to run>]"
+    echo "Usage: $0 -c <compiler> [-l <run label>] [-t <tests to run>]"
     echo "    -c <compiler>     The compiler to use for the test"
+    echo "    -l <run label>    The label to use in output patch file name"
     echo "    -t <tests to run> Runs only the provided, comma-separated tests (filenames including .cpp2)"
     echo "                      If the argument is not used all tests are run"
     exit 1
 }
 
+################
+# Check diff of the provided file using the given diff options
+# If the diff is not empty print it with the provided message
+report_diff () {
+    file="$1"
+    diff_opt="$2"
+    error_msg="$3"
+    patch_file="$4"
+
+    # Compare the content with the reference value checked in git
+    diff_output=$(git diff "$diff_opt" -- "$file")
+    if [[ -n "$diff_output" ]]; then
+        echo "            $error_msg:"
+        echo "                $file"
+        printf "\n$diff_output\n\n" | tee -a "$patch_file"
+        failure=1
+    fi
+}
+
 ################
 # Check file existence and compare its state against the version in git
 check_file () {
@@ -26,34 +46,36 @@ check_file () {
     git ls-files --error-unmatch "$file" > /dev/null 2>&1
     untracked=$?
 
+    patch_file="$label$cxx_compiler.patch"
+
     if [[ $untracked -eq 1 ]]; then
-        echo "            The $description is not tracked by git:"
-        echo "                $file"
         # Add the file to the index to be able to diff it...
         git add "$file"
-        # ... print the diff ...
-        git --no-pager diff HEAD -- "$file" | tee -a "$cxx_compiler-patch.diff"
-        # ... and remove the file from the diff
+        # ... report the diff ...
+        report_diff "$file" \
+            "HEAD" \
+            "The $description is not tracked by git" \
+            "$patch_file"
+        # ... and remove the file from the index
         git rm --cached -- "$file" > /dev/null 2>&1
-        
-        failure=1
     else
         # Compare the content with the reference value checked in git
-        diff_output=$(git diff --ignore-cr-at-eol -- "$file")
-        if [[ -n "$diff_output" ]]; then
-            echo "            Non-matching $description:"
-            printf "\n$diff_output\n\n" | tee -a "$cxx_compiler-patch.diff"
-            failure=1
-        fi
+        report_diff "$file" \
+            "--ignore-cr-at-eol" \
+            "Non-matching $description" \
+            "$patch_file"
     fi
 }
 
-optstring="c:t:"
+optstring="c:l:t:"
 while getopts ${optstring} arg; do
   case "${arg}" in
     c)
         cxx_compiler="${OPTARG}"
         ;;
+    l)
+        label="${OPTARG}-"
+        ;;
     t)
         # Replace commas with spaces
         chosen_tests=${OPTARG/,/ }
@@ -103,11 +125,11 @@ expected_results_dir="test-results"
 ################
 # Get the directory with the exec outputs and compilation command
 if [[ "$cxx_compiler" == *"cl.exe"* ]]; then
-    compiler_cmd='cl.exe -nologo -std:c++latest -MD -EHsc -I ..\include -I ..\..\..\include -Fe:'
+    compiler_cmd='cl.exe -nologo -std:c++latest -MD -EHsc -I ..\..\..\include -Fe:'
     exec_out_dir="$expected_results_dir/msvc-2022"
     compiler_version=$(cl.exe)
 else
-    compiler_cmd="$cxx_compiler -I../include -I../../../include -std=c++20 -pthread -o "
+    compiler_cmd="$cxx_compiler -I../../../include -std=c++20 -pthread -o "
 
     compiler_ver=$("$cxx_compiler" --version)
     if [[ "$compiler_ver" == *"Apple clang version 14.0"* ]]; then

regression-tests/test-results/apple-clang-14/pure2-assert-optional-not-null.cpp.execution

@@ -1,2 +1 @@
 Null safety violation: std::optional does not contain a value
-libc++abi: terminating

regression-tests/test-results/apple-clang-14/pure2-assert-shared-ptr-not-null.cpp.execution

@@ -1,2 +1 @@
 Null safety violation: std::shared_ptr is empty
-libc++abi: terminating

regression-tests/test-results/apple-clang-14/pure2-assert-unique-ptr-not-null.cpp.execution

@@ -1,2 +1 @@
 Null safety violation: std::unique_ptr is empty
-libc++abi: terminating