[Utils] Add --update-tests to lit

This adds a flag to lit for detecting and updating failing tests when
possible to do so automatically. The flag uses a plugin architecture
where config files can add additional auto-updaters for the types of
tests in the test suite. When a test fails with --update-tests enabled
lit passes the test RUN invocation and output to each registered test
updater until one of them signals that it updated the test. As such it
is the responsibility of the test updater to only update tests where it
is reasonably certain that it will actually fix the test, or come close
to doing so.

Also adds two initial test updaters: one built into lit, which automatically
updates the 'expected' file when a 'diff' command fails; and one
specific to clang's Sema test suite, which updates the
expected-[diag-kind] lines in tests using the -verify flag.

Author: hnrklssn

clang/test/Sema/lit.local.cfg

@@ -2,3 +2,15 @@ config.substitutions = list(config.substitutions)
 config.substitutions.insert(
     0, (r"%clang\b", """*** Do not use the driver in Sema tests. ***""")
 )
+
+if lit_config.update_tests:
+    import sys
+    import os
+
+    curdir = os.path.dirname(os.path.realpath(__file__))
+    testdir = os.path.dirname(curdir)
+    clangdir = os.path.dirname(testdir)
+    utilspath = os.path.join(clangdir, "utils")
+    sys.path.append(utilspath)
+    from UpdateVerifyTests.litplugin import verify_test_updater
+    lit_config.test_updaters.append(verify_test_updater)

clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'a'}}
     a = 2; a = 2;

clang/test/utils/update-verify-tests/Inputs/duplicate-diag.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1 2{{use of undeclared identifier 'a'}}
     a = 2; a = 2;

clang/test/utils/update-verify-tests/Inputs/infer-indentation.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
          //     expected-error@+1    2      {{use of undeclared identifier 'a'}}
     a = 2; a = 2; b = 2; b = 2; c = 2;

clang/test/utils/update-verify-tests/Inputs/infer-indentation.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
          //     expected-error@+3          {{use of undeclared identifier 'c'}}
          //     expected-error@+2    2      {{use of undeclared identifier 'b'}}

clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c

@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2;
     // expected-error@-1{{use of undeclared identifier 'a'}}
     b = 2;// expected-error{{use of undeclared identifier 'b'}}
     c = 2;
-    // expected-error@5{{use of undeclared identifier 'c'}}
+    // expected-error@7{{use of undeclared identifier 'c'}}
     d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
 
     e = 2; // error to trigger mismatch

clang/test/utils/update-verify-tests/Inputs/leave-existing-diags.c.expected

@@ -1,9 +1,11 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2;
     // expected-error@-1{{use of undeclared identifier 'a'}}
     b = 2;// expected-error{{use of undeclared identifier 'b'}}
     c = 2;
-    // expected-error@5{{use of undeclared identifier 'c'}}
+    // expected-error@7{{use of undeclared identifier 'c'}}
     d = 2; // expected-error-re{{use of {{.*}} identifier 'd'}}
 
     // expected-error@+1{{use of undeclared identifier 'e'}}

clang/test/utils/update-verify-tests/Inputs/multiple-errors.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2;
     b = 2;

clang/test/utils/update-verify-tests/Inputs/multiple-errors.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'a'}}
     a = 2;

clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2; b = 2; c = 2;
 }

clang/test/utils/update-verify-tests/Inputs/multiple-missing-errors-same-line.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+3{{use of undeclared identifier 'c'}}
     // expected-error@+2{{use of undeclared identifier 'b'}}

clang/test/utils/update-verify-tests/Inputs/no-checks.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     bar = 2;
 }

clang/test/utils/update-verify-tests/Inputs/no-checks.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'bar'}}
     bar = 2;

clang/test/utils/update-verify-tests/Inputs/no-diags.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{asdf}}
     int a = 2;

clang/test/utils/update-verify-tests/Inputs/no-diags.c.expected

@@ -1,4 +1,6 @@
 // expected-no-diagnostics
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     int a = 2;
 }

clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 // expected-no-diagnostics
 void foo() {
     a = 2;

clang/test/utils/update-verify-tests/Inputs/no-expected-diags.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'a'}}
     a = 2;

clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2; // check-error{{asdf}}
            // expected-error@-1{ignored}}

clang/test/utils/update-verify-tests/Inputs/non-default-prefix.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify=check %s
+// RUN: diff %s %s.expected
 void foo() {
     a = 2; // check-error{{use of undeclared identifier 'a'}}
            // expected-error@-1{ignored}}

clang/test/utils/update-verify-tests/Inputs/update-same-line.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     bar = 2;     //   expected-error       {{asdf}}
 }

clang/test/utils/update-verify-tests/Inputs/update-same-line.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     bar = 2;     //   expected-error       {{use of undeclared identifier 'bar'}}
 }

clang/test/utils/update-verify-tests/Inputs/update-single-check.c

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{asdf}}
     bar = 2;

clang/test/utils/update-verify-tests/Inputs/update-single-check.c.expected

@@ -1,3 +1,5 @@
+// RUN: %clang_cc1 -verify %s
+// RUN: diff %s %s.expected
 void foo() {
     // expected-error@+1{{use of undeclared identifier 'bar'}}
     bar = 2;

clang/test/utils/update-verify-tests/LitTests/.gitignore

@@ -0,0 +1,4 @@
+*.c
+*.c.expected
+.lit_test_times.txt
+Output

clang/test/utils/update-verify-tests/LitTests/lit.cfg

@@ -0,0 +1,31 @@
+import lit.formats
+import lit.llvm
+lit.llvm.initialize(lit_config, config)
+from lit.llvm import llvm_config
+from os.path import dirname
+
+config.name = "Lit Update Verify Tests"
+config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
+config.suffixes = [
+    ".c",
+]
+config.test_source_root = dirname(__file__)
+config.target_triple = None
+config.host_triple = None
+
+config.llvm_tools_dir = lit_config.path[0] # --path is explicitly passed by lit-plugin.test
+llvm_config.use_default_substitutions()
+llvm_config.use_clang()
+config.substitutions.append(("%clang_cc1", "%clang -cc1"))
+
+if lit_config.update_tests:
+    import sys
+    import os
+    
+    config.clang_src_dir = dirname(dirname(dirname(dirname(config.test_source_root))))
+    utilspath = os.path.join(config.clang_src_dir, "utils")
+    sys.path.append(utilspath)
+    from UpdateVerifyTests.litplugin import verify_test_updater
+    # normally we'd append to the existing list, but when testing
+    # verify_test_updater we don't want diff_test_updater to accidentally interfere
+    lit_config.test_updaters = [verify_test_updater]

clang/test/utils/update-verify-tests/lit-plugin.test

@@ -0,0 +1,5 @@
+# RUN: cp %S/Inputs/*.c %S/LitTests
+# RUN: cp %S/Inputs/*.c.expected %S/LitTests
+# RUN: not %{lit} %S/LitTests
+# RUN: not %{lit} %S/LitTests --update-tests
+# RUN: %{lit} %S/LitTests

clang/test/utils/update-verify-tests/lit.local.cfg

@@ -12,6 +12,7 @@ if config.standalone_build:
     config.unsupported = True
 else:
     config.suffixes = [".test"]
+    config.excludes.add("LitTests")
 
     script_path = os.path.join(
         config.clang_src_dir, "utils", "update-verify-tests.py"
@@ -23,3 +24,13 @@ else:
             "%s %s" % (python, shell_quote(script_path)),
         )
     )
+    config.substitutions.append(("%{lit}", "%{lit-no-order-opt} --order=lexical"))
+
+    lit_path = os.path.join(config.llvm_tools_dir, "llvm-lit")
+    lit_path = os.path.abspath(lit_path)
+    config.substitutions.append(
+    (
+        "%{lit-no-order-opt}",
+        f"{python} {lit_path} -j1 --path {config.llvm_tools_dir}"
+    )
+)

clang/utils/UpdateVerifyTests/__init__.py

@@ -0,0 +1 @@
+