[cfe] Mention update_expectations.dart in failure log

Change-Id: I692e6aa8571c14817831b7c6a135b2cd6ef0ed2b
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/178991
Reviewed-by: Jens Johansen <[email protected]>
Commit-Queue: Johnni Winther <[email protected]>

Author: johnniwinther

pkg/front_end/test/fasta/testing/suite.dart

@@ -294,6 +294,9 @@ class FastaContext extends ChainContext with MatchContext {
   @override
   String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true';
 
+  @override
+  bool get canBeFixWithUpdateExpectations => true;
+
   @override
   final ExpectationSet expectationSet =
       new ExpectationSet.fromJsonList(jsonDecode(EXPECTATIONS));
@@ -1117,7 +1120,8 @@ class Outline extends Step<TestDescription, ComponentResult, FastaContext> {
                     description, p, userLibraries, options, sourceTarget),
                 context.expectationSet["InstrumentationMismatch"],
                 instrumentation.problemsAsString,
-                autoFixCommand: '${UPDATE_COMMENTS}=true');
+                autoFixCommand: '${UPDATE_COMMENTS}=true',
+                canBeFixWithUpdateExpectations: true);
           }
         }
       }

pkg/front_end/test/fasta/textual_outline_suite.dart

@@ -59,6 +59,9 @@ class Context extends ChainContext with MatchContext {
   @override
   String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true';
 
+  @override
+  bool get canBeFixWithUpdateExpectations => true;
+
   Context(this.updateExpectations);
 
   final List<Step> steps = const <Step>[

pkg/front_end/test/incremental_load_from_dill_suite.dart

@@ -1120,7 +1120,8 @@ Result<TestData> checkExpectFile(TestData data, int worldNum,
           "${extra}Unexpected serialized representation. "
           "Fix or update $uri to contain the below:\n\n"
           "$actualSerialized",
-          autoFixCommand: "updateExpectations=true");
+          autoFixCommand: "updateExpectations=true",
+          canBeFixWithUpdateExpectations: true);
     }
   }
   return null;

pkg/front_end/test/parser_suite.dart

@@ -101,6 +101,9 @@ class Context extends ChainContext with MatchContext {
   @override
   String get updateExpectationsOption => '${UPDATE_EXPECTATIONS}=true';
 
+  @override
+  bool get canBeFixWithUpdateExpectations => true;
+
   final bool addTrace;
   final bool annotateLines;
   final String suiteName;

pkg/front_end/test/unit_test_suites.dart

@@ -138,24 +138,30 @@ class ResultLogger implements Logger {
       "matches": matchedExpectations,
     }));
     if (!matchedExpectations) {
-      String failureLog = result.log;
+      StringBuffer sb = new StringBuffer();
+      sb.write(result.log);
       if (result.error != null) {
-        failureLog = "$failureLog\n\n${result.error}";
+        sb.write("\n\n${result.error}");
       }
       if (result.trace != null) {
-        failureLog = "$failureLog\n\n${result.trace}";
+        sb.write("\n\n${result.trace}");
       }
+      sb.write("\n\nTo re-run this test, run:");
+      sb.write("\n\n   dart pkg/front_end/test/unit_test_suites.dart -p "
+          "$testName");
       if (result.autoFixCommand != null) {
-        failureLog = "$failureLog\n\n"
-            "To re-run this test, run:\n\n"
-            "   dart pkg/front_end/test/unit_test_suites.dart -p $testName\n\n"
-            "To automatically update the test expectations, run:\n\n"
-            "   dart pkg/front_end/test/unit_test_suites.dart -p $testName "
-            "-D${result.autoFixCommand}\n";
-      } else {
-        failureLog = "$failureLog\n\nRe-run this test: dart "
-            "pkg/front_end/test/unit_test_suites.dart -p $testName";
+        sb.write("\n\nTo automatically update the test expectations, run:");
+        sb.write("\n\n   dart pkg/front_end/test/unit_test_suites.dart -p "
+            "$testName -D${result.autoFixCommand}");
+        if (result.canBeFixWithUpdateExpectations) {
+          sb.write('\n\nTo update test expectations for all tests at once, '
+              'run:');
+          sb.write('\n\n  dart pkg/front_end/tool/update_expectations.dart');
+          sb.write('\n\nNote that this takes a long time and should only be '
+              'used when many tests need updating.\n');
+        }
       }
+      String failureLog = sb.toString();
       String outcome = "${result.outcome}";
       logsPort.send(jsonEncode({
         "name": testName,

pkg/front_end/test/utils/kernel_chain.dart

@@ -66,6 +66,8 @@ abstract class MatchContext implements ChainContext {
 
   String get updateExpectationsOption;
 
+  bool get canBeFixWithUpdateExpectations;
+
   ExpectationSet get expectationSet;
 
   Expectation get expectationFileMismatch =>
@@ -96,7 +98,10 @@ abstract class MatchContext implements ChainContext {
             output, onMismatch, "$uri doesn't match ${expectedFile.uri}\n$diff",
             autoFixCommand: onMismatch == expectationFileMismatch
                 ? updateExpectationsOption
-                : null);
+                : null,
+            canBeFixWithUpdateExpectations:
+                onMismatch == expectationFileMismatch &&
+                    canBeFixWithUpdateExpectations);
       } else {
         return new Result<O>.pass(output);
       }

pkg/testing/lib/src/chain.dart

@@ -358,8 +358,16 @@ class Result<O> {
   /// update the test to match new expectations.
   final String autoFixCommand;
 
+  /// If set, the test can be fixed by running
+  ///
+  ///     dart pkg/front_end/tool/update_expectations.dart
+  ///
+  final bool canBeFixWithUpdateExpectations;
+
   Result(this.output, this.outcome, this.error,
-      {this.trace, this.autoFixCommand});
+      {this.trace,
+      this.autoFixCommand,
+      this.canBeFixWithUpdateExpectations: false});
 
   Result.pass(O output) : this(output, Expectation.Pass, null);
 
@@ -384,7 +392,9 @@ class Result<O> {
 
   Result<O2> copyWithOutput<O2>(O2 output) {
     return new Result<O2>(output, outcome, error,
-        trace: trace, autoFixCommand: autoFixCommand)
+        trace: trace,
+        autoFixCommand: autoFixCommand,
+        canBeFixWithUpdateExpectations: canBeFixWithUpdateExpectations)
       ..logs.addAll(logs);
   }
 }