Allow Flutter golden file tests to be flaky (#114450)

* allow marking a golden check as flaky

* add matchesFlutterGolden to analyze.dart; no tags for flutter_goldens/lib

* Pause

* ++

* ++

* Analyzer therapy

* Once more with feeling

* Nits

* Review feedback

* Silly oops

* Test progress

* More tests

* Finish

* Nits

* Analyzer

* Review feedback

Co-authored-by: Yegor Jbanov <[email protected]>

Author: Piinks

dev/automated_tests/flutter_test/flutter_gold_test.dart

@@ -8,7 +8,7 @@ import 'dart:typed_data';
 
 import 'package:file/file.dart';
 import 'package:file/memory.dart';
-import 'package:flutter_goldens/flutter_goldens.dart';
+import 'package:flutter_goldens/src/flutter_goldens_io.dart';
 import 'package:flutter_test/flutter_test.dart';
 import 'package:platform/platform.dart';
 

dev/bots/analyze.dart

@@ -418,8 +418,8 @@ Future<void> verifyNoSyncAsyncStar(String workingDirectory, {int minimumMatches
   }
 }
 
-final RegExp _findGoldenTestPattern = RegExp(r'matchesGoldenFile\(');
-final RegExp _findGoldenDefinitionPattern = RegExp(r'matchesGoldenFile\(Object');
+final RegExp _findGoldenTestPattern = RegExp(r'(matchesGoldenFile|expectFlakyGolden)\(');
+final RegExp _findGoldenDefinitionPattern = RegExp(r'(matchesGoldenFile|expectFlakyGolden)\(Object');
 final RegExp _leadingComment = RegExp(r'//');
 final RegExp _goldenTagPattern1 = RegExp(r'@Tags\(');
 final RegExp _goldenTagPattern2 = RegExp(r"'reduced-test-set'");
@@ -431,8 +431,17 @@ const String _ignoreGoldenTag = '// flutter_ignore: golden_tag (see analyze.dart
 const String _ignoreGoldenTagForFile = '// flutter_ignore_for_file: golden_tag (see analyze.dart)';
 
 Future<void> verifyGoldenTags(String workingDirectory, { int minimumMatches = 2000 }) async {
+  // Skip flutter_goldens/lib because this library uses `matchesGoldenFile`
+  // but is not itself a test that needs tags.
+  final String flutterGoldensPackageLib = path.join(flutterPackages, 'flutter_goldens', 'lib');
+  bool isWithinFlutterGoldenLib(File file) {
+    return path.isWithin(flutterGoldensPackageLib, file.path);
+  }
+
   final List<String> errors = <String>[];
-  await for (final File file in _allFiles(workingDirectory, 'dart', minimumMatches: minimumMatches)) {
+  final Stream<File> allTestFiles = _allFiles(workingDirectory, 'dart', minimumMatches: minimumMatches)
+    .where((File file) => !isWithinFlutterGoldenLib(file));
+  await for (final File file in allTestFiles) {
     bool needsTag = false;
     bool hasTagNotation = false;
     bool hasReducedTag = false;

dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_ignore.dart

@@ -0,0 +1,16 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This would fail analysis, but it is ignored
+// flutter_ignore_for_file: golden_tag (see analyze.dart)
+
+@Tags(<String>['some-other-tag'])
+
+import 'package:test/test.dart';
+
+import 'golden_class.dart';
+
+void main() {
+  expectFlakyGolden('key', 'String');
+}

dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart

@@ -0,0 +1,14 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// The reduced test set tag is missing. This should fail analysis.
+@Tags(<String>['some-other-tag'])
+
+import 'package:test/test.dart';
+
+import 'golden_class.dart';
+
+void main() {
+  expectFlakyGolden('finder', 'missing_tag.png');
+}

examples/api/test/goldens_io.dart renamed to dev/bots/test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart

@@ -2,4 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-export 'package:flutter_goldens/flutter_goldens.dart' show testExecutable;
+// The tag is missing. This should fail analysis.
+
+import 'golden_class.dart';
+
+void main() {
+  expectFlakyGolden('key', 'missing_tag.png');
+}

dev/bots/test/analyze-test-input/root/packages/foo/golden_class.dart

@@ -7,3 +7,7 @@
 void matchesGoldenFile(Object key) {
   return;
 }
+
+void expectFlakyGolden(Object key, String string){
+  return;
+}

dev/bots/test/analyze-test-input/root/packages/foo/golden_doc.dart

@@ -35,8 +35,11 @@
 /// ```
 /// {@end-tool}
 ///
+/// expectFlakyGolden(a, b)
 
 // Other comments
 // matchesGoldenFile('comment.png');
+// expectFlakyGolden(a, b);
 
 String literal = 'matchesGoldenFile()'; // flutter_ignore: golden_tag (see analyze.dart)
+String flakyLiteral = 'expectFlakyGolden';

dev/bots/test/analyze_test.dart

@@ -79,7 +79,9 @@ void main() {
                          'at the top of the file before import statements.';
     const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'.";
     final List<String> lines = <String>[
+        '║ test/analyze-test-input/root/packages/foo/flaky_golden_no_tag.dart: $noTag',
         '║ test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag',
+        '║ test/analyze-test-input/root/packages/foo/flaky_golden_missing_tag.dart: $missingTag',
         '║ test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag',
       ]
       .map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/'))

dev/devicelab/bin/tasks/technical_debt__cost.dart

@@ -17,6 +17,7 @@ const double todoCost = 1009.0; // about two average SWE days, in dollars
 const double ignoreCost = 2003.0; // four average SWE days, in dollars
 const double pythonCost = 3001.0; // six average SWE days, in dollars
 const double skipCost = 2473.0; // 20 hours: 5 to fix the issue we're ignoring, 15 to fix the bugs we missed because the test was off
+const double flakyGoldenCost = 2467.0; // Similar to skip cost
 const double ignoreForFileCost = 2477.0; // similar thinking as skipCost
 const double asDynamicCost = 2011.0; // a few days to refactor the code.
 const double deprecationCost = 233.0; // a few hours to remove the old code.
@@ -69,6 +70,9 @@ Future<double> findCostsForFile(File file) async {
     if (isTest && line.contains('skip:') && !line.contains('[intended]')) {
       total += skipCost;
     }
+    if (isTest && line.contains('expectFlakyGolden(')) {
+      total += flakyGoldenCost;
+    }
     if (isDart && isOptingOutOfNullSafety(line)) {
       total += fileNullSafetyMigrationCost;
     }

examples/api/test/flutter_test_config.dart

@@ -5,7 +5,7 @@
 import 'dart:async';
 
 
-import 'goldens_io.dart' if (dart.library.html) 'goldens_web.dart' as flutter_goldens;
+import 'package:flutter_goldens/flutter_goldens.dart' as flutter_goldens;
 
 Future<void> testExecutable(FutureOr<void> Function() testMain) {
   // Enable golden file testing using Skia Gold.