Run static analyzer during xctest

jmagman · jmagman · commit b37379a23336 · 2021-03-02T17:23:55.000-08:00
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -179,7 +179,7 @@ task:
     - name: build-ipas+drive-examples
       env:
         PATH: $PATH:/usr/local/bin
-        PLUGINS_TO_SKIP_XCTESTS: "battery/battery,camera/camera,connectivity/connectivity,device_info/device_info,espresso,google_maps_flutter/google_maps_flutter,google_sign_in/google_sign_in,in_app_purchase,integration_test,ios_platform_images,local_auth,package_info,path_provider/path_provider,sensors,shared_preferences/shared_preferences,url_launcher/url_launcher,video_player/video_player,webview_flutter,wifi_info_flutter/wifi_info_flutter"
+        PLUGINS_TO_SKIP_XCTESTS: "image_picker/image_picker,integration_test,webview_flutter"
         matrix:
           PLUGIN_SHARDING: "--shardIndex 0 --shardCount 4"
           PLUGIN_SHARDING: "--shardIndex 1 --shardCount 4"
@@ -197,7 +197,7 @@ task:
         - flutter channel $CHANNEL
         - flutter upgrade
         - ./script/incremental_build.sh build-examples --ipa
-        - ./script/incremental_build.sh xctest --target RunnerUITests --skip $PLUGINS_TO_SKIP_XCTESTS --ios-destination "platform=iOS Simulator,name=iPhone 11,OS=14.3"
+        - ./script/incremental_build.sh xctest --skip $PLUGINS_TO_SKIP_XCTESTS --ios-destination "platform=iOS Simulator,name=iPhone 11,OS=latest"
         # `drive-examples` contains integration tests, which changes the UI of the application.
         # This UI change sometimes affects `xctest`.
         # So we run `drive-examples` after `xctest`, changing the order will result ci failure.
diff --git a/.gitignore b/.gitignore
@@ -17,6 +17,7 @@ Pods/
 .symlinks/
 **/Flutter/App.framework/
 **/Flutter/ephemeral/
+**/Flutter/Flutter.podspec
 **/Flutter/Flutter.framework/
 **/Flutter/Generated.xcconfig
 **/Flutter/flutter_assets/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -69,7 +69,7 @@ If XCUITests has not been set up for the plugin, follow these steps to set it up
 1. Open <path_to_plugin>/example/ios/Runner.xcworkspace using XCode.
 1. Create a new "UI Testing Bundle".
 1. In the target options window, populate details as following, then click on "Finish".
-  * In the "product name" field, type in "RunnerUITests" (this is the test target name our CI looks for.).
+  * In the "product name" field, type in "RunnerUITests".
   * In the "Team" field, select "None".
   * In the Organization Name field, type in "Flutter". This should usually be pre-populated.
   * In the organization identifer field, type in "com.google". This should usually be pre-populated.
diff --git a/script/tool/lib/src/lint_podspecs_command.dart b/script/tool/lib/src/lint_podspecs_command.dart
@@ -14,8 +14,7 @@ import 'common.dart';
 
 typedef void Print(Object object);
 
-/// Lint the CocoaPod podspecs, run the static analyzer on iOS/macOS plugin
-/// platform code, and run unit tests.
+/// Lint the CocoaPod podspecs and run unit tests.
 ///
 /// See https://guides.cocoapods.org/terminal/commands.html#pod_lib_lint.
 class LintPodspecsCommand extends PluginCommand {
@@ -35,10 +34,6 @@ class LintPodspecsCommand extends PluginCommand {
         help:
             'Do not pass --allow-warnings flag to "pod lib lint" for podspecs with this basename (example: plugins with known warnings)',
         valueHelp: 'podspec_file_name');
-    argParser.addMultiOption('no-analyze',
-        help:
-            'Do not pass --analyze flag to "pod lib lint" for podspecs with this basename (example: plugins with known analyzer warnings)',
-        valueHelp: 'podspec_file_name');
   }
 
   @override
@@ -102,44 +97,36 @@ class LintPodspecsCommand extends PluginCommand {
   Future<bool> _lintPodspec(File podspec) async {
     // Do not run the static analyzer on plugins with known analyzer issues.
     final String podspecPath = podspec.path;
-    final bool runAnalyzer = !argResults['no-analyze']
-        .contains(p.basenameWithoutExtension(podspecPath));
 
     final String podspecBasename = p.basename(podspecPath);
-    if (runAnalyzer) {
-      _print('Linting and analyzing $podspecBasename');
-    } else {
-      _print('Linting $podspecBasename');
-    }
+    _print('Linting $podspecBasename');
 
     // Lint plugin as framework (use_frameworks!).
-    final ProcessResult frameworkResult = await _runPodLint(podspecPath,
-        runAnalyzer: runAnalyzer, libraryLint: true);
+    final ProcessResult frameworkResult = await _runPodLint(podspecPath, libraryLint: true);
     _print(frameworkResult.stdout);
     _print(frameworkResult.stderr);
 
     // Lint plugin as library.
-    final ProcessResult libraryResult = await _runPodLint(podspecPath,
-        runAnalyzer: runAnalyzer, libraryLint: false);
+    final ProcessResult libraryResult = await _runPodLint(podspecPath, libraryLint: false);
     _print(libraryResult.stdout);
     _print(libraryResult.stderr);
 
     return frameworkResult.exitCode == 0 && libraryResult.exitCode == 0;
   }
 
   Future<ProcessResult> _runPodLint(String podspecPath,
-      {bool runAnalyzer, bool libraryLint}) async {
+      {bool libraryLint}) async {
     final bool allowWarnings = argResults['ignore-warnings']
         .contains(p.basenameWithoutExtension(podspecPath));
     final List<String> arguments = <String>[
       'lib',
       'lint',
       podspecPath,
       if (allowWarnings) '--allow-warnings',
-      if (runAnalyzer) '--analyze',
       if (libraryLint) '--use-libraries'
     ];
 
+    _print('Running "pod ${arguments.join(' ')}"');
     return processRunner.run('pod', arguments,
         workingDir: packagesDir, stdoutEncoding: utf8, stderrEncoding: utf8);
   }
diff --git a/script/tool/lib/src/xctest_command.dart b/script/tool/lib/src/xctest_command.dart
@@ -12,17 +12,15 @@ import 'package:path/path.dart' as p;
 import 'common.dart';
 
 const String _kiOSDestination = 'ios-destination';
-const String _kTarget = 'target';
 const String _kSkip = 'skip';
 const String _kXcodeBuildCommand = 'xcodebuild';
 const String _kXCRunCommand = 'xcrun';
 const String _kFoundNoSimulatorsMessage =
     'Cannot find any available simulators, tests failed';
 
-/// The command to run iOS' XCTests in plugins, this should work for both XCUnitTest and XCUITest targets.
-/// The tests target have to be added to the xcode project of the example app. Usually at "example/ios/Runner.xcodeproj".
-/// The command takes a "-target" argument which has to match the target of the test target.
-/// For information on how to add test target in an xcode project, see https://developer.apple.com/library/archive/documentation/ToolsLanguages/Conceptual/Xcode_Overview/UnitTesting.html
+/// The command to run iOS XCTests in plugins, this should work for both XCUnitTest and XCUITest targets.
+/// The tests target have to be added to the xcode project of the example app. Usually at "example/ios/Runner.xcworkspace".
+/// The static analyzer is also run.
 class XCTestCommand extends PluginCommand {
   XCTestCommand(
     Directory packagesDir,
@@ -36,10 +34,6 @@ class XCTestCommand extends PluginCommand {
           'this is passed to the `-destination` argument in xcodebuild command.\n'
           'See https://developer.apple.com/library/archive/technotes/tn2339/_index.html#//apple_ref/doc/uid/DTS40014588-CH1-UNIT for details on how to specify the destination.',
     );
-    argParser.addOption(_kTarget,
-        help: 'The test target.\n'
-            'This is the xcode project test target. This is passed to the `-scheme` argument in the xcodebuild command. \n'
-            'See https://developer.apple.com/library/archive/technotes/tn2339/_index.html#//apple_ref/doc/uid/DTS40014588-CH1-UNIT for details on how to specify the scheme');
     argParser.addMultiOption(_kSkip,
         help: 'Plugins to skip while running this command. \n');
   }
@@ -49,17 +43,10 @@ class XCTestCommand extends PluginCommand {
 
   @override
   final String description = 'Runs the xctests in the iOS example apps.\n\n'
-      'This command requires "flutter" to be in your path.';
+      'This command requires "flutter" and "xcrun" to be in your path.';
 
   @override
   Future<Null> run() async {
-    if (argResults[_kTarget] == null) {
-      // TODO(cyanglaz): Automatically find all the available testing schemes if this argument is not specified.
-      // https://github.com/flutter/flutter/issues/68419
-      print('--$_kTarget must be specified');
-      throw ToolExit(1);
-    }
-
     String destination = argResults[_kiOSDestination];
     if (destination == null) {
       String simulatorId = await _findAvailableIphoneSimulator();
@@ -72,7 +59,6 @@ class XCTestCommand extends PluginCommand {
 
     checkSharding();
 
-    final String target = argResults[_kTarget];
     final List<String> skipped = argResults[_kSkip];
 
     List<String> failingPackages = <String>[];
@@ -92,57 +78,14 @@ class XCTestCommand extends PluginCommand {
         continue;
       }
       for (Directory example in getExamplesForPlugin(plugin)) {
-        // Look for the test scheme in the example app.
-        print('Look for target named: $_kTarget ...');
-        final List<String> findSchemeArgs = <String>[
-          '-project',
-          'ios/Runner.xcodeproj',
-          '-list',
-          '-json'
-        ];
-        final String completeFindSchemeCommand =
-            '$_kXcodeBuildCommand ${findSchemeArgs.join(' ')}';
-        print(completeFindSchemeCommand);
-        final io.ProcessResult xcodeprojListResult = await processRunner
-            .run(_kXcodeBuildCommand, findSchemeArgs, workingDir: example);
-        if (xcodeprojListResult.exitCode != 0) {
-          print('Error occurred while running "$completeFindSchemeCommand":\n'
-              '${xcodeprojListResult.stderr}');
-          failingPackages.add(packageName);
-          print('\n\n');
-          continue;
-        }
-
-        final String xcodeprojListOutput = xcodeprojListResult.stdout;
-        Map<String, dynamic> xcodeprojListOutputJson =
-            jsonDecode(xcodeprojListOutput);
-        if (!xcodeprojListOutputJson['project']['targets'].contains(target)) {
-          failingPackages.add(packageName);
-          print('$target not configured for $packageName, test failed.');
-          print(
-              'Please check the scheme for the test target if it matches the name $target.\n'
-              'If this plugin does not have an XCTest target, use the $_kSkip flag in the $name command to skip the plugin.');
-          print('\n\n');
-          continue;
+        // Running tests and static analyzer.
+        print('Running tests and analyzer for $packageName ...');
+        int exitCode = await _runTests(true, destination, example);
+        // 66 = there is no test target (this fails fast). Try again with just the analyzer.
+        if (exitCode == 66) {
+          print('Tests not found for $packageName, running analyzer only...');
+          exitCode = await _runTests(false, destination, example);
         }
-        // Found the scheme, running tests
-        print('Running XCTests:$target for $packageName ...');
-        final List<String> xctestArgs = <String>[
-          'test',
-          '-workspace',
-          'ios/Runner.xcworkspace',
-          '-scheme',
-          target,
-          '-destination',
-          destination,
-          'CODE_SIGN_IDENTITY=""',
-          'CODE_SIGNING_REQUIRED=NO'
-        ];
-        final String completeTestCommand =
-            '$_kXcodeBuildCommand ${xctestArgs.join(' ')}';
-        print(completeTestCommand);
-        final int exitCode = await processRunner
-            .runAndStream(_kXcodeBuildCommand, xctestArgs, workingDir: example);
         if (exitCode == 0) {
           print('Successfully ran xctest for $packageName');
         } else {
@@ -164,6 +107,31 @@ class XCTestCommand extends PluginCommand {
     }
   }
 
+  Future<int> _runTests(bool runTests, String destination, Directory example) {
+    final List<String> xctestArgs = <String>[
+      _kXcodeBuildCommand,
+      if (runTests)
+        'test',
+      'analyze',
+      '-workspace',
+      'ios/Runner.xcworkspace',
+      '-configuration',
+      'Debug',
+      '-scheme',
+      'Runner',
+      '-destination',
+      destination,
+      'CODE_SIGN_IDENTITY=""',
+      'CODE_SIGNING_REQUIRED=NO',
+      'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
+    ];
+    final String completeTestCommand =
+        '$_kXCRunCommand ${xctestArgs.join(' ')}';
+    print(completeTestCommand);
+    return processRunner
+        .runAndStream(_kXCRunCommand, xctestArgs, workingDir: example, exitOnError: false);
+  }
+
   Future<String> _findAvailableIphoneSimulator() async {
     // Find the first available destination if not specified.
     final List<String> findSimulatorsArguments = <String>[
diff --git a/script/tool/test/lint_podspecs_command_test.dart b/script/tool/test/lint_podspecs_command_test.dart
@@ -81,7 +81,6 @@ void main() {
                 'lib',
                 'lint',
                 p.join(plugin1Dir.path, 'ios', 'plugin1.podspec'),
-                '--analyze',
                 '--use-libraries'
               ],
               mockPackagesDir.path),
@@ -91,14 +90,13 @@ void main() {
                 'lib',
                 'lint',
                 p.join(plugin1Dir.path, 'ios', 'plugin1.podspec'),
-                '--analyze',
               ],
               mockPackagesDir.path),
         ]),
       );
 
       expect(
-          printedMessages, contains('Linting and analyzing plugin1.podspec'));
+          printedMessages, contains('Linting plugin1.podspec'));
       expect(printedMessages, contains('Foo'));
       expect(printedMessages, contains('Bar'));
     });
@@ -122,41 +120,6 @@ void main() {
       );
     });
 
-    test('skips analyzer for podspecs with known warnings', () async {
-      Directory plugin1Dir =
-          createFakePlugin('plugin1', withExtraFiles: <List<String>>[
-        <String>['plugin1.podspec'],
-      ]);
-
-      await runner.run(<String>['podspecs', '--no-analyze=plugin1']);
-
-      expect(
-        processRunner.recordedCalls,
-        orderedEquals(<ProcessCall>[
-          ProcessCall('which', <String>['pod'], mockPackagesDir.path),
-          ProcessCall(
-              'pod',
-              <String>[
-                'lib',
-                'lint',
-                p.join(plugin1Dir.path, 'plugin1.podspec'),
-                '--use-libraries'
-              ],
-              mockPackagesDir.path),
-          ProcessCall(
-              'pod',
-              <String>[
-                'lib',
-                'lint',
-                p.join(plugin1Dir.path, 'plugin1.podspec'),
-              ],
-              mockPackagesDir.path),
-        ]),
-      );
-
-      expect(printedMessages, contains('Linting plugin1.podspec'));
-    });
-
     test('allow warnings for podspecs with known warnings', () async {
       Directory plugin1Dir =
           createFakePlugin('plugin1', withExtraFiles: <List<String>>[
@@ -176,7 +139,6 @@ void main() {
                 'lint',
                 p.join(plugin1Dir.path, 'plugin1.podspec'),
                 '--allow-warnings',
-                '--analyze',
                 '--use-libraries'
               ],
               mockPackagesDir.path),
@@ -187,14 +149,13 @@ void main() {
                 'lint',
                 p.join(plugin1Dir.path, 'plugin1.podspec'),
                 '--allow-warnings',
-                '--analyze',
               ],
               mockPackagesDir.path),
         ]),
       );
 
       expect(
-          printedMessages, contains('Linting and analyzing plugin1.podspec'));
+          printedMessages, contains('Linting plugin1.podspec'));
     });
   });
 }
diff --git a/script/tool/test/xctest_command_test.dart b/script/tool/test/xctest_command_test.dart
