[tool] Improve check version ci so that it enforces the version in CHANGELOG and pubspec matches. (#3678)

Author: Chris Yang

.cirrus.yml

@@ -6,10 +6,13 @@ root_task_template: &ROOT_TASK_TEMPLATE
   env:
     INTEGRATION_TEST_PATH: "./packages/integration_test"
     CHANNEL: "master" # Default to master when not explicitly set by a task.
+    PLUGIN_TOOLS: "dart run ./script/tool/lib/src/main.dart"
   setup_script:
     - flutter channel $CHANNEL
     - flutter upgrade
     - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work
+    - cd script/tool
+    - pub get
 
 
 # Light-workload Linux tasks.
@@ -24,10 +27,14 @@ task:
     - name: plugin_tools_tests
       script:
         - cd script/tool
-        - pub get
         - CIRRUS_BUILD_ID=null pub run test
     - name: publishable
       script:
+        - if [[ "$CIRRUS_BRANCH" == "master" ]]; then
+        -   $PLUGIN_TOOLS version-check
+        - else
+        -   $PLUGIN_TOOLS version-check --run-on-changed-packages
+        - fi
         - ./script/check_publish.sh
     - name: format
       format_script: ./script/incremental_build.sh format --fail-on-change

script/incremental_build.sh

@@ -49,8 +49,5 @@ else
   else
     echo running "${ACTIONS[@]}"
     (cd "$REPO_DIR" && plugin_tools "${ACTIONS[@]}" --plugins="$CHANGED_PACKAGES" --exclude="$ALL_EXCLUDED" ${PLUGIN_SHARDING[@]})
-    echo "Running version check for changed packages"
-    # TODO(egarciad): Enable this check once in master.
-    # (cd "$REPO_DIR" && $PUB global run flutter_plugin_tools version-check --base_sha="$(get_branch_base_sha)")
   fi
 fi

script/tool/lib/src/common.dart

@@ -7,8 +7,12 @@ import 'dart:io' as io;
 import 'dart:math';
 
 import 'package:args/command_runner.dart';
+import 'package:colorize/colorize.dart';
 import 'package:file/file.dart';
+import 'package:git/git.dart';
+import 'package:meta/meta.dart';
 import 'package:path/path.dart' as p;
+import 'package:pub_semver/pub_semver.dart';
 import 'package:yaml/yaml.dart';
 
 typedef void Print(Object object);
@@ -140,6 +144,13 @@ bool isLinuxPlugin(FileSystemEntity entity, FileSystem fileSystem) {
   return pluginSupportsPlatform(kLinux, entity, fileSystem);
 }
 
+/// Throws a [ToolExit] with `exitCode` and log the `errorMessage` in red.
+void printErrorAndExit({@required String errorMessage, int exitCode = 1}) {
+  final Colorize redError = Colorize(errorMessage)..red();
+  print(redError);
+  throw ToolExit(exitCode);
+}
+
 /// Error thrown when a command needs to exit with a non-zero exit code.
 class ToolExit extends Error {
   ToolExit(this.exitCode);
@@ -152,6 +163,7 @@ abstract class PluginCommand extends Command<Null> {
     this.packagesDir,
     this.fileSystem, {
     this.processRunner = const ProcessRunner(),
+    this.gitDir,
   }) {
     argParser.addMultiOption(
       _pluginsArg,
@@ -179,12 +191,23 @@ abstract class PluginCommand extends Command<Null> {
       help: 'Exclude packages from this command.',
       defaultsTo: <String>[],
     );
+    argParser.addFlag(_runOnChangedPackagesArg,
+        help: 'Run the command on changed packages/plugins.\n'
+            'If the $_pluginsArg is specified, this flag is ignored.\n'
+            'The packages excluded with $_excludeArg is also excluded even if changed.\n'
+            'See $_kBaseSha if a custom base is needed to determine the diff.');
+    argParser.addOption(_kBaseSha,
+        help: 'The base sha used to determine git diff. \n'
+            'This is useful when $_runOnChangedPackagesArg is specified.\n'
+            'If not specified, merge-base is used as base sha.');
   }
 
   static const String _pluginsArg = 'plugins';
   static const String _shardIndexArg = 'shardIndex';
   static const String _shardCountArg = 'shardCount';
   static const String _excludeArg = 'exclude';
+  static const String _runOnChangedPackagesArg = 'run-on-changed-packages';
+  static const String _kBaseSha = 'base-sha';
 
   /// The directory containing the plugin packages.
   final Directory packagesDir;
@@ -199,6 +222,11 @@ abstract class PluginCommand extends Command<Null> {
   /// This can be overridden for testing.
   final ProcessRunner processRunner;
 
+  /// The git directory to use. By default it uses the parent directory.
+  ///
+  /// This can be mocked for testing.
+  final GitDir gitDir;
+
   int _shardIndex;
   int _shardCount;
 
@@ -273,9 +301,13 @@ abstract class PluginCommand extends Command<Null> {
   ///    "client library" package, which declares the API for the plugin, as
   ///    well as one or more platform-specific implementations.
   Stream<Directory> _getAllPlugins() async* {
-    final Set<String> plugins = Set<String>.from(argResults[_pluginsArg]);
+    Set<String> plugins = Set<String>.from(argResults[_pluginsArg]);
     final Set<String> excludedPlugins =
         Set<String>.from(argResults[_excludeArg]);
+    final bool runOnChangedPackages = argResults[_runOnChangedPackagesArg];
+    if (plugins.isEmpty && runOnChangedPackages) {
+      plugins = await _getChangedPackages();
+    }
 
     await for (FileSystemEntity entity
         in packagesDir.list(followLinks: false)) {
@@ -363,6 +395,50 @@ abstract class PluginCommand extends Command<Null> {
             (FileSystemEntity entity) => isFlutterPackage(entity, fileSystem))
         .cast<Directory>();
   }
+
+  /// Retrieve an instance of [GitVersionFinder] based on `_kBaseSha` and [gitDir].
+  ///
+  /// Throws tool exit if [gitDir] nor root directory is a git directory.
+  Future<GitVersionFinder> retrieveVersionFinder() async {
+    final String rootDir = packagesDir.parent.absolute.path;
+    String baseSha = argResults[_kBaseSha];
+
+    GitDir baseGitDir = gitDir;
+    if (baseGitDir == null) {
+      if (!await GitDir.isGitDir(rootDir)) {
+        printErrorAndExit(
+            errorMessage: '$rootDir is not a valid Git repository.',
+            exitCode: 2);
+      }
+      baseGitDir = await GitDir.fromExisting(rootDir);
+    }
+
+    final GitVersionFinder gitVersionFinder =
+        GitVersionFinder(baseGitDir, baseSha);
+    return gitVersionFinder;
+  }
+
+  Future<Set<String>> _getChangedPackages() async {
+    final GitVersionFinder gitVersionFinder = await retrieveVersionFinder();
+
+    final List<String> allChangedFiles =
+        await gitVersionFinder.getChangedFiles();
+    final Set<String> packages = <String>{};
+    allChangedFiles.forEach((String path) {
+      final List<String> pathComponents = path.split('/');
+      final int packagesIndex =
+          pathComponents.indexWhere((String element) => element == 'packages');
+      if (packagesIndex != -1) {
+        packages.add(pathComponents[packagesIndex + 1]);
+      }
+    });
+    if (packages.isNotEmpty) {
+      final String changedPackages = packages.join(',');
+      print(changedPackages);
+    }
+    print('No changed packages.');
+    return packages;
+  }
 }
 
 /// A class used to run processes.
@@ -466,3 +542,68 @@ class ProcessRunner {
     return 'ERROR: Unable to execute "$executable ${args.join(' ')}"$workdir.';
   }
 }
+
+/// Finding diffs based on `baseGitDir` and `baseSha`.
+class GitVersionFinder {
+  /// Constructor
+  GitVersionFinder(this.baseGitDir, this.baseSha);
+
+  /// The top level directory of the git repo.
+  ///
+  /// That is where the .git/ folder exists.
+  final GitDir baseGitDir;
+
+  /// The base sha used to get diff.
+  final String baseSha;
+
+  static bool _isPubspec(String file) {
+    return file.trim().endsWith('pubspec.yaml');
+  }
+
+  /// Get a list of all the pubspec.yaml file that is changed.
+  Future<List<String>> getChangedPubSpecs() async {
+    return (await getChangedFiles()).where(_isPubspec).toList();
+  }
+
+  /// Get a list of all the changed files.
+  Future<List<String>> getChangedFiles() async {
+    final String baseSha = await _getBaseSha();
+    final io.ProcessResult changedFilesCommand = await baseGitDir
+        .runCommand(<String>['diff', '--name-only', '$baseSha', 'HEAD']);
+    print('Determine diff with base sha: $baseSha');
+    final String changedFilesStdout = changedFilesCommand.stdout.toString()  ?? '';
+    if (changedFilesStdout.isEmpty) {
+      return <String>[];
+    }
+    final List<String> changedFiles = changedFilesStdout
+        .split('\n')
+          ..removeWhere((element) => element.isEmpty);
+    return changedFiles.toList();
+  }
+
+  /// Get the package version specified in the pubspec file in `pubspecPath` and at the revision of `gitRef`.
+  Future<Version> getPackageVersion(String pubspecPath, String gitRef) async {
+    final io.ProcessResult gitShow =
+        await baseGitDir.runCommand(<String>['show', '$gitRef:$pubspecPath']);
+    final String fileContent = gitShow.stdout;
+    final String versionString = loadYaml(fileContent)['version'];
+    return versionString == null ? null : Version.parse(versionString);
+  }
+
+  Future<String> _getBaseSha() async {
+    if (baseSha != null && baseSha.isNotEmpty) {
+      return baseSha;
+    }
+
+    io.ProcessResult baseShaFromMergeBase = await baseGitDir.runCommand(
+        <String>['merge-base', '--fork-point', 'FETCH_HEAD', 'HEAD'],
+        throwOnError: false);
+    if (baseShaFromMergeBase == null ||
+        baseShaFromMergeBase.stderr != null ||
+        baseShaFromMergeBase.stdout == null) {
+      baseShaFromMergeBase = await baseGitDir
+          .runCommand(<String>['merge-base', 'FETCH_HEAD', 'HEAD']);
+    }
+    return (baseShaFromMergeBase.stdout as String).trim();
+  }
+}