chore: flutter symbol collector CLI tool (#1673)

Author: vaind

.github/workflows/flutter-symbols.yml

@@ -0,0 +1,58 @@
+name: Flutter symbols collection
+on:
+  schedule:
+    # Run once an hour. It takes just a couple of minutes because of status caching.
+    - cron: "10 * * * *"
+  workflow_dispatch:
+    inputs:
+      flutter_version:
+        description: Flutter version, can be either a specific version (3.17.0) or a wildcard (3.2.*)
+        required: false
+        type: string
+        default: "3.*.*"
+
+defaults:
+  run:
+    working-directory: scripts/flutter_symbol_collector
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dart-lang/setup-dart@6a218f2413a3e78e9087f638a238f6b40893203d # pin@v1
+
+      - run: dart pub get
+
+      - run: dart test
+
+  run:
+    needs: [test]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dart-lang/setup-dart@6a218f2413a3e78e9087f638a238f6b40893203d # pin@v1
+
+      - run: dart pub get
+
+      - name: Download status cache of previously processed files
+        run: |
+          gh run download --name 'flutter-symbol-collector-database' --dir .cache
+          grep -r "" .cache
+        continue-on-error: true
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - run: dart run bin/flutter_symbol_collector.dart --version=${{ inputs.flutter_version || '3.*.*' }}
+        timeout-minutes: 300
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+
+      - name: Upload updated status cache of processed files
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: flutter-symbol-collector-database
+          path: scripts/flutter_symbol_collector/.cache

.github/workflows/update-deps.yml

@@ -35,3 +35,12 @@ jobs:
       changelog-entry: false
     secrets:
       api-token: ${{ secrets.CI_DEPLOY_KEY }}
+
+  symbol-collector:
+    uses: getsentry/github-workflows/.github/workflows/updater.yml@v2
+    with:
+      path: scripts/update-symbol-collector.sh
+      name: Symbol collector CLI
+      changelog-entry: false
+    secrets:
+      api-token: ${{ secrets.CI_DEPLOY_KEY }}

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Enhancements
 
 - Log warning if both tracesSampleRate and tracesSampler are set ([#1701](https://github.com/getsentry/sentry-dart/pull/1701))
+- Better Flutter framework stack traces - we now collect Flutter framework debug symbols for iOS, macOS and Android automatically on the Sentry server ([#1673](https://github.com/getsentry/sentry-dart/pull/1673))
 
 ### Features
 

scripts/flutter_symbol_collector/.gitignore

@@ -0,0 +1,6 @@
+# https://dart.dev/guides/libraries/private-files
+# Created by `dart pub`
+.dart_tool/
+
+.temp
+.cache

scripts/flutter_symbol_collector/README.md

@@ -0,0 +1,4 @@
+# Flutter symbol collector
+
+This is an internal tool to collect Flutter debug symbols and upload them to Sentry.
+This application is not intended for public usage - we're uploading the symbols in CI automatically so you don't have to.

scripts/flutter_symbol_collector/analysis_options.yaml

@@ -0,0 +1,7 @@
+include: package:lints/recommended.yaml
+
+linter:
+  rules:
+    prefer_relative_imports: true
+    unnecessary_brace_in_string_interps: true
+    unawaited_futures: true

scripts/flutter_symbol_collector/bin/flutter_symbol_collector.dart

@@ -0,0 +1,96 @@
+import 'package:args/args.dart';
+import 'package:file/local.dart';
+import 'package:flutter_symbol_collector/flutter_symbol_collector.dart';
+import 'package:github/github.dart';
+import 'package:logging/logging.dart';
+
+const githubToken = String.fromEnvironment('GITHUB_TOKEN');
+final githubAuth = githubToken.isEmpty
+    ? Authentication.anonymous()
+    : Authentication.withToken(githubToken);
+final source = FlutterSymbolSource(githubAuth: githubAuth);
+final fs = LocalFileSystem();
+final tempDir = fs.currentDirectory.childDirectory('.temp');
+final stateCache =
+    DirectoryStatusCache(fs.currentDirectory.childDirectory('.cache'));
+late final SymbolCollectorCli collector;
+
+void main(List<String> arguments) async {
+  Logger.root.level = Level.ALL;
+  Logger.root.onRecord.listen((record) {
+    print('${record.level.name}: ${record.time}: ${record.message}'
+        '${record.error == null ? '' : ': ${record.error}'}');
+  });
+
+  final parser = ArgParser()..addOption('version', defaultsTo: '');
+  final args = parser.parse(arguments);
+  final argVersion = args['version'] as String;
+
+  collector = await SymbolCollectorCli.setup(tempDir);
+
+  // If a specific version was given, run just for this version.
+  if (argVersion.isNotEmpty &&
+      !argVersion.contains('*') &&
+      argVersion.split('.').length == 3) {
+    Logger.root.info('Running for a single flutter version: $argVersion');
+    await processFlutterVersion(FlutterVersion(argVersion));
+  } else {
+    // Otherwise, walk all the versions and run for the matching ones.
+    final versionRegex = RegExp(argVersion.isEmpty
+        ? '.*'
+        : '^${argVersion.replaceAll('.', '\\.').replaceAll('*', '.+')}\$');
+    Logger.root.info('Running for all Flutter versions matching $versionRegex');
+    final versions = await source
+        .listFlutterVersions()
+        .where((v) => !v.isPreRelease)
+        .where((v) => versionRegex.hasMatch(v.tagName))
+        .toList();
+    Logger.root.info(
+        'Found ${versions.length} Flutter versions matching $versionRegex');
+    for (var version in versions) {
+      await processFlutterVersion(version);
+    }
+  }
+}
+
+Future<void> processFlutterVersion(FlutterVersion version) async {
+  if (bool.hasEnvironment('CI')) {
+    print('::group::Processing Flutter ${version.tagName}');
+  }
+  Logger.root.info('Processing Flutter ${version.tagName}');
+  Logger.root.info('Engine version: ${await version.engineVersion}');
+
+  final archives = await source.listSymbolArchives(version);
+  final dir = tempDir.childDirectory(version.tagName);
+  for (final archive in archives) {
+    final status = await stateCache.getStatus(archive);
+    if (status == SymbolArchiveStatus.success) {
+      Logger.root
+          .info('Skipping ${archive.path} - already processed successfully');
+      continue;
+    }
+
+    final archiveDir = dir.childDirectory(archive.platform.operatingSystem);
+    try {
+      if (await source.downloadAndExtractTo(archiveDir, archive.path)) {
+        if (await collector.upload(archiveDir, archive.platform, version)) {
+          await stateCache.setStatus(archive, SymbolArchiveStatus.success);
+          continue;
+        }
+      }
+      await stateCache.setStatus(archive, SymbolArchiveStatus.error);
+    } finally {
+      if (await archiveDir.exists()) {
+        await archiveDir.delete(recursive: true);
+      }
+    }
+  }
+
+  if (await dir.exists()) {
+    await dir.delete(recursive: true);
+  }
+
+  if (bool.hasEnvironment('CI')) {
+    print('::endgroup::');
+  }
+}

scripts/flutter_symbol_collector/lib/flutter_symbol_collector.dart

@@ -0,0 +1,5 @@
+export 'src/flutter_symbol_source.dart';
+export 'src/flutter_version.dart';
+export 'src/symbol_collector_cli.dart';
+export 'src/status_cache.dart';
+export 'src/symbol_archive.dart';

scripts/flutter_symbol_collector/lib/src/flutter_symbol_resolver.dart

@@ -0,0 +1,73 @@
+import 'package:gcloud/storage.dart';
+import 'package:platform/platform.dart';
+
+import 'symbol_archive.dart';
+
+abstract class FlutterSymbolResolver {
+  final String _prefix;
+  final Bucket _bucket;
+  final _resolvedFiles = List<SymbolArchive>.empty(growable: true);
+  Platform get platform;
+
+  FlutterSymbolResolver(this._bucket, String prefix)
+      : _prefix = prefix.endsWith('/')
+            ? prefix.substring(0, prefix.length - 1)
+            : prefix;
+
+  Future<void> tryResolve(String path) async {
+    path = '$_prefix/$path';
+    final matches = await _bucket
+        .list(prefix: path)
+        .where((v) => v.isObject)
+        .where((v) => v.name == path) // because it's a prefix search
+        .map((v) => v.name)
+        .toList();
+    if (matches.isNotEmpty) {
+      _resolvedFiles.add(SymbolArchive(matches.single, platform));
+    }
+  }
+
+  Future<List<SymbolArchive>> listArchives();
+}
+
+class IosSymbolResolver extends FlutterSymbolResolver {
+  IosSymbolResolver(super.bucket, super.prefix);
+
+  @override
+  final platform = FakePlatform(operatingSystem: Platform.iOS);
+
+  @override
+  Future<List<SymbolArchive>> listArchives() async {
+    await tryResolve('ios-release/Flutter.dSYM.zip');
+    return _resolvedFiles;
+  }
+}
+
+class MacOSSymbolResolver extends FlutterSymbolResolver {
+  MacOSSymbolResolver(super.bucket, super.prefix);
+
+  @override
+  final platform = FakePlatform(operatingSystem: Platform.macOS);
+
+  @override
+  Future<List<SymbolArchive>> listArchives() async {
+    // darwin-x64-release directory contains a fat (arm64+x86_64) binary.
+    await tryResolve('darwin-x64-release/FlutterMacOS.dSYM.zip');
+    return _resolvedFiles;
+  }
+}
+
+class AndroidSymbolResolver extends FlutterSymbolResolver {
+  final String architecture;
+
+  AndroidSymbolResolver(super.bucket, super.prefix, this.architecture);
+
+  @override
+  final platform = FakePlatform(operatingSystem: Platform.android);
+
+  @override
+  Future<List<SymbolArchive>> listArchives() async {
+    await tryResolve('android-$architecture-release/symbols.zip');
+    return _resolvedFiles;
+  }
+}