Download and use the goma client from cipd (#41488)

In flutter/flutter#125361 we discovered that a
new version of clang can require updating goma in order for goma to work
properly. This PR adds a dependency on CIPD goma to the DEPS file so
that we can update it for use in local builds when needed. Since CI does
its own management of the goma client and the compiler proxy, and since
goma can only be used by Googlers, the DEPS file download is guarded
behind the `use_cipd_goma` gclient var. To use it one would update the
engine `.gclient` file to be something like:
```
~/flutter/engine/src $ cat ../.gclient
solutions = [
  {
    "managed": False,
    "name": "src/flutter",
    "url": "[email protected]:zanderso/engine.git",
    "custom_deps": {},
    "custom_vars": {
      "use_cipd_goma": True,
    },
    "deps_file": "DEPS",
    "safesync_url": "",
  },
]
```

I'll update the wiki with these instructions after this PR lands.

Author: zanderso

DEPS

@@ -41,6 +41,12 @@ vars = {
   # If there are problems with the toolchain, contact fuchsia-toolchain@.
   'clang_version': 'git_revision:20d06c833d833ef6b2d0f519cc4a7998d49a2803',
 
+  # The goma version and the clang version can be tightly coupled. If goma
+  # stops working on a clang roll, this may need to be updated using the value
+  # from the 'integration' tag of
+  # https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client
+  'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56',
+
   # When updating the Dart revision, ensure that all entries that are
   # dependencies of Dart are also updated to match the entries in the
   # Dart SDK's DEPS file for that revision of Dart. The DEPS file for
@@ -98,7 +104,12 @@ vars = {
   "checkout_llvm": False,
 
   # Setup Git hooks by default.
-  "setup_githooks": True,
+  'setup_githooks': True,
+
+  # When this is true, the goma client will be downloaded from cipd, and
+  # the engine build will prefer to use this client over a client that is
+  # specified by GOMA_DIR, or installed in the default goma install location.
+  'use_cipd_goma': False,
 
   # This is not downloaded be default because it increases the
   # `gclient sync` time by between 1 and 3 minutes. This option is enabled
@@ -830,6 +841,40 @@ deps = {
     'dep_type': 'cipd',
   },
 
+  # GOMA
+  'src/buildtools/mac-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/mac-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'use_cipd_goma and host_os == "mac"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/linux-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/linux-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'use_cipd_goma and host_os == "linux"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/windows-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/windows-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'use_cipd_goma and download_windows_deps',
+    'dep_type': 'cipd',
+  },
+
   # Get the SDK from https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core at the 'latest' tag
   # Get the toolchain from https://chrome-infra-packages.appspot.com/p/fuchsia/clang at the 'goma' tag
    'src/fuchsia/sdk/mac': {
@@ -965,6 +1010,36 @@ hooks = [
       'src/flutter/tools/activate_emsdk.py',
     ]
   },
+  {
+    'name': 'Start compiler proxy',
+    'pattern': '.',
+    'condition': 'use_cipd_goma and host_os == "mac"',
+    'action': [
+      'python3',
+      'src/buildtools/mac-x64/goma/goma_ctl.py',
+      'ensure_start'
+    ]
+  },
+  {
+    'name': 'Start compiler proxy',
+    'pattern': '.',
+    'condition': 'use_cipd_goma and host_os == "linux"',
+    'action': [
+      'python3',
+      'src/buildtools/linux-x64/goma/goma_ctl.py',
+      'ensure_start'
+    ]
+  },
+  {
+    'name': 'Start compiler proxy',
+    'pattern': '.',
+    'condition': 'use_cipd_goma and download_windows_deps',
+    'action': [
+      'python3',
+      'src/buildtools/windows-x64/goma/goma_ctl.py',
+      'ensure_start'
+    ]
+  },
   {
     'name': 'Setup githooks',
     'pattern': '.',

tools/gn

@@ -199,12 +199,32 @@ def get_target_cpu(args):
   return 'x64'
 
 
+def buildtools_dir():
+  host_os = get_host_os()
+  host_cpu = get_host_cpu()
+  if host_os == 'win':
+    host_os = 'windows'
+  if host_os == 'mac' and host_cpu == 'arm64':
+    host_cpu = 'x64'
+  return '%s-%s' % (host_os, host_cpu)
+
+
 def setup_goma(args):
   goma_gn_args = {}
 
+  # When running in CI, the recipes use their own goma install, and take
+  # care of starting and stopping the compiler proxy.
+  running_on_luci = os.environ.get('LUCI_CONTEXT') is not None
+
+  # Prefer the goma fetched by gclient if it exists.
+  cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma')
+
+  # Next, if GOMA_DIR is set, use that install.
   goma_dir = os.environ.get('GOMA_DIR')
-  goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma')
 
+  # Finally, look for goma in the install location recommended in our
+  # documentation.
+  goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma')
   # GOMA has a different default (home) path on gWindows.
   if not os.path.exists(goma_home_dir) and sys.platform.startswith(
       ('cygwin', 'win')):
@@ -214,6 +234,9 @@ def setup_goma(args):
     goma_gn_args['use_goma'] = False
     goma_gn_args['goma_dir'] = None
     print('Disabling GOMA for wasm builds, it is not supported yet.')
+  elif args.goma and not running_on_luci and os.path.exists(cipd_goma_dir):
+    goma_gn_args['use_goma'] = True
+    goma_gn_args['goma_dir'] = cipd_goma_dir
   elif args.goma and goma_dir and os.path.exists(goma_dir):
     goma_gn_args['use_goma'] = True
     goma_gn_args['goma_dir'] = goma_dir
@@ -233,7 +256,7 @@ def setup_goma(args):
     goma_gn_args['goma_dir'] = None
 
   if goma_gn_args['use_goma'] and sys.platform == 'darwin':
-    if (os.environ.get('LUCI_CONTEXT') is None or args.xcode_symlinks or
+    if (not running_on_luci or args.xcode_symlinks or
         os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'):
       goma_gn_args['create_xcode_symlinks'] = True