fix: invoking code-server in integrated terminal (#5360)

* Include bin scripts for all platforms

These will get symlinked as part of the postinstall.  These scripts
provide everything ours does inside the integrated terminal plus more.

* Improve OS detection

Specifically for Windows although we do not yet support Windows.

Also standardize the duplicate arch functions since they had drifted
from each other bit.

* Remove duplicate asar symlink

Since standalone releases run the postinstall they will get the asar
symlink there.  That means the symlink will not exist for the npm
package and we will not need to ignore it.

The symlink portion is split out so it can be re-used for other
symlinks (for example linking bin scripts).

* Add symlinks to bin scripts

* Add test for opening a file from the terminal

* Add global Playwright timeout

Otherwise it will exceed the Actions timeout and get rudely killed
without any output.

* Make sed work on macOS

* Fix Node path in bin scripts

* Disable shellcheck expansion error

* Make scripts executable

* Remove .bak files created by sed

* Include Code build script in cache hash

Otherwise if we change the script it will not rebuild Code.

* Make sure the terminal opens

The selector was timing out even though it matched more than one element
but matching on the focused one appears to work.

In addition add a loop so it can keep trying to open the terminal
if something goes wrong with the focus.

Author: code-asher

.github/workflows/ci.yaml

@@ -156,7 +156,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: lib/vscode-reh-web-*
-          key: vscode-reh-package-${{ secrets.VSCODE_CACHE_VERSION }}-${{ steps.vscode-rev.outputs.rev }}-${{ steps.version.outputs.version }}-${{ hashFiles('patches/*.diff') }}
+          key: vscode-reh-package-${{ secrets.VSCODE_CACHE_VERSION }}-${{ steps.vscode-rev.outputs.rev }}-${{ steps.version.outputs.version }}-${{ hashFiles('patches/*.diff', 'ci/build/build-vscode.sh') }}
 
       - name: Build vscode
         if: steps.cache-vscode.outputs.cache-hit != 'true'
@@ -499,7 +499,7 @@ jobs:
           ./test/node_modules/.bin/playwright install
 
       - name: Run end-to-end tests
-        run: yarn test:e2e
+        run: yarn test:e2e --global-timeout 840000
 
       - name: Upload test artifacts
         if: always()

ci/build/build-release.sh

@@ -110,10 +110,6 @@ bundle_vscode() {
   rsync "$VSCODE_SRC_PATH/extensions/package.json" "$VSCODE_OUT_PATH/extensions/package.json"
   rsync "$VSCODE_SRC_PATH/extensions/yarn.lock" "$VSCODE_OUT_PATH/extensions/yarn.lock"
   rsync "$VSCODE_SRC_PATH/extensions/postinstall.mjs" "$VSCODE_OUT_PATH/extensions/postinstall.mjs"
-
-  pushd "$VSCODE_OUT_PATH"
-  symlink_asar
-  popd
 }
 
 main "$@"

ci/build/build-standalone-release.sh

@@ -27,8 +27,9 @@ main() {
   ln -s "./bin/code-server" "$RELEASE_PATH/code-server"
   ln -s "./lib/node" "$RELEASE_PATH/node"
 
-  cd "$RELEASE_PATH"
+  pushd "$RELEASE_PATH"
   yarn --production --frozen-lockfile
+  popd
 }
 
 main "$@"

ci/build/build-vscode.sh

@@ -6,12 +6,38 @@ set -euo pipefail
 # MINIFY controls whether a minified version of vscode is built.
 MINIFY=${MINIFY-true}
 
+delete-bin-script() {
+  rm -f "lib/vscode-reh-web-linux-x64/bin/$1"
+}
+
+copy-bin-script() {
+  local script="$1"
+  local dest="lib/vscode-reh-web-linux-x64/bin/$script"
+  cp "lib/vscode/resources/server/bin/$script" "$dest"
+  sed -i.bak "s/@@VERSION@@/$(vscode_version)/g" "$dest"
+  sed -i.bak "s/@@COMMIT@@/$VSCODE_DISTRO_COMMIT/g" "$dest"
+  sed -i.bak "s/@@APPNAME@@/code-server/g" "$dest"
+
+  # Fix Node path on Darwin and Linux.
+  # We do not want expansion here; this text should make it to the file as-is.
+  # shellcheck disable=SC2016
+  sed -i.bak 's/^ROOT=\(.*\)$/VSROOT=\1\nROOT="$(dirname "$(dirname "$VSROOT")")"/g' "$dest"
+  sed -i.bak 's/ROOT\/out/VSROOT\/out/g' "$dest"
+
+  # Fix Node path on Windows.
+  sed -i.bak 's/^set ROOT_DIR=\(.*\)$/set ROOT_DIR=%~dp0..\\..\\..\\..\r\nset VSROOT_DIR=\1/g' "$dest"
+  sed -i.bak 's/%ROOT_DIR%\\out/%VSROOT_DIR%\\out/g' "$dest"
+
+  chmod +x "$dest"
+  rm "$dest.bak"
+}
+
 main() {
   cd "$(dirname "${0}")/../.."
 
   source ./ci/lib.sh
 
-  cd lib/vscode
+  pushd lib/vscode
 
   # Set the commit Code will embed into the product.json.  We need to do this
   # since Code tries to get the commit from the `.git` directory which will fail
@@ -58,13 +84,31 @@ main() {
 EOF
   ) > product.json
 
-  # Any platform works since we have our own packaging step (for now).
+  # Any platform here works since we will do our own packaging.  We have to do
+  # this because we have an NPM package that could be installed on any platform.
+  # The correct platform dependencies and scripts will be installed as part of
+  # the post-install during `npm install` or when building a standalone release.
   yarn gulp "vscode-reh-web-linux-x64${MINIFY:+-min}"
 
   # Reset so if you develop after building you will not be stuck with the wrong
   # commit (the dev client will use `oss-dev` but the dev server will still use
   # product.json which will have `stable-$commit`).
   git checkout product.json
+
+  popd
+
+  # These provide a `code-server` command in the integrated terminal to open
+  # files in the current instance.
+  delete-bin-script remote-cli/code-server
+  copy-bin-script remote-cli/code-darwin.sh
+  copy-bin-script remote-cli/code-linux.sh
+  copy-bin-script remote-cli/code.cmd
+
+  # These provide a way for terminal applications to open browser windows.
+  delete-bin-script helpers/browser.sh
+  copy-bin-script helpers/browser-darwin.sh
+  copy-bin-script helpers/browser-linux.sh
+  copy-bin-script helpers/browser.cmd
 }
 
 main "$@"

ci/build/npm-postinstall.sh

@@ -1,23 +1,69 @@
 #!/usr/bin/env sh
 set -eu
 
-# Copied from arch() in ci/lib.sh.
-detect_arch() {
-  case "$(uname -m)" in
-    aarch64)
-      echo arm64
-      ;;
-    x86_64 | amd64)
-      echo amd64
-      ;;
-    *)
-      # This will cause the download to fail, but is intentional
-      uname -m
-      ;;
+# Copied from ../lib.sh.
+arch() {
+  cpu="$(uname -m)"
+  case "$cpu" in
+    aarch64) cpu=arm64 ;;
+    x86_64) cpu=amd64 ;;
   esac
+  echo "$cpu"
 }
 
-ARCH="${NPM_CONFIG_ARCH:-$(detect_arch)}"
+# Copied from ../lib.sh except we do not rename Darwin since the cloud agent
+# uses "darwin" in the release names and we do not need to detect Alpine.
+os() {
+  osname=$(uname | tr '[:upper:]' '[:lower:]')
+  case $osname in
+    cygwin* | mingw*) osname="windows" ;;
+  esac
+  echo "$osname"
+}
+
+# Create a symlink at $2 pointing to $1 on any platform.  Anything that
+# currently exists at $2 will be deleted.
+symlink() {
+  source="$1"
+  dest="$2"
+  rm -rf "$dest"
+  case $OS in
+    windows) mklink /J "$dest" "$source" ;;
+    *) ln -s "$source" "$dest" ;;
+  esac
+}
+
+# VS Code bundles some modules into an asar which is an archive format that
+# works like tar. It then seems to get unpacked into node_modules.asar.
+#
+# I don't know why they do this but all the dependencies they bundle already
+# exist in node_modules so just symlink it. We have to do this since not only
+# Code itself but also extensions will look specifically in this directory for
+# files (like the ripgrep binary or the oniguruma wasm).
+symlink_asar() {
+  symlink node_modules node_modules.asar
+}
+
+# Make a symlink at bin/$1/$3 pointing to the platform-specific version of the
+# script in $2.  The extension of the link will be .cmd for Windows otherwise it
+# will be whatever is in $4 (or no extension if $4 is not set).
+symlink_bin_script() {
+  oldpwd="$(pwd)"
+  cd "bin/$1"
+  source="$2"
+  dest="$3"
+  ext="${4-}"
+  case $OS in
+    windows) symlink "$source.cmd" "$dest.cmd" ;;
+    darwin | macos) symlink "$source-darwin.sh" "$dest$ext" ;;
+    *) symlink "$source-linux.sh" "$dest$ext" ;;
+  esac
+  cd "$oldpwd"
+}
+
+ARCH="${NPM_CONFIG_ARCH:-$(arch)}"
+OS="$(os)"
+
 # This is due to an upstream issue with RHEL7/CentOS 7 comptability with node-argon2
 # See: https://github.com/cdr/code-server/pull/3422#pullrequestreview-677765057
 export npm_config_build_from_source=true
@@ -56,8 +102,6 @@ main() {
     ;;
   esac
 
-  OS="$(uname | tr '[:upper:]' '[:lower:]')"
-
   mkdir -p ./lib
 
   if curl -fsSL "https://github.com/coder/cloud-agent/releases/latest/download/cloud-agent-$OS-$ARCH" -o ./lib/coder-cloud-agent; then
@@ -79,22 +123,14 @@ main() {
   fi
 }
 
-# This is a copy of symlink_asar in ../lib.sh. Look there for details.
-symlink_asar() {
-  rm -rf node_modules.asar
-  if [ "${WINDIR-}" ]; then
-    mklink /J node_modules.asar node_modules
-  else
-    ln -s node_modules node_modules.asar
-  fi
-}
-
 vscode_yarn() {
   echo 'Installing Code dependencies...'
   cd lib/vscode
   yarn --production --frozen-lockfile --no-default-rc
 
   symlink_asar
+  symlink_bin_script remote-cli code code-server
+  symlink_bin_script helpers browser browser .sh
 
   cd extensions
   yarn --production --frozen-lockfile

ci/lib.sh

@@ -18,35 +18,30 @@ vscode_version() {
 }
 
 os() {
-  local os
-  os=$(uname | tr '[:upper:]' '[:lower:]')
-  if [[ $os == "linux" ]]; then
-    # Alpine's ldd doesn't have a version flag but if you use an invalid flag
-    # (like --version) it outputs the version to stderr and exits with 1.
-    local ldd_output
-    ldd_output=$(ldd --version 2>&1 || true)
-    if echo "$ldd_output" | grep -iq musl; then
-      os="alpine"
-    fi
-  elif [[ $os == "darwin" ]]; then
-    os="macos"
-  fi
-  echo "$os"
+  osname=$(uname | tr '[:upper:]' '[:lower:]')
+  case $osname in
+    linux)
+      # Alpine's ldd doesn't have a version flag but if you use an invalid flag
+      # (like --version) it outputs the version to stderr and exits with 1.
+      # TODO: Better to check /etc/os-release; see ../install.sh.
+      ldd_output=$(ldd --version 2>&1 || true)
+      if echo "$ldd_output" | grep -iq musl; then
+        osname="alpine"
+      fi
+      ;;
+    darwin) osname="macos" ;;
+    cygwin* | mingw*) osname="windows" ;;
+  esac
+  echo "$osname"
 }
 
 arch() {
   cpu="$(uname -m)"
   case "$cpu" in
-    aarch64)
-      echo arm64
-      ;;
-    x86_64 | amd64)
-      echo amd64
-      ;;
-    *)
-      echo "$cpu"
-      ;;
+    aarch64) cpu=arm64 ;;
+    x86_64) cpu=amd64 ;;
   esac
+  echo "$cpu"
 }
 
 # Grabs the most recent ci.yaml github workflow run that was triggered from the
@@ -104,21 +99,3 @@ export OS
 # RELEASE_PATH is the destination directory for the release from the root.
 # Defaults to release
 RELEASE_PATH="${RELEASE_PATH-release}"
-
-# VS Code bundles some modules into an asar which is an archive format that
-# works like tar. It then seems to get unpacked into node_modules.asar.
-#
-# I don't know why they do this but all the dependencies they bundle already
-# exist in node_modules so just symlink it. We have to do this since not only VS
-# Code itself but also extensions will look specifically in this directory for
-# files (like the ripgrep binary or the oniguruma wasm).
-symlink_asar() {
-  rm -rf node_modules.asar
-  if [ "${WINDIR-}" ]; then
-    # mklink takes the link name first.
-    mklink /J node_modules.asar node_modules
-  else
-    # ln takes the link name second.
-    ln -s node_modules node_modules.asar
-  fi
-}

ci/steps/publish-npm.sh

@@ -81,10 +81,6 @@ main() {
   # https://github.com/actions/upload-artifact/issues/38
   tar -xzf release-npm-package/package.tar.gz
 
-  # Ignore symlink when publishing npm package
-  # See: https://github.com/coder/code-server/pull/3935
-  echo "node_modules.asar" > release/.npmignore
-
   # We use this to set the name of the package in the
   # package.json
   PACKAGE_NAME="code-server"

test/e2e/models/CodeServer.ts

@@ -283,19 +283,31 @@ export class CodeServerPage {
   }
 
   /**
-   * Focuses Integrated Terminal
-   * by using "Terminal: Focus Terminal"
-   * from the Command Palette
+   * Focuses the integrated terminal by navigating through the command palette.
    *
-   * This should focus the terminal no matter
-   * if it already has focus and/or is or isn't
-   * visible already.
+   * This should focus the terminal no matter if it already has focus and/or is
+   * or isn't visible already.  It will always create a new terminal to avoid
+   * clobbering parallel tests.
    */
   async focusTerminal() {
-    await this.executeCommandViaMenus("Terminal: Focus Terminal")
+    const doFocus = async (): Promise<boolean> => {
+      await this.executeCommandViaMenus("Terminal: Create New Terminal")
+      try {
+        await this.page.waitForLoadState("load")
+        await this.page.waitForSelector("textarea.xterm-helper-textarea:focus-within", { timeout: 5000 })
+        return true
+      } catch (error) {
+        return false
+      }
+    }
+
+    let attempts = 1
+    while (!(await doFocus())) {
+      ++attempts
+      this.codeServer.logger.debug(`no focused terminal textarea, retrying (${attempts}/∞)`)
+    }
 
-    // Wait for terminal textarea to show up
-    await this.page.waitForSelector("textarea.xterm-helper-textarea")
+    this.codeServer.logger.debug(`opening terminal took ${attempts} ${plural(attempts, "attempt")}`)
   }
 
   /**
@@ -423,7 +435,7 @@ export class CodeServerPage {
     let context = new Context()
     while (!(await Promise.race([openThenWaitClose(context), navigate(context)]))) {
       ++attempts
-      logger.debug("closed, retrying (${attempt}/∞)")
+      logger.debug(`closed, retrying (${attempts}/∞)`)
       context.cancel()
       context = new Context()
     }