diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 3283d3281e5b..cd0d8ba44c6a 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -33,497 +33,3 @@ jobs: node_version: ${{ env.NODE_VERSION}} vsix_name: ${{ env.VSIX_NAME }} artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Lint - uses: ./.github/actions/lint - with: - node_version: ${{ env.NODE_VERSION }} - - check-types: - name: Check Python types - runs-on: ubuntu-latest - steps: - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - - - name: Checkout - uses: actions/checkout@v4 - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --no-cache-dir --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --no-cache-dir --implementation py' - - - name: Install other Python requirements - run: | - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - python -m pip install --upgrade -r build/test-requirements.txt - - - name: Run Pyright - uses: jakebailey/pyright-action@v1 - with: - version: 1.1.308 - working-directory: 'pythonFiles' - - python-tests: - name: Python Tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ${{ env.special-working-directory }} - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. - os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.8', '3.x', '3.12-dev'] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - path: ${{ env.special-working-directory-relative }} - - - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Run Python unit tests - run: python pythonFiles/tests/run_all.py - - tests: - name: Tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - defaults: - run: - working-directory: ${{ env.special-working-directory }} - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. - os: [ubuntu-latest, windows-latest] - # Run the tests on the oldest and most recent versions of Python. - python: ['3.x'] - test-suite: [ts-unit, venv, single-workspace, debugger, functional] - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - path: ${{ env.special-working-directory-relative }} - - - name: Install Node - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - cache-dependency-path: ${{ env.special-working-directory-relative }}/package-lock.json - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade --pre debugpy - - - name: Download get-pip.py - run: | - python -m pip install wheel - python -m pip install -r build/build-install-requirements.txt - python ./pythonFiles/download_get_pip.py - shell: bash - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/python" --no-cache-dir --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: '"${{ env.special-working-directory-relative }}/pythonFiles/jedilsp_requirements/requirements.txt"' - options: '-t "${{ env.special-working-directory-relative }}/pythonFiles/lib/jedilsp" --no-cache-dir --implementation py' - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install debugpy wheels (Python ${{ matrix.python }}) - run: | - python -m pip install wheel - python -m pip --disable-pip-version-check install -r build/build-install-requirements.txt - python ./pythonFiles/install_debugpy.py - shell: bash - if: matrix.test-suite == 'debugger' - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - if: matrix.test-suite == 'functional' - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' && startsWith(matrix.python, 3.) - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - if: matrix.test-suite == 'venv' - run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Set CI_PYTHON_PATH and CI_DISABLE_AUTO_SELECTION - run: | - echo "CI_PYTHON_PATH=python" >> $GITHUB_ENV - echo "CI_DISABLE_AUTO_SELECTION=1" >> $GITHUB_ENV - shell: bash - if: matrix.test-suite != 'ts-unit' - - # Run TypeScript unit tests only for Python 3.X. - - name: Run TypeScript unit tests - run: npm run test:unittests - if: matrix.test-suite == 'ts-unit' && startsWith(matrix.python, 3.) - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'venv' - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'single-workspace' - - - name: Run debugger tests - env: - CI_PYTHON_VERSION: ${{ matrix.python }} - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testDebugger - working-directory: ${{ env.special-working-directory }} - if: matrix.test-suite == 'debugger' - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - run: npm run test:functional - if: matrix.test-suite == 'functional' - - smoke-tests: - name: Smoke tests - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - needs: [build-vsix] - strategy: - fail-fast: false - matrix: - # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, - # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case. - os: [ubuntu-latest, windows-latest] - steps: - # Need the source to have the tests available. - - name: Checkout - uses: actions/checkout@v4 - - - name: Smoke tests - uses: ./.github/actions/smoke-tests - with: - node_version: ${{ env.NODE_VERSION }} - artifact_name: ${{ env.ARTIFACT_NAME_VSIX }} - - ### Coverage run - coverage: - name: Coverage - # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # Only run coverage on linux for PRs - os: [ubuntu-latest] - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install Node - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_VERSION }} - cache: 'npm' - - - name: Install dependencies (npm ci) - run: npm ci - - - name: Compile - run: npx gulp prePublishNonBundle - - - name: Use Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v4 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - cache-dependency-path: | - requirements.txt - pythonFiles/jedilsp_requirements/requirements.txt - build/test-requirements.txt - build/functional-test-requirements.txt - - - name: Install base Python requirements - uses: brettcannon/pip-secure-install@v1 - with: - options: '-t ./pythonFiles/lib/python --implementation py' - - - name: Install Jedi requirements - uses: brettcannon/pip-secure-install@v1 - with: - requirements-file: './pythonFiles/jedilsp_requirements/requirements.txt' - options: '-t ./pythonFiles/lib/jedilsp --implementation py' - - - name: Install debugpy - run: | - # We need to have debugpy so that tests relying on it keep passing, but we don't need install_debugpy's logic in the test phase. - python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --implementation py --no-deps --upgrade --pre debugpy - - - name: Install test requirements - run: python -m pip install --upgrade -r build/test-requirements.txt - - - name: Install functional test requirements - run: python -m pip install --upgrade -r ./build/functional-test-requirements.txt - - - name: Prepare pipenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install pipenv - python -m pipenv run python ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} pipenvPath - - - name: Prepare poetry for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - shell: pwsh - run: | - python -m pip install poetry - Move-Item -Path ".\build\ci\pyproject.toml" -Destination . - poetry env use python - - - name: Prepare virtualenv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m pip install virtualenv - python -m virtualenv .virtualenv/ - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".virtualenv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } else { - & ".virtualenv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} virtualEnvPath - } - - - name: Prepare venv for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - python -m venv .venv - if ('${{ matrix.os }}' -match 'windows-latest') { - & ".venv/Scripts/python.exe" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } else { - & ".venv/bin/python" ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} venvPath - } - - - name: Prepare conda for venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - PYTHON_VIRTUAL_ENVS_LOCATION: './src/tmp/envPaths.json' - shell: pwsh - run: | - # 1. For `terminalActivation.testvirtualenvs.test.ts` - if ('${{ matrix.os }}' -match 'windows-latest') { - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath python.exe - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath Scripts | Join-Path -ChildPath conda - } else{ - $condaPythonPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath python - $condaExecPath = Join-Path -Path $Env:CONDA -ChildPath bin | Join-Path -ChildPath conda - } - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaExecPath $condaExecPath - & $condaPythonPath ./build/ci/addEnvPath.py ${{ env.PYTHON_VIRTUAL_ENVS_LOCATION }} condaPath - & $condaExecPath init --all - - - name: Run TypeScript unit tests - run: npm run test:unittests:cover - - - name: Run Python unit tests - run: | - python pythonFiles/tests/run_all.py - - # The virtual environment based tests use the `testSingleWorkspace` set of tests - # with the environment variable `TEST_FILES_SUFFIX` set to `testvirtualenvs`, - # which is set in the "Prepare environment for venv tests" step. - # We also use a third-party GitHub Action to install xvfb on Linux, - # run tests and then clean up the process once the tests ran. - # See https://github.com/GabrielBB/xvfb-action - - name: Run venv tests - env: - TEST_FILES_SUFFIX: testvirtualenvs - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - - name: Run single-workspace tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - uses: GabrielBB/xvfb-action@v1.6 - with: - run: npm run testSingleWorkspace:cover - - # Enable these tests when coverage is setup for multiroot workspace tests - # - name: Run multi-workspace tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testMultiWorkspace:cover - - # Enable these tests when coverage is setup for debugger tests - # - name: Run debugger tests - # env: - # CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - # CI_DISABLE_AUTO_SELECTION: 1 - # uses: GabrielBB/xvfb-action@v1.6 - # with: - # run: npm run testDebugger:cover - - # Run TypeScript functional tests - - name: Run TypeScript functional tests - env: - CI_PYTHON_VERSION: ${{ env.PYTHON_VERSION }} - CI_DISABLE_AUTO_SELECTION: 1 - run: npm run test:functional:cover - - - name: Generate coverage reports - run: npm run test:cover:report - - - name: Upload HTML report - uses: actions/upload-artifact@v3 - with: - name: ${{ runner.os }}-coverage-report-html - path: ./coverage - retention-days: 1 diff --git a/src/client/common/process/worker/main.ts b/src/client/common/process/worker/main.ts new file mode 100644 index 000000000000..d8f158a73c70 --- /dev/null +++ b/src/client/common/process/worker/main.ts @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { Worker } from 'worker_threads'; +import { traceError } from '../../../logging'; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types +export async function executeWorkerFile(workerFileName: string, workerData: any): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker(workerFileName, { workerData }); + worker.on('message', (res: { err: Error; res: unknown }) => { + if (res.err) { + reject(res.err); + } + resolve(res.res); + }); + worker.on('error', (ex: Error) => { + traceError(`Error in worker ${workerFileName}`, ex); + reject(ex); + }); + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker ${workerFileName} stopped with exit code ${code}`)); + } + }); + }); +} diff --git a/src/client/common/process/worker/plainExecWorker.ts b/src/client/common/process/worker/plainExecWorker.ts new file mode 100644 index 000000000000..69374eba53d7 --- /dev/null +++ b/src/client/common/process/worker/plainExecWorker.ts @@ -0,0 +1,16 @@ +import { parentPort, workerData } from 'worker_threads'; +import { workerPlainExec } from './workerRawProcessApis'; + +workerPlainExec(workerData.file, workerData.args, workerData.options, workerData.defaultEnv, workerData.disposables) + .then((res) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage({ res }); + }) + .catch((err) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage({ err }); + }); diff --git a/src/client/common/process/worker/rawProcessApiWrapper.ts b/src/client/common/process/worker/rawProcessApiWrapper.ts new file mode 100644 index 000000000000..afd08dae52a7 --- /dev/null +++ b/src/client/common/process/worker/rawProcessApiWrapper.ts @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { SpawnOptions } from 'child_process'; +import * as path from 'path'; +import { executeWorkerFile } from './main'; +import { EnvironmentVariables, ExecutionResult, IDisposable, ShellOptions } from './types'; + +export function shellExec( + command: string, + options: ShellOptions, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + return executeWorkerFile(path.join(__dirname, 'shellExecWorker.js'), { + command, + options, + defaultEnv, + disposables, + }); +} + +export function plainExec( + file: string, + args: string[], + options: SpawnOptions & { doNotLog?: boolean } = {}, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + return executeWorkerFile(path.join(__dirname, 'plainExecWorker.js'), { + file, + args, + options, + defaultEnv, + disposables, + }); +} diff --git a/src/client/common/process/worker/shellExecWorker.ts b/src/client/common/process/worker/shellExecWorker.ts new file mode 100644 index 000000000000..739d0bc1b6d5 --- /dev/null +++ b/src/client/common/process/worker/shellExecWorker.ts @@ -0,0 +1,16 @@ +import { parentPort, workerData } from 'worker_threads'; +import { workerShellExec } from './workerRawProcessApis'; + +workerShellExec(workerData.command, workerData.options, workerData.defaultEnv, workerData.disposables) + .then((res) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage(res); + }) + .catch((ex) => { + if (!parentPort) { + throw new Error('Not in a worker thread'); + } + parentPort.postMessage(ex); + }); diff --git a/src/client/common/process/worker/types.ts b/src/client/common/process/worker/types.ts new file mode 100644 index 000000000000..5c58aec10214 --- /dev/null +++ b/src/client/common/process/worker/types.ts @@ -0,0 +1,38 @@ +/* eslint-disable @typescript-eslint/no-empty-function */ +/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ +import { ExecOptions, SpawnOptions as ChildProcessSpawnOptions } from 'child_process'; + +export function noop() {} +export interface IDisposable { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + dispose(): void | undefined | Promise; +} +export type EnvironmentVariables = Record; +export class StdErrError extends Error { + // eslint-disable-next-line @typescript-eslint/no-useless-constructor + constructor(message: string) { + super(message); + } +} + +export type SpawnOptions = ChildProcessSpawnOptions & { + encoding?: string; + // /** + // * Can't use `CancellationToken` here as it comes from vscode which is not available in worker threads. + // */ + // token?: CancellationToken; + mergeStdOutErr?: boolean; + throwOnStdErr?: boolean; + extraVariables?: NodeJS.ProcessEnv; + // /** + // * Can't use `OutputChannel` here as it comes from vscode which is not available in worker threads. + // */ + // outputChannel?: OutputChannel; + stdinStr?: string; +}; +export type ShellOptions = ExecOptions & { throwOnStdErr?: boolean }; + +export type ExecutionResult = { + stdout: T; + stderr?: T; +}; diff --git a/src/client/common/process/worker/workerRawProcessApis.ts b/src/client/common/process/worker/workerRawProcessApis.ts new file mode 100644 index 000000000000..1c1c6ca5b46b --- /dev/null +++ b/src/client/common/process/worker/workerRawProcessApis.ts @@ -0,0 +1,213 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// !!!! IMPORTANT: DO NOT IMPORT FROM VSCODE MODULE AS IT IS NOT AVAILABLE INSIDE WORKER THREADS !!!! + +import { exec, execSync, spawn } from 'child_process'; +import { Readable } from 'stream'; +import { createDeferred } from '../../utils/async'; +import { DEFAULT_ENCODING } from '../constants'; +import { decodeBuffer } from '../decoder'; +import { + ShellOptions, + SpawnOptions, + EnvironmentVariables, + IDisposable, + noop, + StdErrError, + ExecutionResult, +} from './types'; + +const PS_ERROR_SCREEN_BOGUS = /your [0-9]+x[0-9]+ screen size is bogus\. expect trouble/; + +function getDefaultOptions(options: T, defaultEnv?: EnvironmentVariables): T { + const defaultOptions = { ...options }; + const execOptions = defaultOptions as SpawnOptions; + if (execOptions) { + execOptions.encoding = + typeof execOptions.encoding === 'string' && execOptions.encoding.length > 0 + ? execOptions.encoding + : DEFAULT_ENCODING; + const { encoding } = execOptions; + delete execOptions.encoding; + execOptions.encoding = encoding; + } + if (!defaultOptions.env || Object.keys(defaultOptions.env).length === 0) { + const env = defaultEnv || process.env; + defaultOptions.env = { ...env }; + } else { + defaultOptions.env = { ...defaultOptions.env }; + } + + if (execOptions && execOptions.extraVariables) { + defaultOptions.env = { ...defaultOptions.env, ...execOptions.extraVariables }; + } + + // Always ensure we have unbuffered output. + defaultOptions.env.PYTHONUNBUFFERED = '1'; + if (!defaultOptions.env.PYTHONIOENCODING) { + defaultOptions.env.PYTHONIOENCODING = 'utf-8'; + } + + return defaultOptions; +} + +export function workerShellExec( + command: string, + options: ShellOptions, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + const shellOptions = getDefaultOptions(options, defaultEnv); + return new Promise((resolve, reject) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const callback = (e: any, stdout: any, stderr: any) => { + if (e && e !== null) { + reject(e); + } else if (shellOptions.throwOnStdErr && stderr && stderr.length) { + reject(new Error(stderr)); + } else { + stdout = filterOutputUsingCondaRunMarkers(stdout); + // Make sure stderr is undefined if we actually had none. This is checked + // elsewhere because that's how exec behaves. + resolve({ stderr: stderr && stderr.length > 0 ? stderr : undefined, stdout }); + } + }; + let procExited = false; + const proc = exec(command, shellOptions, callback); // NOSONAR + proc.once('close', () => { + procExited = true; + }); + proc.once('exit', () => { + procExited = true; + }); + proc.once('error', () => { + procExited = true; + }); + const disposable: IDisposable = { + dispose: () => { + // If process has not exited nor killed, force kill it. + if (!procExited && !proc.killed) { + if (proc.pid) { + killPid(proc.pid); + } else { + proc.kill(); + } + } + }, + }; + if (disposables) { + disposables.add(disposable); + } + }); +} + +export function workerPlainExec( + file: string, + args: string[], + options: SpawnOptions & { doNotLog?: boolean } = {}, + defaultEnv?: EnvironmentVariables, + disposables?: Set, +): Promise> { + const spawnOptions = getDefaultOptions(options, defaultEnv); + const encoding = spawnOptions.encoding ? spawnOptions.encoding : 'utf8'; + const proc = spawn(file, args, spawnOptions); + // Listen to these errors (unhandled errors in streams tears down the process). + // Errors will be bubbled up to the `error` event in `proc`, hence no need to log. + proc.stdout?.on('error', noop); + proc.stderr?.on('error', noop); + const deferred = createDeferred>(); + const disposable: IDisposable = { + dispose: () => { + // If process has not exited nor killed, force kill it. + if (!proc.killed && !deferred.completed) { + if (proc.pid) { + killPid(proc.pid); + } else { + proc.kill(); + } + } + }, + }; + disposables?.add(disposable); + const internalDisposables: IDisposable[] = []; + + // eslint-disable-next-line @typescript-eslint/ban-types + const on = (ee: Readable | null, name: string, fn: Function) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ee?.on(name, fn as any); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + internalDisposables.push({ dispose: () => ee?.removeListener(name, fn as any) as any }); + }; + + // Tokens not supported yet as they come from vscode module which is not available. + // if (options.token) { + // internalDisposables.push(options.token.onCancellationRequested(disposable.dispose)); + // } + + const stdoutBuffers: Buffer[] = []; + on(proc.stdout, 'data', (data: Buffer) => { + stdoutBuffers.push(data); + }); + const stderrBuffers: Buffer[] = []; + on(proc.stderr, 'data', (data: Buffer) => { + if (options.mergeStdOutErr) { + stdoutBuffers.push(data); + stderrBuffers.push(data); + } else { + stderrBuffers.push(data); + } + }); + + proc.once('close', () => { + if (deferred.completed) { + return; + } + const stderr: string | undefined = + stderrBuffers.length === 0 ? undefined : decodeBuffer(stderrBuffers, encoding); + if ( + stderr && + stderr.length > 0 && + options.throwOnStdErr && + // ignore this specific error silently; see this issue for context: https://github.com/microsoft/vscode/issues/75932 + !(PS_ERROR_SCREEN_BOGUS.test(stderr) && stderr.replace(PS_ERROR_SCREEN_BOGUS, '').trim().length === 0) + ) { + deferred.reject(new StdErrError(stderr)); + } else { + let stdout = decodeBuffer(stdoutBuffers, encoding); + stdout = filterOutputUsingCondaRunMarkers(stdout); + deferred.resolve({ stdout, stderr }); + } + internalDisposables.forEach((d) => d.dispose()); + disposable.dispose(); + }); + proc.once('error', (ex) => { + deferred.reject(ex); + internalDisposables.forEach((d) => d.dispose()); + disposable.dispose(); + }); + + return deferred.promise; +} + +function filterOutputUsingCondaRunMarkers(stdout: string) { + // These markers are added if conda run is used or `interpreterInfo.py` is + // run, see `get_output_via_markers.py`. + const regex = />>>PYTHON-EXEC-OUTPUT([\s\S]*)<<= 2 ? match[1].trim() : undefined; + return filteredOut !== undefined ? filteredOut : stdout; +} + +function killPid(pid: number): void { + try { + if (process.platform === 'win32') { + // Windows doesn't support SIGTERM, so execute taskkill to kill the process + execSync(`taskkill /pid ${pid} /T /F`); // NOSONAR + } else { + process.kill(pid); + } + } catch { + console.warn('Unable to kill process with pid', pid); + } +}