diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index 0c91e3c94bdc..44ab3746dde4 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -98,6 +98,7 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { deferredExec.resolve({ stdout: '', stderr: '' }); deferred.resolve(); }); + await deferredExec.promise; } } diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 653de36a3e23..4a9a57b16fed 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -175,7 +175,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { this.outputChannel?.append(data); }); - result?.proc?.on('close', () => { + result?.proc?.on('exit', () => { deferredExec.resolve({ stdout: '', stderr: '' }); deferred.resolve(); disposeDataReceiver?.(this.testServer); diff --git a/src/test/linters/lint.functional.test.ts b/src/test/linters/lint.functional.test.ts index a3dc70b7c21e..9887cbc5605a 100644 --- a/src/test/linters/lint.functional.test.ts +++ b/src/test/linters/lint.functional.test.ts @@ -4,7 +4,6 @@ 'use strict'; import * as assert from 'assert'; -import * as childProcess from 'child_process'; import * as fs from 'fs-extra'; import * as os from 'os'; import * as path from 'path'; @@ -780,11 +779,6 @@ suite('Linting Functional Tests', () => { teardown(() => { sinon.restore(); }); - - const pythonPath = childProcess.execSync(`"${PYTHON_PATH}" -c "import sys;print(sys.executable)"`); - - console.log(`Testing linter with python ${pythonPath}`); - // These are integration tests that mock out everything except // the filesystem and process execution. diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts new file mode 100644 index 000000000000..5c92c7cf3941 --- /dev/null +++ b/src/test/testing/common/testingAdapter.test.ts @@ -0,0 +1,428 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { TestRun, Uri } from 'vscode'; +import * as typeMoq from 'typemoq'; +import * as path from 'path'; +import * as assert from 'assert'; +import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; +import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; +import { PythonTestServer } from '../../../client/testing/testController/common/server'; +import { IPythonExecutionFactory } from '../../../client/common/process/types'; +import { ITestDebugLauncher } from '../../../client/testing/common/types'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { EXTENSION_ROOT_DIR_FOR_TESTS, initialize } from '../../initialize'; +import { traceError, traceLog } from '../../../client/logging'; +import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; +import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; + +suite('End to End Tests: test adapters', () => { + let resultResolver: typeMoq.IMock; + let pythonTestServer: ITestServer; + let pythonExecFactory: IPythonExecutionFactory; + let debugLauncher: ITestDebugLauncher; + let configService: IConfigurationService; + let testOutputChannel: ITestOutputChannel; + let serviceContainer: IServiceContainer; + let workspaceUri: Uri; + const rootPathSmallWorkspace = path.join( + EXTENSION_ROOT_DIR_FOR_TESTS, + 'src', + 'testTestingRootWkspc', + 'smallWorkspace', + ); + const rootPathLargeWorkspace = path.join( + EXTENSION_ROOT_DIR_FOR_TESTS, + 'src', + 'testTestingRootWkspc', + 'largeWorkspace', + ); + suiteSetup(async () => { + serviceContainer = (await initialize()).serviceContainer; + }); + + setup(async () => { + // create objects that were injected + configService = serviceContainer.get(IConfigurationService); + pythonExecFactory = serviceContainer.get(IPythonExecutionFactory); + debugLauncher = serviceContainer.get(ITestDebugLauncher); + testOutputChannel = serviceContainer.get(ITestOutputChannel); + + // create mock resultResolver object + resultResolver = typeMoq.Mock.ofType(); + + // create objects that were not injected + pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); + await pythonTestServer.serverReady(); + }); + test('unittest discovery adapter small workspace', async () => { + // result resolver and saved data for assertions + let actualData: { + status: unknown; + error: string | any[]; + tests: unknown; + }; + resultResolver + .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceLog(`resolveDiscovery ${data}`); + actualData = data; + return Promise.resolve(); + }); + + // set workspace to test workspace folder and set up settings + workspaceUri = Uri.parse(rootPathSmallWorkspace); + configService.getSettings(workspaceUri).testing.unittestArgs = ['-s', '.', '-p', '*test*.py']; + + // run unittest discovery + const discoveryAdapter = new UnittestTestDiscoveryAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + + await discoveryAdapter.discoverTests(workspaceUri).finally(() => { + // verification after discovery is complete + resultResolver.verify( + (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.once(), + ); + + // 1. Check the status is "success" + assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + // 2. Confirm no errors + assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(actualData.tests, 'Expected tests to be present'); + }); + }); + + test('unittest discovery adapter large workspace', async () => { + // result resolver and saved data for assertions + let actualData: { + status: unknown; + error: string | any[]; + tests: unknown; + }; + resultResolver + .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceLog(`resolveDiscovery ${data}`); + actualData = data; + return Promise.resolve(); + }); + + // set settings to work for the given workspace + workspaceUri = Uri.parse(rootPathLargeWorkspace); + configService.getSettings(workspaceUri).testing.unittestArgs = ['-s', '.', '-p', '*test*.py']; + // run discovery + const discoveryAdapter = new UnittestTestDiscoveryAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + + await discoveryAdapter.discoverTests(workspaceUri).finally(() => { + // verification after discovery is complete + resultResolver.verify( + (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.once(), + ); + + // 1. Check the status is "success" + assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + // 2. Confirm no errors + assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(actualData.tests, 'Expected tests to be present'); + }); + }); + test('pytest discovery adapter small workspace', async () => { + // result resolver and saved data for assertions + let actualData: { + status: unknown; + error: string | any[]; + tests: unknown; + }; + resultResolver + .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceLog(`resolveDiscovery ${data}`); + actualData = data; + return Promise.resolve(); + }); + // run pytest discovery + const discoveryAdapter = new PytestTestDiscoveryAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathSmallWorkspace); + + await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { + // verification after discovery is complete + resultResolver.verify( + (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.once(), + ); + + // 1. Check the status is "success" + assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + // 2. Confirm no errors + assert.strictEqual(actualData.error.length, 0, "Expected no errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(actualData.tests, 'Expected tests to be present'); + }); + }); + test('pytest discovery adapter large workspace', async () => { + // result resolver and saved data for assertions + let actualData: { + status: unknown; + error: string | any[]; + tests: unknown; + }; + resultResolver + .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceLog(`resolveDiscovery ${data}`); + actualData = data; + return Promise.resolve(); + }); + // run pytest discovery + const discoveryAdapter = new PytestTestDiscoveryAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathLargeWorkspace); + + await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { + // verification after discovery is complete + resultResolver.verify( + (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.once(), + ); + + // 1. Check the status is "success" + assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + // 2. Confirm no errors + assert.strictEqual(actualData.error.length, 0, "Expected no errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(actualData.tests, 'Expected tests to be present'); + }); + }); + test('unittest execution adapter small workspace', async () => { + // result resolver and saved data for assertions + let actualData: { + status: unknown; + error: string | any[]; + result: unknown; + }; + resultResolver + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceLog(`resolveExecution ${data}`); + actualData = data; + return Promise.resolve(); + }); + + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathSmallWorkspace); + configService.getSettings(workspaceUri).testing.unittestArgs = ['-s', '.', '-p', '*test*.py']; + // run execution + const executionAdapter = new UnittestTestExecutionAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + await executionAdapter + .runTests(workspaceUri, ['test_simple.SimpleClass.test_simple_unit'], false, testRun.object) + .finally(() => { + // verification after execution is complete + resultResolver.verify( + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.once(), + ); + + // 1. Check the status is "success" + assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + // 2. Confirm tests are found + assert.ok(actualData.result, 'Expected results to be present'); + }); + }); + test('unittest execution adapter large workspace', async () => { + // result resolver and saved data for assertions + resultResolver + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceError(`resolveExecution ${data}`); + console.log(`resolveExecution ${data}`); + traceLog(`resolveExecution ${data}`); + // do the following asserts for each time resolveExecution is called, should be called once per test. + // 1. Check the status, can be subtest success or failure + assert( + data.status === 'subtest-success' || data.status === 'subtest-failure', + "Expected status to be 'subtest-success' or 'subtest-failure'", + ); + // 2. Confirm tests are found + assert.ok(data.result, 'Expected results to be present'); + return Promise.resolve(); + }); + + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathLargeWorkspace); + configService.getSettings(workspaceUri).testing.unittestArgs = ['-s', '.', '-p', '*test*.py']; + + // run unittest execution + const executionAdapter = new UnittestTestExecutionAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + await executionAdapter + .runTests(workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], false, testRun.object) + .finally(() => { + // verification after discovery is complete + resultResolver.verify( + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.exactly(200), + ); + }); + }); + test('pytest execution adapter small workspace', async () => { + // result resolver and saved data for assertions + let actualData: { + status: unknown; + error: string | any[]; + result: unknown; + }; + resultResolver + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceLog(`resolveExecution ${data}`); + actualData = data; + return Promise.resolve(); + }); + + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathSmallWorkspace); + + // run pytest execution + const executionAdapter = new PytestTestExecutionAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + await executionAdapter + .runTests( + workspaceUri, + [`${rootPathSmallWorkspace}/test_simple.py::test_a`], + false, + testRun.object, + pythonExecFactory, + ) + .finally(() => { + // verification after discovery is complete + resultResolver.verify( + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.once(), + ); + + // 1. Check the status is "success" + assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + // 2. Confirm no errors + assert.strictEqual(actualData.error, null, "Expected no errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(actualData.result, 'Expected results to be present'); + }); + }); + test('pytest execution adapter large workspace', async () => { + resultResolver + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((data) => { + traceLog(`resolveExecution ${data}`); + // do the following asserts for each time resolveExecution is called, should be called once per test. + // 1. Check the status is "success" + assert.strictEqual(data.status, 'success', "Expected status to be 'success'"); + // 2. Confirm no errors + assert.strictEqual(data.error, null, "Expected no errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(data.result, 'Expected results to be present'); + return Promise.resolve(); + }); + + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathLargeWorkspace); + + // generate list of test_ids + const testIds: string[] = []; + for (let i = 0; i < 200; i = i + 1) { + const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; + testIds.push(testId); + } + + // run pytest execution + const executionAdapter = new PytestTestExecutionAdapter( + pythonTestServer, + configService, + testOutputChannel, + resultResolver.object, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { + // resolve execution should be called 200 times since there are 200 tests run. + resultResolver.verify( + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.exactly(200), + ); + }); + }); +}); diff --git a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py new file mode 100644 index 000000000000..3e84df0a2d9f --- /dev/null +++ b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py @@ -0,0 +1,16 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import pytest +import unittest + + +@pytest.mark.parametrize("num", range(0, 200)) +def test_odd_even(num): + return num % 2 == 0 + + +class NumbersTest(unittest.TestCase): + def test_even(self): + for i in range(0, 200): + with self.subTest(i=i): + self.assertEqual(i % 2, 0) diff --git a/src/testTestingRootWkspc/smallWorkspace/test_simple.py b/src/testTestingRootWkspc/smallWorkspace/test_simple.py new file mode 100644 index 000000000000..6b4f7bd2f8a6 --- /dev/null +++ b/src/testTestingRootWkspc/smallWorkspace/test_simple.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import unittest + + +def test_a(): + assert 1 == 1 + + +class SimpleClass(unittest.TestCase): + def test_simple_unit(self): + assert True