From 87d299a28ffb765148e18d58ad958920a81ffcb4 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 14 Aug 2023 20:58:22 -0700 Subject: [PATCH 01/55] add path parser --- src/test/testing/common/testingAdapter.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 5af7e59f6a46..802cdb856383 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -402,11 +402,14 @@ suite('End to End Tests: test adapters', () => { // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathLargeWorkspace); + const parts = rootPathLargeWorkspace.split(path.sep); + + const rootPathLargeWorkspaceAsId = parts.join('/'); // generate list of test_ids const testIds: string[] = []; for (let i = 0; i < 200; i = i + 1) { - const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; + const testId = `${rootPathLargeWorkspaceAsId}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } From 5566ad3ad60561207e8dab2ca49a54098dfcaed7 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 14 Aug 2023 22:02:43 -0700 Subject: [PATCH 02/55] remove other parser --- src/test/testing/common/testingAdapter.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 802cdb856383..5af7e59f6a46 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -402,14 +402,11 @@ suite('End to End Tests: test adapters', () => { // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathLargeWorkspace); - const parts = rootPathLargeWorkspace.split(path.sep); - - const rootPathLargeWorkspaceAsId = parts.join('/'); // generate list of test_ids const testIds: string[] = []; for (let i = 0; i < 200; i = i + 1) { - const testId = `${rootPathLargeWorkspaceAsId}/test_parameterized_subtest.py::test_odd_even[${i}]`; + const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } From f2a4bd131490a829e3bbf992de2607395efb557b Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 15 Aug 2023 10:42:43 -0700 Subject: [PATCH 03/55] switching to at least to reduce flakiness --- src/test/testing/common/testingAdapter.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 5af7e59f6a46..a5ed0ac7d7c3 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -327,7 +327,7 @@ suite('End to End Tests: test adapters', () => { // verification after discovery is complete resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeastOnce(), + typeMoq.Times.atLeast(200), ); }); }); @@ -430,7 +430,7 @@ suite('End to End Tests: test adapters', () => { // resolve execution should be called 200 times since there are 200 tests run. resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(200), + typeMoq.Times.atLeast(200), ); }); }); From cf814acecb8be1dda11a381f2aa2c39ebd72afda Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 15 Aug 2023 12:31:29 -0700 Subject: [PATCH 04/55] add in then not finally --- .../testing/common/testingAdapter.test.ts | 57 +++++++++++++++---- .../test_parameterized_subtest.py | 4 +- 2 files changed, 48 insertions(+), 13 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index a5ed0ac7d7c3..0a3ada89d422 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -17,6 +17,7 @@ import { traceLog } from '../../../client/logging'; import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; +import { createDeferred } from '../../../client/common/utils/async'; suite('End to End Tests: test adapters', () => { let resultResolver: typeMoq.IMock; @@ -282,9 +283,11 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); // 2. Confirm tests are found assert.ok(actualData.result, 'Expected results to be present'); + console.log('finally p1'); }); }); test('unittest execution adapter large workspace', async () => { + const count = 0; // result resolver and saved data for assertions resultResolver .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) @@ -321,15 +324,23 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); + console.log('FROM TEST, unit large'); + const deferred = createDeferred(); await executionAdapter .runTests(workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], false, testRun.object) - .finally(() => { + .then(() => { // verification after discovery is complete resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeast(200), + typeMoq.Times.atLeast(2000), ); + console.log('hit then'); + }) + .finally(() => { + deferred.resolve(); }); + await deferred.promise; + console.log('reached the very end', count); }); test('pytest execution adapter small workspace', async () => { // result resolver and saved data for assertions @@ -364,6 +375,8 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); + // const deferred = createDeferred(); + console.log('run tests for small workspace'); await executionAdapter .runTests( workspaceUri, @@ -372,8 +385,9 @@ suite('End to End Tests: test adapters', () => { testRun.object, pythonExecFactory, ) - .finally(() => { + .then(() => { // verification after discovery is complete + console.log('hit then'); resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), typeMoq.Times.once(), @@ -384,9 +398,18 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(actualData.error, null, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.result, 'Expected results to be present'); + }) + .catch((err) => { + console.log('hit catch'); + console.log(err); + }) + .finally(() => { + console.log('hit finally'); }); + console.log('at very end of pytest small'); }); test('pytest execution adapter large workspace', async () => { + const errorMessages: string[] = []; resultResolver .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) .returns((data) => { @@ -405,7 +428,7 @@ suite('End to End Tests: test adapters', () => { // generate list of test_ids const testIds: string[] = []; - for (let i = 0; i < 200; i = i + 1) { + for (let i = 0; i < 2000; i = i + 1) { const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } @@ -426,13 +449,25 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { - // resolve execution should be called 200 times since there are 200 tests run. - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeast(200), - ); - }); + console.log('FROM TEST, do the run large'); + await executionAdapter + .runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory) + .then(() => { + // resolve execution should be called 200 times since there are 200 tests run. + console.log('hit then'); + assert.strictEqual( + errorMessages.length, + 0, + ['Test run was unsuccessful, the following errors were produced: \n', ...errorMessages].join('\n'), + ); + resultResolver.verify( + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.atLeast(2000), + ); + }) + .finally(() => { + console.log('hit finally large'); + }); }); test('unittest execution adapter seg fault error handling', async () => { const testId = `test_seg_fault.TestSegmentationFault.test_segfault`; diff --git a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py index 8c6a29adf495..a76856ebb929 100644 --- a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py +++ b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py @@ -4,13 +4,13 @@ import unittest -@pytest.mark.parametrize("num", range(0, 200)) +@pytest.mark.parametrize("num", range(0, 2000)) def test_odd_even(num): assert num % 2 == 0 class NumbersTest(unittest.TestCase): def test_even(self): - for i in range(0, 200): + for i in range(0, 2000): with self.subTest(i=i): self.assertEqual(i % 2, 0) From 083871705b4720c6d39a07ae473827d583be3667 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 16 Aug 2023 12:12:23 -0700 Subject: [PATCH 05/55] change to keep socket open --- pythonFiles/testing_tools/socket_manager.py | 8 +++- pythonFiles/unittestadapter/execution.py | 22 +++++++-- pythonFiles/vscode_pytest/__init__.py | 25 ++++++++-- .../testing/common/testingAdapter.test.ts | 47 ++++++++----------- 4 files changed, 64 insertions(+), 38 deletions(-) diff --git a/pythonFiles/testing_tools/socket_manager.py b/pythonFiles/testing_tools/socket_manager.py index 372a50b5e012..b2afbf0e5a17 100644 --- a/pythonFiles/testing_tools/socket_manager.py +++ b/pythonFiles/testing_tools/socket_manager.py @@ -23,6 +23,12 @@ def __init__(self, addr): self.socket = None def __enter__(self): + return self.connect() + + def __exit__(self, *_): + self.close() + + def connect(self): self.socket = socket.socket( socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP ) @@ -35,7 +41,7 @@ def __enter__(self): return self - def __exit__(self, *_): + def close(self): if self.socket: try: self.socket.shutdown(socket.SHUT_RDWR) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index f239f81c2d87..8de0de5bfcf0 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import argparse +import atexit import enum import json import os @@ -224,9 +225,20 @@ def run_tests( return payload +__socket = None +atexit.register(lambda: __socket.close() if __socket else None) def send_run_data(raw_data, port, uuid): # Build the request data (it has to be a POST request or the Node side will not process it), and send it. + addr = ("localhost", port) + global __socket + if __socket is None: + try: + __socket = socket_manager.SocketManager(addr) + __socket.connect() + except Exception as e: + print(f"Plugin error connection error[vscode-pytest]: {e}") + __socket = None status = raw_data["outcome"] cwd = os.path.abspath(START_DIR) if raw_data["subtest"]: @@ -236,7 +248,7 @@ def send_run_data(raw_data, port, uuid): test_dict = {} test_dict[test_id] = raw_data payload: PayloadDict = {"cwd": cwd, "status": status, "result": test_dict} - addr = ("localhost", port) + data = json.dumps(payload) request = f"""Content-Length: {len(data)} Content-Type: application/json @@ -245,10 +257,10 @@ def send_run_data(raw_data, port, uuid): {data}""" try: with socket_manager.SocketManager(addr) as s: - if s.socket is not None: - s.socket.sendall(request.encode("utf-8")) - except Exception as e: - print(f"Error sending response: {e}") + if __socket.socket is not None: + __socket.socket.sendall(request.encode("utf-8")) + except Exception as ex: + print(f"Error sending response: {ex}") print(f"Request data: {request}") diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 3d5bde44204f..2ca4b4e0ade2 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -1,3 +1,7 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import atexit import json import os import pathlib @@ -606,6 +610,8 @@ class ExecutionPayloadDict(Dict): def get_node_path(node: Any) -> pathlib.Path: return getattr(node, "path", pathlib.Path(node.fspath)) +__socket = None +atexit.register(lambda: __socket.close() if __socket else None) def execution_post( cwd: str, @@ -621,13 +627,20 @@ def execution_post( """ testPort = os.getenv("TEST_PORT", 45454) testuuid = os.getenv("TEST_UUID") + addr = ("localhost", int(testPort)) + global __socket + if __socket is None: + try: + __socket = socket_manager.SocketManager(addr) + __socket.connect() + except Exception as e: + print(f"Plugin error connection error[vscode-pytest]: {e}") + __socket = None payload: ExecutionPayloadDict = ExecutionPayloadDict( cwd=cwd, status=status, result=tests, not_found=None, error=None ) if ERRORS: payload["error"] = ERRORS - - addr = ("localhost", int(testPort)) data = json.dumps(payload) request = f"""Content-Length: {len(data)} Content-Type: application/json @@ -635,9 +648,11 @@ def execution_post( {data}""" try: - with socket_manager.SocketManager(addr) as s: - if s.socket is not None: - s.socket.sendall(request.encode("utf-8")) + if __socket.socket is not None: + __socket.socket.sendall(request.encode("utf-8")) + else: + print(f"Plugin error connection error[vscode-pytest]") + print(f"[vscode-pytest] data: {request}") except Exception as e: print(f"Plugin error connection error[vscode-pytest]: {e}") print(f"[vscode-pytest] data: {request}") diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 0a3ada89d422..1a0cc614db3b 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -6,7 +6,7 @@ import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; -import { ITestResultResolver, ITestServer } from '../../../client/testing/testController/common/types'; +import { ITestResultResolver } from '../../../client/testing/testController/common/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { IPythonExecutionFactory } from '../../../client/common/process/types'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; @@ -21,7 +21,7 @@ import { createDeferred } from '../../../client/common/utils/async'; suite('End to End Tests: test adapters', () => { let resultResolver: typeMoq.IMock; - let pythonTestServer: ITestServer; + let pythonTestServer: PythonTestServer; let pythonExecFactory: IPythonExecutionFactory; let debugLauncher: ITestDebugLauncher; let configService: IConfigurationService; @@ -64,6 +64,9 @@ suite('End to End Tests: test adapters', () => { pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); await pythonTestServer.serverReady(); }); + teardown(async () => { + pythonTestServer.dispose(); + }); test('unittest discovery adapter small workspace', async () => { // result resolver and saved data for assertions let actualData: { @@ -283,24 +286,23 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); // 2. Confirm tests are found assert.ok(actualData.result, 'Expected results to be present'); - console.log('finally p1'); }); }); test('unittest execution adapter large workspace', async () => { - const count = 0; + let count = 0; + const errorMessages: string[] = []; // result resolver and saved data for assertions resultResolver .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) .returns((data) => { - traceLog(`resolveExecution ${data}`); - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status, can be subtest success or failure - assert( - data.status === 'subtest-success' || data.status === 'subtest-failure', - "Expected status to be 'subtest-success' or 'subtest-failure'", - ); - // 2. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); + count = count + 1; + if (data.status !== 'subtest-success' && data.status !== 'subtest-failure') { + errorMessages.push("Expected status to be 'subtest-success' or 'subtest-failure'"); + errorMessages.push(data.message); + } + if (data.result === null) { + errorMessages.push('Expected results to be present'); + } return Promise.resolve(); }); @@ -324,23 +326,25 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - console.log('FROM TEST, unit large'); const deferred = createDeferred(); await executionAdapter .runTests(workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], false, testRun.object) .then(() => { // verification after discovery is complete + assert.strictEqual( + errorMessages.length, + 0, + ['Test run was unsuccessful, the following errors were produced: \n', ...errorMessages].join('\n'), + ); resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), typeMoq.Times.atLeast(2000), ); - console.log('hit then'); }) .finally(() => { deferred.resolve(); }); await deferred.promise; - console.log('reached the very end', count); }); test('pytest execution adapter small workspace', async () => { // result resolver and saved data for assertions @@ -375,8 +379,6 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - // const deferred = createDeferred(); - console.log('run tests for small workspace'); await executionAdapter .runTests( workspaceUri, @@ -387,7 +389,6 @@ suite('End to End Tests: test adapters', () => { ) .then(() => { // verification after discovery is complete - console.log('hit then'); resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), typeMoq.Times.once(), @@ -398,15 +399,7 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(actualData.error, null, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.result, 'Expected results to be present'); - }) - .catch((err) => { - console.log('hit catch'); - console.log(err); - }) - .finally(() => { - console.log('hit finally'); }); - console.log('at very end of pytest small'); }); test('pytest execution adapter large workspace', async () => { const errorMessages: string[] = []; From 78819b80affc662fe458a4b90e1978145c886018 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 16 Aug 2023 13:25:22 -0700 Subject: [PATCH 06/55] switch to 200 --- src/test/testing/common/testingAdapter.test.ts | 4 ++-- .../largeWorkspace/test_parameterized_subtest.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 1a0cc614db3b..3eba7527e68d 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -338,7 +338,7 @@ suite('End to End Tests: test adapters', () => { ); resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeast(2000), + typeMoq.Times.atLeast(200), ); }) .finally(() => { @@ -421,7 +421,7 @@ suite('End to End Tests: test adapters', () => { // generate list of test_ids const testIds: string[] = []; - for (let i = 0; i < 2000; i = i + 1) { + for (let i = 0; i < 200; i = i + 1) { const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } diff --git a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py index a76856ebb929..8c6a29adf495 100644 --- a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py +++ b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py @@ -4,13 +4,13 @@ import unittest -@pytest.mark.parametrize("num", range(0, 2000)) +@pytest.mark.parametrize("num", range(0, 200)) def test_odd_even(num): assert num % 2 == 0 class NumbersTest(unittest.TestCase): def test_even(self): - for i in range(0, 2000): + for i in range(0, 200): with self.subTest(i=i): self.assertEqual(i % 2, 0) From 7e25d5036238a52269490dabbf48dfc85710e441 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 16 Aug 2023 14:11:42 -0700 Subject: [PATCH 07/55] remove old line --- pythonFiles/unittestadapter/execution.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 8de0de5bfcf0..dbde465c9d46 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -256,9 +256,8 @@ def send_run_data(raw_data, port, uuid): {data}""" try: - with socket_manager.SocketManager(addr) as s: - if __socket.socket is not None: - __socket.socket.sendall(request.encode("utf-8")) + if __socket.socket is not None: + __socket.socket.sendall(request.encode("utf-8")) except Exception as ex: print(f"Error sending response: {ex}") print(f"Request data: {request}") From d07bda57d6845bf71c4372c8915d2fe4c76c7a9b Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 16 Aug 2023 14:20:07 -0700 Subject: [PATCH 08/55] reformat + pyright + change to 20 --- pythonFiles/unittestadapter/execution.py | 4 +++- pythonFiles/vscode_pytest/__init__.py | 4 +++- src/test/testing/common/testingAdapter.test.ts | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index dbde465c9d46..e38f831726dc 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -225,9 +225,11 @@ def run_tests( return payload + __socket = None atexit.register(lambda: __socket.close() if __socket else None) + def send_run_data(raw_data, port, uuid): # Build the request data (it has to be a POST request or the Node side will not process it), and send it. addr = ("localhost", port) @@ -256,7 +258,7 @@ def send_run_data(raw_data, port, uuid): {data}""" try: - if __socket.socket is not None: + if __socket is not None and __socket.socket is not None: __socket.socket.sendall(request.encode("utf-8")) except Exception as ex: print(f"Error sending response: {ex}") diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 2ca4b4e0ade2..6ba91c724e95 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -610,9 +610,11 @@ class ExecutionPayloadDict(Dict): def get_node_path(node: Any) -> pathlib.Path: return getattr(node, "path", pathlib.Path(node.fspath)) + __socket = None atexit.register(lambda: __socket.close() if __socket else None) + def execution_post( cwd: str, status: Literal["success", "error"], @@ -648,7 +650,7 @@ def execution_post( {data}""" try: - if __socket.socket is not None: + if __socket is not None and __socket.socket is not None: __socket.socket.sendall(request.encode("utf-8")) else: print(f"Plugin error connection error[vscode-pytest]") diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 3eba7527e68d..7bdad0fd0595 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -338,7 +338,7 @@ suite('End to End Tests: test adapters', () => { ); resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeast(200), + typeMoq.Times.atLeast(20), ); }) .finally(() => { @@ -421,7 +421,7 @@ suite('End to End Tests: test adapters', () => { // generate list of test_ids const testIds: string[] = []; - for (let i = 0; i < 200; i = i + 1) { + for (let i = 0; i < 20; i = i + 1) { const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } From 0986aa6034d57cefbbb76d29c3fc8b98121d34d7 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 16 Aug 2023 14:25:56 -0700 Subject: [PATCH 09/55] linting --- pythonFiles/vscode_pytest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 6ba91c724e95..4f68ff47228f 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -653,7 +653,7 @@ def execution_post( if __socket is not None and __socket.socket is not None: __socket.socket.sendall(request.encode("utf-8")) else: - print(f"Plugin error connection error[vscode-pytest]") + print("Plugin error connection error[vscode-pytest]") print(f"[vscode-pytest] data: {request}") except Exception as e: print(f"Plugin error connection error[vscode-pytest]: {e}") From 87f30366e1b0a1d12a32fa59698f356d18ec0ba3 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 16 Aug 2023 15:08:18 -0700 Subject: [PATCH 10/55] change to assert --- .../largeWorkspace/test_parameterized_subtest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py index 8c6a29adf495..654cdffd25d4 100644 --- a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py +++ b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py @@ -4,13 +4,13 @@ import unittest -@pytest.mark.parametrize("num", range(0, 200)) +@pytest.mark.parametrize("num", range(0, 20)) def test_odd_even(num): assert num % 2 == 0 class NumbersTest(unittest.TestCase): def test_even(self): - for i in range(0, 200): + for i in range(0, 20): with self.subTest(i=i): self.assertEqual(i % 2, 0) From 19a4e06b67ebf09b3f257d8e6620da216155dee4 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 23 Aug 2023 14:20:05 -0700 Subject: [PATCH 11/55] switch to mock output channel --- .../testing/common/testingAdapter.test.ts | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 7bdad0fd0595..d9d437e4f173 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -25,9 +25,9 @@ suite('End to End Tests: test adapters', () => { let pythonExecFactory: IPythonExecutionFactory; let debugLauncher: ITestDebugLauncher; let configService: IConfigurationService; - let testOutputChannel: ITestOutputChannel; let serviceContainer: IServiceContainer; let workspaceUri: Uri; + let testOutputChannel: typeMoq.IMock; const rootPathSmallWorkspace = path.join( EXTENSION_ROOT_DIR_FOR_TESTS, 'src', @@ -55,7 +55,6 @@ suite('End to End Tests: test adapters', () => { configService = serviceContainer.get(IConfigurationService); pythonExecFactory = serviceContainer.get(IPythonExecutionFactory); debugLauncher = serviceContainer.get(ITestDebugLauncher); - testOutputChannel = serviceContainer.get(ITestOutputChannel); // create mock resultResolver object resultResolver = typeMoq.Mock.ofType(); @@ -63,6 +62,24 @@ suite('End to End Tests: test adapters', () => { // create objects that were not injected pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); await pythonTestServer.serverReady(); + + testOutputChannel = typeMoq.Mock.ofType(); + testOutputChannel + .setup((x) => x.append(typeMoq.It.isAny())) + .callback((appendVal: any) => { + console.log('out - ', appendVal); + }) + .returns(() => { + // Whatever you need to return + }); + testOutputChannel + .setup((x) => x.appendLine(typeMoq.It.isAny())) + .callback((appendVal: any) => { + console.log('outL - ', appendVal); + }) + .returns(() => { + // Whatever you need to return + }); }); teardown(async () => { pythonTestServer.dispose(); @@ -90,7 +107,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); @@ -143,7 +160,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); @@ -180,7 +197,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); @@ -219,7 +236,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); @@ -262,7 +279,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); @@ -314,7 +331,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); @@ -367,7 +384,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); @@ -430,7 +447,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, + testOutputChannel.object, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); From f4e1b5530d04c2a20d609645bc7f82a1646bf2dd Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 23 Aug 2023 14:32:16 -0700 Subject: [PATCH 12/55] extra log --- pythonFiles/vscode_pytest/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 4f68ff47228f..d088979b6bc0 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -627,6 +627,7 @@ def execution_post( session_node -- the status of running the tests tests -- the tests that were run and their status. """ + print("sending execution post!") testPort = os.getenv("TEST_PORT", 45454) testuuid = os.getenv("TEST_UUID") addr = ("localhost", int(testPort)) From f11502595d56b2e7405084bf3a0518acee451999 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 23 Aug 2023 14:32:45 -0700 Subject: [PATCH 13/55] add in tests --- pythonFiles/vscode_pytest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index d088979b6bc0..50a5b2bb4242 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -627,7 +627,7 @@ def execution_post( session_node -- the status of running the tests tests -- the tests that were run and their status. """ - print("sending execution post!") + print("sending execution post!", tests) testPort = os.getenv("TEST_PORT", 45454) testuuid = os.getenv("TEST_UUID") addr = ("localhost", int(testPort)) From 4df1d11ac0f841497b10332d0e598494f8c07255 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 23 Aug 2023 14:40:46 -0700 Subject: [PATCH 14/55] add timeout --- .../testing/common/testingAdapter.test.ts | 41 ++++++------------- 1 file changed, 12 insertions(+), 29 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index d9d437e4f173..3eba7527e68d 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -25,9 +25,9 @@ suite('End to End Tests: test adapters', () => { let pythonExecFactory: IPythonExecutionFactory; let debugLauncher: ITestDebugLauncher; let configService: IConfigurationService; + let testOutputChannel: ITestOutputChannel; let serviceContainer: IServiceContainer; let workspaceUri: Uri; - let testOutputChannel: typeMoq.IMock; const rootPathSmallWorkspace = path.join( EXTENSION_ROOT_DIR_FOR_TESTS, 'src', @@ -55,6 +55,7 @@ suite('End to End Tests: test adapters', () => { configService = serviceContainer.get(IConfigurationService); pythonExecFactory = serviceContainer.get(IPythonExecutionFactory); debugLauncher = serviceContainer.get(ITestDebugLauncher); + testOutputChannel = serviceContainer.get(ITestOutputChannel); // create mock resultResolver object resultResolver = typeMoq.Mock.ofType(); @@ -62,24 +63,6 @@ suite('End to End Tests: test adapters', () => { // create objects that were not injected pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); await pythonTestServer.serverReady(); - - testOutputChannel = typeMoq.Mock.ofType(); - testOutputChannel - .setup((x) => x.append(typeMoq.It.isAny())) - .callback((appendVal: any) => { - console.log('out - ', appendVal); - }) - .returns(() => { - // Whatever you need to return - }); - testOutputChannel - .setup((x) => x.appendLine(typeMoq.It.isAny())) - .callback((appendVal: any) => { - console.log('outL - ', appendVal); - }) - .returns(() => { - // Whatever you need to return - }); }); teardown(async () => { pythonTestServer.dispose(); @@ -107,7 +90,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); @@ -160,7 +143,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); @@ -197,7 +180,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); @@ -236,7 +219,7 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); @@ -279,7 +262,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); @@ -331,7 +314,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); @@ -355,7 +338,7 @@ suite('End to End Tests: test adapters', () => { ); resultResolver.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeast(20), + typeMoq.Times.atLeast(200), ); }) .finally(() => { @@ -384,7 +367,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); @@ -438,7 +421,7 @@ suite('End to End Tests: test adapters', () => { // generate list of test_ids const testIds: string[] = []; - for (let i = 0; i < 20; i = i + 1) { + for (let i = 0; i < 200; i = i + 1) { const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } @@ -447,7 +430,7 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel.object, + testOutputChannel, resultResolver.object, ); const testRun = typeMoq.Mock.ofType(); From 3638b950d0b5716e82e8a8728f413e9702805aad Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 23 Aug 2023 15:00:54 -0700 Subject: [PATCH 15/55] implement retries --- pythonFiles/vscode_pytest/__init__.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 50a5b2bb4242..a657dd3cd3be 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -632,6 +632,7 @@ def execution_post( testuuid = os.getenv("TEST_UUID") addr = ("localhost", int(testPort)) global __socket + if __socket is None: try: __socket = socket_manager.SocketManager(addr) @@ -650,6 +651,28 @@ def execution_post( Request-uuid: {testuuid} {data}""" + + max_retries = 3 + retries = 0 + while retries < max_retries: + try: + if __socket is not None and __socket.socket is not None: + __socket.socket.sendall(request.encode("utf-8")) + print("Execution post sent successfully!") + break # Exit the loop if the send was successful + else: + print("Plugin error connection error[vscode-pytest]") + print(f"[vscode-pytest] data: {request}") + except Exception as e: + print(f"Plugin error connection error[vscode-pytest]: {e}") + print(f"[vscode-pytest] data: {request}") + retries += 1 # Increment retry counter + if retries < max_retries: + print(f"Retrying ({retries}/{max_retries}) in 2 seconds...") + time.sleep(2) # Wait for a short duration before retrying + else: + print("Maximum retry attempts reached. Cannot send execution post.") + try: if __socket is not None and __socket.socket is not None: __socket.socket.sendall(request.encode("utf-8")) From 03c37819eac653e4094d0f442983f52dd3169242 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 23 Aug 2023 15:05:12 -0700 Subject: [PATCH 16/55] import time --- pythonFiles/vscode_pytest/__init__.py | 1 + src/test/testing/common/testingAdapter.test.ts | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index a657dd3cd3be..cde5fa8b9e0d 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -6,6 +6,7 @@ import os import pathlib import sys +import time import traceback import pytest diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 3eba7527e68d..18fc9b89ee5e 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -63,6 +63,24 @@ suite('End to End Tests: test adapters', () => { // create objects that were not injected pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); await pythonTestServer.serverReady(); + + testOutputChannel = typeMoq.Mock.ofType(); + testOutputChannel + .setup((x) => x.append(typeMoq.It.isAny())) + .callback((appendVal: any) => { + console.log('out - ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); + testOutputChannel + .setup((x) => x.appendLine(typeMoq.It.isAny())) + .callback((appendVal: any) => { + console.log('outL - ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); }); teardown(async () => { pythonTestServer.dispose(); From 3207dc7fa9d1d4eb1866e2303667d7375427a001 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 23 Aug 2023 16:07:29 -0700 Subject: [PATCH 17/55] try on ts side --- src/client/testing/testController/common/server.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index d0a225ae5712..9069f9e02387 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -36,6 +36,7 @@ export class PythonTestServer implements ITestServer, Disposable { let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data socket.on('data', (data: Buffer) => { try { + console.log('raw Data: ', data.toString()); let rawData: string = data.toString(); buffer = Buffer.concat([buffer, data]); while (buffer.length > 0) { @@ -65,6 +66,7 @@ export class PythonTestServer implements ITestServer, Disposable { }); // if the rawData includes result then this is a run request } else if (rawData.includes(`"result":`)) { + console.log('fire run data received'); this._onRunDataReceived.fire({ uuid, data: rpcContent.extractedJSON, From fb589f64e6dc5fb859ac630515a2879c3bcb2de6 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 24 Aug 2023 08:49:48 -0700 Subject: [PATCH 18/55] add logging typescript side --- pythonFiles/vscode_pytest/__init__.py | 33 ++++++++++--------- .../testing/testController/common/server.ts | 2 +- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index cde5fa8b9e0d..54ca9264fa6f 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -306,12 +306,12 @@ def pytest_sessionfinish(session, exitstatus): 4: Pytest encountered an internal error or exception during test execution. 5: Pytest was unable to find any tests to run. """ - print( - "pytest session has finished, exit status: ", - exitstatus, - "in discovery? ", - IS_DISCOVERY, - ) + # print( + # "pytest session has finished, exit status: ", + # exitstatus, + # "in discovery? ", + # IS_DISCOVERY, + # ) cwd = pathlib.Path.cwd() if IS_DISCOVERY: if not (exitstatus == 0 or exitstatus == 1 or exitstatus == 5): @@ -628,7 +628,7 @@ def execution_post( session_node -- the status of running the tests tests -- the tests that were run and their status. """ - print("sending execution post!", tests) + # print("sending execution post!", tests) testPort = os.getenv("TEST_PORT", 45454) testuuid = os.getenv("TEST_UUID") addr = ("localhost", int(testPort)) @@ -660,6 +660,7 @@ def execution_post( if __socket is not None and __socket.socket is not None: __socket.socket.sendall(request.encode("utf-8")) print("Execution post sent successfully!") + print("data sent", tests, "end of data") break # Exit the loop if the send was successful else: print("Plugin error connection error[vscode-pytest]") @@ -674,15 +675,15 @@ def execution_post( else: print("Maximum retry attempts reached. Cannot send execution post.") - try: - if __socket is not None and __socket.socket is not None: - __socket.socket.sendall(request.encode("utf-8")) - else: - print("Plugin error connection error[vscode-pytest]") - print(f"[vscode-pytest] data: {request}") - except Exception as e: - print(f"Plugin error connection error[vscode-pytest]: {e}") - print(f"[vscode-pytest] data: {request}") + # try: + # if __socket is not None and __socket.socket is not None: + # __socket.socket.sendall(request.encode("utf-8")) + # else: + # print("Plugin error connection error[vscode-pytest]") + # print(f"[vscode-pytest] data: {request}") + # except Exception as e: + # print(f"Plugin error connection error[vscode-pytest]: {e}") + # print(f"[vscode-pytest] data: {request}") class PathEncoder(json.JSONEncoder): diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 9069f9e02387..f2fb474ee87b 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -36,7 +36,7 @@ export class PythonTestServer implements ITestServer, Disposable { let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data socket.on('data', (data: Buffer) => { try { - console.log('raw Data: ', data.toString()); + console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); let rawData: string = data.toString(); buffer = Buffer.concat([buffer, data]); while (buffer.length > 0) { From 499ecbcfa25cdf8004a26440253246ed72baf140 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 24 Aug 2023 09:04:23 -0700 Subject: [PATCH 19/55] log on run data recieved --- src/client/testing/testController/common/server.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index f2fb474ee87b..d9c316e7dbc1 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -66,7 +66,11 @@ export class PythonTestServer implements ITestServer, Disposable { }); // if the rawData includes result then this is a run request } else if (rawData.includes(`"result":`)) { - console.log('fire run data received'); + console.log( + '\n *** fire run data received: \n', + rpcContent.extractedJSON, + '\n *** end', + ); this._onRunDataReceived.fire({ uuid, data: rpcContent.extractedJSON, From b4f9204503fac534517fe831cac31bfa6f21dba5 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 24 Aug 2023 16:02:43 -0700 Subject: [PATCH 20/55] multi payload per chunk support --- .../testing/testController/common/server.ts | 89 +- .../testing/testController/common/utils.ts | 94 ++- .../testController/payloadTestCases.ts | 113 +++ .../testController/server.unit.test.ts | 779 ++++++++++-------- .../testing/testController/utils.unit.test.ts | 22 +- 5 files changed, 691 insertions(+), 406 deletions(-) create mode 100644 src/test/testing/testController/payloadTestCases.ts diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index d9c316e7dbc1..4fef100b657b 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -15,7 +15,7 @@ import { traceError, traceInfo, traceLog } from '../../../logging'; import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; -import { jsonRPCHeaders, jsonRPCContent, JSONRPC_UUID_HEADER, createExecutionErrorPayload } from './utils'; +import { containsHeaders, extractJsonPayload } from './utils'; import { createDeferred } from '../../../common/utils/async'; export class PythonTestServer implements ITestServer, Disposable { @@ -37,56 +37,34 @@ export class PythonTestServer implements ITestServer, Disposable { socket.on('data', (data: Buffer) => { try { console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); - let rawData: string = data.toString(); buffer = Buffer.concat([buffer, data]); while (buffer.length > 0) { - const rpcHeaders = jsonRPCHeaders(buffer.toString()); - const uuid = rpcHeaders.headers.get(JSONRPC_UUID_HEADER); - const totalContentLength = rpcHeaders.headers.get('Content-Length'); - if (!uuid) { - traceError('On data received: Error occurred because payload UUID is undefined'); - this._onDataReceived.fire({ uuid: '', data: '' }); - return; - } - if (!this.uuids.includes(uuid)) { - traceError('On data received: Error occurred because the payload UUID is not recognized'); + try { + const extractedJsonPayload = extractJsonPayload(buffer.toString(), this.uuids); + if ( + extractedJsonPayload.uuid !== undefined && + extractedJsonPayload.cleanedJsonData !== undefined + ) { + // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. + this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); + } + buffer = Buffer.from(extractedJsonPayload.remainingRawData); + if (!containsHeaders(extractedJsonPayload.remainingRawData)) { + // if the remaining data does not contain headers, then there is no more data to process. + // break to get more data from the socket. + break; + } + if (buffer.length === 0) { + // if the buffer is empty, then there is no more data to process. + // break to get more data from the socket. + buffer = Buffer.alloc(0); + break; + } + } catch (ex) { + traceError(`Error:: ${ex}`); this._onDataReceived.fire({ uuid: '', data: '' }); return; } - rawData = rpcHeaders.remainingRawData; - const rpcContent = jsonRPCContent(rpcHeaders.headers, rawData); - const extractedData = rpcContent.extractedJSON; - // do not send until we have the full content - if (extractedData.length === Number(totalContentLength)) { - // if the rawData includes tests then this is a discovery request - if (rawData.includes(`"tests":`)) { - this._onDiscoveryDataReceived.fire({ - uuid, - data: rpcContent.extractedJSON, - }); - // if the rawData includes result then this is a run request - } else if (rawData.includes(`"result":`)) { - console.log( - '\n *** fire run data received: \n', - rpcContent.extractedJSON, - '\n *** end', - ); - this._onRunDataReceived.fire({ - uuid, - data: rpcContent.extractedJSON, - }); - } else { - traceLog( - `Error processing test server request: request is not recognized as discovery or run.`, - ); - this._onDataReceived.fire({ uuid: '', data: '' }); - return; - } - // this.uuids = this.uuids.filter((u) => u !== uuid); WHERE DOES THIS GO?? - buffer = Buffer.alloc(0); - } else { - break; - } } } catch (ex) { traceError(`Error processing test server request: ${ex} observe`); @@ -113,6 +91,25 @@ export class PythonTestServer implements ITestServer, Disposable { }); } + private _fireDataReceived(uuid: string, extractedJSON: string): void { + if (extractedJSON.includes(`"tests":`)) { + this._onDiscoveryDataReceived.fire({ + uuid, + data: extractedJSON, + }); + // if the rawData includes result then this is a run request + } else if (extractedJSON.includes(`"result":`)) { + console.log('\n *** fire run data received: \n', extractedJSON, '\n *** end'); + this._onRunDataReceived.fire({ + uuid, + data: extractedJSON, + }); + } else { + traceLog(`Error processing test server request: request is not recognized as discovery or run.`); + this._onDataReceived.fire({ uuid: '', data: '' }); + } + } + public serverReady(): Promise { return this.ready; } diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index dd1b51551a45..5b814e432954 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -15,21 +15,92 @@ export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); return `${lines.join('\r\n')}\r\n`; } -export interface IJSONRPCContent { +export interface IJSONRPCData { extractedJSON: string; remainingRawData: string; } -export interface IJSONRPCHeaders { +export interface ParsedRPCHeadersAndData { headers: Map; remainingRawData: string; } +export interface ExtractOutput { + uuid: string | undefined; + cleanedJsonData: string | undefined; + remainingRawData: string; +} + export const JSONRPC_UUID_HEADER = 'Request-uuid'; export const JSONRPC_CONTENT_LENGTH_HEADER = 'Content-Length'; export const JSONRPC_CONTENT_TYPE_HEADER = 'Content-Type'; -export function jsonRPCHeaders(rawData: string): IJSONRPCHeaders { +export function extractJsonPayload(rawData: string, uuids: Array): ExtractOutput { + /** + * Extracts JSON-RPC payload from the provided raw data. + * @param {string} rawData - The raw string data from which the JSON payload will be extracted. + * @param {Array} uuids - The list of UUIDs that are active. + * @returns {string} The remaining raw data after the JSON payload is extracted. + */ + + const rpcHeaders: ParsedRPCHeadersAndData = parseJsonRPCHeadersAndData(rawData); + + // verify the RPC has a UUID and that it is recognized + let uuid = rpcHeaders.headers.get(JSONRPC_UUID_HEADER); + uuid = checkUuid(uuid, uuids); + + const payloadLength = rpcHeaders.headers.get('Content-Length'); + + // separate out the data within context length of the given payload from the remaining data in the buffer + const rpcContent: IJSONRPCData = ExtractJsonRPCData(payloadLength, rpcHeaders.remainingRawData); + const cleanedJsonData = rpcContent.extractedJSON; + const { remainingRawData } = rpcContent; + + // if the given payload has the complete json, process it otherwise wait for the rest in the buffer + if (cleanedJsonData.length === Number(payloadLength)) { + // call to process this data + // remove this data from the buffer + return { uuid, cleanedJsonData, remainingRawData }; + } + // wait for the remaining + return { uuid: undefined, cleanedJsonData: undefined, remainingRawData: rawData }; +} + +export function containsHeaders(rawData: string): boolean { + /** + * Checks if the provided raw data contains JSON-RPC specific headers. + */ + return ( + rawData.includes(JSONRPC_CONTENT_LENGTH_HEADER) && + rawData.includes(JSONRPC_CONTENT_TYPE_HEADER) && + rawData.includes(JSONRPC_UUID_HEADER) + ); +} + +export function checkUuid(uuid: string | undefined, uuids: Array): string { + if (!uuid) { + throw new Error('On data received: Error occurred because payload UUID is undefined'); + } + if (!uuids.includes(uuid)) { + throw new Error('On data received: Error occurred because the payload UUID is not recognized'); + } + return uuid; +} + +export function parseJsonRPCHeadersAndData(rawData: string): ParsedRPCHeadersAndData { + /** + * Parses the provided raw data to extract JSON-RPC specific headers and remaining data. + * + * This function aims to extract specific JSON-RPC headers (like UUID, content length, + * and content type) from the provided raw string data. Headers are expected to be + * delimited by newlines and the format should be "key:value". The function stops parsing + * once it encounters an empty line, and the rest of the data after this line is treated + * as the remaining raw data. + * + * @param {string} rawData - The raw string containing headers and possibly other data. + * @returns {ParsedRPCHeadersAndData} An object containing the parsed headers as a map and the + * remaining raw data after the headers. + */ const lines = rawData.split('\n'); let remainingRawData = ''; const headerMap = new Map(); @@ -51,8 +122,21 @@ export function jsonRPCHeaders(rawData: string): IJSONRPCHeaders { }; } -export function jsonRPCContent(headers: Map, rawData: string): IJSONRPCContent { - const length = parseInt(headers.get('Content-Length') ?? '0', 10); +export function ExtractJsonRPCData(payloadLength: string | undefined, rawData: string): IJSONRPCData { + /** + * Extracts JSON-RPC content based on provided headers and raw data. + * + * This function uses the `Content-Length` header from the provided headers map + * to determine how much of the rawData string represents the actual JSON content. + * After extracting the expected content, it also returns any remaining data + * that comes after the extracted content as remaining raw data. + * + * @param {string | undefined} payloadLength - The value of the `Content-Length` header. + * @param {string} rawData - The raw string data from which the JSON content will be extracted. + * + * @returns {IJSONRPCContent} An object containing the extracted JSON content and any remaining raw data. + */ + const length = parseInt(payloadLength ?? '0', 10); const data = rawData.slice(0, length); const remainingRawData = rawData.slice(length); return { diff --git a/src/test/testing/testController/payloadTestCases.ts b/src/test/testing/testController/payloadTestCases.ts new file mode 100644 index 000000000000..2180a82ebca4 --- /dev/null +++ b/src/test/testing/testController/payloadTestCases.ts @@ -0,0 +1,113 @@ +export interface PayloadChunk { + payload: string[]; + data: string; +} + +export function PAYLOAD_SINGLE_CHUNK(uuid: string): PayloadChunk { + const val = `{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}}`; + const payload = `Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +${val}`; + return { + payload: [payload], + data: val, + }; +} + +export function PAYLOAD_MULTI_CHUNK(uuid: string): PayloadChunk { + const val = `{abc}`; + return { + payload: [ + `Content-Length: 5 +Content-Type: application/json +Request-uuid: ${uuid} + +${val}Content-Length: 5 +Content-Type: application/json +Request-uuid: ${uuid} + +${val}Content-Length: 5 +Content-Type: application/json +Request-uuid: ${uuid} + +${val}Content-Length: 5 +Content-Type: application/json +Request-uuid: ${uuid} + +${val}`, + ], + data: val + val + val + val, + }; +} + +export function PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid: string): Array { + return [ + `Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vsc`, + `ode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.te`, + `st_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}}`, + ]; +} + +export function PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY(uuid: string): Array { + return [ + `Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}} + +Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src" + +Content-Length: 959 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-failure", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=1)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-failure", "message": "(, AssertionError('1 != 0'), )", "traceback": " File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 57, in testPartExecutor\n yield\n File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 538, in subTest\n yield\n File \"/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py\", line 16, in test_even\n self.assertEqual(i % 2, 0)\nAssertionError: 1 != 0\n", "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=1)"}}} +Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/ru`, + `nner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.Numbers`, + `Test.test_even (i=2)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=2)"}}}`, + ]; +} + +export function PAYLOAD_SPLIT_MULTI_CHUNK_RAN_ORDER_ARRAY(uuid: string): Array { + return [ + `Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}} + +Content-Length: 411 +Content-Type: application/json +Request-uuid: 9${uuid} + +{"cwd": "/home/runner/work/vscode-`, + `python/vscode-python/path with`, + ` spaces/src" + +Content-Length: 959 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-failure", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=1)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-failure", "message": "(, AssertionError('1 != 0'), )", "traceback": " File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 57, in testPartExecutor\n yield\n File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 538, in subTest\n yield\n File \"/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py\", line 16, in test_even\n self.assertEqual(i % 2, 0)\nAssertionError: 1 != 0\n", "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=1)"}}} +Content-Length: 411 +Content-Type: application/json +Request-uuid: ${uuid} + +{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=2)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=2)"}}}`, + ]; +} diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index 53c2b72e40f7..594c4ed287b1 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -6,58 +6,43 @@ import * as assert from 'assert'; import * as net from 'net'; import * as sinon from 'sinon'; import * as crypto from 'crypto'; -import { OutputChannel, Uri } from 'vscode'; import { Observable } from 'rxjs'; import * as typeMoq from 'typemoq'; -import { - IPythonExecutionFactory, - IPythonExecutionService, - Output, - SpawnOptions, -} from '../../../client/common/process/types'; +import { Uri } from 'vscode'; +import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; import { Deferred, createDeferred } from '../../../client/common/utils/async'; import { MockChildProcess } from '../../mocks/mockChildProcess'; +import { PAYLOAD_MULTI_CHUNK, PAYLOAD_SINGLE_CHUNK } from './payloadTestCases'; suite('Python Test Server', () => { - const fakeUuid = 'fake-uuid'; - - let stubExecutionFactory: IPythonExecutionFactory; - let stubExecutionService: IPythonExecutionService; + const FAKE_UUID = 'fake-uuid'; let server: PythonTestServer; - let sandbox: sinon.SinonSandbox; let v4Stub: sinon.SinonStub; let debugLauncher: ITestDebugLauncher; let mockProc: MockChildProcess; let execService: typeMoq.IMock; let deferred: Deferred; - let execFactory = typeMoq.Mock.ofType(); + const sandbox = sinon.createSandbox(); - setup(() => { - sandbox = sinon.createSandbox(); - v4Stub = sandbox.stub(crypto, 'randomUUID'); + setup(async () => { + // set up test command options - v4Stub.returns(fakeUuid); - stubExecutionService = ({ - execObservable: () => Promise.resolve({ stdout: '', stderr: '' }), - } as unknown) as IPythonExecutionService; - - stubExecutionFactory = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService), - } as unknown) as IPythonExecutionFactory; + v4Stub = sandbox.stub(crypto, 'randomUUID'); + v4Stub.returns(FAKE_UUID); // set up exec service with child process mockProc = new MockChildProcess('', ['']); execService = typeMoq.Mock.ofType(); - const output = new Observable>(() => { + const outputObservable = new Observable>(() => { /* no op */ }); execService .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) .returns(() => ({ proc: mockProc, - out: output, + out: outputObservable, dispose: () => { /* no-body */ }, @@ -69,185 +54,13 @@ suite('Python Test Server', () => { sandbox.restore(); server.dispose(); }); - - test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { - const options = { - command: { - script: 'myscript', - args: ['-foo', 'foo'], - }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - const expectedSpawnOptions = { - cwd: '/foo/bar', - outputChannel: undefined, - token: undefined, - throwOnStdErr: true, - extraVariables: { - PYTHONPATH: '/foo/bar', - RUN_TEST_IDS_PORT: '56789', - }, - } as SpawnOptions; - const deferred2 = createDeferred(); - execFactory = typeMoq.Mock.ofType(); - execFactory - .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - .returns(() => { - deferred2.resolve(); - return Promise.resolve(execService.object); - }); - - server = new PythonTestServer(execFactory.object, debugLauncher); - await server.serverReady(); - - server.sendCommand(options, '56789'); - // add in await and trigger - await deferred2.promise; - mockProc.trigger('close'); - - const port = server.getPort(); - const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo']; - execService.verify((x) => x.execObservable(expectedArgs, expectedSpawnOptions), typeMoq.Times.once()); - }); - - test('sendCommand should write to an output channel if it is provided as an option', async () => { - const output: string[] = []; - const outChannel = { - appendLine: (str: string) => { - output.push(str); - }, - } as OutputChannel; - const options = { - command: { - script: 'myscript', - args: ['-foo', 'foo'], - }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - outChannel, - }; - deferred = createDeferred(); - execFactory = typeMoq.Mock.ofType(); - execFactory - .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - .returns(() => { - deferred.resolve(); - return Promise.resolve(execService.object); - }); - - server = new PythonTestServer(execFactory.object, debugLauncher); - await server.serverReady(); - - server.sendCommand(options); - // add in await and trigger - await deferred.promise; - mockProc.trigger('close'); - - const port = server.getPort(); - const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', fakeUuid, '-foo', 'foo'].join(' '); - - assert.deepStrictEqual(output, [expected]); - }); - - test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { - let eventData: { status: string; errors: string[] } | undefined; - stubExecutionService = ({ - execObservable: () => { - throw new Error('Failed to execute'); - }, - } as unknown) as IPythonExecutionService; - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - - server = new PythonTestServer(stubExecutionFactory, debugLauncher); - await server.serverReady(); - - server.onDataReceived(({ data }) => { - eventData = JSON.parse(data); - }); - - await server.sendCommand(options); - - assert.notEqual(eventData, undefined); - assert.deepStrictEqual(eventData?.status, 'error'); - assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); - }); - - test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { - let eventData: string | undefined; + test('basic payload', async () => { + let eventData = ''; const client = new net.Socket(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - - deferred = createDeferred(); - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write('malformed data'); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - // add in await and trigger - await deferred.promise; - mockProc.trigger('close'); - - assert.deepStrictEqual(eventData, ''); - }); - - test('If the server doesnt recognize the UUID it should ignore it', async () => { - let eventData: string | undefined; - const client = new net.Socket(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; deferred = createDeferred(); mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { + const output2 = new Observable>(() => { /* no op */ }); const stubExecutionService2 = ({ @@ -255,7 +68,7 @@ suite('Python Test Server', () => { client.connect(server.getPort()); return { proc: mockProc, - out: output, + out: output2, dispose: () => { /* no-body */ }, @@ -267,69 +80,26 @@ suite('Python Test Server', () => { createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), } as unknown) as IPythonExecutionFactory; server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write('{"Request-uuid": "unknown-uuid"}'); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - assert.deepStrictEqual(eventData, ''); - }); - - // required to have "tests" or "results" - // the heading length not being equal and yes being equal - // multiple payloads - test('Error if payload does not have a content length header', async () => { - let eventData: string | undefined; - const client = new net.Socket(); + const uuid = server.createUUID(); const options = { command: { script: 'myscript', args: ['-foo', 'foo'] }, workspaceFolder: Uri.file('/foo/bar'), cwd: '/foo/bar', - uuid: fakeUuid, + uuid, }; - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; + const payloadChunk = PAYLOAD_SINGLE_CHUNK(uuid); - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); await server.serverReady(); - server.onDataReceived(({ data }) => { - eventData = data; + + server.onRunDataReceived(({ data }) => { + eventData = eventData + data; deferred.resolve(); }); - client.on('connect', () => { console.log('Socket connected, local port:', client.localPort); - client.write('{"not content length": "5"}'); + for (const line of payloadChunk.payload) { + client.write(line); + } client.end(); }); client.on('error', (error) => { @@ -338,86 +108,16 @@ suite('Python Test Server', () => { server.sendCommand(options); await deferred.promise; - assert.deepStrictEqual(eventData, ''); - }); - - const testData = [ - { - testName: 'fires discovery correctly on test payload', - payload: `Content-Length: 52 -Content-Type: application/json -Request-uuid: UUID_HERE - -{"cwd": "path", "status": "success", "tests": "xyz"}`, - expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', - }, - // Add more test data as needed - ]; - - testData.forEach(({ testName, payload, expectedResult }) => { - test(`test: ${testName}`, async () => { - // Your test logic here - let eventData: string | undefined; - const client = new net.Socket(); - - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid: fakeUuid, - }; - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - await server.serverReady(); - const uuid = server.createUUID(); - payload = payload.replace('UUID_HERE', uuid); - server.onDiscoveryDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); - }); - - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - client.write(payload); - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - assert.deepStrictEqual(eventData, expectedResult); - }); + const expectedResult = payloadChunk.data; + assert.deepStrictEqual(eventData, expectedResult); }); - - test('Calls run resolver if the result header is in the payload', async () => { - let eventData: string | undefined; + test('second payload', async () => { + let eventData = ''; const client = new net.Socket(); deferred = createDeferred(); mockProc = new MockChildProcess('', ['']); - const output = new Observable>(() => { + const output2 = new Observable>(() => { /* no op */ }); const stubExecutionService2 = ({ @@ -425,7 +125,7 @@ Request-uuid: UUID_HERE client.connect(server.getPort()); return { proc: mockProc, - out: output, + out: output2, dispose: () => { /* no-body */ }, @@ -437,39 +137,430 @@ Request-uuid: UUID_HERE createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), } as unknown) as IPythonExecutionFactory; server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + const uuid = server.createUUID(); const options = { command: { script: 'myscript', args: ['-foo', 'foo'] }, workspaceFolder: Uri.file('/foo/bar'), cwd: '/foo/bar', - uuid: fakeUuid, + uuid, }; + const payloadChunk = PAYLOAD_MULTI_CHUNK(uuid); await server.serverReady(); - const uuid = server.createUUID(); + server.onRunDataReceived(({ data }) => { - eventData = data; - deferred.resolve(); + eventData = eventData + data; }); - - const payload = `Content-Length: 87 -Content-Type: application/json -Request-uuid: ${uuid} - -{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; - client.on('connect', () => { console.log('Socket connected, local port:', client.localPort); - client.write(payload); + for (const line of payloadChunk.payload) { + client.write(line); + } client.end(); }); + client.on('close', () => { + console.log('Socket connection exit:'); + deferred.resolve(); + }); client.on('error', (error) => { console.log('Socket connection error:', error); }); server.sendCommand(options); await deferred.promise; - const expectedResult = - '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; + const expectedResult = payloadChunk.data; assert.deepStrictEqual(eventData, expectedResult); }); + + // test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { + // deferred2 = createDeferred(); + // execFactory = typeMoq.Mock.ofType(); + // execFactory + // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + // .returns(() => { + // deferred2.resolve(); + // return Promise.resolve(execService.object); + // }); + // server = new PythonTestServer(execFactory.object, debugLauncher); + // await server.serverReady(); + // server.sendCommand(BASE_TEST_COMMAND_OPTIONS, '56789'); + // // add in await and trigger + // await deferred2.promise; + // mockProc.trigger('close'); + + // const port = server.getPort(); + // const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; + // execService.verify((x) => x.execObservable(expectedArgs, BASE_SPAWN_OPTIONS), typeMoq.Times.once()); + // }); + + // test('sendCommand should write to an output channel if it is provided as an option', async () => { + // const output2: string[] = []; + // const outChannel = { + // appendLine: (str: string) => { + // output2.push(str); + // }, + // } as OutputChannel; + // const options = { + // command: { + // script: 'myscript', + // args: ['-foo', 'foo'], + // }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // outChannel, + // }; + // deferred = createDeferred(); + // execFactory = typeMoq.Mock.ofType(); + // execFactory + // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + // .returns(() => { + // deferred.resolve(); + // return Promise.resolve(execService.object); + // }); + + // server = new PythonTestServer(execFactory.object, debugLauncher); + // await server.serverReady(); + + // server.sendCommand(options); + // // add in await and trigger + // await deferred.promise; + // mockProc.trigger('close'); + + // const port = server.getPort(); + // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); + + // assert.deepStrictEqual(output2, [expected]); + // }); + + // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { + // let eventData: { status: string; errors: string[] } | undefined; + // stubExecutionService = ({ + // execObservable: () => { + // throw new Error('Failed to execute'); + // }, + // } as unknown) as IPythonExecutionService; + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + + // server = new PythonTestServer(stubExecutionFactory, debugLauncher); + // await server.serverReady(); + + // server.onDataReceived(({ data }) => { + // eventData = JSON.parse(data); + // }); + + // await server.sendCommand(options); + + // assert.notEqual(eventData, undefined); + // assert.deepStrictEqual(eventData?.status, 'error'); + // assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); + // }); + + // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { + // deferred2 = createDeferred(); + // execFactory = typeMoq.Mock.ofType(); + // execFactory + // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + // .returns(() => { + // deferred2.resolve(); + // return Promise.resolve(execService.object); + // }); + // server = new PythonTestServer(execFactory.object, debugLauncher); + // await server.serverReady(); + // let eventData: string | undefined; + // const client = new net.Socket(); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + // mockProc = new MockChildProcess('', ['']); + // const output2 = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output2, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + + // deferred = createDeferred(); + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // server.onDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write('malformed data'); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // // add in await and trigger + // await deferred.promise; + // mockProc.trigger('close'); + + // assert.deepStrictEqual(eventData, ''); + // }); + + // test('If the server doesnt recognize the UUID it should ignore it', async () => { + // let eventData: string | undefined; + // const client = new net.Socket(); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // server.onDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write('{"Request-uuid": "unknown-uuid"}'); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // assert.deepStrictEqual(eventData, ''); + // }); + + // // required to have "tests" or "results" + // // the heading length not being equal and yes being equal + // // multiple payloads + // test('Error if payload does not have a content length header', async () => { + // let eventData: string | undefined; + // const client = new net.Socket(); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // server.onDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write('{"not content length": "5"}'); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // assert.deepStrictEqual(eventData, ''); + // }); + + // const testData = [ + // { + // testName: 'fires discovery correctly on test payload', + // payload: `Content-Length: 52 + // Content-Type: application/json + // Request-uuid: UUID_HERE + + // {"cwd": "path", "status": "success", "tests": "xyz"}`, + // expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', + // }, + // // Add more test data as needed + // ]; + + // testData.forEach(({ testName, payload, expectedResult }) => { + // test(`test: ${testName}`, async () => { + // // Your test logic here + // let eventData: string | undefined; + // const client = new net.Socket(); + + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // const uuid = server.createUUID(); + // payload = payload.replace('UUID_HERE', uuid); + // server.onDiscoveryDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write(payload); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // assert.deepStrictEqual(eventData, expectedResult); + // }); + // }); + + // test('Calls run resolver if the result header is in the payload', async () => { + // let eventData: string | undefined; + // const client = new net.Socket(); + + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + + // await server.serverReady(); + // const uuid = server.createUUID(); + // server.onRunDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // const payload = `Content-Length: 87 + // Content-Type: application/json + // Request-uuid: ${uuid} + + // {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write(payload); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // const expectedResult = + // '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; + // assert.deepStrictEqual(eventData, expectedResult); + // }); }); diff --git a/src/test/testing/testController/utils.unit.test.ts b/src/test/testing/testController/utils.unit.test.ts index d971c7d37c9f..9168abc7041f 100644 --- a/src/test/testing/testController/utils.unit.test.ts +++ b/src/test/testing/testController/utils.unit.test.ts @@ -6,15 +6,15 @@ import { JSONRPC_CONTENT_LENGTH_HEADER, JSONRPC_CONTENT_TYPE_HEADER, JSONRPC_UUID_HEADER, - jsonRPCContent, - jsonRPCHeaders, + ExtractJsonRPCData, + parseJsonRPCHeadersAndData, } from '../../../client/testing/testController/common/utils'; suite('Test Controller Utils: JSON RPC', () => { test('Empty raw data string', async () => { const rawDataString = ''; - const output = jsonRPCHeaders(rawDataString); + const output = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(output.headers.size, 0); assert.deepStrictEqual(output.remainingRawData, ''); }); @@ -22,20 +22,20 @@ suite('Test Controller Utils: JSON RPC', () => { test('Valid data empty JSON', async () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: 2\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n{}`; - const rpcHeaders = jsonRPCHeaders(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(rpcHeaders.headers.size, 3); assert.deepStrictEqual(rpcHeaders.remainingRawData, '{}'); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, '{}'); }); test('Valid data NO JSON', async () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: 0\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n`; - const rpcHeaders = jsonRPCHeaders(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(rpcHeaders.headers.size, 3); assert.deepStrictEqual(rpcHeaders.remainingRawData, ''); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, ''); }); @@ -45,10 +45,10 @@ suite('Test Controller Utils: JSON RPC', () => { '{"jsonrpc": "2.0", "method": "initialize", "params": {"processId": 1234, "rootPath": "/home/user/project", "rootUri": "file:///home/user/project", "capabilities": {}}, "id": 0}'; const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: ${json.length}\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n${json}`; - const rpcHeaders = jsonRPCHeaders(rawDataString); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString); assert.deepStrictEqual(rpcHeaders.headers.size, 3); assert.deepStrictEqual(rpcHeaders.remainingRawData, json); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, json); }); @@ -58,9 +58,9 @@ suite('Test Controller Utils: JSON RPC', () => { const rawDataString = `${JSONRPC_CONTENT_LENGTH_HEADER}: ${json.length}\n${JSONRPC_CONTENT_TYPE_HEADER}: application/json\n${JSONRPC_UUID_HEADER}: 1234\n\n${json}`; const rawDataString2 = rawDataString + rawDataString; - const rpcHeaders = jsonRPCHeaders(rawDataString2); + const rpcHeaders = parseJsonRPCHeadersAndData(rawDataString2); assert.deepStrictEqual(rpcHeaders.headers.size, 3); - const rpcContent = jsonRPCContent(rpcHeaders.headers, rpcHeaders.remainingRawData); + const rpcContent = ExtractJsonRPCData(rpcHeaders.headers.get('Content-Length'), rpcHeaders.remainingRawData); assert.deepStrictEqual(rpcContent.extractedJSON, json); assert.deepStrictEqual(rpcContent.remainingRawData, rawDataString); }); From 87c0ef1cd8d65c890a2c58560d3ae4ac40dada5f Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Sun, 27 Aug 2023 11:22:18 -0700 Subject: [PATCH 21/55] from stash --- .vscode/launch.json | 2 +- .vscode/settings.json | 10 +- .../tests/pytestadapter/test_discovery.py | 4 + .../tests/pytestadapter/test_execution.py | 7 +- pythonFiles/vscode_pytest/__init__.py | 118 +++++++----------- .../terminalEnvVarCollectionService.ts | 2 +- .../pytest/pytestExecutionAdapter.ts | 10 +- 7 files changed, 70 insertions(+), 83 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 82981a93305d..e448dd15b13b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -101,7 +101,7 @@ "--extensionTestsPath=${workspaceFolder}/out/test" ], "env": { - "VSC_PYTHON_CI_TEST_GREP": "" // Modify this to run a subset of the single workspace tests + "VSC_PYTHON_CI_TEST_GREP": "End to End Tests: test adapters" // Modify this to run a subset of the single workspace tests }, "stopOnEntry": false, "sourceMaps": true, diff --git a/.vscode/settings.json b/.vscode/settings.json index 06011b3d13cd..596a17f64b85 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -66,13 +66,5 @@ ], "typescript.preferences.importModuleSpecifier": "relative", "debug.javascript.usePreview": false, - // Branch name suggestion. - "git.branchRandomName.enable": true, - "git.branchProtection": [ - "main", - "release/*" - ], - "git.pullBeforeCheckout": true, - // Open merge editor for resolving conflicts. - "git.mergeEditor": true, + "python.defaultInterpreterPath": ".venv/bin/python" } diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 8d785be27c8b..333e6cc2477e 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -33,6 +33,7 @@ def test_import_error(tmp_path): ["--collect-only", os.fspath(p)] ) assert actual_list + assert actual_list.pop(-1).get("eot") for actual in actual_list: assert all(item in actual for item in ("status", "cwd", "error")) assert actual["status"] == "error" @@ -60,6 +61,7 @@ def test_syntax_error(tmp_path): p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) actual = runner(["--collect-only", os.fspath(p)]) + assert actual.pop(-1).get("eot") if actual: actual = actual[0] assert actual @@ -76,6 +78,7 @@ def test_parameterized_error_collect(): """ file_path_str = "error_parametrize_discovery.py" actual = runner(["--collect-only", file_path_str]) + assert actual.pop(-1).get("eot") if actual: actual = actual[0] assert all(item in actual for item in ("status", "cwd", "error")) @@ -146,6 +149,7 @@ def test_pytest_collect(file, expected_const): os.fspath(TEST_DATA_PATH / file), ] ) + assert actual.pop(-1).get("eot") if actual: actual = actual[0] assert actual diff --git a/pythonFiles/tests/pytestadapter/test_execution.py b/pythonFiles/tests/pytestadapter/test_execution.py index 07354b01709b..55952059b44e 100644 --- a/pythonFiles/tests/pytestadapter/test_execution.py +++ b/pythonFiles/tests/pytestadapter/test_execution.py @@ -23,6 +23,7 @@ def test_config_file(): expected_execution_test_output.config_file_pytest_expected_execution_output ) assert actual + assert actual.pop(-1).get("eot") assert len(actual) == len(expected_const) actual_result_dict = dict() for a in actual: @@ -43,6 +44,7 @@ def test_rootdir_specified(): expected_execution_test_output.config_file_pytest_expected_execution_output ) assert actual + assert actual.pop(-1).get("eot") assert len(actual) == len(expected_const) actual_result_dict = dict() for a in actual: @@ -73,6 +75,7 @@ def test_syntax_error_execution(tmp_path): p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) actual = runner(["error_syntax_discover.py::test_function"]) + assert actual.pop(-1).get("eot") if actual: actual = actual[0] assert actual @@ -88,6 +91,7 @@ def test_bad_id_error_execution(): The json should still be returned but the errors list should be present. """ actual = runner(["not/a/real::test_id"]) + assert actual.pop(-1).get("eot") if actual: actual = actual[0] assert actual @@ -209,7 +213,8 @@ def test_pytest_execution(test_ids, expected_const): args = test_ids actual = runner(args) assert actual - print(actual) + # print(actual) + assert actual.pop(-1).get("eot") assert len(actual) == len(expected_const) actual_result_dict = dict() for a in actual: diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 54ca9264fa6f..b72662d376fa 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -357,6 +357,8 @@ def pytest_sessionfinish(session, exitstatus): exitstatus_bool, None, ) + # send end of transmission token + send_post_request({"eot": True}) def build_test_tree(session: pytest.Session) -> TestNode: @@ -608,6 +610,12 @@ class ExecutionPayloadDict(Dict): error: Union[str, None] # Currently unused need to check +class EOTPayloadDict(TypedDict): + """A dictionary that is used to send a end of transmission post request to the server.""" + + eot: bool + + def get_node_path(node: Any) -> pathlib.Path: return getattr(node, "path", pathlib.Path(node.fspath)) @@ -617,18 +625,45 @@ def get_node_path(node: Any) -> pathlib.Path: def execution_post( - cwd: str, - status: Literal["success", "error"], - tests: Union[testRunResultDict, None], + cwd: str, status: Literal["success", "error"], tests: Union[testRunResultDict, None] ): + payload: ExecutionPayloadDict = ExecutionPayloadDict( + cwd=cwd, status=status, result=tests, not_found=None, error=None + ) + if ERRORS: + payload["error"] = ERRORS + send_post_request(payload) + + +def post_response(cwd: str, session_node: TestNode) -> None: + payload: DiscoveryPayloadDict = { + "cwd": cwd, + "status": "success" if not ERRORS else "error", + "tests": session_node, + "error": [], + } + if ERRORS is not None: + payload["error"] = ERRORS + send_post_request(payload, cls_encoder=PathEncoder) + + +class PathEncoder(json.JSONEncoder): + """A custom JSON encoder that encodes pathlib.Path objects as strings.""" + + def default(self, obj): + if isinstance(obj, pathlib.Path): + return os.fspath(obj) + return super().default(obj) + + +def send_post_request(payload: dict, cls_encoder=None): """ - Sends a post request to the server after the tests have been executed. + Sends a post request to the server. + Keyword arguments: - cwd -- the current working directory. - session_node -- the status of running the tests - tests -- the tests that were run and their status. + payload -- the payload data to be sent. + cls_encoder -- a custom encoder if needed. """ - # print("sending execution post!", tests) testPort = os.getenv("TEST_PORT", 45454) testuuid = os.getenv("TEST_UUID") addr = ("localhost", int(testPort)) @@ -641,12 +676,8 @@ def execution_post( except Exception as e: print(f"Plugin error connection error[vscode-pytest]: {e}") __socket = None - payload: ExecutionPayloadDict = ExecutionPayloadDict( - cwd=cwd, status=status, result=tests, not_found=None, error=None - ) - if ERRORS: - payload["error"] = ERRORS - data = json.dumps(payload) + + data = json.dumps(payload, cls=cls_encoder) request = f"""Content-Length: {len(data)} Content-Type: application/json Request-uuid: {testuuid} @@ -659,8 +690,8 @@ def execution_post( try: if __socket is not None and __socket.socket is not None: __socket.socket.sendall(request.encode("utf-8")) - print("Execution post sent successfully!") - print("data sent", tests, "end of data") + print("Post request sent successfully!") + print("data sent", payload, "end of data") break # Exit the loop if the send was successful else: print("Plugin error connection error[vscode-pytest]") @@ -673,57 +704,4 @@ def execution_post( print(f"Retrying ({retries}/{max_retries}) in 2 seconds...") time.sleep(2) # Wait for a short duration before retrying else: - print("Maximum retry attempts reached. Cannot send execution post.") - - # try: - # if __socket is not None and __socket.socket is not None: - # __socket.socket.sendall(request.encode("utf-8")) - # else: - # print("Plugin error connection error[vscode-pytest]") - # print(f"[vscode-pytest] data: {request}") - # except Exception as e: - # print(f"Plugin error connection error[vscode-pytest]: {e}") - # print(f"[vscode-pytest] data: {request}") - - -class PathEncoder(json.JSONEncoder): - """A custom JSON encoder that encodes pathlib.Path objects as strings.""" - - def default(self, obj): - if isinstance(obj, pathlib.Path): - return os.fspath(obj) - return super().default(obj) - - -def post_response(cwd: str, session_node: TestNode) -> None: - """Sends a post request to the server. - - Keyword arguments: - cwd -- the current working directory. - session_node -- the session node, which is the top of the testing tree. - errors -- a list of errors that occurred during test collection. - """ - payload: DiscoveryPayloadDict = { - "cwd": cwd, - "status": "success" if not ERRORS else "error", - "tests": session_node, - "error": [], - } - if ERRORS is not None: - payload["error"] = ERRORS - test_port: Union[str, int] = os.getenv("TEST_PORT", 45454) - test_uuid: Union[str, None] = os.getenv("TEST_UUID") - addr = "localhost", int(test_port) - data = json.dumps(payload, cls=PathEncoder) - request = f"""Content-Length: {len(data)} -Content-Type: application/json -Request-uuid: {test_uuid} - -{data}""" - try: - with socket_manager.SocketManager(addr) as s: - if s.socket is not None: - s.socket.sendall(request.encode("utf-8")) - except Exception as e: - print(f"Plugin error connection error[vscode-pytest]: {e}") - print(f"[vscode-pytest] data: {request}") + print("Maximum retry attempts reached. Cannot send post request.") diff --git a/src/client/interpreter/activation/terminalEnvVarCollectionService.ts b/src/client/interpreter/activation/terminalEnvVarCollectionService.ts index 75ef8168484b..654594e0af98 100644 --- a/src/client/interpreter/activation/terminalEnvVarCollectionService.ts +++ b/src/client/interpreter/activation/terminalEnvVarCollectionService.ts @@ -214,7 +214,7 @@ export class TerminalEnvVarCollectionService implements IExtensionActivationServ const displayPath = this.pathUtils.getDisplayName(settings.pythonPath, workspaceFolder?.uri.fsPath); const description = new MarkdownString(`${Interpreters.activateTerminalDescription} \`${displayPath}\``); - envVarCollection.description = description; + console.log(description); await this.trackTerminalPrompt(shell, resource, env); } diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 9705374d74af..46623ed9158a 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -44,8 +44,16 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { const uuid = this.testServer.createUUID(uri.fsPath); traceVerbose(uri, testIds, debugBool); const dataReceivedDisposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { + console.log('data received'); if (runInstance) { - this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + const eParsed = JSON.parse(e.data); + console.log('ee', eParsed); + console.log('eot', eParsed.eot); + this.resultResolver?.resolveExecution(eParsed, runInstance); + if (eParsed.eot === true) { + this.testServer.deleteUUID(uuid); + dataReceivedDisposable.dispose(); + } } }); const disposeDataReceiver = function (testServer: ITestServer) { From b98bf18991af0fc8c4658587908af399b383899a Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 28 Aug 2023 09:26:54 -0700 Subject: [PATCH 22/55] add doc string, fix pyright --- pythonFiles/vscode_pytest/__init__.py | 34 ++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index b72662d376fa..6c268c3a2d8a 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -358,7 +358,8 @@ def pytest_sessionfinish(session, exitstatus): None, ) # send end of transmission token - send_post_request({"eot": True}) + payload: EOTPayloadDict = {"eot": True} + send_post_request(payload) def build_test_tree(session: pytest.Session) -> TestNode: @@ -617,6 +618,7 @@ class EOTPayloadDict(TypedDict): def get_node_path(node: Any) -> pathlib.Path: + """A function that returns the path of a node given the switch to pathlib.Path.""" return getattr(node, "path", pathlib.Path(node.fspath)) @@ -627,6 +629,15 @@ def get_node_path(node: Any) -> pathlib.Path: def execution_post( cwd: str, status: Literal["success", "error"], tests: Union[testRunResultDict, None] ): + """ + Sends a POST request with execution payload details. + + Args: + cwd (str): Current working directory. + status (Literal["success", "error"]): Execution status indicating success or error. + tests (Union[testRunResultDict, None]): Test run results, if available. + """ + payload: ExecutionPayloadDict = ExecutionPayloadDict( cwd=cwd, status=status, result=tests, not_found=None, error=None ) @@ -636,6 +647,14 @@ def execution_post( def post_response(cwd: str, session_node: TestNode) -> None: + """ + Sends a POST request with test session details in payload. + + Args: + cwd (str): Current working directory. + session_node (TestNode): Node information of the test session. + """ + payload: DiscoveryPayloadDict = { "cwd": cwd, "status": "success" if not ERRORS else "error", @@ -656,7 +675,10 @@ def default(self, obj): return super().default(obj) -def send_post_request(payload: dict, cls_encoder=None): +def send_post_request( + payload: ExecutionPayloadDict | DiscoveryPayloadDict | EOTPayloadDict, + cls_encoder=None, +): """ Sends a post request to the server. @@ -673,8 +695,8 @@ def send_post_request(payload: dict, cls_encoder=None): try: __socket = socket_manager.SocketManager(addr) __socket.connect() - except Exception as e: - print(f"Plugin error connection error[vscode-pytest]: {e}") + except Exception as error: + print(f"Plugin error connection error[vscode-pytest]: {error}") __socket = None data = json.dumps(payload, cls=cls_encoder) @@ -696,8 +718,8 @@ def send_post_request(payload: dict, cls_encoder=None): else: print("Plugin error connection error[vscode-pytest]") print(f"[vscode-pytest] data: {request}") - except Exception as e: - print(f"Plugin error connection error[vscode-pytest]: {e}") + except Exception as error: + print(f"Plugin error connection error[vscode-pytest]: {error}") print(f"[vscode-pytest] data: {request}") retries += 1 # Increment retry counter if retries < max_retries: From e7fd57fefd5391005da9292a0e0dc2f47a73b0a3 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 28 Aug 2023 09:36:21 -0700 Subject: [PATCH 23/55] revert incorrectly staged changes --- .vscode/launch.json | 2 +- .vscode/settings.json | 3 +-- .../interpreter/activation/terminalEnvVarCollectionService.ts | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index e448dd15b13b..82981a93305d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -101,7 +101,7 @@ "--extensionTestsPath=${workspaceFolder}/out/test" ], "env": { - "VSC_PYTHON_CI_TEST_GREP": "End to End Tests: test adapters" // Modify this to run a subset of the single workspace tests + "VSC_PYTHON_CI_TEST_GREP": "" // Modify this to run a subset of the single workspace tests }, "stopOnEntry": false, "sourceMaps": true, diff --git a/.vscode/settings.json b/.vscode/settings.json index 596a17f64b85..174a850c901e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -65,6 +65,5 @@ "--max-line-length=88" ], "typescript.preferences.importModuleSpecifier": "relative", - "debug.javascript.usePreview": false, - "python.defaultInterpreterPath": ".venv/bin/python" + "debug.javascript.usePreview": false } diff --git a/src/client/interpreter/activation/terminalEnvVarCollectionService.ts b/src/client/interpreter/activation/terminalEnvVarCollectionService.ts index 654594e0af98..75ef8168484b 100644 --- a/src/client/interpreter/activation/terminalEnvVarCollectionService.ts +++ b/src/client/interpreter/activation/terminalEnvVarCollectionService.ts @@ -214,7 +214,7 @@ export class TerminalEnvVarCollectionService implements IExtensionActivationServ const displayPath = this.pathUtils.getDisplayName(settings.pythonPath, workspaceFolder?.uri.fsPath); const description = new MarkdownString(`${Interpreters.activateTerminalDescription} \`${displayPath}\``); - console.log(description); + envVarCollection.description = description; await this.trackTerminalPrompt(shell, resource, env); } From 60e46a237a2e09bf1f36751d36774511c1893349 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 28 Aug 2023 10:59:46 -0700 Subject: [PATCH 24/55] pytest discovery --- pythonFiles/vscode_pytest/__init__.py | 4 +++- .../testController/common/resultResolver.ts | 22 ++++++++++++++----- .../testing/testController/common/server.ts | 12 +++++----- .../testing/testController/common/types.ts | 18 +++++++++++++-- .../pytest/pytestDiscoveryAdapter.ts | 9 +++++--- 5 files changed, 48 insertions(+), 17 deletions(-) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 6c268c3a2d8a..5e2ceb58cd94 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -358,7 +358,8 @@ def pytest_sessionfinish(session, exitstatus): None, ) # send end of transmission token - payload: EOTPayloadDict = {"eot": True} + command_type = "discovery" if IS_DISCOVERY else "execution" + payload: EOTPayloadDict = {"command_type": command_type, "eot": True} send_post_request(payload) @@ -614,6 +615,7 @@ class ExecutionPayloadDict(Dict): class EOTPayloadDict(TypedDict): """A dictionary that is used to send a end of transmission post request to the server.""" + command_type: Literal["discovery"] | Literal["execution"] eot: bool diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index aa9f2a541f51..6e707f2eae69 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -3,7 +3,7 @@ import { CancellationToken, TestController, TestItem, Uri, TestMessage, Location, TestRun } from 'vscode'; import * as util from 'util'; -import { DiscoveredTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; +import { DiscoveredTestPayload, EOTTestPayload, ExecutionTestPayload, ITestResultResolver } from './types'; import { TestProvider } from '../../types'; import { traceError, traceLog } from '../../../logging'; import { Testing } from '../../../common/utils/localize'; @@ -12,6 +12,7 @@ import { sendTelemetryEvent } from '../../../telemetry'; import { EventName } from '../../../telemetry/constants'; import { splitLines } from '../../../common/stringUtils'; import { buildErrorNodeOptions, fixLogLines, populateTestTree } from './utils'; +import { Deferred } from '../../../common/utils/async'; export class PythonResultResolver implements ITestResultResolver { testController: TestController; @@ -35,16 +36,27 @@ export class PythonResultResolver implements ITestResultResolver { this.vsIdToRunId = new Map(); } - public resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise { + public resolveDiscovery( + payload: DiscoveredTestPayload | EOTTestPayload, + deferredTillEOT: Deferred, + token?: CancellationToken, + ): Promise { const workspacePath = this.workspaceUri.fsPath; traceLog('Using result resolver for discovery'); - const rawTestData = payload; - if (!rawTestData) { + if (!payload) { // No test data is available return Promise.resolve(); } - + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + const rawTestData = payload as DiscoveredTestPayload; // Check if there were any errors in the discovery process. if (rawTestData.status === 'error') { const testingErrorConst = diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 4fef100b657b..cd4c78f14bae 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -49,17 +49,17 @@ export class PythonTestServer implements ITestServer, Disposable { this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); } buffer = Buffer.from(extractedJsonPayload.remainingRawData); - if (!containsHeaders(extractedJsonPayload.remainingRawData)) { - // if the remaining data does not contain headers, then there is no more data to process. - // break to get more data from the socket. - break; - } if (buffer.length === 0) { // if the buffer is empty, then there is no more data to process. // break to get more data from the socket. buffer = Buffer.alloc(0); break; } + if (!containsHeaders(extractedJsonPayload.remainingRawData)) { + // if the remaining data does not contain headers, then there is no more data to process. + // break to get more data from the socket. + break; + } } catch (ex) { traceError(`Error:: ${ex}`); this._onDataReceived.fire({ uuid: '', data: '' }); @@ -92,7 +92,7 @@ export class PythonTestServer implements ITestServer, Disposable { } private _fireDataReceived(uuid: string, extractedJSON: string): void { - if (extractedJSON.includes(`"tests":`)) { + if (extractedJSON.includes(`"tests":`) || extractedJSON.includes(`"command_type": "discovery"`)) { this._onDiscoveryDataReceived.fire({ uuid, data: extractedJSON, diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index 610c3d76d17b..ddbb83e3063c 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -14,6 +14,7 @@ import { } from 'vscode'; import { ITestDebugLauncher, TestDiscoveryOptions } from '../../common/types'; import { IPythonExecutionFactory } from '../../../common/process/types'; +import { Deferred } from '../../../common/utils/async'; export type TestRunInstanceOptions = TestRunOptions & { exclude?: readonly TestItem[]; @@ -191,8 +192,16 @@ export interface ITestResultResolver { runIdToVSid: Map; runIdToTestItem: Map; vsIdToRunId: Map; - resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise; - resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise; + resolveDiscovery( + payload: DiscoveredTestPayload | EOTTestPayload, + deferredTillEOT: Deferred, + token?: CancellationToken, + ): Promise; + resolveExecution( + payload: ExecutionTestPayload | EOTTestPayload, + runInstance: TestRun, + deferredTillEOT: Deferred, + ): Promise; } export interface ITestDiscoveryAdapter { // ** first line old method signature, second line new method signature @@ -241,6 +250,11 @@ export type DiscoveredTestPayload = { error?: string[]; }; +export type EOTTestPayload = { + commandType: 'discovery' | 'execution'; + eot: boolean; +}; + export type ExecutionTestPayload = { cwd: string; status: 'success' | 'error'; diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index 450e2ef1edf2..3409914b8e74 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -2,6 +2,7 @@ // Licensed under the MIT License. import * as path from 'path'; import { Uri } from 'vscode'; +import { Disposable } from 'vscode-languageserver-protocol'; import { ExecutionFactoryCreateWithEnvironmentOptions, ExecutionResult, @@ -9,7 +10,7 @@ import { SpawnOptions, } from '../../../common/process/types'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred } from '../../../common/utils/async'; +import { Deferred, createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { traceVerbose } from '../../../logging'; import { @@ -36,8 +37,9 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { const uuid = this.testServer.createUUID(uri.fsPath); const { pytestArgs } = settings.testing; traceVerbose(pytestArgs); - const dataReceivedDisposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { - this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + const deferredTillEOT: Deferred = createDeferred(); + const dataReceivedDisposable = this.testServer.onDiscoveryDataReceived(async (e: DataReceivedEvent) => { + this.resultResolver?.resolveDiscovery(JSON.parse(e.data), deferredTillEOT); }); const disposeDataReceiver = function (testServer: ITestServer) { testServer.deleteUUID(uuid); @@ -45,6 +47,7 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { }; try { await this.runPytestDiscovery(uri, uuid, executionFactory); + await deferredTillEOT.promise; } finally { disposeDataReceiver(this.testServer); } From 0e6ddf47b84eab68130f0458ff6669c50c7af409 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 28 Aug 2023 11:29:14 -0700 Subject: [PATCH 25/55] tests passing for pytest execution --- .../testController/common/resultResolver.ts | 16 ++++++++-- .../testing/testController/common/server.ts | 2 +- .../pytest/pytestExecutionAdapter.ts | 32 +++++++------------ 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 6e707f2eae69..23db6f21e16e 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -99,8 +99,20 @@ export class PythonResultResolver implements ITestResultResolver { return Promise.resolve(); } - public resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise { - const rawTestExecData = payload; + public resolveExecution( + payload: ExecutionTestPayload | EOTTestPayload, + runInstance: TestRun, + deferredTillEOT: Deferred, + ): Promise { + if (payload !== undefined && 'eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + const rawTestExecData = payload as ExecutionTestPayload; if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { // Map which holds the subtest information for each test item. diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index cd4c78f14bae..5ddd4969674a 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -98,7 +98,7 @@ export class PythonTestServer implements ITestServer, Disposable { data: extractedJSON, }); // if the rawData includes result then this is a run request - } else if (extractedJSON.includes(`"result":`)) { + } else if (extractedJSON.includes(`"result":`) || extractedJSON.includes(`"command_type": "execution"`)) { console.log('\n *** fire run data received: \n', extractedJSON, '\n *** end'); this._onRunDataReceived.fire({ uuid, diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 46623ed9158a..89f8455fcaf2 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -4,8 +4,8 @@ import { TestRun, Uri } from 'vscode'; import * as path from 'path'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred } from '../../../common/utils/async'; -import { traceError, traceInfo, traceVerbose } from '../../../logging'; +import { Deferred, createDeferred } from '../../../common/utils/async'; +import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; import { DataReceivedEvent, ExecutionTestPayload, @@ -43,17 +43,12 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { ): Promise { const uuid = this.testServer.createUUID(uri.fsPath); traceVerbose(uri, testIds, debugBool); + const deferredTillEOT: Deferred = createDeferred(); const dataReceivedDisposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { console.log('data received'); if (runInstance) { const eParsed = JSON.parse(e.data); - console.log('ee', eParsed); - console.log('eot', eParsed.eot); - this.resultResolver?.resolveExecution(eParsed, runInstance); - if (eParsed.eot === true) { - this.testServer.deleteUUID(uuid); - dataReceivedDisposable.dispose(); - } + this.resultResolver?.resolveExecution(eParsed, runInstance, deferredTillEOT); } }); const disposeDataReceiver = function (testServer: ITestServer) { @@ -63,16 +58,12 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { runInstance?.token.onCancellationRequested(() => { disposeDataReceiver(this.testServer); }); - await this.runTestsNew( - uri, - testIds, - uuid, - runInstance, - debugBool, - executionFactory, - debugLauncher, - disposeDataReceiver, - ); + try { + await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, executionFactory, debugLauncher); + } finally { + await deferredTillEOT.promise; + disposeDataReceiver(this.testServer); + } // placeholder until after the rewrite is adopted // TODO: remove after adoption. @@ -92,7 +83,6 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { debugBool?: boolean, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, - disposeDataReceiver?: (testServer: ITestServer) => void, ): Promise { const deferred = createDeferred(); const relativePathToPytest = 'pythonFiles'; @@ -193,7 +183,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { } deferredExec.resolve({ stdout: '', stderr: '' }); deferred.resolve(); - disposeDataReceiver?.(this.testServer); + // disposeDataReceiver?.(this.testServer); }); await deferredExec.promise; } From 53f4306c141824ba428aac309b1f05f36f58709e Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 28 Aug 2023 14:12:26 -0700 Subject: [PATCH 26/55] unittest functioning- all test.ts --- pythonFiles/unittestadapter/discovery.py | 36 +++- pythonFiles/unittestadapter/execution.py | 34 +++- .../testing/testController/common/server.ts | 1 - .../pytest/pytestDiscoveryAdapter.ts | 2 +- .../unittest/testDiscoveryAdapter.ts | 11 +- .../unittest/testExecutionAdapter.ts | 13 +- .../testing/common/testingAdapter.test.ts | 188 +++++++++++------- .../test_parameterized_subtest.py | 4 +- 8 files changed, 188 insertions(+), 101 deletions(-) diff --git a/pythonFiles/unittestadapter/discovery.py b/pythonFiles/unittestadapter/discovery.py index cbad40ad1838..6097e52ef587 100644 --- a/pythonFiles/unittestadapter/discovery.py +++ b/pythonFiles/unittestadapter/discovery.py @@ -48,6 +48,13 @@ class PayloadDict(TypedDict): error: NotRequired[List[str]] +class EOTPayloadDict(TypedDict): + """A dictionary that is used to send a end of transmission post request to the server.""" + + command_type: Literal["discovery"] | Literal["execution"] + eot: bool + + def discover_tests( start_dir: str, pattern: str, top_level_dir: Optional[str], uuid: Optional[str] ) -> PayloadDict: @@ -106,17 +113,7 @@ def discover_tests( return payload -if __name__ == "__main__": - # Get unittest discovery arguments. - argv = sys.argv[1:] - index = argv.index("--udiscovery") - - start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) - - # Perform test discovery. - port, uuid = parse_discovery_cli_args(argv[:index]) - payload = discover_tests(start_dir, pattern, top_level_dir, uuid) - +def post_response(payload: PayloadDict | EOTPayloadDict, port: int, uuid: str) -> None: # Build the request data (it has to be a POST request or the Node side will not process it), and send it. addr = ("localhost", port) data = json.dumps(payload) @@ -132,3 +129,20 @@ def discover_tests( except Exception as e: print(f"Error sending response: {e}") print(f"Request data: {request}") + + +if __name__ == "__main__": + # Get unittest discovery arguments. + argv = sys.argv[1:] + index = argv.index("--udiscovery") + + start_dir, pattern, top_level_dir = parse_unittest_args(argv[index + 1 :]) + + # Perform test discovery. + port, uuid = parse_discovery_cli_args(argv[:index]) + # Post this discovery payload. + payload = discover_tests(start_dir, pattern, top_level_dir, uuid) + post_response(payload, port, uuid) + # Post EOT token. + eot_payload: EOTPayloadDict = {"command_type": "discovery", "eot": True} + post_response(eot_payload, port, uuid) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index e38f831726dc..869199af9fa6 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -18,7 +18,7 @@ sys.path.append(os.fspath(script_dir)) sys.path.insert(0, os.fspath(script_dir / "lib" / "python")) -from typing_extensions import NotRequired, TypeAlias, TypedDict +from typing_extensions import Literal, NotRequired, TypeAlias, TypedDict from testing_tools import process_json_util, socket_manager from unittestadapter.utils import parse_unittest_args @@ -169,6 +169,13 @@ class PayloadDict(TypedDict): error: NotRequired[str] +class EOTPayloadDict(TypedDict): + """A dictionary that is used to send a end of transmission post request to the server.""" + + command_type: Literal["discovery"] | Literal["execution"] + eot: bool + + # Args: start_path path to a directory or a file, list of ids that may be empty. # Edge cases: # - if tests got deleted since the VS Code side last ran discovery and the current test run, @@ -231,16 +238,6 @@ def run_tests( def send_run_data(raw_data, port, uuid): - # Build the request data (it has to be a POST request or the Node side will not process it), and send it. - addr = ("localhost", port) - global __socket - if __socket is None: - try: - __socket = socket_manager.SocketManager(addr) - __socket.connect() - except Exception as e: - print(f"Plugin error connection error[vscode-pytest]: {e}") - __socket = None status = raw_data["outcome"] cwd = os.path.abspath(START_DIR) if raw_data["subtest"]: @@ -250,7 +247,20 @@ def send_run_data(raw_data, port, uuid): test_dict = {} test_dict[test_id] = raw_data payload: PayloadDict = {"cwd": cwd, "status": status, "result": test_dict} + post_response(payload, port, uuid) + +def post_response(payload: PayloadDict | EOTPayloadDict, port: int, uuid: str) -> None: + # Build the request data (it has to be a POST request or the Node side will not process it), and send it. + addr = ("localhost", port) + global __socket + if __socket is None: + try: + __socket = socket_manager.SocketManager(addr) + __socket.connect() + except Exception as error: + print(f"Plugin error connection error[vscode-pytest]: {error}") + __socket = None data = json.dumps(payload) request = f"""Content-Length: {len(data)} Content-Type: application/json @@ -325,3 +335,5 @@ def send_run_data(raw_data, port, uuid): "error": "No test ids received from buffer", "result": None, } + eot_payload: EOTPayloadDict = {"command_type": "execution", "eot": True} + post_response(eot_payload, PORT, UUID) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 5ddd4969674a..860017944665 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -236,7 +236,6 @@ export class PythonTestServer implements ITestServer, Disposable { }); } deferred.resolve({ stdout: '', stderr: '' }); - callback?.(); }); await deferred.promise; } diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index 3409914b8e74..ca053f02c9c3 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -47,8 +47,8 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { }; try { await this.runPytestDiscovery(uri, uuid, executionFactory); - await deferredTillEOT.promise; } finally { + await deferredTillEOT.promise; disposeDataReceiver(this.testServer); } // this is only a placeholder to handle function overloading until rewrite is finished diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts index 9820aa89626c..17939cb62d47 100644 --- a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -14,6 +14,7 @@ import { TestCommandOptions, TestDiscoveryCommand, } from '../common/types'; +import { Deferred, createDeferred } from '../../../common/utils/async'; /** * Wrapper class for unittest test discovery. This is where we call `runTestCommand`. @@ -34,7 +35,7 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { const command = buildDiscoveryCommand(unittestArgs); const uuid = this.testServer.createUUID(uri.fsPath); - + const deferredTillEOT: Deferred = createDeferred(); const options: TestCommandOptions = { workspaceFolder: uri, command, @@ -44,16 +45,16 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { }; const dataReceivedDisposable = this.testServer.onDiscoveryDataReceived((e: DataReceivedEvent) => { - this.resultResolver?.resolveDiscovery(JSON.parse(e.data)); + this.resultResolver?.resolveDiscovery(JSON.parse(e.data), deferredTillEOT); }); const disposeDataReceiver = function (testServer: ITestServer) { testServer.deleteUUID(uuid); dataReceivedDisposable.dispose(); }; - await this.callSendCommand(options, () => { - disposeDataReceiver(this.testServer); - }); + await this.callSendCommand(options); + await deferredTillEOT.promise; + disposeDataReceiver(this.testServer); // placeholder until after the rewrite is adopted // TODO: remove after adoption. const discoveryPayload: DiscoveredTestPayload = { diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index bc5e41d19d9d..7aa7e21acc5e 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -4,7 +4,7 @@ import * as path from 'path'; import { TestRun, Uri } from 'vscode'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; -import { createDeferred } from '../../../common/utils/async'; +import { Deferred, createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; import { DataReceivedEvent, @@ -37,9 +37,10 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { runInstance?: TestRun, ): Promise { const uuid = this.testServer.createUUID(uri.fsPath); + const deferredTillEOT: Deferred = createDeferred(); const disposedDataReceived = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { if (runInstance) { - this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance); + this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance, deferredTillEOT); } }); const disposeDataReceiver = function (testServer: ITestServer) { @@ -49,7 +50,13 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { runInstance?.token.onCancellationRequested(() => { disposeDataReceiver(this.testServer); }); - await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, disposeDataReceiver); + try { + await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, disposeDataReceiver); + await deferredTillEOT.promise; + disposeDataReceiver(this.testServer); + } catch (error) { + traceLog(error); + } const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; return executionPayload; } diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 18fc9b89ee5e..ae88dcd200f9 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -6,7 +6,7 @@ import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; -import { ITestResultResolver } from '../../../client/testing/testController/common/types'; +import { EOTTestPayload, ITestResultResolver } from '../../../client/testing/testController/common/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { IPythonExecutionFactory } from '../../../client/common/process/types'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; @@ -94,9 +94,17 @@ suite('End to End Tests: test adapters', () => { }; resultResolver .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - traceLog(`resolveDiscovery ${data}`); - actualData = data; + .returns((payload, deferredTillEOT) => { + traceLog(`resolveDiscovery ${payload}`); + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + actualData = payload; return Promise.resolve(); }); @@ -114,21 +122,10 @@ suite('End to End Tests: test adapters', () => { await discoveryAdapter.discoverTests(workspaceUri).finally(() => { // verification after discovery is complete - // resultResolver.verify( - // (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - // typeMoq.Times.once(), - // ); - - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(actualData.tests, 'Expected tests to be present'); - }); - - await discoveryAdapter.discoverTests(Uri.parse(rootPathErrorWorkspace)).finally(() => { - // verification after discovery is complete + resultResolver.verify( + (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.exactly(2), + ); // 1. Check the status is "success" assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); @@ -148,9 +145,17 @@ suite('End to End Tests: test adapters', () => { }; resultResolver .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - traceLog(`resolveDiscovery ${data}`); - actualData = data; + .returns((payload, deferredTillEOT) => { + traceLog(`resolveDiscovery ${payload}`); + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + actualData = payload; return Promise.resolve(); }); @@ -169,7 +174,7 @@ suite('End to End Tests: test adapters', () => { // verification after discovery is complete resultResolver.verify( (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), + typeMoq.Times.exactly(2), ); // 1. Check the status is "success" @@ -188,10 +193,18 @@ suite('End to End Tests: test adapters', () => { tests: unknown; }; resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - traceLog(`resolveDiscovery ${data}`); - actualData = data; + .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((payload, deferredTillEOT) => { + traceLog(`resolveDiscovery ${payload}`); + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + actualData = payload; return Promise.resolve(); }); // run pytest discovery @@ -209,7 +222,7 @@ suite('End to End Tests: test adapters', () => { // verification after discovery is complete resultResolver.verify( (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), + typeMoq.Times.exactly(2), ); // 1. Check the status is "success" @@ -228,9 +241,18 @@ suite('End to End Tests: test adapters', () => { tests: unknown; }; resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - actualData = data; + .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((payload, deferredTillEOT) => { + traceLog(`resolveDiscovery ${payload}`); + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + actualData = payload; return Promise.resolve(); }); // run pytest discovery @@ -248,7 +270,7 @@ suite('End to End Tests: test adapters', () => { // verification after discovery is complete resultResolver.verify( (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), + typeMoq.Times.exactly(2), ); // 1. Check the status is "success" @@ -267,9 +289,18 @@ suite('End to End Tests: test adapters', () => { result: unknown; }; resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - actualData = data; + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((payload, _run, deferredTillEOT) => { + traceLog(`resolveExecution ${payload}`); + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + actualData = payload; return Promise.resolve(); }); @@ -297,8 +328,8 @@ suite('End to End Tests: test adapters', () => { .finally(() => { // verification after execution is complete resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.exactly(2), ); // 1. Check the status is "success" assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); @@ -311,14 +342,22 @@ suite('End to End Tests: test adapters', () => { const errorMessages: string[] = []; // result resolver and saved data for assertions resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((payload, _run, deferredTillEOT) => { + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } count = count + 1; - if (data.status !== 'subtest-success' && data.status !== 'subtest-failure') { + if (payload.status !== 'subtest-success' && payload.status !== 'subtest-failure') { errorMessages.push("Expected status to be 'subtest-success' or 'subtest-failure'"); - errorMessages.push(data.message); + errorMessages.push(payload.message); } - if (data.result === null) { + if (payload.result === null) { errorMessages.push('Expected results to be present'); } return Promise.resolve(); @@ -344,7 +383,6 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - const deferred = createDeferred(); await executionAdapter .runTests(workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], false, testRun.object) .then(() => { @@ -355,14 +393,10 @@ suite('End to End Tests: test adapters', () => { ['Test run was unsuccessful, the following errors were produced: \n', ...errorMessages].join('\n'), ); resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeast(200), + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.exactly(4), ); - }) - .finally(() => { - deferred.resolve(); }); - await deferred.promise; }); test('pytest execution adapter small workspace', async () => { // result resolver and saved data for assertions @@ -372,12 +406,20 @@ suite('End to End Tests: test adapters', () => { result: unknown; }; resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - actualData = data; + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((payload, _run, deferredTillEOT) => { + traceLog(`resolveExecution ${payload}`); + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + actualData = payload; return Promise.resolve(); }); - // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathSmallWorkspace); @@ -408,8 +450,8 @@ suite('End to End Tests: test adapters', () => { .then(() => { // verification after discovery is complete resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.once(), + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.exactly(2), ); // 1. Check the status is "success" assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); @@ -422,15 +464,27 @@ suite('End to End Tests: test adapters', () => { test('pytest execution adapter large workspace', async () => { const errorMessages: string[] = []; resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(data.error, null, "Expected no errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((payload, _run, deferredTillEOT) => { + traceLog(`resolveExecution ${payload}`); + if ('eot' in payload) { + // the payload is an EOT payload, so resolve the deferred promise. + const eotPayload = payload as EOTTestPayload; + if (eotPayload.eot === true) { + deferredTillEOT.resolve(); + return Promise.resolve(); + } + } + // Check the following for each time resolveExecution is called to collect any errors. + if (payload.status !== 'success') { + errorMessages.push("Expected status to be 'success'"); + } + if (payload.error !== null) { + errorMessages.push("Expected no errors in 'error' field"); + } + if (payload.result === null) { + errorMessages.push('Expected results to be present'); + } return Promise.resolve(); }); @@ -439,7 +493,7 @@ suite('End to End Tests: test adapters', () => { // generate list of test_ids const testIds: string[] = []; - for (let i = 0; i < 200; i = i + 1) { + for (let i = 0; i < 3; i = i + 1) { const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } @@ -571,8 +625,8 @@ suite('End to End Tests: test adapters', () => { ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(1), + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), + typeMoq.Times.exactly(4), ); }); }); diff --git a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py index 654cdffd25d4..ebbfa7393cb7 100644 --- a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py +++ b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py @@ -4,13 +4,13 @@ import unittest -@pytest.mark.parametrize("num", range(0, 20)) +@pytest.mark.parametrize("num", range(0, 3)) def test_odd_even(num): assert num % 2 == 0 class NumbersTest(unittest.TestCase): def test_even(self): - for i in range(0, 20): + for i in range(0, 3): with self.subTest(i=i): self.assertEqual(i % 2, 0) From 55945661428b4d86aff7e827ebbcc407f4662ac5 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 28 Aug 2023 15:28:16 -0700 Subject: [PATCH 27/55] remove result resolver mock --- .../testController/common/resultResolver.ts | 14 +- .../testing/testController/common/types.ts | 2 + .../testing/common/testingAdapter.test.ts | 351 +++++++----------- 3 files changed, 144 insertions(+), 223 deletions(-) diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index 23db6f21e16e..c4ad53c226cc 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -41,21 +41,24 @@ export class PythonResultResolver implements ITestResultResolver { deferredTillEOT: Deferred, token?: CancellationToken, ): Promise { - const workspacePath = this.workspaceUri.fsPath; - traceLog('Using result resolver for discovery'); - if (!payload) { // No test data is available return Promise.resolve(); } if ('eot' in payload) { // the payload is an EOT payload, so resolve the deferred promise. + traceLog('ResultResolver EOT received for discovery.'); const eotPayload = payload as EOTTestPayload; if (eotPayload.eot === true) { deferredTillEOT.resolve(); return Promise.resolve(); } } + return this._resolveDiscovery(payload as DiscoveredTestPayload, token); + } + + public _resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise { + const workspacePath = this.workspaceUri.fsPath; const rawTestData = payload as DiscoveredTestPayload; // Check if there were any errors in the discovery process. if (rawTestData.status === 'error') { @@ -106,12 +109,17 @@ export class PythonResultResolver implements ITestResultResolver { ): Promise { if (payload !== undefined && 'eot' in payload) { // the payload is an EOT payload, so resolve the deferred promise. + traceLog('ResultResolver EOT received for discovery.'); const eotPayload = payload as EOTTestPayload; if (eotPayload.eot === true) { deferredTillEOT.resolve(); return Promise.resolve(); } } + return this._resolveExecution(payload as ExecutionTestPayload, runInstance); + } + + public _resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise { const rawTestExecData = payload as ExecutionTestPayload; if (rawTestExecData !== undefined && rawTestExecData.result !== undefined) { // Map which holds the subtest information for each test item. diff --git a/src/client/testing/testController/common/types.ts b/src/client/testing/testController/common/types.ts index ddbb83e3063c..386c397b310c 100644 --- a/src/client/testing/testController/common/types.ts +++ b/src/client/testing/testController/common/types.ts @@ -202,6 +202,8 @@ export interface ITestResultResolver { runInstance: TestRun, deferredTillEOT: Deferred, ): Promise; + _resolveDiscovery(payload: DiscoveredTestPayload, token?: CancellationToken): Promise; + _resolveExecution(payload: ExecutionTestPayload, runInstance: TestRun): Promise; } export interface ITestDiscoveryAdapter { // ** first line old method signature, second line new method signature diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index ae88dcd200f9..8b0452204a18 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -1,12 +1,17 @@ /* eslint-disable @typescript-eslint/no-explicit-any */ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { TestRun, Uri } from 'vscode'; +import { TestController, TestRun, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; -import { EOTTestPayload, ITestResultResolver } from '../../../client/testing/testController/common/types'; +import { + EOTTestPayload, + ExecutionTestPayload, + ITestController, + ITestResultResolver, +} from '../../../client/testing/testController/common/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { IPythonExecutionFactory } from '../../../client/common/process/types'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; @@ -17,10 +22,12 @@ import { traceLog } from '../../../client/logging'; import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; -import { createDeferred } from '../../../client/common/utils/async'; +import { PythonResultResolver } from '../../../client/testing/testController/common/resultResolver'; +import { TestProvider } from '../../../client/testing/types'; +import { PYTEST_PROVIDER, UNITTEST_PROVIDER } from '../../../client/testing/common/constants'; suite('End to End Tests: test adapters', () => { - let resultResolver: typeMoq.IMock; + let resultResolver: ITestResultResolver; let pythonTestServer: PythonTestServer; let pythonExecFactory: IPythonExecutionFactory; let debugLauncher: ITestDebugLauncher; @@ -56,9 +63,9 @@ suite('End to End Tests: test adapters', () => { pythonExecFactory = serviceContainer.get(IPythonExecutionFactory); debugLauncher = serviceContainer.get(ITestDebugLauncher); testOutputChannel = serviceContainer.get(ITestOutputChannel); + testController = serviceContainer.get(ITestController); // create mock resultResolver object - resultResolver = typeMoq.Mock.ofType(); // create objects that were not injected pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); @@ -88,44 +95,35 @@ suite('End to End Tests: test adapters', () => { test('unittest discovery adapter small workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + workspaceUri = Uri.parse(rootPathSmallWorkspace); + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, deferredTillEOT) => { - traceLog(`resolveDiscovery ${payload}`); - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - actualData = payload; - return Promise.resolve(); - }); // set workspace to test workspace folder and set up settings - workspaceUri = Uri.parse(rootPathSmallWorkspace); + configService.getSettings(workspaceUri).testing.unittestArgs = ['-s', '.', '-p', '*test*.py']; // run unittest discovery const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); await discoveryAdapter.discoverTests(workspaceUri).finally(() => { // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(2), - ); // 1. Check the status is "success" assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); @@ -133,31 +131,27 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); + + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('unittest discovery adapter large workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, deferredTillEOT) => { - traceLog(`resolveDiscovery ${payload}`); - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - actualData = payload; - return Promise.resolve(); - }); // set settings to work for the given workspace workspaceUri = Uri.parse(rootPathLargeWorkspace); @@ -166,53 +160,43 @@ suite('End to End Tests: test adapters', () => { const discoveryAdapter = new UnittestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); await discoveryAdapter.discoverTests(workspaceUri).finally(() => { - // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(2), - ); - // 1. Check the status is "success" assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); // 2. Confirm no errors assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); + + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('pytest discovery adapter small workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, deferredTillEOT) => { - traceLog(`resolveDiscovery ${payload}`); - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - actualData = payload; - return Promise.resolve(); - }); // run pytest discovery const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); // set workspace to test workspace folder @@ -220,47 +204,39 @@ suite('End to End Tests: test adapters', () => { await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(2), - ); // 1. Check the status is "success" assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); // 2. Confirm no errors - assert.strictEqual(actualData.error.length, 0, "Expected no errors in 'error' field"); + assert.strictEqual(actualData.error?.length, 0, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); + + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('pytest discovery adapter large workspace', async () => { // result resolver and saved data for assertions let actualData: { - status: unknown; - error: string | any[]; - tests: unknown; + cwd: string; + tests?: unknown; + status: 'success' | 'error'; + error?: string[]; + }; + resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveDiscovery = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + actualData = payload; + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, deferredTillEOT) => { - traceLog(`resolveDiscovery ${payload}`); - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - actualData = payload; - return Promise.resolve(); - }); // run pytest discovery const discoveryAdapter = new PytestTestDiscoveryAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); // set workspace to test workspace folder @@ -268,41 +244,28 @@ suite('End to End Tests: test adapters', () => { await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveDiscovery(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(2), - ); - // 1. Check the status is "success" assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); // 2. Confirm no errors - assert.strictEqual(actualData.error.length, 0, "Expected no errors in 'error' field"); + assert.strictEqual(actualData.error?.length, 0, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); + + assert.strictEqual(callCount, 1, 'Expected _resolveDiscovery to be called once'); }); }); test('unittest execution adapter small workspace', async () => { // result resolver and saved data for assertions - let actualData: { - status: unknown; - error: string | any[]; - result: unknown; + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, _run, deferredTillEOT) => { - traceLog(`resolveExecution ${payload}`); - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - actualData = payload; - return Promise.resolve(); - }); // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathSmallWorkspace); @@ -311,8 +274,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -326,42 +289,22 @@ suite('End to End Tests: test adapters', () => { await executionAdapter .runTests(workspaceUri, ['test_simple.SimpleClass.test_simple_unit'], false, testRun.object) .finally(() => { - // verification after execution is complete - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(2), - ); - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm tests are found - assert.ok(actualData.result, 'Expected results to be present'); + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); }); }); test('unittest execution adapter large workspace', async () => { - let count = 0; - const errorMessages: string[] = []; // result resolver and saved data for assertions - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, _run, deferredTillEOT) => { - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - count = count + 1; - if (payload.status !== 'subtest-success' && payload.status !== 'subtest-failure') { - errorMessages.push("Expected status to be 'subtest-success' or 'subtest-failure'"); - errorMessages.push(payload.message); - } - if (payload.result === null) { - errorMessages.push('Expected results to be present'); - } - return Promise.resolve(); - }); + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + return Promise.resolve(); + }; // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathLargeWorkspace); @@ -371,8 +314,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -386,40 +329,22 @@ suite('End to End Tests: test adapters', () => { await executionAdapter .runTests(workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], false, testRun.object) .then(() => { - // verification after discovery is complete - assert.strictEqual( - errorMessages.length, - 0, - ['Test run was unsuccessful, the following errors were produced: \n', ...errorMessages].join('\n'), - ); - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(4), - ); + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 3, 'Expected _resolveExecution to be called once'); }); }); test('pytest execution adapter small workspace', async () => { // result resolver and saved data for assertions - let actualData: { - status: unknown; - error: string | any[]; - result: unknown; + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + return Promise.resolve(); }; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, _run, deferredTillEOT) => { - traceLog(`resolveExecution ${payload}`); - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - actualData = payload; - return Promise.resolve(); - }); // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathSmallWorkspace); @@ -427,8 +352,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -462,31 +387,17 @@ suite('End to End Tests: test adapters', () => { }); }); test('pytest execution adapter large workspace', async () => { - const errorMessages: string[] = []; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((payload, _run, deferredTillEOT) => { - traceLog(`resolveExecution ${payload}`); - if ('eot' in payload) { - // the payload is an EOT payload, so resolve the deferred promise. - const eotPayload = payload as EOTTestPayload; - if (eotPayload.eot === true) { - deferredTillEOT.resolve(); - return Promise.resolve(); - } - } - // Check the following for each time resolveExecution is called to collect any errors. - if (payload.status !== 'success') { - errorMessages.push("Expected status to be 'success'"); - } - if (payload.error !== null) { - errorMessages.push("Expected no errors in 'error' field"); - } - if (payload.result === null) { - errorMessages.push('Expected results to be present'); - } - return Promise.resolve(); - }); + // result resolver and saved data for assertions + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + return Promise.resolve(); + }; // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathLargeWorkspace); @@ -502,8 +413,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun From 22dc7814a38a99427ccd40c727ac7f40c4b0be75 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Mon, 28 Aug 2023 15:29:22 -0700 Subject: [PATCH 28/55] fix imports --- src/test/testing/common/testingAdapter.test.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 8b0452204a18..ace6b4ec90ef 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -6,12 +6,7 @@ import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; -import { - EOTTestPayload, - ExecutionTestPayload, - ITestController, - ITestResultResolver, -} from '../../../client/testing/testController/common/types'; +import { ITestController, ITestResultResolver } from '../../../client/testing/testController/common/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { IPythonExecutionFactory } from '../../../client/common/process/types'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; From 397f3c5149ff4b71151af781bd5ee93b97d396af Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 29 Aug 2023 08:10:33 -0700 Subject: [PATCH 29/55] add sep function for server data process --- .../testing/testController/common/server.ts | 73 ++++++++++--------- 1 file changed, 37 insertions(+), 36 deletions(-) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 860017944665..a6e6b19525bc 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -33,43 +33,9 @@ export class PythonTestServer implements ITestServer, Disposable { constructor(private executionFactory: IPythonExecutionFactory, private debugLauncher: ITestDebugLauncher) { this.server = net.createServer((socket: net.Socket) => { - let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data + const buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data socket.on('data', (data: Buffer) => { - try { - console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); - buffer = Buffer.concat([buffer, data]); - while (buffer.length > 0) { - try { - const extractedJsonPayload = extractJsonPayload(buffer.toString(), this.uuids); - if ( - extractedJsonPayload.uuid !== undefined && - extractedJsonPayload.cleanedJsonData !== undefined - ) { - // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. - this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); - } - buffer = Buffer.from(extractedJsonPayload.remainingRawData); - if (buffer.length === 0) { - // if the buffer is empty, then there is no more data to process. - // break to get more data from the socket. - buffer = Buffer.alloc(0); - break; - } - if (!containsHeaders(extractedJsonPayload.remainingRawData)) { - // if the remaining data does not contain headers, then there is no more data to process. - // break to get more data from the socket. - break; - } - } catch (ex) { - traceError(`Error:: ${ex}`); - this._onDataReceived.fire({ uuid: '', data: '' }); - return; - } - } - } catch (ex) { - traceError(`Error processing test server request: ${ex} observe`); - this._onDataReceived.fire({ uuid: '', data: '' }); - } + this._resolveData(buffer, data); }); }); this.ready = new Promise((resolve, _reject) => { @@ -91,6 +57,41 @@ export class PythonTestServer implements ITestServer, Disposable { }); } + public _resolveData(buffer: Buffer, data: Buffer): void { + try { + console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); + buffer = Buffer.concat([buffer, data]); + while (buffer.length > 0) { + try { + const extractedJsonPayload = extractJsonPayload(buffer.toString(), this.uuids); + if (extractedJsonPayload.uuid !== undefined && extractedJsonPayload.cleanedJsonData !== undefined) { + // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. + this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); + } + buffer = Buffer.from(extractedJsonPayload.remainingRawData); + if (buffer.length === 0) { + // if the buffer is empty, then there is no more data to process. + // break to get more data from the socket. + buffer = Buffer.alloc(0); + break; + } + if (!containsHeaders(extractedJsonPayload.remainingRawData)) { + // if the remaining data does not contain headers, then there is no more data to process. + // break to get more data from the socket. + break; + } + } catch (ex) { + traceError(`Error:: ${ex}`); + this._onDataReceived.fire({ uuid: '', data: '' }); + return; + } + } + } catch (ex) { + traceError(`Error processing test server request: ${ex} observe`); + this._onDataReceived.fire({ uuid: '', data: '' }); + } + } + private _fireDataReceived(uuid: string, extractedJSON: string): void { if (extractedJSON.includes(`"tests":`) || extractedJSON.includes(`"command_type": "discovery"`)) { this._onDiscoveryDataReceived.fire({ From 0a45e5b91612435567e970d9c89ce9dc92edba93 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 7 Sep 2023 13:15:30 -0400 Subject: [PATCH 30/55] most tests working, need to fix the buffer --- .vscode/launch.json | 5 +- .../testing/testController/common/server.ts | 11 +- .../pytest/pytestDiscoveryAdapter.ts | 1 - .../pytest/pytestExecutionAdapter.ts | 2 +- src/test/.vscode/settings.json | 3 +- .../testing/common/testingPayloadsEot.test.ts | 199 +++ .../testing/testController/helper.server.ts | 568 ++++++++ .../testController/payloadTestCases.ts | 164 ++- .../testController/server.unit.test.ts | 1141 +++++++++-------- 9 files changed, 1515 insertions(+), 579 deletions(-) create mode 100644 src/test/testing/common/testingPayloadsEot.test.ts create mode 100644 src/test/testing/testController/helper.server.ts diff --git a/.vscode/launch.json b/.vscode/launch.json index 82981a93305d..801fb604ba22 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -101,7 +101,7 @@ "--extensionTestsPath=${workspaceFolder}/out/test" ], "env": { - "VSC_PYTHON_CI_TEST_GREP": "" // Modify this to run a subset of the single workspace tests + "VSC_PYTHON_CI_TEST_GREP": "EOT tests" // Modify this to run a subset of the single workspace tests }, "stopOnEntry": false, "sourceMaps": true, @@ -160,7 +160,8 @@ "--ui=tdd", "--recursive", "--colors", - //"--grep", "", + "--grep", + "Python Test Server", "--timeout=300000" ], "outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"], diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index a6e6b19525bc..73d0bd5d4562 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -57,13 +57,17 @@ export class PythonTestServer implements ITestServer, Disposable { }); } + savedBuffer = ''; + public _resolveData(buffer: Buffer, data: Buffer): void { try { console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); buffer = Buffer.concat([buffer, data]); while (buffer.length > 0) { try { - const extractedJsonPayload = extractJsonPayload(buffer.toString(), this.uuids); + const bufferFromString = this.savedBuffer + buffer.toString(); + const extractedJsonPayload = extractJsonPayload(bufferFromString, this.uuids); + // what payload is so small it doesn't include the whole UUID if (extractedJsonPayload.uuid !== undefined && extractedJsonPayload.cleanedJsonData !== undefined) { // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); @@ -72,12 +76,15 @@ export class PythonTestServer implements ITestServer, Disposable { if (buffer.length === 0) { // if the buffer is empty, then there is no more data to process. // break to get more data from the socket. + this.savedBuffer = ''; buffer = Buffer.alloc(0); break; } - if (!containsHeaders(extractedJsonPayload.remainingRawData)) { + if (containsHeaders(extractedJsonPayload.remainingRawData)) { // if the remaining data does not contain headers, then there is no more data to process. // break to get more data from the socket. + // buffer = Buffer.alloc(0); + this.savedBuffer = extractedJsonPayload.remainingRawData; break; } } catch (ex) { diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index ca053f02c9c3..9c7c5fbb4400 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -2,7 +2,6 @@ // Licensed under the MIT License. import * as path from 'path'; import { Uri } from 'vscode'; -import { Disposable } from 'vscode-languageserver-protocol'; import { ExecutionFactoryCreateWithEnvironmentOptions, ExecutionResult, diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 89f8455fcaf2..0435e9e35181 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -59,7 +59,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { disposeDataReceiver(this.testServer); }); try { - await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, executionFactory, debugLauncher); + this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, executionFactory, debugLauncher); } finally { await deferredTillEOT.promise; disposeDataReceiver(this.testServer); diff --git a/src/test/.vscode/settings.json b/src/test/.vscode/settings.json index ef9292849a9d..250ed86de0e3 100644 --- a/src/test/.vscode/settings.json +++ b/src/test/.vscode/settings.json @@ -15,5 +15,6 @@ "python.formatting.provider": "yapf", // Don't set this to `Pylance`, for CI we want to use the LS that ships with the extension. "python.languageServer": "Jedi", - "python.pythonPath": "C:\\GIT\\s p\\vscode-python\\.venv\\Scripts\\python.exe" + "python.pythonPath": "C:\\GIT\\s p\\vscode-python\\.venv\\Scripts\\python.exe", + "python.defaultInterpreterPath": "python" } diff --git a/src/test/testing/common/testingPayloadsEot.test.ts b/src/test/testing/common/testingPayloadsEot.test.ts new file mode 100644 index 000000000000..7c1f969e39c2 --- /dev/null +++ b/src/test/testing/common/testingPayloadsEot.test.ts @@ -0,0 +1,199 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { TestController, TestRun, Uri } from 'vscode'; +import * as typeMoq from 'typemoq'; +import * as path from 'path'; +import * as assert from 'assert'; +import * as net from 'net'; +import { Observable } from 'rxjs'; +import * as crypto from 'crypto'; +// import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; +import * as sinon from 'sinon'; +import { ITestController, ITestResultResolver } from '../../../client/testing/testController/common/types'; +import { PythonTestServer } from '../../../client/testing/testController/common/server'; +import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; +import { ITestDebugLauncher } from '../../../client/testing/common/types'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { IServiceContainer } from '../../../client/ioc/types'; +import { EXTENSION_ROOT_DIR_FOR_TESTS, initialize } from '../../initialize'; +import { traceLog } from '../../../client/logging'; +import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; +// import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; +// import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; +import { PythonResultResolver } from '../../../client/testing/testController/common/resultResolver'; +// import { TestProvider } from '../../../client/testing/types'; +import { PYTEST_PROVIDER } from '../../../client/testing/common/constants'; +import { MockChildProcess } from '../../mocks/mockChildProcess'; +import { + PAYLOAD_SINGLE_CHUNK, + PAYLOAD_MULTI_CHUNK, + PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY, + DataWithPayloadChunks, + PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY, +} from '../testController/payloadTestCases'; +import { sleep } from '../../core'; + +const FAKE_UUID = 'fake-u-u-i-d'; +export interface TestCase { + name: string; + value: DataWithPayloadChunks; +} + +const testCases: Array = [ + // { name: 'single payload single chunk', value: PAYLOAD_SINGLE_CHUNK(FAKE_UUID) }, + // { name: 'multiple payloads per buffer chunk', value: PAYLOAD_MULTI_CHUNK(FAKE_UUID) }, + // { name: 'single payload across multiple buffer chunks', value: PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(FAKE_UUID) }, + { + name: 'two chunks, payload split and two payloads in a chunk', + value: PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY(FAKE_UUID), + }, +]; + +suite('EOT tests', () => { + let resultResolver: ITestResultResolver; + let pythonTestServer: PythonTestServer; + let debugLauncher: ITestDebugLauncher; + let configService: IConfigurationService; + let serviceContainer: IServiceContainer; + let workspaceUri: Uri; + let testOutputChannel: typeMoq.IMock; + let testController: TestController; + let stubExecutionFactory: typeMoq.IMock; + let client: net.Socket; + const sandbox = sinon.createSandbox(); + // const unittestProvider: TestProvider = UNITTEST_PROVIDER; + // const pytestProvider: TestProvider = PYTEST_PROVIDER; + const rootPathSmallWorkspace = path.join('src'); + suiteSetup(async () => { + serviceContainer = (await initialize()).serviceContainer; + }); + + setup(async () => { + // create objects that were injected + configService = serviceContainer.get(IConfigurationService); + debugLauncher = serviceContainer.get(ITestDebugLauncher); + testController = serviceContainer.get(ITestController); + + // create client to act as python server which sends testing result response + client = new net.Socket(); + client.on('error', (error) => { + console.log('Socket connection error:', error); + }); + + const mockProc = new MockChildProcess('', ['']); + const output2 = new Observable>(() => { + /* no op */ + }); + + // stub out execution service and factory so mock data is returned from client. + const stubExecutionService = ({ + execObservable: () => { + client.connect(pythonTestServer.getPort()); + return { + proc: mockProc, + out: output2, + dispose: () => { + /* no-body */ + }, + }; + }, + } as unknown) as IPythonExecutionService; + + stubExecutionFactory = typeMoq.Mock.ofType(); + stubExecutionFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(stubExecutionService)); + + // stub create UUID + + const v4Stub = sandbox.stub(crypto, 'randomUUID'); + v4Stub.returns(FAKE_UUID); + + // create python test server + pythonTestServer = new PythonTestServer(stubExecutionFactory.object, debugLauncher); + await pythonTestServer.serverReady(); + // handles output from client + testOutputChannel = typeMoq.Mock.ofType(); + testOutputChannel + .setup((x) => x.append(typeMoq.It.isAny())) + .callback((appendVal: any) => { + console.log('out - ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); + testOutputChannel + .setup((x) => x.appendLine(typeMoq.It.isAny())) + .callback((appendVal: any) => { + console.log('outL - ', appendVal.toString()); + }) + .returns(() => { + // Whatever you need to return + }); + }); + teardown(async () => { + pythonTestServer.dispose(); + sandbox.restore(); + }); + testCases.forEach((testCase) => { + test(`Testing Payloads: ${testCase.name}`, async () => { + let actualCollectedResult = ''; + client.on('connect', async () => { + console.log('socket connected, sending stubbed data'); + // payload is a string array, each string represents one line written to the buffer + const { payloadArray } = testCase.value; + for (let i = 0; i < payloadArray.length; i = i + 1) { + client.write(payloadArray[i]); + await sleep(3); + } + client.end(); + }); + + resultResolver = new PythonResultResolver(testController, PYTEST_PROVIDER, workspaceUri); + resultResolver._resolveExecution = async (payload, _token?) => { + // the payloads that get to the _resolveExecution are all data and should be successful. + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + actualCollectedResult = actualCollectedResult + JSON.stringify(payload.result); + return Promise.resolve(); + }; + // set workspace to test workspace folder + workspaceUri = Uri.parse(rootPathSmallWorkspace); + + // run pytest execution + const executionAdapter = new PytestTestExecutionAdapter( + pythonTestServer, + configService, + testOutputChannel.object, + resultResolver, + ); + const testRun = typeMoq.Mock.ofType(); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); + await executionAdapter + .runTests( + workspaceUri, + [`${rootPathSmallWorkspace}/test_simple.py::test_a`], + false, + testRun.object, + stubExecutionFactory.object, + ) + .then(() => { + assert.strictEqual( + testCase.value.data, + actualCollectedResult, + "Expected collected result to match 'data'", + ); + // verify that the _resolveExecution was called once per test + // assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + }); + }); + }); +}); diff --git a/src/test/testing/testController/helper.server.ts b/src/test/testing/testController/helper.server.ts new file mode 100644 index 000000000000..d6c9467e9988 --- /dev/null +++ b/src/test/testing/testController/helper.server.ts @@ -0,0 +1,568 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import * as assert from 'assert'; +import * as net from 'net'; +import * as sinon from 'sinon'; +import * as crypto from 'crypto'; +import { Observable } from 'rxjs'; +import * as typeMoq from 'typemoq'; +import { Uri } from 'vscode'; +import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; +import { PythonTestServer } from '../../../client/testing/testController/common/server'; +import { ITestDebugLauncher } from '../../../client/testing/common/types'; +import { Deferred, createDeferred } from '../../../client/common/utils/async'; +import { MockChildProcess } from '../../mocks/mockChildProcess'; +import { PAYLOAD_MULTI_CHUNK, PAYLOAD_SINGLE_CHUNK, PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY } from './payloadTestCases'; + +suite('Python Test Server', () => { + const FAKE_UUID = 'fake-uuid'; + let server: PythonTestServer; + let v4Stub: sinon.SinonStub; + let debugLauncher: ITestDebugLauncher; + let mockProc: MockChildProcess; + let execService: typeMoq.IMock; + let deferred: Deferred; + const sandbox = sinon.createSandbox(); + + setup(async () => { + // set up test command options + + v4Stub = sandbox.stub(crypto, 'randomUUID'); + v4Stub.returns(FAKE_UUID); + + // set up exec service with child process + mockProc = new MockChildProcess('', ['']); + execService = typeMoq.Mock.ofType(); + const outputObservable = new Observable>(() => { + /* no op */ + }); + execService + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + proc: mockProc, + out: outputObservable, + dispose: () => { + /* no-body */ + }, + })); + execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + }); + + teardown(() => { + sandbox.restore(); + server.dispose(); + }); + test('basic payload', async () => { + let eventData = ''; + const client = new net.Socket(); + + deferred = createDeferred(); + mockProc = new MockChildProcess('', ['']); + const output2 = new Observable>(() => { + /* no op */ + }); + const stubExecutionService2 = ({ + execObservable: () => { + client.connect(server.getPort()); + return { + proc: mockProc, + out: output2, + dispose: () => { + /* no-body */ + }, + }; + }, + } as unknown) as IPythonExecutionService; + + const stubExecutionFactory2 = ({ + createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + } as unknown) as IPythonExecutionFactory; + server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + const uuid = server.createUUID(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid, + }; + + const dataWithPayloadChunks = PAYLOAD_SINGLE_CHUNK(uuid); + + await server.serverReady(); + + server.onRunDataReceived(({ data }) => { + eventData = eventData + data; + deferred.resolve(); + }); + client.on('connect', () => { + console.log('Socket connected, local port:', client.localPort); + // since this test is a single payload as a single chunk there should be a single line in the payload. + for (const line of dataWithPayloadChunks.payloadArray) { + client.write(line); + } + client.end(); + }); + client.on('error', (error) => { + console.log('Socket connection error:', error); + }); + + server.sendCommand(options); + await deferred.promise; + const expectedResult = dataWithPayloadChunks.data; + assert.deepStrictEqual(eventData, expectedResult); + }); + test('second payload', async () => { + let eventData = ''; + const client = new net.Socket(); + + deferred = createDeferred(); + mockProc = new MockChildProcess('', ['']); + const output2 = new Observable>(() => { + /* no op */ + }); + const stubExecutionService2 = ({ + execObservable: () => { + client.connect(server.getPort()); + return { + proc: mockProc, + out: output2, + dispose: () => { + /* no-body */ + }, + }; + }, + } as unknown) as IPythonExecutionService; + + const stubExecutionFactory2 = ({ + createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + } as unknown) as IPythonExecutionFactory; + server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + const uuid = server.createUUID(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid, + }; + const payloadChunk = PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid); + + await server.serverReady(); + + server.onRunDataReceived(({ data }) => { + eventData = eventData + data; + }); + client.on('connect', () => { + console.log('Socket connected, local port:', client.localPort); + for (const line of payloadChunk.payloadArray) { + client.write(line); + } + client.end(); + }); + client.on('close', () => { + console.log('Socket connection exit:'); + deferred.resolve(); + }); + client.on('error', (error) => { + console.log('Socket connection error:', error); + }); + + server.sendCommand(options); + await deferred.promise; + const expectedResult = payloadChunk.data; + assert.deepStrictEqual(eventData, expectedResult); + }); + + // test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { + // deferred2 = createDeferred(); + // execFactory = typeMoq.Mock.ofType(); + // execFactory + // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + // .returns(() => { + // deferred2.resolve(); + // return Promise.resolve(execService.object); + // }); + // server = new PythonTestServer(execFactory.object, debugLauncher); + // await server.serverReady(); + // server.sendCommand(BASE_TEST_COMMAND_OPTIONS, '56789'); + // // add in await and trigger + // await deferred2.promise; + // mockProc.trigger('close'); + + // const port = server.getPort(); + // const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; + // execService.verify((x) => x.execObservable(expectedArgs, BASE_SPAWN_OPTIONS), typeMoq.Times.once()); + // }); + + // test('sendCommand should write to an output channel if it is provided as an option', async () => { + // const output2: string[] = []; + // const outChannel = { + // appendLine: (str: string) => { + // output2.push(str); + // }, + // } as OutputChannel; + // const options = { + // command: { + // script: 'myscript', + // args: ['-foo', 'foo'], + // }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // outChannel, + // }; + // deferred = createDeferred(); + // execFactory = typeMoq.Mock.ofType(); + // execFactory + // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + // .returns(() => { + // deferred.resolve(); + // return Promise.resolve(execService.object); + // }); + + // server = new PythonTestServer(execFactory.object, debugLauncher); + // await server.serverReady(); + + // server.sendCommand(options); + // // add in await and trigger + // await deferred.promise; + // mockProc.trigger('close'); + + // const port = server.getPort(); + // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); + + // assert.deepStrictEqual(output2, [expected]); + // }); + + // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { + // let eventData: { status: string; errors: string[] } | undefined; + // stubExecutionService = ({ + // execObservable: () => { + // throw new Error('Failed to execute'); + // }, + // } as unknown) as IPythonExecutionService; + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + + // server = new PythonTestServer(stubExecutionFactory, debugLauncher); + // await server.serverReady(); + + // server.onDataReceived(({ data }) => { + // eventData = JSON.parse(data); + // }); + + // await server.sendCommand(options); + + // assert.notEqual(eventData, undefined); + // assert.deepStrictEqual(eventData?.status, 'error'); + // assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); + // }); + + // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { + // deferred2 = createDeferred(); + // execFactory = typeMoq.Mock.ofType(); + // execFactory + // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + // .returns(() => { + // deferred2.resolve(); + // return Promise.resolve(execService.object); + // }); + // server = new PythonTestServer(execFactory.object, debugLauncher); + // await server.serverReady(); + // let eventData: string | undefined; + // const client = new net.Socket(); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + // mockProc = new MockChildProcess('', ['']); + // const output2 = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output2, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + + // deferred = createDeferred(); + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // server.onDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write('malformed data'); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // // add in await and trigger + // await deferred.promise; + // mockProc.trigger('close'); + + // assert.deepStrictEqual(eventData, ''); + // }); + + // test('If the server doesnt recognize the UUID it should ignore it', async () => { + // let eventData: string | undefined; + // const client = new net.Socket(); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // server.onDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write('{"Request-uuid": "unknown-uuid"}'); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // assert.deepStrictEqual(eventData, ''); + // }); + + // // required to have "tests" or "results" + // // the heading length not being equal and yes being equal + // // multiple payloads + // test('Error if payload does not have a content length header', async () => { + // let eventData: string | undefined; + // const client = new net.Socket(); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // server.onDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write('{"not content length": "5"}'); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // assert.deepStrictEqual(eventData, ''); + // }); + + // const testData = [ + // { + // testName: 'fires discovery correctly on test payload', + // payload: `Content-Length: 52 + // Content-Type: application/json + // Request-uuid: UUID_HERE + + // {"cwd": "path", "status": "success", "tests": "xyz"}`, + // expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', + // }, + // // Add more test data as needed + // ]; + + // testData.forEach(({ testName, payload, expectedResult }) => { + // test(`test: ${testName}`, async () => { + // // Your test logic here + // let eventData: string | undefined; + // const client = new net.Socket(); + + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // await server.serverReady(); + // const uuid = server.createUUID(); + // payload = payload.replace('UUID_HERE', uuid); + // server.onDiscoveryDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write(payload); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // assert.deepStrictEqual(eventData, expectedResult); + // }); + // }); + + // test('Calls run resolver if the result header is in the payload', async () => { + // let eventData: string | undefined; + // const client = new net.Socket(); + + // deferred = createDeferred(); + // mockProc = new MockChildProcess('', ['']); + // const output = new Observable>(() => { + // /* no op */ + // }); + // const stubExecutionService2 = ({ + // execObservable: () => { + // client.connect(server.getPort()); + // return { + // proc: mockProc, + // out: output, + // dispose: () => { + // /* no-body */ + // }, + // }; + // }, + // } as unknown) as IPythonExecutionService; + + // const stubExecutionFactory2 = ({ + // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + // } as unknown) as IPythonExecutionFactory; + // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + // const options = { + // command: { script: 'myscript', args: ['-foo', 'foo'] }, + // workspaceFolder: Uri.file('/foo/bar'), + // cwd: '/foo/bar', + // uuid: FAKE_UUID, + // }; + + // await server.serverReady(); + // const uuid = server.createUUID(); + // server.onRunDataReceived(({ data }) => { + // eventData = data; + // deferred.resolve(); + // }); + + // const payload = `Content-Length: 87 + // Content-Type: application/json + // Request-uuid: ${uuid} + + // {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; + + // client.on('connect', () => { + // console.log('Socket connected, local port:', client.localPort); + // client.write(payload); + // client.end(); + // }); + // client.on('error', (error) => { + // console.log('Socket connection error:', error); + // }); + + // server.sendCommand(options); + // await deferred.promise; + // const expectedResult = + // '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; + // assert.deepStrictEqual(eventData, expectedResult); + // }); +}); diff --git a/src/test/testing/testController/payloadTestCases.ts b/src/test/testing/testController/payloadTestCases.ts index 2180a82ebca4..5ddcc0edecf9 100644 --- a/src/test/testing/testController/payloadTestCases.ts +++ b/src/test/testing/testController/payloadTestCases.ts @@ -1,86 +1,122 @@ -export interface PayloadChunk { - payload: string[]; +export interface DataWithPayloadChunks { + payloadArray: string[]; data: string; } -export function PAYLOAD_SINGLE_CHUNK(uuid: string): PayloadChunk { - const val = `{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}}`; - const payload = `Content-Length: 411 +const EOT_PAYLOAD = `Content-Length: 42 +Content-Type: application/json +Request-uuid: fake-u-u-i-d + +{"command_type": "execution", "eot": true}`; + +const SINGLE_UNITTEST_SUBTEST = { + cwd: '/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace', + status: 'success', + result: { + 'test_parameterized_subtest.NumbersTest.test_even (i=0)': { + test: 'test_parameterized_subtest.NumbersTest.test_even', + outcome: 'success', + message: 'None', + traceback: null, + subtest: 'test_parameterized_subtest.NumbersTest.test_even (i=0)', + }, + }, +}; + +const SINGLE_PYTEST_PAYLOAD = { + cwd: 'path/to', + status: 'success', + result: { + 'path/to/file.py::test_funct': { + test: 'path/to/file.py::test_funct', + outcome: 'success', + message: 'None', + traceback: null, + subtest: 'path/to/file.py::test_funct', + }, + }, +}; + +const SINGLE_PYTEST_PAYLOAD_TWO = { + cwd: 'path/to/second', + status: 'success', + result: { + 'path/to/workspace/parametrize_tests.py::test_adding[3+5-8]': { + test: 'path/to/workspace/parametrize_tests.py::test_adding[3+5-8]', + outcome: 'success', + message: 'None', + traceback: null, + }, + }, +}; + +export function createPayload(uuid: string, data: unknown): string { + return `Content-Length: ${JSON.stringify(data).length} Content-Type: application/json Request-uuid: ${uuid} -${val}`; +${JSON.stringify(data)}`; +} + +export function PAYLOAD_SINGLE_CHUNK(uuid: string): DataWithPayloadChunks { + const payload = createPayload(uuid, SINGLE_UNITTEST_SUBTEST); + return { - payload: [payload], - data: val, + payloadArray: [payload, EOT_PAYLOAD], + data: JSON.stringify(SINGLE_UNITTEST_SUBTEST.result), }; } -export function PAYLOAD_MULTI_CHUNK(uuid: string): PayloadChunk { - const val = `{abc}`; +// more than one payload (item with header) per chunk sent +// payload has 3 SINGLE_UNITTEST_SUBTEST +export function PAYLOAD_MULTI_CHUNK(uuid: string): DataWithPayloadChunks { + let payload = ''; + let result = ''; + for (let i = 0; i < 3; i = i + 1) { + payload += createPayload(uuid, SINGLE_UNITTEST_SUBTEST); + result += JSON.stringify(SINGLE_UNITTEST_SUBTEST.result); + } return { - payload: [ - `Content-Length: 5 -Content-Type: application/json -Request-uuid: ${uuid} - -${val}Content-Length: 5 -Content-Type: application/json -Request-uuid: ${uuid} - -${val}Content-Length: 5 -Content-Type: application/json -Request-uuid: ${uuid} - -${val}Content-Length: 5 -Content-Type: application/json -Request-uuid: ${uuid} - -${val}`, - ], - data: val + val + val + val, + payloadArray: [payload, EOT_PAYLOAD], + data: result, }; } -export function PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid: string): Array { - return [ - `Content-Length: 411 -Content-Type: application/json -Request-uuid: ${uuid} - -{"cwd": "/home/runner/work/vsc`, - `ode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.te`, - `st_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}}`, +// single payload divided by an arbitrary character and split across payloads +export function PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid: string): DataWithPayloadChunks { + const payload = createPayload(uuid, SINGLE_PYTEST_PAYLOAD); + // payload length is know to be >200 + const splitPayload: Array = [ + payload.substring(0, 50), + payload.substring(50, 100), + payload.substring(100, 150), + payload.substring(150), ]; + const finalResult = JSON.stringify(SINGLE_PYTEST_PAYLOAD.result); + splitPayload.push(EOT_PAYLOAD); + return { + payloadArray: splitPayload, + data: finalResult, + }; } -export function PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY(uuid: string): Array { - return [ - `Content-Length: 411 -Content-Type: application/json -Request-uuid: ${uuid} - -{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=0)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=0)"}}} - -Content-Length: 411 -Content-Type: application/json -Request-uuid: ${uuid} - -{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src" +// here a payload is split across the buffer chunks and there are multiple payloads in a single buffer chunk +export function PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY(uuid: string): DataWithPayloadChunks { + // payload1 length is know to be >200 + const payload1 = createPayload(uuid, SINGLE_PYTEST_PAYLOAD); + const payload2 = createPayload(uuid, SINGLE_PYTEST_PAYLOAD_TWO); -Content-Length: 959 -Content-Type: application/json -Request-uuid: ${uuid} + // chunk 1 is 50 char of payload1, chunk 2 is 50-end of payload1 and all of payload2 + const splitPayload: Array = [payload1.substring(0, 100), payload1.substring(100).concat(payload2)]; + const finalResult = JSON.stringify(SINGLE_PYTEST_PAYLOAD.result).concat( + JSON.stringify(SINGLE_PYTEST_PAYLOAD_TWO.result), + ); -{"cwd": "/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-failure", "result": {"test_parameterized_subtest.NumbersTest.test_even (i=1)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-failure", "message": "(, AssertionError('1 != 0'), )", "traceback": " File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 57, in testPartExecutor\n yield\n File \"/opt/hostedtoolcache/Python/3.11.4/x64/lib/python3.11/unittest/case.py\", line 538, in subTest\n yield\n File \"/home/runner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py\", line 16, in test_even\n self.assertEqual(i % 2, 0)\nAssertionError: 1 != 0\n", "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=1)"}}} -Content-Length: 411 -Content-Type: application/json -Request-uuid: ${uuid} - -{"cwd": "/home/ru`, - `nner/work/vscode-python/vscode-python/path with spaces/src/testTestingRootWkspc/largeWorkspace", "status": "subtest-success", "result": {"test_parameterized_subtest.Numbers`, - `Test.test_even (i=2)": {"test": "test_parameterized_subtest.NumbersTest.test_even", "outcome": "subtest-success", "message": "None", "traceback": null, "subtest": "test_parameterized_subtest.NumbersTest.test_even (i=2)"}}}`, - ]; + splitPayload.push(EOT_PAYLOAD); + return { + payloadArray: splitPayload, + data: finalResult, + }; } export function PAYLOAD_SPLIT_MULTI_CHUNK_RAN_ORDER_ARRAY(uuid: string): Array { diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index 594c4ed287b1..563f6b5bed38 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -12,9 +12,26 @@ import { Uri } from 'vscode'; import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; -import { Deferred, createDeferred } from '../../../client/common/utils/async'; +import { createDeferred } from '../../../client/common/utils/async'; import { MockChildProcess } from '../../mocks/mockChildProcess'; -import { PAYLOAD_MULTI_CHUNK, PAYLOAD_SINGLE_CHUNK } from './payloadTestCases'; +import { + PAYLOAD_MULTI_CHUNK, + PAYLOAD_SINGLE_CHUNK, + PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY, + DataWithPayloadChunks, +} from './payloadTestCases'; + +const testCases = [ + { + val: () => PAYLOAD_SINGLE_CHUNK('fake-uuid'), + }, + { + val: () => PAYLOAD_MULTI_CHUNK('fake-uuid'), + }, + { + val: () => PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY('fake-uuid'), + }, +]; suite('Python Test Server', () => { const FAKE_UUID = 'fake-uuid'; @@ -54,513 +71,621 @@ suite('Python Test Server', () => { sandbox.restore(); server.dispose(); }); - test('basic payload', async () => { - let eventData = ''; - const client = new net.Socket(); - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output2 = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output2, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - const uuid = server.createUUID(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid, - }; - const payloadChunk = PAYLOAD_SINGLE_CHUNK(uuid); - - await server.serverReady(); - - server.onRunDataReceived(({ data }) => { - eventData = eventData + data; - deferred.resolve(); - }); - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - for (const line of payloadChunk.payload) { - client.write(line); - } - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - const expectedResult = payloadChunk.data; - assert.deepStrictEqual(eventData, expectedResult); - }); - test('second payload', async () => { - let eventData = ''; - const client = new net.Socket(); - - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output2 = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output2, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - const uuid = server.createUUID(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid, - }; - const payloadChunk = PAYLOAD_MULTI_CHUNK(uuid); - - await server.serverReady(); - - server.onRunDataReceived(({ data }) => { - eventData = eventData + data; - }); - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - for (const line of payloadChunk.payload) { - client.write(line); - } - client.end(); - }); - client.on('close', () => { - console.log('Socket connection exit:'); - deferred.resolve(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); + testCases.forEach((testCase) => { + test(`run correctly`, async () => { + const testCaseDataObj: DataWithPayloadChunks = testCase.val(); + let eventData = ''; + const client = new net.Socket(); + + deferred = createDeferred(); + mockProc = new MockChildProcess('', ['']); + const output2 = new Observable>(() => { + /* no op */ + }); + const stubExecutionService2 = ({ + execObservable: () => { + client.connect(server.getPort()); + return { + proc: mockProc, + out: output2, + dispose: () => { + /* no-body */ + }, + }; + }, + } as unknown) as IPythonExecutionService; + + const stubExecutionFactory2 = ({ + createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), + } as unknown) as IPythonExecutionFactory; + server = new PythonTestServer(stubExecutionFactory2, debugLauncher); + const uuid = server.createUUID(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid, + }; + + const dataWithPayloadChunks = testCaseDataObj; + + await server.serverReady(); + + server.onRunDataReceived(({ data }) => { + eventData = eventData + data; + deferred.resolve(); + }); + client.on('connect', () => { + console.log('Socket connected, local port:', client.localPort); + // since this test is a single payload as a single chunk there should be a single line in the payload. + for (const line of dataWithPayloadChunks.payloadArray) { + client.write(line); + } + client.end(); + }); + client.on('error', (error) => { + console.log('Socket connection error:', error); + }); + + server.sendCommand(options); + await deferred.promise; + const expectedResult = dataWithPayloadChunks.data; + assert.deepStrictEqual(eventData, expectedResult); }); - - server.sendCommand(options); - await deferred.promise; - const expectedResult = payloadChunk.data; - assert.deepStrictEqual(eventData, expectedResult); }); - - // test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { - // deferred2 = createDeferred(); - // execFactory = typeMoq.Mock.ofType(); - // execFactory - // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - // .returns(() => { - // deferred2.resolve(); - // return Promise.resolve(execService.object); - // }); - // server = new PythonTestServer(execFactory.object, debugLauncher); - // await server.serverReady(); - // server.sendCommand(BASE_TEST_COMMAND_OPTIONS, '56789'); - // // add in await and trigger - // await deferred2.promise; - // mockProc.trigger('close'); - - // const port = server.getPort(); - // const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; - // execService.verify((x) => x.execObservable(expectedArgs, BASE_SPAWN_OPTIONS), typeMoq.Times.once()); - // }); - - // test('sendCommand should write to an output channel if it is provided as an option', async () => { - // const output2: string[] = []; - // const outChannel = { - // appendLine: (str: string) => { - // output2.push(str); - // }, - // } as OutputChannel; - // const options = { - // command: { - // script: 'myscript', - // args: ['-foo', 'foo'], - // }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // outChannel, - // }; - // deferred = createDeferred(); - // execFactory = typeMoq.Mock.ofType(); - // execFactory - // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - // .returns(() => { - // deferred.resolve(); - // return Promise.resolve(execService.object); - // }); - - // server = new PythonTestServer(execFactory.object, debugLauncher); - // await server.serverReady(); - - // server.sendCommand(options); - // // add in await and trigger - // await deferred.promise; - // mockProc.trigger('close'); - - // const port = server.getPort(); - // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); - - // assert.deepStrictEqual(output2, [expected]); - // }); - - // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { - // let eventData: { status: string; errors: string[] } | undefined; - // stubExecutionService = ({ - // execObservable: () => { - // throw new Error('Failed to execute'); - // }, - // } as unknown) as IPythonExecutionService; - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - - // server = new PythonTestServer(stubExecutionFactory, debugLauncher); - // await server.serverReady(); - - // server.onDataReceived(({ data }) => { - // eventData = JSON.parse(data); - // }); - - // await server.sendCommand(options); - - // assert.notEqual(eventData, undefined); - // assert.deepStrictEqual(eventData?.status, 'error'); - // assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); - // }); - - // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { - // deferred2 = createDeferred(); - // execFactory = typeMoq.Mock.ofType(); - // execFactory - // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - // .returns(() => { - // deferred2.resolve(); - // return Promise.resolve(execService.object); - // }); - // server = new PythonTestServer(execFactory.object, debugLauncher); - // await server.serverReady(); - // let eventData: string | undefined; - // const client = new net.Socket(); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - // mockProc = new MockChildProcess('', ['']); - // const output2 = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output2, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - - // deferred = createDeferred(); - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // server.onDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write('malformed data'); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // // add in await and trigger - // await deferred.promise; - // mockProc.trigger('close'); - - // assert.deepStrictEqual(eventData, ''); - // }); - - // test('If the server doesnt recognize the UUID it should ignore it', async () => { - // let eventData: string | undefined; - // const client = new net.Socket(); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // server.onDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write('{"Request-uuid": "unknown-uuid"}'); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // assert.deepStrictEqual(eventData, ''); - // }); - - // // required to have "tests" or "results" - // // the heading length not being equal and yes being equal - // // multiple payloads - // test('Error if payload does not have a content length header', async () => { - // let eventData: string | undefined; - // const client = new net.Socket(); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // server.onDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write('{"not content length": "5"}'); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // assert.deepStrictEqual(eventData, ''); - // }); - - // const testData = [ - // { - // testName: 'fires discovery correctly on test payload', - // payload: `Content-Length: 52 - // Content-Type: application/json - // Request-uuid: UUID_HERE - - // {"cwd": "path", "status": "success", "tests": "xyz"}`, - // expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', - // }, - // // Add more test data as needed - // ]; - - // testData.forEach(({ testName, payload, expectedResult }) => { - // test(`test: ${testName}`, async () => { - // // Your test logic here - // let eventData: string | undefined; - // const client = new net.Socket(); - - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // const uuid = server.createUUID(); - // payload = payload.replace('UUID_HERE', uuid); - // server.onDiscoveryDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write(payload); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // assert.deepStrictEqual(eventData, expectedResult); - // }); - // }); - - // test('Calls run resolver if the result header is in the payload', async () => { - // let eventData: string | undefined; - // const client = new net.Socket(); - - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - - // await server.serverReady(); - // const uuid = server.createUUID(); - // server.onRunDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // const payload = `Content-Length: 87 - // Content-Type: application/json - // Request-uuid: ${uuid} - - // {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write(payload); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // const expectedResult = - // '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; - // assert.deepStrictEqual(eventData, expectedResult); - // }); }); + +// need to add EOT token +// why is it finishing for single chunk without the EOT token? + +// suite('Python Test Server', () => { +// const FAKE_UUID = 'fake-uuid'; +// let server: PythonTestServer; +// let v4Stub: sinon.SinonStub; +// let debugLauncher: ITestDebugLauncher; +// let mockProc: MockChildProcess; +// let execService: typeMoq.IMock; +// let deferred: Deferred; +// const sandbox = sinon.createSandbox(); + +// setup(async () => { +// // set up test command options + +// v4Stub = sandbox.stub(crypto, 'randomUUID'); +// v4Stub.returns(FAKE_UUID); + +// // set up exec service with child process +// mockProc = new MockChildProcess('', ['']); +// execService = typeMoq.Mock.ofType(); +// const outputObservable = new Observable>(() => { +// /* no op */ +// }); +// execService +// .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) +// .returns(() => ({ +// proc: mockProc, +// out: outputObservable, +// dispose: () => { +// /* no-body */ +// }, +// })); +// execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); +// }); + +// teardown(() => { +// sandbox.restore(); +// server.dispose(); +// }); +// test('basic payload', async () => { +// let eventData = ''; +// const client = new net.Socket(); + +// deferred = createDeferred(); +// mockProc = new MockChildProcess('', ['']); +// const output2 = new Observable>(() => { +// /* no op */ +// }); +// const stubExecutionService2 = ({ +// execObservable: () => { +// client.connect(server.getPort()); +// return { +// proc: mockProc, +// out: output2, +// dispose: () => { +// /* no-body */ +// }, +// }; +// }, +// } as unknown) as IPythonExecutionService; + +// const stubExecutionFactory2 = ({ +// createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), +// } as unknown) as IPythonExecutionFactory; +// server = new PythonTestServer(stubExecutionFactory2, debugLauncher); +// const uuid = server.createUUID(); +// const options = { +// command: { script: 'myscript', args: ['-foo', 'foo'] }, +// workspaceFolder: Uri.file('/foo/bar'), +// cwd: '/foo/bar', +// uuid, +// }; + +// const dataWithPayloadChunks = PAYLOAD_SINGLE_CHUNK(uuid); + +// await server.serverReady(); + +// server.onRunDataReceived(({ data }) => { +// eventData = eventData + data; +// deferred.resolve(); +// }); +// client.on('connect', () => { +// console.log('Socket connected, local port:', client.localPort); +// // since this test is a single payload as a single chunk there should be a single line in the payload. +// for (const line of dataWithPayloadChunks.payload) { +// client.write(line); +// } +// client.end(); +// }); +// client.on('error', (error) => { +// console.log('Socket connection error:', error); +// }); + +// server.sendCommand(options); +// await deferred.promise; +// const expectedResult = dataWithPayloadChunks.data; +// assert.deepStrictEqual(eventData, expectedResult); +// }); +// test('second payload', async () => { +// let eventData = ''; +// const client = new net.Socket(); + +// deferred = createDeferred(); +// mockProc = new MockChildProcess('', ['']); +// const output2 = new Observable>(() => { +// /* no op */ +// }); +// const stubExecutionService2 = ({ +// execObservable: () => { +// client.connect(server.getPort()); +// return { +// proc: mockProc, +// out: output2, +// dispose: () => { +// /* no-body */ +// }, +// }; +// }, +// } as unknown) as IPythonExecutionService; + +// const stubExecutionFactory2 = ({ +// createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), +// } as unknown) as IPythonExecutionFactory; +// server = new PythonTestServer(stubExecutionFactory2, debugLauncher); +// const uuid = server.createUUID(); +// const options = { +// command: { script: 'myscript', args: ['-foo', 'foo'] }, +// workspaceFolder: Uri.file('/foo/bar'), +// cwd: '/foo/bar', +// uuid, +// }; +// const payloadChunk = PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid); + +// await server.serverReady(); + +// server.onRunDataReceived(({ data }) => { +// eventData = eventData + data; +// }); +// client.on('connect', () => { +// console.log('Socket connected, local port:', client.localPort); +// for (const line of payloadChunk.payload) { +// client.write(line); +// } +// client.end(); +// }); +// client.on('close', () => { +// console.log('Socket connection exit:'); +// deferred.resolve(); +// }); +// client.on('error', (error) => { +// console.log('Socket connection error:', error); +// }); + +// server.sendCommand(options); +// await deferred.promise; +// const expectedResult = payloadChunk.data; +// assert.deepStrictEqual(eventData, expectedResult); +// }); + +// // test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { +// // deferred2 = createDeferred(); +// // execFactory = typeMoq.Mock.ofType(); +// // execFactory +// // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) +// // .returns(() => { +// // deferred2.resolve(); +// // return Promise.resolve(execService.object); +// // }); +// // server = new PythonTestServer(execFactory.object, debugLauncher); +// // await server.serverReady(); +// // server.sendCommand(BASE_TEST_COMMAND_OPTIONS, '56789'); +// // // add in await and trigger +// // await deferred2.promise; +// // mockProc.trigger('close'); + +// // const port = server.getPort(); +// // const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; +// // execService.verify((x) => x.execObservable(expectedArgs, BASE_SPAWN_OPTIONS), typeMoq.Times.once()); +// // }); + +// // test('sendCommand should write to an output channel if it is provided as an option', async () => { +// // const output2: string[] = []; +// // const outChannel = { +// // appendLine: (str: string) => { +// // output2.push(str); +// // }, +// // } as OutputChannel; +// // const options = { +// // command: { +// // script: 'myscript', +// // args: ['-foo', 'foo'], +// // }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: FAKE_UUID, +// // outChannel, +// // }; +// // deferred = createDeferred(); +// // execFactory = typeMoq.Mock.ofType(); +// // execFactory +// // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) +// // .returns(() => { +// // deferred.resolve(); +// // return Promise.resolve(execService.object); +// // }); + +// // server = new PythonTestServer(execFactory.object, debugLauncher); +// // await server.serverReady(); + +// // server.sendCommand(options); +// // // add in await and trigger +// // await deferred.promise; +// // mockProc.trigger('close'); + +// // const port = server.getPort(); +// // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); + +// // assert.deepStrictEqual(output2, [expected]); +// // }); + +// // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { +// // let eventData: { status: string; errors: string[] } | undefined; +// // stubExecutionService = ({ +// // execObservable: () => { +// // throw new Error('Failed to execute'); +// // }, +// // } as unknown) as IPythonExecutionService; +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: FAKE_UUID, +// // }; + +// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); +// // await server.serverReady(); + +// // server.onDataReceived(({ data }) => { +// // eventData = JSON.parse(data); +// // }); + +// // await server.sendCommand(options); + +// // assert.notEqual(eventData, undefined); +// // assert.deepStrictEqual(eventData?.status, 'error'); +// // assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); +// // }); + +// // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { +// // deferred2 = createDeferred(); +// // execFactory = typeMoq.Mock.ofType(); +// // execFactory +// // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) +// // .returns(() => { +// // deferred2.resolve(); +// // return Promise.resolve(execService.object); +// // }); +// // server = new PythonTestServer(execFactory.object, debugLauncher); +// // await server.serverReady(); +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: FAKE_UUID, +// // }; +// // mockProc = new MockChildProcess('', ['']); +// // const output2 = new Observable>(() => { +// // /* no op */ +// // }); +// // const stubExecutionService2 = ({ +// // execObservable: () => { +// // client.connect(server.getPort()); +// // return { +// // proc: mockProc, +// // out: output2, +// // dispose: () => { +// // /* no-body */ +// // }, +// // }; +// // }, +// // } as unknown) as IPythonExecutionService; + +// // const stubExecutionFactory2 = ({ +// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), +// // } as unknown) as IPythonExecutionFactory; + +// // deferred = createDeferred(); +// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('malformed data'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // server.sendCommand(options); +// // // add in await and trigger +// // await deferred.promise; +// // mockProc.trigger('close'); + +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// // test('If the server doesnt recognize the UUID it should ignore it', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: FAKE_UUID, +// // }; + +// // deferred = createDeferred(); +// // mockProc = new MockChildProcess('', ['']); +// // const output = new Observable>(() => { +// // /* no op */ +// // }); +// // const stubExecutionService2 = ({ +// // execObservable: () => { +// // client.connect(server.getPort()); +// // return { +// // proc: mockProc, +// // out: output, +// // dispose: () => { +// // /* no-body */ +// // }, +// // }; +// // }, +// // } as unknown) as IPythonExecutionService; + +// // const stubExecutionFactory2 = ({ +// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), +// // } as unknown) as IPythonExecutionFactory; +// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('{"Request-uuid": "unknown-uuid"}'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// // // required to have "tests" or "results" +// // // the heading length not being equal and yes being equal +// // // multiple payloads +// // test('Error if payload does not have a content length header', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: FAKE_UUID, +// // }; +// // deferred = createDeferred(); +// // mockProc = new MockChildProcess('', ['']); +// // const output = new Observable>(() => { +// // /* no op */ +// // }); +// // const stubExecutionService2 = ({ +// // execObservable: () => { +// // client.connect(server.getPort()); +// // return { +// // proc: mockProc, +// // out: output, +// // dispose: () => { +// // /* no-body */ +// // }, +// // }; +// // }, +// // } as unknown) as IPythonExecutionService; + +// // const stubExecutionFactory2 = ({ +// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), +// // } as unknown) as IPythonExecutionFactory; +// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); +// // await server.serverReady(); +// // server.onDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write('{"not content length": "5"}'); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, ''); +// // }); + +// // const testData = [ +// // { +// // testName: 'fires discovery correctly on test payload', +// // payload: `Content-Length: 52 +// // Content-Type: application/json +// // Request-uuid: UUID_HERE + +// // {"cwd": "path", "status": "success", "tests": "xyz"}`, +// // expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', +// // }, +// // // Add more test data as needed +// // ]; + +// // testData.forEach(({ testName, payload, expectedResult }) => { +// // test(`test: ${testName}`, async () => { +// // // Your test logic here +// // let eventData: string | undefined; +// // const client = new net.Socket(); + +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: FAKE_UUID, +// // }; +// // deferred = createDeferred(); +// // mockProc = new MockChildProcess('', ['']); +// // const output = new Observable>(() => { +// // /* no op */ +// // }); +// // const stubExecutionService2 = ({ +// // execObservable: () => { +// // client.connect(server.getPort()); +// // return { +// // proc: mockProc, +// // out: output, +// // dispose: () => { +// // /* no-body */ +// // }, +// // }; +// // }, +// // } as unknown) as IPythonExecutionService; +// // const stubExecutionFactory2 = ({ +// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), +// // } as unknown) as IPythonExecutionFactory; + +// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); +// // await server.serverReady(); +// // const uuid = server.createUUID(); +// // payload = payload.replace('UUID_HERE', uuid); +// // server.onDiscoveryDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write(payload); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // server.sendCommand(options); +// // await deferred.promise; +// // assert.deepStrictEqual(eventData, expectedResult); +// // }); +// // }); + +// // test('Calls run resolver if the result header is in the payload', async () => { +// // let eventData: string | undefined; +// // const client = new net.Socket(); + +// // deferred = createDeferred(); +// // mockProc = new MockChildProcess('', ['']); +// // const output = new Observable>(() => { +// // /* no op */ +// // }); +// // const stubExecutionService2 = ({ +// // execObservable: () => { +// // client.connect(server.getPort()); +// // return { +// // proc: mockProc, +// // out: output, +// // dispose: () => { +// // /* no-body */ +// // }, +// // }; +// // }, +// // } as unknown) as IPythonExecutionService; + +// // const stubExecutionFactory2 = ({ +// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), +// // } as unknown) as IPythonExecutionFactory; +// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); +// // const options = { +// // command: { script: 'myscript', args: ['-foo', 'foo'] }, +// // workspaceFolder: Uri.file('/foo/bar'), +// // cwd: '/foo/bar', +// // uuid: FAKE_UUID, +// // }; + +// // await server.serverReady(); +// // const uuid = server.createUUID(); +// // server.onRunDataReceived(({ data }) => { +// // eventData = data; +// // deferred.resolve(); +// // }); + +// // const payload = `Content-Length: 87 +// // Content-Type: application/json +// // Request-uuid: ${uuid} + +// // {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; + +// // client.on('connect', () => { +// // console.log('Socket connected, local port:', client.localPort); +// // client.write(payload); +// // client.end(); +// // }); +// // client.on('error', (error) => { +// // console.log('Socket connection error:', error); +// // }); + +// // server.sendCommand(options); +// // await deferred.promise; +// // const expectedResult = +// // '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; +// // assert.deepStrictEqual(eventData, expectedResult); +// // }); +// }); From 505740bdb018ca3cf80711c850f643f56cd293b1 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 7 Sep 2023 14:07:24 -0400 Subject: [PATCH 31/55] 4 tests working --- .../testing/testController/common/server.ts | 67 +++++++++---------- .../testing/testController/common/utils.ts | 6 +- .../testing/common/testingPayloadsEot.test.ts | 32 ++++++--- 3 files changed, 57 insertions(+), 48 deletions(-) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 73d0bd5d4562..3aa6b5b80388 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -33,9 +33,26 @@ export class PythonTestServer implements ITestServer, Disposable { constructor(private executionFactory: IPythonExecutionFactory, private debugLauncher: ITestDebugLauncher) { this.server = net.createServer((socket: net.Socket) => { - const buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data + let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data socket.on('data', (data: Buffer) => { - this._resolveData(buffer, data); + console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); + buffer = Buffer.concat([buffer, data]); // get the new data and add it to the buffer + while (buffer.length > 0) { + try { + // try to resolve data, returned unresolved data + const remainingBuffer = this._resolveData(buffer); + if (remainingBuffer.length === buffer.length) { + // if the remaining buffer is exactly the same as the buffer before processing, + // then there is no more data to process so loop should be exited. + break; + } + buffer = remainingBuffer; + } catch (ex) { + traceError(`Error processing test server request: ${ex} observe`); + buffer = Buffer.alloc(0); + this._onDataReceived.fire({ uuid: '', data: '' }); + } + } }); }); this.ready = new Promise((resolve, _reject) => { @@ -59,44 +76,24 @@ export class PythonTestServer implements ITestServer, Disposable { savedBuffer = ''; - public _resolveData(buffer: Buffer, data: Buffer): void { + public _resolveData(buffer: Buffer): Buffer { try { - console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); - buffer = Buffer.concat([buffer, data]); - while (buffer.length > 0) { - try { - const bufferFromString = this.savedBuffer + buffer.toString(); - const extractedJsonPayload = extractJsonPayload(bufferFromString, this.uuids); - // what payload is so small it doesn't include the whole UUID - if (extractedJsonPayload.uuid !== undefined && extractedJsonPayload.cleanedJsonData !== undefined) { - // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. - this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); - } - buffer = Buffer.from(extractedJsonPayload.remainingRawData); - if (buffer.length === 0) { - // if the buffer is empty, then there is no more data to process. - // break to get more data from the socket. - this.savedBuffer = ''; - buffer = Buffer.alloc(0); - break; - } - if (containsHeaders(extractedJsonPayload.remainingRawData)) { - // if the remaining data does not contain headers, then there is no more data to process. - // break to get more data from the socket. - // buffer = Buffer.alloc(0); - this.savedBuffer = extractedJsonPayload.remainingRawData; - break; - } - } catch (ex) { - traceError(`Error:: ${ex}`); - this._onDataReceived.fire({ uuid: '', data: '' }); - return; - } + const extractedJsonPayload = extractJsonPayload(buffer.toString(), this.uuids); + // what payload is so small it doesn't include the whole UUID think got this + if (extractedJsonPayload.uuid !== undefined && extractedJsonPayload.cleanedJsonData !== undefined) { + // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. + this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); + } + buffer = Buffer.from(extractedJsonPayload.remainingRawData); + if (buffer.length === 0) { + // if the buffer is empty, then there is no more data to process so buffer should be cleared. + buffer = Buffer.alloc(0); } } catch (ex) { - traceError(`Error processing test server request: ${ex} observe`); + traceError(`Error:: ${ex}`); this._onDataReceived.fire({ uuid: '', data: '' }); } + return buffer; } private _fireDataReceived(uuid: string, extractedJSON: string): void { diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 5b814e432954..012ebfbfe140 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -77,11 +77,13 @@ export function containsHeaders(rawData: string): boolean { ); } -export function checkUuid(uuid: string | undefined, uuids: Array): string { +export function checkUuid(uuid: string | undefined, uuids: Array): string | undefined { if (!uuid) { - throw new Error('On data received: Error occurred because payload UUID is undefined'); + // no UUID found, this could occurred if the payload is full yet so send back without erroring + return undefined; } if (!uuids.includes(uuid)) { + // no UUID found, this could occurred if the payload is full yet so send back without erroring throw new Error('On data received: Error occurred because the payload UUID is not recognized'); } return uuid; diff --git a/src/test/testing/common/testingPayloadsEot.test.ts b/src/test/testing/common/testingPayloadsEot.test.ts index 7c1f969e39c2..84fbe8ee64f2 100644 --- a/src/test/testing/common/testingPayloadsEot.test.ts +++ b/src/test/testing/common/testingPayloadsEot.test.ts @@ -16,13 +16,9 @@ import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../ import { ITestDebugLauncher } from '../../../client/testing/common/types'; import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; import { IServiceContainer } from '../../../client/ioc/types'; -import { EXTENSION_ROOT_DIR_FOR_TESTS, initialize } from '../../initialize'; -import { traceLog } from '../../../client/logging'; +import { initialize } from '../../initialize'; import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; -// import { UnittestTestDiscoveryAdapter } from '../../../client/testing/testController/unittest/testDiscoveryAdapter'; -// import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; import { PythonResultResolver } from '../../../client/testing/testController/common/resultResolver'; -// import { TestProvider } from '../../../client/testing/types'; import { PYTEST_PROVIDER } from '../../../client/testing/common/constants'; import { MockChildProcess } from '../../mocks/mockChildProcess'; import { @@ -32,7 +28,6 @@ import { DataWithPayloadChunks, PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY, } from '../testController/payloadTestCases'; -import { sleep } from '../../core'; const FAKE_UUID = 'fake-u-u-i-d'; export interface TestCase { @@ -41,9 +36,18 @@ export interface TestCase { } const testCases: Array = [ - // { name: 'single payload single chunk', value: PAYLOAD_SINGLE_CHUNK(FAKE_UUID) }, - // { name: 'multiple payloads per buffer chunk', value: PAYLOAD_MULTI_CHUNK(FAKE_UUID) }, - // { name: 'single payload across multiple buffer chunks', value: PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(FAKE_UUID) }, + { + name: 'single payload single chunk', + value: PAYLOAD_SINGLE_CHUNK(FAKE_UUID), + }, + { + name: 'multiple payloads per buffer chunk', + value: PAYLOAD_MULTI_CHUNK(FAKE_UUID), + }, + { + name: 'single payload across multiple buffer chunks', + value: PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(FAKE_UUID), + }, { name: 'two chunks, payload split and two payloads in a chunk', value: PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY(FAKE_UUID), @@ -144,8 +148,12 @@ suite('EOT tests', () => { // payload is a string array, each string represents one line written to the buffer const { payloadArray } = testCase.value; for (let i = 0; i < payloadArray.length; i = i + 1) { - client.write(payloadArray[i]); - await sleep(3); + await (async (clientSub, payloadSub) => { + if (!clientSub.write(payloadSub)) { + // If write returns false, wait for the 'drain' event before proceeding + await new Promise((resolve) => clientSub.once('drain', resolve)); + } + })(client, payloadArray[i]); } client.end(); }); @@ -191,6 +199,8 @@ suite('EOT tests', () => { actualCollectedResult, "Expected collected result to match 'data'", ); + // nervous about this not testing race conditions correctly + // client.end(); // verify that the _resolveExecution was called once per test // assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); }); From 3a0c5858d32f996c3d4e2fcb6e89db720c3a62b9 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 12 Sep 2023 15:30:47 -0700 Subject: [PATCH 32/55] revert files which shouldn't be touched --- .vscode/launch.json | 5 ++--- src/test/.vscode/settings.json | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 801fb604ba22..82981a93305d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -101,7 +101,7 @@ "--extensionTestsPath=${workspaceFolder}/out/test" ], "env": { - "VSC_PYTHON_CI_TEST_GREP": "EOT tests" // Modify this to run a subset of the single workspace tests + "VSC_PYTHON_CI_TEST_GREP": "" // Modify this to run a subset of the single workspace tests }, "stopOnEntry": false, "sourceMaps": true, @@ -160,8 +160,7 @@ "--ui=tdd", "--recursive", "--colors", - "--grep", - "Python Test Server", + //"--grep", "", "--timeout=300000" ], "outFiles": ["${workspaceFolder}/out/**/*.js", "!${workspaceFolder}/**/node_modules**/*"], diff --git a/src/test/.vscode/settings.json b/src/test/.vscode/settings.json index 250ed86de0e3..ef9292849a9d 100644 --- a/src/test/.vscode/settings.json +++ b/src/test/.vscode/settings.json @@ -15,6 +15,5 @@ "python.formatting.provider": "yapf", // Don't set this to `Pylance`, for CI we want to use the LS that ships with the extension. "python.languageServer": "Jedi", - "python.pythonPath": "C:\\GIT\\s p\\vscode-python\\.venv\\Scripts\\python.exe", - "python.defaultInterpreterPath": "python" + "python.pythonPath": "C:\\GIT\\s p\\vscode-python\\.venv\\Scripts\\python.exe" } From 5be4238b70a658a3fd86a27b0894bf5dbcbbd6bd Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 12 Sep 2023 15:39:52 -0700 Subject: [PATCH 33/55] fix test_discovery typing --- .../tests/pytestadapter/test_discovery.py | 133 +++++++++++------- 1 file changed, 86 insertions(+), 47 deletions(-) diff --git a/pythonFiles/tests/pytestadapter/test_discovery.py b/pythonFiles/tests/pytestadapter/test_discovery.py index 333e6cc2477e..674d92ac0545 100644 --- a/pythonFiles/tests/pytestadapter/test_discovery.py +++ b/pythonFiles/tests/pytestadapter/test_discovery.py @@ -29,16 +29,26 @@ def test_import_error(tmp_path): temp_dir.mkdir() p = temp_dir / "error_pytest_import.py" shutil.copyfile(file_path, p) - actual_list: Optional[List[Dict[str, Any]]] = runner( - ["--collect-only", os.fspath(p)] - ) - assert actual_list - assert actual_list.pop(-1).get("eot") - for actual in actual_list: - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 2 + actual: Optional[List[Dict[str, Any]]] = runner(["--collect-only", os.fspath(p)]) + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + + # Ensure that 'error' is a list and then check its length + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 2 + else: + assert False def test_syntax_error(tmp_path): @@ -61,14 +71,25 @@ def test_syntax_error(tmp_path): p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) actual = runner(["--collect-only", os.fspath(p)]) - assert actual.pop(-1).get("eot") - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 2 + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + + # Ensure that 'error' is a list and then check its length + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 2 + else: + assert False def test_parameterized_error_collect(): @@ -78,13 +99,25 @@ def test_parameterized_error_collect(): """ file_path_str = "error_parametrize_discovery.py" actual = runner(["--collect-only", file_path_str]) - assert actual.pop(-1).get("eot") - if actual: - actual = actual[0] - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 2 + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + + # Ensure that 'error' is a list and then check its length + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 2 + else: + assert False @pytest.mark.parametrize( @@ -149,14 +182,16 @@ def test_pytest_collect(file, expected_const): os.fspath(TEST_DATA_PATH / file), ] ) - assert actual.pop(-1).get("eot") - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "tests")) - assert actual["status"] == "success" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert actual["tests"] == expected_const + + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + actual_item = actual_list.pop(0) + assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + assert actual_item.get("tests") == expected_const def test_pytest_root_dir(): @@ -172,14 +207,16 @@ def test_pytest_root_dir(): ], TEST_DATA_PATH / "root", ) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "tests")) - assert actual["status"] == "success" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "root") + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + actual_item = actual_list.pop(0) + assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") assert ( - actual["tests"] + actual_item.get("tests") == expected_discovery_test_output.root_with_config_expected_output ) @@ -197,13 +234,15 @@ def test_pytest_config_file(): ], TEST_DATA_PATH / "root", ) - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "tests")) - assert actual["status"] == "success" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH / "root") + assert actual + actual_list: List[Dict[str, Any]] = actual + if actual_list is not None: + assert actual_list.pop(-1).get("eot") + actual_item = actual_list.pop(0) + assert all(item in actual_item.keys() for item in ("status", "cwd", "error")) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH / "root") assert ( - actual["tests"] + actual_item.get("tests") == expected_discovery_test_output.root_with_config_expected_output ) From 95348ae4242f4612b1fc9f03895d324688271198 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 12 Sep 2023 16:09:57 -0700 Subject: [PATCH 34/55] pyright --- pythonFiles/tests/pytestadapter/helpers.py | 4 + .../tests/pytestadapter/test_execution.py | 112 +++++++++++------- 2 files changed, 75 insertions(+), 41 deletions(-) diff --git a/pythonFiles/tests/pytestadapter/helpers.py b/pythonFiles/tests/pytestadapter/helpers.py index 7195cfe43ea5..b534e950945a 100644 --- a/pythonFiles/tests/pytestadapter/helpers.py +++ b/pythonFiles/tests/pytestadapter/helpers.py @@ -12,6 +12,10 @@ import uuid from typing import Any, Dict, List, Optional, Tuple +script_dir = pathlib.Path(__file__).parent.parent.parent +sys.path.append(os.fspath(script_dir)) +sys.path.append(os.fspath(script_dir / "lib" / "python")) + TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data" from typing_extensions import TypedDict diff --git a/pythonFiles/tests/pytestadapter/test_execution.py b/pythonFiles/tests/pytestadapter/test_execution.py index 55952059b44e..bf4347d14c78 100644 --- a/pythonFiles/tests/pytestadapter/test_execution.py +++ b/pythonFiles/tests/pytestadapter/test_execution.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. import os import shutil +from typing import Any, Dict, List, Optional import pytest @@ -23,15 +24,19 @@ def test_config_file(): expected_execution_test_output.config_file_pytest_expected_execution_output ) assert actual - assert actual.pop(-1).get("eot") - assert len(actual) == len(expected_const) + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + assert len(actual_list) == len(expected_const) actual_result_dict = dict() - for a in actual: - assert all(item in a for item in ("status", "cwd", "result")) - assert a["status"] == "success" - assert a["cwd"] == os.fspath(new_cwd) - actual_result_dict.update(a["result"]) - assert actual_result_dict == expected_const + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "result") + ) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(new_cwd) + actual_result_dict.update(actual_item["result"]) + assert actual_result_dict == expected_const def test_rootdir_specified(): @@ -44,15 +49,19 @@ def test_rootdir_specified(): expected_execution_test_output.config_file_pytest_expected_execution_output ) assert actual - assert actual.pop(-1).get("eot") - assert len(actual) == len(expected_const) + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + assert len(actual_list) == len(expected_const) actual_result_dict = dict() - for a in actual: - assert all(item in a for item in ("status", "cwd", "result")) - assert a["status"] == "success" - assert a["cwd"] == os.fspath(new_cwd) - actual_result_dict.update(a["result"]) - assert actual_result_dict == expected_const + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "result") + ) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(new_cwd) + actual_result_dict.update(actual_item["result"]) + assert actual_result_dict == expected_const def test_syntax_error_execution(tmp_path): @@ -75,14 +84,23 @@ def test_syntax_error_execution(tmp_path): p = temp_dir / "error_syntax_discovery.py" shutil.copyfile(file_path, p) actual = runner(["error_syntax_discover.py::test_function"]) - assert actual.pop(-1).get("eot") - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 1 + assert actual + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 1 + else: + assert False def test_bad_id_error_execution(): @@ -91,14 +109,23 @@ def test_bad_id_error_execution(): The json should still be returned but the errors list should be present. """ actual = runner(["not/a/real::test_id"]) - assert actual.pop(-1).get("eot") - if actual: - actual = actual[0] - assert actual - assert all(item in actual for item in ("status", "cwd", "error")) - assert actual["status"] == "error" - assert actual["cwd"] == os.fspath(TEST_DATA_PATH) - assert len(actual["error"]) == 1 + assert actual + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "error") + ) + assert actual_item.get("status") == "error" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + error_content = actual_item.get("error") + if error_content is not None and isinstance( + error_content, (list, tuple, str) + ): # You can add other types if needed + assert len(error_content) == 1 + else: + assert False @pytest.mark.parametrize( @@ -209,19 +236,22 @@ def test_pytest_execution(test_ids, expected_const): Keyword arguments: test_ids -- an array of test_ids to run. expected_const -- a dictionary of the expected output from running pytest discovery on the files. - """ # noqa: E501 + """ args = test_ids actual = runner(args) assert actual - # print(actual) - assert actual.pop(-1).get("eot") - assert len(actual) == len(expected_const) + actual_list: List[Dict[str, Any]] = actual + assert actual_list.pop(-1).get("eot") + assert len(actual_list) == len(expected_const) actual_result_dict = dict() - for a in actual: - assert all(item in a for item in ("status", "cwd", "result")) - assert a["status"] == "success" - assert a["cwd"] == os.fspath(TEST_DATA_PATH) - actual_result_dict.update(a["result"]) + if actual_list is not None: + for actual_item in actual_list: + assert all( + item in actual_item.keys() for item in ("status", "cwd", "result") + ) + assert actual_item.get("status") == "success" + assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) + actual_result_dict.update(actual_item["result"]) for key in actual_result_dict: if ( actual_result_dict[key]["outcome"] == "failure" From cd1a4c825af8ad26820ffbbc1cda50dbdad2e0a2 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Tue, 12 Sep 2023 16:15:31 -0700 Subject: [PATCH 35/55] pyright p2 --- pythonFiles/unittestadapter/discovery.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/pythonFiles/unittestadapter/discovery.py b/pythonFiles/unittestadapter/discovery.py index 6097e52ef587..6208d24bee9f 100644 --- a/pythonFiles/unittestadapter/discovery.py +++ b/pythonFiles/unittestadapter/discovery.py @@ -141,8 +141,13 @@ def post_response(payload: PayloadDict | EOTPayloadDict, port: int, uuid: str) - # Perform test discovery. port, uuid = parse_discovery_cli_args(argv[:index]) # Post this discovery payload. - payload = discover_tests(start_dir, pattern, top_level_dir, uuid) - post_response(payload, port, uuid) - # Post EOT token. - eot_payload: EOTPayloadDict = {"command_type": "discovery", "eot": True} - post_response(eot_payload, port, uuid) + if uuid is not None: + payload = discover_tests(start_dir, pattern, top_level_dir, uuid) + post_response(payload, port, uuid) + # Post EOT token. + eot_payload: EOTPayloadDict = {"command_type": "discovery", "eot": True} + post_response(eot_payload, port, uuid) + else: + print("Error: no uuid provided or parsed.") + eot_payload: EOTPayloadDict = {"command_type": "discovery", "eot": True} + post_response(eot_payload, port, "") From 3c3f33981b2f33b54b26a8e89d7a200a4e45fbc3 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 13 Sep 2023 09:02:23 -0700 Subject: [PATCH 36/55] pytestExecutionAdapter tests --- .../testing/testController/common/server.ts | 2 +- .../testing/testController/common/utils.ts | 16 ++--- .../pytest/pytestExecutionAdapter.ts | 4 +- .../unittest/testDiscoveryAdapter.ts | 4 +- .../testing/common/testingAdapter.test.ts | 70 +++++++------------ .../testing/testController/helper.server.ts | 2 +- .../pytestExecutionAdapter.unit.test.ts | 44 +++++++----- .../resultResolver.unit.test.ts | 28 +++++--- .../testController/server.unit.test.ts | 2 +- 9 files changed, 84 insertions(+), 88 deletions(-) diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 3aa6b5b80388..a7368f1fe01e 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -15,7 +15,7 @@ import { traceError, traceInfo, traceLog } from '../../../logging'; import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; -import { containsHeaders, extractJsonPayload } from './utils'; +import { createExecutionErrorPayload, extractJsonPayload } from './utils'; import { createDeferred } from '../../../common/utils/async'; export class PythonTestServer implements ITestServer, Disposable { diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 012ebfbfe140..3286c072ec66 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -10,6 +10,7 @@ import { IExperimentService } from '../../../common/types'; import { IServiceContainer } from '../../../ioc/types'; import { DebugTestTag, ErrorTestItemOptions, RunTestTag } from './testItemUtilities'; import { DiscoveredTestItem, DiscoveredTestNode, ExecutionTestPayload, ITestResultResolver } from './types'; +import { Deferred, createDeferred } from '../../../common/utils/async'; export function fixLogLines(content: string): string { const lines = content.split(/\r?\n/g); @@ -35,6 +36,10 @@ export const JSONRPC_UUID_HEADER = 'Request-uuid'; export const JSONRPC_CONTENT_LENGTH_HEADER = 'Content-Length'; export const JSONRPC_CONTENT_TYPE_HEADER = 'Content-Type'; +export function createEOTDeferred(): Deferred { + return createDeferred(); +} + export function extractJsonPayload(rawData: string, uuids: Array): ExtractOutput { /** * Extracts JSON-RPC payload from the provided raw data. @@ -66,17 +71,6 @@ export function extractJsonPayload(rawData: string, uuids: Array): Extra return { uuid: undefined, cleanedJsonData: undefined, remainingRawData: rawData }; } -export function containsHeaders(rawData: string): boolean { - /** - * Checks if the provided raw data contains JSON-RPC specific headers. - */ - return ( - rawData.includes(JSONRPC_CONTENT_LENGTH_HEADER) && - rawData.includes(JSONRPC_CONTENT_TYPE_HEADER) && - rawData.includes(JSONRPC_UUID_HEADER) - ); -} - export function checkUuid(uuid: string | undefined, uuids: Array): string | undefined { if (!uuid) { // no UUID found, this could occurred if the payload is full yet so send back without erroring diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 0435e9e35181..031209841ec1 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -5,7 +5,7 @@ import { TestRun, Uri } from 'vscode'; import * as path from 'path'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { Deferred, createDeferred } from '../../../common/utils/async'; -import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../logging'; import { DataReceivedEvent, ExecutionTestPayload, @@ -43,7 +43,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { ): Promise { const uuid = this.testServer.createUUID(uri.fsPath); traceVerbose(uri, testIds, debugBool); - const deferredTillEOT: Deferred = createDeferred(); + const deferredTillEOT: Deferred = utils.createEOTDeferred(); const dataReceivedDisposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { console.log('data received'); if (runInstance) { diff --git a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts index 17939cb62d47..440df4f94dc6 100644 --- a/src/client/testing/testController/unittest/testDiscoveryAdapter.ts +++ b/src/client/testing/testController/unittest/testDiscoveryAdapter.ts @@ -52,7 +52,9 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter { dataReceivedDisposable.dispose(); }; - await this.callSendCommand(options); + await this.callSendCommand(options, () => { + disposeDataReceiver?.(this.testServer); + }); await deferredTillEOT.promise; disposeDataReceiver(this.testServer); // placeholder until after the rewrite is adopted diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index ace6b4ec90ef..34b66340982d 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -27,9 +27,12 @@ suite('End to End Tests: test adapters', () => { let pythonExecFactory: IPythonExecutionFactory; let debugLauncher: ITestDebugLauncher; let configService: IConfigurationService; - let testOutputChannel: ITestOutputChannel; let serviceContainer: IServiceContainer; let workspaceUri: Uri; + let testOutputChannel: typeMoq.IMock; + let testController: TestController; + const unittestProvider: TestProvider = UNITTEST_PROVIDER; + const pytestProvider: TestProvider = PYTEST_PROVIDER; const rootPathSmallWorkspace = path.join( EXTENSION_ROOT_DIR_FOR_TESTS, 'src', @@ -57,11 +60,8 @@ suite('End to End Tests: test adapters', () => { configService = serviceContainer.get(IConfigurationService); pythonExecFactory = serviceContainer.get(IPythonExecutionFactory); debugLauncher = serviceContainer.get(ITestDebugLauncher); - testOutputChannel = serviceContainer.get(ITestOutputChannel); testController = serviceContainer.get(ITestController); - // create mock resultResolver object - // create objects that were not injected pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher); await pythonTestServer.serverReady(); @@ -368,17 +368,8 @@ suite('End to End Tests: test adapters', () => { pythonExecFactory, ) .then(() => { - // verification after discovery is complete - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(2), - ); - // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors - assert.strictEqual(actualData.error, null, "Expected no errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(actualData.result, 'Expected results to be present'); + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); }); }); test('pytest execution adapter large workspace', async () => { @@ -420,31 +411,17 @@ suite('End to End Tests: test adapters', () => { onCancellationRequested: () => undefined, } as any), ); - console.log('FROM TEST, do the run large'); - await executionAdapter - .runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory) - .then(() => { - // resolve execution should be called 200 times since there are 200 tests run. - console.log('hit then'); - assert.strictEqual( - errorMessages.length, - 0, - ['Test run was unsuccessful, the following errors were produced: \n', ...errorMessages].join('\n'), - ); - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.atLeast(2000), - ); - }) - .finally(() => { - console.log('hit finally large'); - }); + await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).then(() => { + // verify that the _resolveExecution was called once per test + assert.strictEqual(callCount, 3, 'Expected _resolveExecution to be called once'); + }); }); test('unittest execution adapter seg fault error handling', async () => { + const resultResolverMock: typeMoq.IMock = typeMoq.Mock.ofType(); const testId = `test_seg_fault.TestSegmentationFault.test_segfault`; const testIds: string[] = [testId]; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) + resultResolverMock + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) .returns((data) => { // do the following asserts for each time resolveExecution is called, should be called once per test. // 1. Check the status is "success" @@ -469,8 +446,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new UnittestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolverMock.object, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -482,17 +459,18 @@ suite('End to End Tests: test adapters', () => { } as any), ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object).finally(() => { - resultResolver.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()), + resultResolverMock.verify( + (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), typeMoq.Times.exactly(1), ); }); }); test('pytest execution adapter seg fault error handling', async () => { + const resultResolverMock: typeMoq.IMock = typeMoq.Mock.ofType(); const testId = `${rootPathErrorWorkspace}/test_seg_fault.py::TestSegmentationFault::test_segfault`; const testIds: string[] = [testId]; - resultResolver - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny())) + resultResolverMock + .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) .returns((data) => { // do the following asserts for each time resolveExecution is called, should be called once per test. // 1. Check the status is "success" @@ -517,8 +495,8 @@ suite('End to End Tests: test adapters', () => { const executionAdapter = new PytestTestExecutionAdapter( pythonTestServer, configService, - testOutputChannel, - resultResolver.object, + testOutputChannel.object, + resultResolverMock.object, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -530,9 +508,9 @@ suite('End to End Tests: test adapters', () => { } as any), ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { - resultResolver.verify( + resultResolverMock.verify( (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(4), + typeMoq.Times.exactly(1), ); }); }); diff --git a/src/test/testing/testController/helper.server.ts b/src/test/testing/testController/helper.server.ts index d6c9467e9988..b5077aa7f202 100644 --- a/src/test/testing/testController/helper.server.ts +++ b/src/test/testing/testController/helper.server.ts @@ -14,7 +14,7 @@ import { PythonTestServer } from '../../../client/testing/testController/common/ import { ITestDebugLauncher } from '../../../client/testing/common/types'; import { Deferred, createDeferred } from '../../../client/common/utils/async'; import { MockChildProcess } from '../../mocks/mockChildProcess'; -import { PAYLOAD_MULTI_CHUNK, PAYLOAD_SINGLE_CHUNK, PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY } from './payloadTestCases'; +import { PAYLOAD_SINGLE_CHUNK, PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY } from './payloadTestCases'; suite('Python Test Server', () => { const FAKE_UUID = 'fake-uuid'; diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index 43b763f56e6c..9cc428ab0a4c 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -21,6 +21,7 @@ import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/co import * as util from '../../../../client/testing/testController/common/utils'; import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; import { MockChildProcess } from '../../../mocks/mockChildProcess'; +import { traceInfo } from '../../../../client/logging'; suite('pytest test execution adapter', () => { let testServer: typeMoq.IMock; @@ -33,7 +34,7 @@ suite('pytest test execution adapter', () => { (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; let myTestPath: string; let mockProc: MockChildProcess; - let utilsStub: sinon.SinonStub; + let utilsStartServerStub: sinon.SinonStub; setup(() => { testServer = typeMoq.Mock.ofType(); testServer.setup((t) => t.getPort()).returns(() => 12345); @@ -51,6 +52,8 @@ suite('pytest test execution adapter', () => { isTestExecution: () => false, } as unknown) as IConfigurationService; + // mock out the result resolver + // set up exec service with child process mockProc = new MockChildProcess('', ['']); const output = new Observable>(() => { @@ -67,7 +70,7 @@ suite('pytest test execution adapter', () => { }, })); execFactory = typeMoq.Mock.ofType(); - utilsStub = sinon.stub(util, 'startTestIdServer'); + utilsStartServerStub = sinon.stub(util, 'startTestIdServer'); debugLauncher = typeMoq.Mock.ofType(); execFactory .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) @@ -79,13 +82,6 @@ suite('pytest test execution adapter', () => { deferred.resolve(); return Promise.resolve({ stdout: '{}' }); }); - debugLauncher - .setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns(() => { - deferred.resolve(); - return Promise.resolve(); - }); - execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); @@ -104,7 +100,7 @@ suite('pytest test execution adapter', () => { deferred2.resolve(); return Promise.resolve(execService.object); }); - utilsStub.callsFake(() => { + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); @@ -131,7 +127,7 @@ suite('pytest test execution adapter', () => { mockProc.trigger('close'); // assert - sinon.assert.calledWithExactly(utilsStub, testIds); + sinon.assert.calledWithExactly(utilsStartServerStub, testIds); }); test('pytest execution called with correct args', async () => { const deferred2 = createDeferred(); @@ -143,7 +139,7 @@ suite('pytest test execution adapter', () => { deferred2.resolve(); return Promise.resolve(execService.object); }); - utilsStub.callsFake(() => { + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); @@ -175,7 +171,6 @@ suite('pytest test execution adapter', () => { TEST_UUID: 'uuid123', TEST_PORT: '12345', }; - // execService.verify((x) => x.exec(expectedArgs, typeMoq.It.isAny()), typeMoq.Times.once()); execService.verify( (x) => x.execObservable( @@ -203,7 +198,7 @@ suite('pytest test execution adapter', () => { deferred2.resolve(); return Promise.resolve(execService.object); }); - utilsStub.callsFake(() => { + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); @@ -262,12 +257,28 @@ suite('pytest test execution adapter', () => { }); test('Debug launched correctly for pytest', async () => { const deferred3 = createDeferred(); - utilsStub.callsFake(() => { + const deferredEOT = createDeferred(); + utilsStartServerStub.callsFake(() => { deferred3.resolve(); return Promise.resolve(54321); }); + debugLauncher + .setup((dl) => dl.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(async () => { + traceInfo('stubs launch debugger'); + deferredEOT.resolve(); + }); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); const testRun = typeMoq.Mock.ofType(); - testRun.setup((t) => t.token).returns(() => ({ onCancellationRequested: () => undefined } as any)); + testRun + .setup((t) => t.token) + .returns( + () => + ({ + onCancellationRequested: () => undefined, + } as any), + ); const uri = Uri.file(myTestPath); const uuid = 'uuid123'; testServer @@ -298,5 +309,6 @@ suite('pytest test execution adapter', () => { ), typeMoq.Times.once(), ); + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); }); }); diff --git a/src/test/testing/testController/resultResolver.unit.test.ts b/src/test/testing/testController/resultResolver.unit.test.ts index 694fdacb8049..56322cc700bc 100644 --- a/src/test/testing/testController/resultResolver.unit.test.ts +++ b/src/test/testing/testController/resultResolver.unit.test.ts @@ -14,6 +14,7 @@ import { import * as testItemUtilities from '../../../client/testing/testController/common/testItemUtilities'; import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; import * as util from '../../../client/testing/testController/common/utils'; +import { Deferred, createDeferred } from '../../../client/common/utils/async'; suite('Result Resolver tests', () => { suite('Test discovery', () => { @@ -87,7 +88,8 @@ suite('Result Resolver tests', () => { const populateTestTreeStub = sinon.stub(util, 'populateTestTree').returns(); // call resolve discovery - resultResolver.resolveDiscovery(payload, cancelationToken); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveDiscovery(payload, deferredTillEOT, cancelationToken); // assert the stub functions were called with the correct parameters @@ -126,7 +128,8 @@ suite('Result Resolver tests', () => { const createErrorTestItemStub = sinon.stub(testItemUtilities, 'createErrorTestItem').returns(blankTestItem); // call resolve discovery - resultResolver.resolveDiscovery(payload); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveDiscovery(payload, deferredTillEOT, cancelationToken); // assert the stub functions were called with the correct parameters @@ -171,7 +174,8 @@ suite('Result Resolver tests', () => { // stub out functionality of populateTestTreeStub which is called in resolveDiscovery const populateTestTreeStub = sinon.stub(util, 'populateTestTree').returns(); // call resolve discovery - resultResolver.resolveDiscovery(payload, cancelationToken); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveDiscovery(payload, deferredTillEOT, cancelationToken); // assert the stub functions were called with the correct parameters @@ -307,7 +311,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item assert.ok(generatedId); @@ -347,7 +352,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.failed(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once()); @@ -386,7 +392,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.once()); @@ -425,7 +432,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.errored(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once()); @@ -464,7 +472,8 @@ suite('Result Resolver tests', () => { }; // call resolveExecution - resultResolver.resolveExecution(successPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(successPayload, runInstance.object, deferredTillEOT); // verify that the passed function was called for the single test item runInstance.verify((r) => r.passed(typemoq.It.isAny()), typemoq.Times.once()); @@ -485,7 +494,8 @@ suite('Result Resolver tests', () => { error: 'error', }; - resultResolver.resolveExecution(errorPayload, runInstance.object); + const deferredTillEOT: Deferred = createDeferred(); + resultResolver.resolveExecution(errorPayload, runInstance.object, deferredTillEOT); // verify that none of these functions are called diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index 563f6b5bed38..d09b6e9224c4 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -12,7 +12,7 @@ import { Uri } from 'vscode'; import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; -import { createDeferred } from '../../../client/common/utils/async'; +import { Deferred, createDeferred } from '../../../client/common/utils/async'; import { MockChildProcess } from '../../mocks/mockChildProcess'; import { PAYLOAD_MULTI_CHUNK, From c8d80fe0491d8014e65e70eca613bd18704b3334 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 13 Sep 2023 10:53:18 -0700 Subject: [PATCH 37/55] fix server tests --- .../testController/server.unit.test.ts | 699 ++++-------------- 1 file changed, 155 insertions(+), 544 deletions(-) diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index d09b6e9224c4..cd4abc6832c2 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -8,8 +8,13 @@ import * as sinon from 'sinon'; import * as crypto from 'crypto'; import { Observable } from 'rxjs'; import * as typeMoq from 'typemoq'; -import { Uri } from 'vscode'; -import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; +import { OutputChannel, Uri } from 'vscode'; +import { + IPythonExecutionFactory, + IPythonExecutionService, + ObservableExecutionResult, + Output, +} from '../../../client/common/process/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; import { ITestDebugLauncher } from '../../../client/testing/common/types'; import { Deferred, createDeferred } from '../../../client/common/utils/async'; @@ -33,7 +38,7 @@ const testCases = [ }, ]; -suite('Python Test Server', () => { +suite('Python Test Server, DataWithPayloadChunks', () => { const FAKE_UUID = 'fake-uuid'; let server: PythonTestServer; let v4Stub: sinon.SinonStub; @@ -113,7 +118,12 @@ suite('Python Test Server', () => { await server.serverReady(); server.onRunDataReceived(({ data }) => { - eventData = eventData + data; + try { + const resultData = JSON.parse(data).result; + eventData = eventData + JSON.stringify(resultData); + } catch (e) { + assert(false, 'Error parsing data'); + } deferred.resolve(); }); client.on('connect', () => { @@ -136,556 +146,157 @@ suite('Python Test Server', () => { }); }); -// need to add EOT token -// why is it finishing for single chunk without the EOT token? - -// suite('Python Test Server', () => { -// const FAKE_UUID = 'fake-uuid'; -// let server: PythonTestServer; -// let v4Stub: sinon.SinonStub; -// let debugLauncher: ITestDebugLauncher; -// let mockProc: MockChildProcess; -// let execService: typeMoq.IMock; -// let deferred: Deferred; -// const sandbox = sinon.createSandbox(); - -// setup(async () => { -// // set up test command options - -// v4Stub = sandbox.stub(crypto, 'randomUUID'); -// v4Stub.returns(FAKE_UUID); - -// // set up exec service with child process -// mockProc = new MockChildProcess('', ['']); -// execService = typeMoq.Mock.ofType(); -// const outputObservable = new Observable>(() => { -// /* no op */ -// }); -// execService -// .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) -// .returns(() => ({ -// proc: mockProc, -// out: outputObservable, -// dispose: () => { -// /* no-body */ -// }, -// })); -// execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); -// }); - -// teardown(() => { -// sandbox.restore(); -// server.dispose(); -// }); -// test('basic payload', async () => { -// let eventData = ''; -// const client = new net.Socket(); - -// deferred = createDeferred(); -// mockProc = new MockChildProcess('', ['']); -// const output2 = new Observable>(() => { -// /* no op */ -// }); -// const stubExecutionService2 = ({ -// execObservable: () => { -// client.connect(server.getPort()); -// return { -// proc: mockProc, -// out: output2, -// dispose: () => { -// /* no-body */ -// }, -// }; -// }, -// } as unknown) as IPythonExecutionService; - -// const stubExecutionFactory2 = ({ -// createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), -// } as unknown) as IPythonExecutionFactory; -// server = new PythonTestServer(stubExecutionFactory2, debugLauncher); -// const uuid = server.createUUID(); -// const options = { -// command: { script: 'myscript', args: ['-foo', 'foo'] }, -// workspaceFolder: Uri.file('/foo/bar'), -// cwd: '/foo/bar', -// uuid, -// }; - -// const dataWithPayloadChunks = PAYLOAD_SINGLE_CHUNK(uuid); - -// await server.serverReady(); - -// server.onRunDataReceived(({ data }) => { -// eventData = eventData + data; -// deferred.resolve(); -// }); -// client.on('connect', () => { -// console.log('Socket connected, local port:', client.localPort); -// // since this test is a single payload as a single chunk there should be a single line in the payload. -// for (const line of dataWithPayloadChunks.payload) { -// client.write(line); -// } -// client.end(); -// }); -// client.on('error', (error) => { -// console.log('Socket connection error:', error); -// }); - -// server.sendCommand(options); -// await deferred.promise; -// const expectedResult = dataWithPayloadChunks.data; -// assert.deepStrictEqual(eventData, expectedResult); -// }); -// test('second payload', async () => { -// let eventData = ''; -// const client = new net.Socket(); - -// deferred = createDeferred(); -// mockProc = new MockChildProcess('', ['']); -// const output2 = new Observable>(() => { -// /* no op */ -// }); -// const stubExecutionService2 = ({ -// execObservable: () => { -// client.connect(server.getPort()); -// return { -// proc: mockProc, -// out: output2, -// dispose: () => { -// /* no-body */ -// }, -// }; -// }, -// } as unknown) as IPythonExecutionService; - -// const stubExecutionFactory2 = ({ -// createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), -// } as unknown) as IPythonExecutionFactory; -// server = new PythonTestServer(stubExecutionFactory2, debugLauncher); -// const uuid = server.createUUID(); -// const options = { -// command: { script: 'myscript', args: ['-foo', 'foo'] }, -// workspaceFolder: Uri.file('/foo/bar'), -// cwd: '/foo/bar', -// uuid, -// }; -// const payloadChunk = PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid); - -// await server.serverReady(); - -// server.onRunDataReceived(({ data }) => { -// eventData = eventData + data; -// }); -// client.on('connect', () => { -// console.log('Socket connected, local port:', client.localPort); -// for (const line of payloadChunk.payload) { -// client.write(line); -// } -// client.end(); -// }); -// client.on('close', () => { -// console.log('Socket connection exit:'); -// deferred.resolve(); -// }); -// client.on('error', (error) => { -// console.log('Socket connection error:', error); -// }); - -// server.sendCommand(options); -// await deferred.promise; -// const expectedResult = payloadChunk.data; -// assert.deepStrictEqual(eventData, expectedResult); -// }); - -// // test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { -// // deferred2 = createDeferred(); -// // execFactory = typeMoq.Mock.ofType(); -// // execFactory -// // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) -// // .returns(() => { -// // deferred2.resolve(); -// // return Promise.resolve(execService.object); -// // }); -// // server = new PythonTestServer(execFactory.object, debugLauncher); -// // await server.serverReady(); -// // server.sendCommand(BASE_TEST_COMMAND_OPTIONS, '56789'); -// // // add in await and trigger -// // await deferred2.promise; -// // mockProc.trigger('close'); - -// // const port = server.getPort(); -// // const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; -// // execService.verify((x) => x.execObservable(expectedArgs, BASE_SPAWN_OPTIONS), typeMoq.Times.once()); -// // }); - -// // test('sendCommand should write to an output channel if it is provided as an option', async () => { -// // const output2: string[] = []; -// // const outChannel = { -// // appendLine: (str: string) => { -// // output2.push(str); -// // }, -// // } as OutputChannel; -// // const options = { -// // command: { -// // script: 'myscript', -// // args: ['-foo', 'foo'], -// // }, -// // workspaceFolder: Uri.file('/foo/bar'), -// // cwd: '/foo/bar', -// // uuid: FAKE_UUID, -// // outChannel, -// // }; -// // deferred = createDeferred(); -// // execFactory = typeMoq.Mock.ofType(); -// // execFactory -// // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) -// // .returns(() => { -// // deferred.resolve(); -// // return Promise.resolve(execService.object); -// // }); - -// // server = new PythonTestServer(execFactory.object, debugLauncher); -// // await server.serverReady(); - -// // server.sendCommand(options); -// // // add in await and trigger -// // await deferred.promise; -// // mockProc.trigger('close'); - -// // const port = server.getPort(); -// // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); - -// // assert.deepStrictEqual(output2, [expected]); -// // }); - -// // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { -// // let eventData: { status: string; errors: string[] } | undefined; -// // stubExecutionService = ({ -// // execObservable: () => { -// // throw new Error('Failed to execute'); -// // }, -// // } as unknown) as IPythonExecutionService; -// // const options = { -// // command: { script: 'myscript', args: ['-foo', 'foo'] }, -// // workspaceFolder: Uri.file('/foo/bar'), -// // cwd: '/foo/bar', -// // uuid: FAKE_UUID, -// // }; - -// // server = new PythonTestServer(stubExecutionFactory, debugLauncher); -// // await server.serverReady(); - -// // server.onDataReceived(({ data }) => { -// // eventData = JSON.parse(data); -// // }); - -// // await server.sendCommand(options); - -// // assert.notEqual(eventData, undefined); -// // assert.deepStrictEqual(eventData?.status, 'error'); -// // assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); -// // }); - -// // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { -// // deferred2 = createDeferred(); -// // execFactory = typeMoq.Mock.ofType(); -// // execFactory -// // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) -// // .returns(() => { -// // deferred2.resolve(); -// // return Promise.resolve(execService.object); -// // }); -// // server = new PythonTestServer(execFactory.object, debugLauncher); -// // await server.serverReady(); -// // let eventData: string | undefined; -// // const client = new net.Socket(); -// // const options = { -// // command: { script: 'myscript', args: ['-foo', 'foo'] }, -// // workspaceFolder: Uri.file('/foo/bar'), -// // cwd: '/foo/bar', -// // uuid: FAKE_UUID, -// // }; -// // mockProc = new MockChildProcess('', ['']); -// // const output2 = new Observable>(() => { -// // /* no op */ -// // }); -// // const stubExecutionService2 = ({ -// // execObservable: () => { -// // client.connect(server.getPort()); -// // return { -// // proc: mockProc, -// // out: output2, -// // dispose: () => { -// // /* no-body */ -// // }, -// // }; -// // }, -// // } as unknown) as IPythonExecutionService; - -// // const stubExecutionFactory2 = ({ -// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), -// // } as unknown) as IPythonExecutionFactory; - -// // deferred = createDeferred(); -// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); -// // await server.serverReady(); -// // server.onDataReceived(({ data }) => { -// // eventData = data; -// // deferred.resolve(); -// // }); - -// // client.on('connect', () => { -// // console.log('Socket connected, local port:', client.localPort); -// // client.write('malformed data'); -// // client.end(); -// // }); -// // client.on('error', (error) => { -// // console.log('Socket connection error:', error); -// // }); - -// // server.sendCommand(options); -// // // add in await and trigger -// // await deferred.promise; -// // mockProc.trigger('close'); - -// // assert.deepStrictEqual(eventData, ''); -// // }); - -// // test('If the server doesnt recognize the UUID it should ignore it', async () => { -// // let eventData: string | undefined; -// // const client = new net.Socket(); -// // const options = { -// // command: { script: 'myscript', args: ['-foo', 'foo'] }, -// // workspaceFolder: Uri.file('/foo/bar'), -// // cwd: '/foo/bar', -// // uuid: FAKE_UUID, -// // }; - -// // deferred = createDeferred(); -// // mockProc = new MockChildProcess('', ['']); -// // const output = new Observable>(() => { -// // /* no op */ -// // }); -// // const stubExecutionService2 = ({ -// // execObservable: () => { -// // client.connect(server.getPort()); -// // return { -// // proc: mockProc, -// // out: output, -// // dispose: () => { -// // /* no-body */ -// // }, -// // }; -// // }, -// // } as unknown) as IPythonExecutionService; - -// // const stubExecutionFactory2 = ({ -// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), -// // } as unknown) as IPythonExecutionFactory; -// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); -// // await server.serverReady(); -// // server.onDataReceived(({ data }) => { -// // eventData = data; -// // deferred.resolve(); -// // }); - -// // client.on('connect', () => { -// // console.log('Socket connected, local port:', client.localPort); -// // client.write('{"Request-uuid": "unknown-uuid"}'); -// // client.end(); -// // }); -// // client.on('error', (error) => { -// // console.log('Socket connection error:', error); -// // }); - -// // server.sendCommand(options); -// // await deferred.promise; -// // assert.deepStrictEqual(eventData, ''); -// // }); - -// // // required to have "tests" or "results" -// // // the heading length not being equal and yes being equal -// // // multiple payloads -// // test('Error if payload does not have a content length header', async () => { -// // let eventData: string | undefined; -// // const client = new net.Socket(); -// // const options = { -// // command: { script: 'myscript', args: ['-foo', 'foo'] }, -// // workspaceFolder: Uri.file('/foo/bar'), -// // cwd: '/foo/bar', -// // uuid: FAKE_UUID, -// // }; -// // deferred = createDeferred(); -// // mockProc = new MockChildProcess('', ['']); -// // const output = new Observable>(() => { -// // /* no op */ -// // }); -// // const stubExecutionService2 = ({ -// // execObservable: () => { -// // client.connect(server.getPort()); -// // return { -// // proc: mockProc, -// // out: output, -// // dispose: () => { -// // /* no-body */ -// // }, -// // }; -// // }, -// // } as unknown) as IPythonExecutionService; - -// // const stubExecutionFactory2 = ({ -// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), -// // } as unknown) as IPythonExecutionFactory; -// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); -// // await server.serverReady(); -// // server.onDataReceived(({ data }) => { -// // eventData = data; -// // deferred.resolve(); -// // }); - -// // client.on('connect', () => { -// // console.log('Socket connected, local port:', client.localPort); -// // client.write('{"not content length": "5"}'); -// // client.end(); -// // }); -// // client.on('error', (error) => { -// // console.log('Socket connection error:', error); -// // }); - -// // server.sendCommand(options); -// // await deferred.promise; -// // assert.deepStrictEqual(eventData, ''); -// // }); - -// // const testData = [ -// // { -// // testName: 'fires discovery correctly on test payload', -// // payload: `Content-Length: 52 -// // Content-Type: application/json -// // Request-uuid: UUID_HERE - -// // {"cwd": "path", "status": "success", "tests": "xyz"}`, -// // expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', -// // }, -// // // Add more test data as needed -// // ]; +suite('Python Test Server, Send command etc', () => { + const FAKE_UUID = 'fake-uuid'; + let server: PythonTestServer; + let v4Stub: sinon.SinonStub; + let debugLauncher: ITestDebugLauncher; + let mockProc: MockChildProcess; + let execService: typeMoq.IMock; + let deferred: Deferred; + const sandbox = sinon.createSandbox(); -// // testData.forEach(({ testName, payload, expectedResult }) => { -// // test(`test: ${testName}`, async () => { -// // // Your test logic here -// // let eventData: string | undefined; -// // const client = new net.Socket(); + setup(async () => { + // set up test command options -// // const options = { -// // command: { script: 'myscript', args: ['-foo', 'foo'] }, -// // workspaceFolder: Uri.file('/foo/bar'), -// // cwd: '/foo/bar', -// // uuid: FAKE_UUID, -// // }; -// // deferred = createDeferred(); -// // mockProc = new MockChildProcess('', ['']); -// // const output = new Observable>(() => { -// // /* no op */ -// // }); -// // const stubExecutionService2 = ({ -// // execObservable: () => { -// // client.connect(server.getPort()); -// // return { -// // proc: mockProc, -// // out: output, -// // dispose: () => { -// // /* no-body */ -// // }, -// // }; -// // }, -// // } as unknown) as IPythonExecutionService; -// // const stubExecutionFactory2 = ({ -// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), -// // } as unknown) as IPythonExecutionFactory; + v4Stub = sandbox.stub(crypto, 'randomUUID'); + v4Stub.returns(FAKE_UUID); -// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); -// // await server.serverReady(); -// // const uuid = server.createUUID(); -// // payload = payload.replace('UUID_HERE', uuid); -// // server.onDiscoveryDataReceived(({ data }) => { -// // eventData = data; -// // deferred.resolve(); -// // }); + // set up exec service with child process + mockProc = new MockChildProcess('', ['']); + execService = typeMoq.Mock.ofType(); + execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + }); -// // client.on('connect', () => { -// // console.log('Socket connected, local port:', client.localPort); -// // client.write(payload); -// // client.end(); -// // }); -// // client.on('error', (error) => { -// // console.log('Socket connection error:', error); -// // }); + teardown(() => { + sandbox.restore(); + server.dispose(); + }); + test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { + const deferred2 = createDeferred(); + const RUN_TEST_IDS_PORT_CONST = '5678'; + execService + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns((_args, options2) => { + try { + assert.strictEqual( + options2.extraVariables.PYTHONPATH, + '/foo/bar', + 'Expect python path to exist as extra variable and be set correctly', + ); + assert.strictEqual( + options2.extraVariables.RUN_TEST_IDS_PORT, + RUN_TEST_IDS_PORT_CONST, + 'Expect test id port to be in extra variables and set correctly', + ); + } catch (e) { + assert(false, 'Error parsing data, extra variables do not match'); + } + return typeMoq.Mock.ofType>().object; + }); + const execFactory = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => { + deferred2.resolve(); + return Promise.resolve(execService.object); + }); + server = new PythonTestServer(execFactory.object, debugLauncher); + await server.serverReady(); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid: FAKE_UUID, + }; + server.sendCommand(options, RUN_TEST_IDS_PORT_CONST); + // add in await and trigger + await deferred2.promise; + mockProc.trigger('close'); + + const port = server.getPort(); + const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; + execService.verify((x) => x.execObservable(expectedArgs, typeMoq.It.isAny()), typeMoq.Times.once()); + }); -// // server.sendCommand(options); -// // await deferred.promise; -// // assert.deepStrictEqual(eventData, expectedResult); -// // }); -// // }); + test('sendCommand should write to an output channel if it is provided as an option', async () => { + const output2: string[] = []; + const outChannel = { + appendLine: (str: string) => { + output2.push(str); + }, + } as OutputChannel; + const options = { + command: { + script: 'myscript', + args: ['-foo', 'foo'], + }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid: FAKE_UUID, + outChannel, + }; + deferred = createDeferred(); + const execFactory = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => { + deferred.resolve(); + return Promise.resolve(execService.object); + }); -// // test('Calls run resolver if the result header is in the payload', async () => { -// // let eventData: string | undefined; -// // const client = new net.Socket(); + server = new PythonTestServer(execFactory.object, debugLauncher); + await server.serverReady(); -// // deferred = createDeferred(); -// // mockProc = new MockChildProcess('', ['']); -// // const output = new Observable>(() => { -// // /* no op */ -// // }); -// // const stubExecutionService2 = ({ -// // execObservable: () => { -// // client.connect(server.getPort()); -// // return { -// // proc: mockProc, -// // out: output, -// // dispose: () => { -// // /* no-body */ -// // }, -// // }; -// // }, -// // } as unknown) as IPythonExecutionService; + server.sendCommand(options); + // add in await and trigger + await deferred.promise; + mockProc.trigger('close'); -// // const stubExecutionFactory2 = ({ -// // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), -// // } as unknown) as IPythonExecutionFactory; -// // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); -// // const options = { -// // command: { script: 'myscript', args: ['-foo', 'foo'] }, -// // workspaceFolder: Uri.file('/foo/bar'), -// // cwd: '/foo/bar', -// // uuid: FAKE_UUID, -// // }; + const port = server.getPort(); + const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); -// // await server.serverReady(); -// // const uuid = server.createUUID(); -// // server.onRunDataReceived(({ data }) => { -// // eventData = data; -// // deferred.resolve(); -// // }); + assert.deepStrictEqual(output2, [expected]); + }); -// // const payload = `Content-Length: 87 -// // Content-Type: application/json -// // Request-uuid: ${uuid} + test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { + let eventData: { status: string; errors: string[] } | undefined; + const deferred2 = createDeferred(); + const deferred3 = createDeferred(); + const stubExecutionService = typeMoq.Mock.ofType(); + stubExecutionService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + stubExecutionService + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred3.resolve(); + throw new Error('Failed to execute'); + }); + const options = { + command: { script: 'myscript', args: ['-foo', 'foo'] }, + workspaceFolder: Uri.file('/foo/bar'), + cwd: '/foo/bar', + uuid: FAKE_UUID, + }; + const stubExecutionFactory = typeMoq.Mock.ofType(); + stubExecutionFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => { + deferred2.resolve(); + return Promise.resolve(stubExecutionService.object); + }); -// // {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; + server = new PythonTestServer(stubExecutionFactory.object, debugLauncher); + await server.serverReady(); -// // client.on('connect', () => { -// // console.log('Socket connected, local port:', client.localPort); -// // client.write(payload); -// // client.end(); -// // }); -// // client.on('error', (error) => { -// // console.log('Socket connection error:', error); -// // }); + server.onDataReceived(({ data }) => { + eventData = JSON.parse(data); + }); -// // server.sendCommand(options); -// // await deferred.promise; -// // const expectedResult = -// // '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; -// // assert.deepStrictEqual(eventData, expectedResult); -// // }); -// }); + server.sendCommand(options); + await deferred2.promise; + await deferred3.promise; + assert.notEqual(eventData, undefined); + assert.deepStrictEqual(eventData?.status, 'error'); + assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); + }); +}); From 10bd3e3e9f580b78190a5cce492050e2f831f52b Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 13 Sep 2023 11:02:19 -0700 Subject: [PATCH 38/55] fix linting --- pythonFiles/tests/pytestadapter/test_execution.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pythonFiles/tests/pytestadapter/test_execution.py b/pythonFiles/tests/pytestadapter/test_execution.py index bf4347d14c78..37a392f66d4b 100644 --- a/pythonFiles/tests/pytestadapter/test_execution.py +++ b/pythonFiles/tests/pytestadapter/test_execution.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import os import shutil -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List import pytest @@ -226,7 +226,8 @@ def test_pytest_execution(test_ids, expected_const): 3. uf_single_method_execution_expected_output: test run on a single method in a file. 4. uf_non_adjacent_tests_execution_expected_output: test run on unittests in two files with single selection in test explorer. 5. unit_pytest_same_file_execution_expected_output: test run on a file with both unittest and pytest tests. - 6. dual_level_nested_folder_execution_expected_output: test run on a file with one test file at the top level and one test file in a nested folder. + 6. dual_level_nested_folder_execution_expected_output: test run on a file with one test file + at the top level and one test file in a nested folder. 7. double_nested_folder_expected_execution_output: test run on a double nested folder. 8. parametrize_tests_expected_execution_output: test run on a parametrize test with 3 inputs. 9. single_parametrize_tests_expected_execution_output: test run on single parametrize test. From 8ac9df1408027640f8004abf563540f352e4c230 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 13 Sep 2023 11:05:07 -0700 Subject: [PATCH 39/55] null uuid error handling --- pythonFiles/unittestadapter/execution.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pythonFiles/unittestadapter/execution.py b/pythonFiles/unittestadapter/execution.py index 869199af9fa6..7bbc97e78f31 100644 --- a/pythonFiles/unittestadapter/execution.py +++ b/pythonFiles/unittestadapter/execution.py @@ -336,4 +336,8 @@ def post_response(payload: PayloadDict | EOTPayloadDict, port: int, uuid: str) - "result": None, } eot_payload: EOTPayloadDict = {"command_type": "execution", "eot": True} - post_response(eot_payload, PORT, UUID) + if UUID is None: + print("Error sending response, uuid unknown to python server.") + post_response(eot_payload, PORT, "unknown") + else: + post_response(eot_payload, PORT, UUID) From 00a8fd6d1395cf09dc92f2c8cc73fe5f7d88b16f Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 13 Sep 2023 11:39:55 -0700 Subject: [PATCH 40/55] seg fault tests working --- .../testing/testController/common/server.ts | 9 +- .../testing/testController/common/utils.ts | 15 +- .../pytest/pytestExecutionAdapter.ts | 6 +- .../testing/common/testingAdapter.test.ts | 98 ++- .../testing/common/testingPayloadsEot.test.ts | 9 +- .../testing/testController/helper.server.ts | 568 ------------------ .../resultResolver.unit.test.ts | 3 +- .../testController/server.unit.test.ts | 5 +- 8 files changed, 82 insertions(+), 631 deletions(-) delete mode 100644 src/test/testing/testController/helper.server.ts diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index a7368f1fe01e..80d93586d17a 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -15,7 +15,7 @@ import { traceError, traceInfo, traceLog } from '../../../logging'; import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types'; import { ITestDebugLauncher, LaunchOptions } from '../../common/types'; import { UNITTEST_PROVIDER } from '../../common/constants'; -import { createExecutionErrorPayload, extractJsonPayload } from './utils'; +import { createEOTPayload, createExecutionErrorPayload, extractJsonPayload } from './utils'; import { createDeferred } from '../../../common/utils/async'; export class PythonTestServer implements ITestServer, Disposable { @@ -35,7 +35,6 @@ export class PythonTestServer implements ITestServer, Disposable { this.server = net.createServer((socket: net.Socket) => { let buffer: Buffer = Buffer.alloc(0); // Buffer to accumulate received data socket.on('data', (data: Buffer) => { - console.log('\n&&&& raw Data: ', data.toString(), '&&&& \n'); buffer = Buffer.concat([buffer, data]); // get the new data and add it to the buffer while (buffer.length > 0) { try { @@ -104,7 +103,6 @@ export class PythonTestServer implements ITestServer, Disposable { }); // if the rawData includes result then this is a run request } else if (extractedJSON.includes(`"result":`) || extractedJSON.includes(`"command_type": "execution"`)) { - console.log('\n *** fire run data received: \n', extractedJSON, '\n *** end'); this._onRunDataReceived.fire({ uuid, data: extractedJSON, @@ -239,6 +237,11 @@ export class PythonTestServer implements ITestServer, Disposable { uuid, data: JSON.stringify(createExecutionErrorPayload(code, signal, testIds, options.cwd)), }); + // then send a EOT payload + this._onRunDataReceived.fire({ + uuid, + data: JSON.stringify(createEOTPayload(true)), + }); } deferred.resolve({ stdout: '', stderr: '' }); }); diff --git a/src/client/testing/testController/common/utils.ts b/src/client/testing/testController/common/utils.ts index 3286c072ec66..572863ecdbfe 100644 --- a/src/client/testing/testController/common/utils.ts +++ b/src/client/testing/testController/common/utils.ts @@ -9,7 +9,13 @@ import { EnableTestAdapterRewrite } from '../../../common/experiments/groups'; import { IExperimentService } from '../../../common/types'; import { IServiceContainer } from '../../../ioc/types'; import { DebugTestTag, ErrorTestItemOptions, RunTestTag } from './testItemUtilities'; -import { DiscoveredTestItem, DiscoveredTestNode, ExecutionTestPayload, ITestResultResolver } from './types'; +import { + DiscoveredTestItem, + DiscoveredTestNode, + EOTTestPayload, + ExecutionTestPayload, + ITestResultResolver, +} from './types'; import { Deferred, createDeferred } from '../../../common/utils/async'; export function fixLogLines(content: string): string { @@ -292,3 +298,10 @@ export function createExecutionErrorPayload( } return etp; } + +export function createEOTPayload(executionBool: boolean): EOTTestPayload { + return { + commandType: executionBool ? 'execution' : 'discovery', + eot: true, + } as EOTTestPayload; +} diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 031209841ec1..f5043a3c0458 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -45,7 +45,6 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { traceVerbose(uri, testIds, debugBool); const deferredTillEOT: Deferred = utils.createEOTDeferred(); const dataReceivedDisposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { - console.log('data received'); if (runInstance) { const eParsed = JSON.parse(e.data); this.resultResolver?.resolveExecution(eParsed, runInstance, deferredTillEOT); @@ -180,6 +179,11 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { uuid, data: JSON.stringify(utils.createExecutionErrorPayload(code, signal, testIds, cwd)), }); + // then send a EOT payload + this.testServer.triggerRunDataReceivedEvent({ + uuid, + data: JSON.stringify(utils.createEOTPayload(true)), + }); } deferredExec.resolve({ stdout: '', stderr: '' }); deferred.resolve(); diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 34b66340982d..47263c7638a5 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -70,7 +70,7 @@ suite('End to End Tests: test adapters', () => { testOutputChannel .setup((x) => x.append(typeMoq.It.isAny())) .callback((appendVal: any) => { - console.log('out - ', appendVal.toString()); + traceLog('output channel - ', appendVal.toString()); }) .returns(() => { // Whatever you need to return @@ -78,7 +78,7 @@ suite('End to End Tests: test adapters', () => { testOutputChannel .setup((x) => x.appendLine(typeMoq.It.isAny())) .callback((appendVal: any) => { - console.log('outL - ', appendVal.toString()); + traceLog('output channel ', appendVal.toString()); }) .returns(() => { // Whatever you need to return @@ -416,28 +416,29 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(callCount, 3, 'Expected _resolveExecution to be called once'); }); }); - test('unittest execution adapter seg fault error handling', async () => { - const resultResolverMock: typeMoq.IMock = typeMoq.Mock.ofType(); + test('UNITTEST execution adapter seg fault error handling', async () => { + resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveExecution = async (data, _token?) => { + // do the following asserts for each time resolveExecution is called, should be called once per test. + // 1. Check the status is "success" + assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); + // 2. Confirm no errors + assert.ok(data.error, "Expected errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(data.result, 'Expected results to be present'); + // 4. make sure the testID is found in the results + assert.notDeepEqual( + JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'), + -1, + 'Expected testId to be present', + ); + callCount = callCount + 1; + return Promise.resolve(); + }; + const testId = `test_seg_fault.TestSegmentationFault.test_segfault`; const testIds: string[] = [testId]; - resultResolverMock - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results - assert.notDeepEqual( - JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'), - -1, - 'Expected testId to be present', - ); - return Promise.resolve(); - }); // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathErrorWorkspace); @@ -447,7 +448,7 @@ suite('End to End Tests: test adapters', () => { pythonTestServer, configService, testOutputChannel.object, - resultResolverMock.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -459,34 +460,32 @@ suite('End to End Tests: test adapters', () => { } as any), ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object).finally(() => { - resultResolverMock.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(1), - ); + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); }); }); test('pytest execution adapter seg fault error handling', async () => { - const resultResolverMock: typeMoq.IMock = typeMoq.Mock.ofType(); + resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); + let callCount = 0; + resultResolver._resolveExecution = async (data, _token?) => { + // do the following asserts for each time resolveExecution is called, should be called once per test. + // 1. Check the status is "success" + assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); + // 2. Confirm no errors + assert.ok(data.error, "Expected errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(data.result, 'Expected results to be present'); + // 4. make sure the testID is found in the results + assert.notDeepEqual( + JSON.stringify(data).search('test_seg_fault.py::TestSegmentationFault::test_segfault'), + -1, + 'Expected testId to be present', + ); + callCount = callCount + 1; + return Promise.resolve(); + }; + const testId = `${rootPathErrorWorkspace}/test_seg_fault.py::TestSegmentationFault::test_segfault`; const testIds: string[] = [testId]; - resultResolverMock - .setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns((data) => { - // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results - assert.notDeepEqual( - JSON.stringify(data).search('test_seg_fault.py::TestSegmentationFault::test_segfault'), - -1, - 'Expected testId to be present', - ); - return Promise.resolve(); - }); // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathErrorWorkspace); @@ -496,7 +495,7 @@ suite('End to End Tests: test adapters', () => { pythonTestServer, configService, testOutputChannel.object, - resultResolverMock.object, + resultResolver, ); const testRun = typeMoq.Mock.ofType(); testRun @@ -508,10 +507,7 @@ suite('End to End Tests: test adapters', () => { } as any), ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { - resultResolverMock.verify( - (x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()), - typeMoq.Times.exactly(1), - ); + assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); }); }); }); diff --git a/src/test/testing/common/testingPayloadsEot.test.ts b/src/test/testing/common/testingPayloadsEot.test.ts index 84fbe8ee64f2..227ad5fa1697 100644 --- a/src/test/testing/common/testingPayloadsEot.test.ts +++ b/src/test/testing/common/testingPayloadsEot.test.ts @@ -28,6 +28,7 @@ import { DataWithPayloadChunks, PAYLOAD_SPLIT_MULTI_CHUNK_ARRAY, } from '../testController/payloadTestCases'; +import { traceLog } from '../../../client/logging'; const FAKE_UUID = 'fake-u-u-i-d'; export interface TestCase { @@ -82,7 +83,7 @@ suite('EOT tests', () => { // create client to act as python server which sends testing result response client = new net.Socket(); client.on('error', (error) => { - console.log('Socket connection error:', error); + traceLog('Socket connection error:', error); }); const mockProc = new MockChildProcess('', ['']); @@ -122,7 +123,7 @@ suite('EOT tests', () => { testOutputChannel .setup((x) => x.append(typeMoq.It.isAny())) .callback((appendVal: any) => { - console.log('out - ', appendVal.toString()); + traceLog('out - ', appendVal.toString()); }) .returns(() => { // Whatever you need to return @@ -130,7 +131,7 @@ suite('EOT tests', () => { testOutputChannel .setup((x) => x.appendLine(typeMoq.It.isAny())) .callback((appendVal: any) => { - console.log('outL - ', appendVal.toString()); + traceLog('outL - ', appendVal.toString()); }) .returns(() => { // Whatever you need to return @@ -144,7 +145,7 @@ suite('EOT tests', () => { test(`Testing Payloads: ${testCase.name}`, async () => { let actualCollectedResult = ''; client.on('connect', async () => { - console.log('socket connected, sending stubbed data'); + traceLog('socket connected, sending stubbed data'); // payload is a string array, each string represents one line written to the buffer const { payloadArray } = testCase.value; for (let i = 0; i < payloadArray.length; i = i + 1) { diff --git a/src/test/testing/testController/helper.server.ts b/src/test/testing/testController/helper.server.ts deleted file mode 100644 index b5077aa7f202..000000000000 --- a/src/test/testing/testController/helper.server.ts +++ /dev/null @@ -1,568 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. -/* eslint-disable @typescript-eslint/no-explicit-any */ - -import * as assert from 'assert'; -import * as net from 'net'; -import * as sinon from 'sinon'; -import * as crypto from 'crypto'; -import { Observable } from 'rxjs'; -import * as typeMoq from 'typemoq'; -import { Uri } from 'vscode'; -import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; -import { PythonTestServer } from '../../../client/testing/testController/common/server'; -import { ITestDebugLauncher } from '../../../client/testing/common/types'; -import { Deferred, createDeferred } from '../../../client/common/utils/async'; -import { MockChildProcess } from '../../mocks/mockChildProcess'; -import { PAYLOAD_SINGLE_CHUNK, PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY } from './payloadTestCases'; - -suite('Python Test Server', () => { - const FAKE_UUID = 'fake-uuid'; - let server: PythonTestServer; - let v4Stub: sinon.SinonStub; - let debugLauncher: ITestDebugLauncher; - let mockProc: MockChildProcess; - let execService: typeMoq.IMock; - let deferred: Deferred; - const sandbox = sinon.createSandbox(); - - setup(async () => { - // set up test command options - - v4Stub = sandbox.stub(crypto, 'randomUUID'); - v4Stub.returns(FAKE_UUID); - - // set up exec service with child process - mockProc = new MockChildProcess('', ['']); - execService = typeMoq.Mock.ofType(); - const outputObservable = new Observable>(() => { - /* no op */ - }); - execService - .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns(() => ({ - proc: mockProc, - out: outputObservable, - dispose: () => { - /* no-body */ - }, - })); - execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); - }); - - teardown(() => { - sandbox.restore(); - server.dispose(); - }); - test('basic payload', async () => { - let eventData = ''; - const client = new net.Socket(); - - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output2 = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output2, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - const uuid = server.createUUID(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid, - }; - - const dataWithPayloadChunks = PAYLOAD_SINGLE_CHUNK(uuid); - - await server.serverReady(); - - server.onRunDataReceived(({ data }) => { - eventData = eventData + data; - deferred.resolve(); - }); - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - // since this test is a single payload as a single chunk there should be a single line in the payload. - for (const line of dataWithPayloadChunks.payloadArray) { - client.write(line); - } - client.end(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - const expectedResult = dataWithPayloadChunks.data; - assert.deepStrictEqual(eventData, expectedResult); - }); - test('second payload', async () => { - let eventData = ''; - const client = new net.Socket(); - - deferred = createDeferred(); - mockProc = new MockChildProcess('', ['']); - const output2 = new Observable>(() => { - /* no op */ - }); - const stubExecutionService2 = ({ - execObservable: () => { - client.connect(server.getPort()); - return { - proc: mockProc, - out: output2, - dispose: () => { - /* no-body */ - }, - }; - }, - } as unknown) as IPythonExecutionService; - - const stubExecutionFactory2 = ({ - createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - } as unknown) as IPythonExecutionFactory; - server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - const uuid = server.createUUID(); - const options = { - command: { script: 'myscript', args: ['-foo', 'foo'] }, - workspaceFolder: Uri.file('/foo/bar'), - cwd: '/foo/bar', - uuid, - }; - const payloadChunk = PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY(uuid); - - await server.serverReady(); - - server.onRunDataReceived(({ data }) => { - eventData = eventData + data; - }); - client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); - for (const line of payloadChunk.payloadArray) { - client.write(line); - } - client.end(); - }); - client.on('close', () => { - console.log('Socket connection exit:'); - deferred.resolve(); - }); - client.on('error', (error) => { - console.log('Socket connection error:', error); - }); - - server.sendCommand(options); - await deferred.promise; - const expectedResult = payloadChunk.data; - assert.deepStrictEqual(eventData, expectedResult); - }); - - // test('sendCommand should add the port to the command being sent and add the correct extra spawn variables', async () => { - // deferred2 = createDeferred(); - // execFactory = typeMoq.Mock.ofType(); - // execFactory - // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - // .returns(() => { - // deferred2.resolve(); - // return Promise.resolve(execService.object); - // }); - // server = new PythonTestServer(execFactory.object, debugLauncher); - // await server.serverReady(); - // server.sendCommand(BASE_TEST_COMMAND_OPTIONS, '56789'); - // // add in await and trigger - // await deferred2.promise; - // mockProc.trigger('close'); - - // const port = server.getPort(); - // const expectedArgs = ['myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo']; - // execService.verify((x) => x.execObservable(expectedArgs, BASE_SPAWN_OPTIONS), typeMoq.Times.once()); - // }); - - // test('sendCommand should write to an output channel if it is provided as an option', async () => { - // const output2: string[] = []; - // const outChannel = { - // appendLine: (str: string) => { - // output2.push(str); - // }, - // } as OutputChannel; - // const options = { - // command: { - // script: 'myscript', - // args: ['-foo', 'foo'], - // }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // outChannel, - // }; - // deferred = createDeferred(); - // execFactory = typeMoq.Mock.ofType(); - // execFactory - // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - // .returns(() => { - // deferred.resolve(); - // return Promise.resolve(execService.object); - // }); - - // server = new PythonTestServer(execFactory.object, debugLauncher); - // await server.serverReady(); - - // server.sendCommand(options); - // // add in await and trigger - // await deferred.promise; - // mockProc.trigger('close'); - - // const port = server.getPort(); - // const expected = ['python', 'myscript', '--port', `${port}`, '--uuid', FAKE_UUID, '-foo', 'foo'].join(' '); - - // assert.deepStrictEqual(output2, [expected]); - // }); - - // test('If script execution fails during sendCommand, an onDataReceived event should be fired with the "error" status', async () => { - // let eventData: { status: string; errors: string[] } | undefined; - // stubExecutionService = ({ - // execObservable: () => { - // throw new Error('Failed to execute'); - // }, - // } as unknown) as IPythonExecutionService; - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - - // server = new PythonTestServer(stubExecutionFactory, debugLauncher); - // await server.serverReady(); - - // server.onDataReceived(({ data }) => { - // eventData = JSON.parse(data); - // }); - - // await server.sendCommand(options); - - // assert.notEqual(eventData, undefined); - // assert.deepStrictEqual(eventData?.status, 'error'); - // assert.deepStrictEqual(eventData?.errors, ['Failed to execute']); - // }); - - // test('If the server receives malformed data, it should display a log message, and not fire an event', async () => { - // deferred2 = createDeferred(); - // execFactory = typeMoq.Mock.ofType(); - // execFactory - // .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - // .returns(() => { - // deferred2.resolve(); - // return Promise.resolve(execService.object); - // }); - // server = new PythonTestServer(execFactory.object, debugLauncher); - // await server.serverReady(); - // let eventData: string | undefined; - // const client = new net.Socket(); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - // mockProc = new MockChildProcess('', ['']); - // const output2 = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output2, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - - // deferred = createDeferred(); - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // server.onDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write('malformed data'); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // // add in await and trigger - // await deferred.promise; - // mockProc.trigger('close'); - - // assert.deepStrictEqual(eventData, ''); - // }); - - // test('If the server doesnt recognize the UUID it should ignore it', async () => { - // let eventData: string | undefined; - // const client = new net.Socket(); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // server.onDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write('{"Request-uuid": "unknown-uuid"}'); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // assert.deepStrictEqual(eventData, ''); - // }); - - // // required to have "tests" or "results" - // // the heading length not being equal and yes being equal - // // multiple payloads - // test('Error if payload does not have a content length header', async () => { - // let eventData: string | undefined; - // const client = new net.Socket(); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // server.onDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write('{"not content length": "5"}'); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // assert.deepStrictEqual(eventData, ''); - // }); - - // const testData = [ - // { - // testName: 'fires discovery correctly on test payload', - // payload: `Content-Length: 52 - // Content-Type: application/json - // Request-uuid: UUID_HERE - - // {"cwd": "path", "status": "success", "tests": "xyz"}`, - // expectedResult: '{"cwd": "path", "status": "success", "tests": "xyz"}', - // }, - // // Add more test data as needed - // ]; - - // testData.forEach(({ testName, payload, expectedResult }) => { - // test(`test: ${testName}`, async () => { - // // Your test logic here - // let eventData: string | undefined; - // const client = new net.Socket(); - - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // await server.serverReady(); - // const uuid = server.createUUID(); - // payload = payload.replace('UUID_HERE', uuid); - // server.onDiscoveryDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write(payload); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // assert.deepStrictEqual(eventData, expectedResult); - // }); - // }); - - // test('Calls run resolver if the result header is in the payload', async () => { - // let eventData: string | undefined; - // const client = new net.Socket(); - - // deferred = createDeferred(); - // mockProc = new MockChildProcess('', ['']); - // const output = new Observable>(() => { - // /* no op */ - // }); - // const stubExecutionService2 = ({ - // execObservable: () => { - // client.connect(server.getPort()); - // return { - // proc: mockProc, - // out: output, - // dispose: () => { - // /* no-body */ - // }, - // }; - // }, - // } as unknown) as IPythonExecutionService; - - // const stubExecutionFactory2 = ({ - // createActivatedEnvironment: () => Promise.resolve(stubExecutionService2), - // } as unknown) as IPythonExecutionFactory; - // server = new PythonTestServer(stubExecutionFactory2, debugLauncher); - // const options = { - // command: { script: 'myscript', args: ['-foo', 'foo'] }, - // workspaceFolder: Uri.file('/foo/bar'), - // cwd: '/foo/bar', - // uuid: FAKE_UUID, - // }; - - // await server.serverReady(); - // const uuid = server.createUUID(); - // server.onRunDataReceived(({ data }) => { - // eventData = data; - // deferred.resolve(); - // }); - - // const payload = `Content-Length: 87 - // Content-Type: application/json - // Request-uuid: ${uuid} - - // {"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}`; - - // client.on('connect', () => { - // console.log('Socket connected, local port:', client.localPort); - // client.write(payload); - // client.end(); - // }); - // client.on('error', (error) => { - // console.log('Socket connection error:', error); - // }); - - // server.sendCommand(options); - // await deferred.promise; - // const expectedResult = - // '{"cwd": "path", "status": "success", "result": "xyz", "not_found": null, "error": null}'; - // assert.deepStrictEqual(eventData, expectedResult); - // }); -}); diff --git a/src/test/testing/testController/resultResolver.unit.test.ts b/src/test/testing/testController/resultResolver.unit.test.ts index 56322cc700bc..2078c72e8cf6 100644 --- a/src/test/testing/testController/resultResolver.unit.test.ts +++ b/src/test/testing/testController/resultResolver.unit.test.ts @@ -15,6 +15,7 @@ import * as testItemUtilities from '../../../client/testing/testController/commo import * as ResultResolver from '../../../client/testing/testController/common/resultResolver'; import * as util from '../../../client/testing/testController/common/utils'; import { Deferred, createDeferred } from '../../../client/common/utils/async'; +import { traceLog } from '../../../client/logging'; suite('Result Resolver tests', () => { suite('Test discovery', () => { @@ -290,7 +291,7 @@ suite('Result Resolver tests', () => { .setup((t) => t.createTestItem(typemoq.It.isAny(), typemoq.It.isAny())) .callback((id: string) => { generatedId = id; - console.log('createTestItem function called with id:', id); + traceLog('createTestItem function called with id:', id); }) .returns(() => ({ id: 'id_this', label: 'label_this', uri: workspaceUri } as TestItem)); diff --git a/src/test/testing/testController/server.unit.test.ts b/src/test/testing/testController/server.unit.test.ts index cd4abc6832c2..92a9a1135f55 100644 --- a/src/test/testing/testController/server.unit.test.ts +++ b/src/test/testing/testController/server.unit.test.ts @@ -25,6 +25,7 @@ import { PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY, DataWithPayloadChunks, } from './payloadTestCases'; +import { traceLog } from '../../../client/logging'; const testCases = [ { @@ -127,7 +128,7 @@ suite('Python Test Server, DataWithPayloadChunks', () => { deferred.resolve(); }); client.on('connect', () => { - console.log('Socket connected, local port:', client.localPort); + traceLog('Socket connected, local port:', client.localPort); // since this test is a single payload as a single chunk there should be a single line in the payload. for (const line of dataWithPayloadChunks.payloadArray) { client.write(line); @@ -135,7 +136,7 @@ suite('Python Test Server, DataWithPayloadChunks', () => { client.end(); }); client.on('error', (error) => { - console.log('Socket connection error:', error); + traceLog('Socket connection error:', error); }); server.sendCommand(options); From 2eee564b70f744ab5918dd1e9af9fc8db178315a Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 13 Sep 2023 13:20:30 -0700 Subject: [PATCH 41/55] logging fixing --- pythonFiles/vscode_pytest/__init__.py | 10 ++-------- pythonFiles/vscode_pytest/run_pytest_script.py | 2 +- src/client/testing/testController/controller.ts | 1 + 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/pythonFiles/vscode_pytest/__init__.py b/pythonFiles/vscode_pytest/__init__.py index 5e2ceb58cd94..86f753315604 100644 --- a/pythonFiles/vscode_pytest/__init__.py +++ b/pythonFiles/vscode_pytest/__init__.py @@ -306,12 +306,6 @@ def pytest_sessionfinish(session, exitstatus): 4: Pytest encountered an internal error or exception during test execution. 5: Pytest was unable to find any tests to run. """ - # print( - # "pytest session has finished, exit status: ", - # exitstatus, - # "in discovery? ", - # IS_DISCOVERY, - # ) cwd = pathlib.Path.cwd() if IS_DISCOVERY: if not (exitstatus == 0 or exitstatus == 1 or exitstatus == 5): @@ -714,8 +708,8 @@ def send_post_request( try: if __socket is not None and __socket.socket is not None: __socket.socket.sendall(request.encode("utf-8")) - print("Post request sent successfully!") - print("data sent", payload, "end of data") + # print("Post request sent successfully!") + # print("data sent", payload, "end of data") break # Exit the loop if the send was successful else: print("Plugin error connection error[vscode-pytest]") diff --git a/pythonFiles/vscode_pytest/run_pytest_script.py b/pythonFiles/vscode_pytest/run_pytest_script.py index ffb4d0c55b16..d10a1ace9d01 100644 --- a/pythonFiles/vscode_pytest/run_pytest_script.py +++ b/pythonFiles/vscode_pytest/run_pytest_script.py @@ -53,7 +53,7 @@ buffer = b"" # Process the JSON data - print(f"Received JSON data: {test_ids_from_buffer}") + print(f"Received JSON data in run script") break except json.JSONDecodeError: # JSON decoding error, the complete JSON object is not yet received diff --git a/src/client/testing/testController/controller.ts b/src/client/testing/testController/controller.ts index 1550323ff8f8..0c7db5594004 100644 --- a/src/client/testing/testController/controller.ts +++ b/src/client/testing/testController/controller.ts @@ -371,6 +371,7 @@ export class PythonTestController implements ITestController, IExtensionSingleAc ); const dispose = token.onCancellationRequested(() => { + runInstance.appendOutput(`Run instance cancelled.\r\n`); runInstance.end(); }); From b064d3754e0b58b0e7ae81fe2a14e9cf2747eda5 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Wed, 13 Sep 2023 15:41:27 -0700 Subject: [PATCH 42/55] add cancellation token tests --- .../pytest/pytestExecutionAdapter.ts | 19 +- .../unittest/testExecutionAdapter.ts | 11 +- .../pytestExecutionAdapter.unit.test.ts | 129 +++++++++- .../testCancellationRunAdapters.unit.test.ts | 224 ++++++++++++++++++ 4 files changed, 369 insertions(+), 14 deletions(-) create mode 100644 src/test/testing/testController/testCancellationRunAdapters.unit.test.ts diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index f5043a3c0458..1f8936f772e9 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -55,10 +55,19 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { dataReceivedDisposable.dispose(); }; runInstance?.token.onCancellationRequested(() => { - disposeDataReceiver(this.testServer); + deferredTillEOT.resolve(); }); try { - this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, executionFactory, debugLauncher); + this.runTestsNew( + uri, + testIds, + uuid, + runInstance, + debugBool, + executionFactory, + debugLauncher, + deferredTillEOT, + ); } finally { await deferredTillEOT.promise; disposeDataReceiver(this.testServer); @@ -82,8 +91,8 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { debugBool?: boolean, executionFactory?: IPythonExecutionFactory, debugLauncher?: ITestDebugLauncher, + deferredTillEOT?: Deferred, ): Promise { - const deferred = createDeferred(); const relativePathToPytest = 'pythonFiles'; const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); this.configSettings.isTestExecution(); @@ -146,8 +155,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { }; traceInfo(`Running DEBUG pytest with arguments: ${testArgs.join(' ')}\r\n`); await debugLauncher!.launchDebugger(launchOptions, () => { - deferred.resolve(); - this.testServer.deleteUUID(uuid); + deferredTillEOT?.resolve(); }); } else { // combine path to run script with run args @@ -186,7 +194,6 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { }); } deferredExec.resolve({ stdout: '', stderr: '' }); - deferred.resolve(); // disposeDataReceiver?.(this.testServer); }); await deferredExec.promise; diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index 7aa7e21acc5e..373dc626fbcc 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -48,10 +48,10 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { disposedDataReceived.dispose(); }; runInstance?.token.onCancellationRequested(() => { - disposeDataReceiver(this.testServer); + deferredTillEOT.resolve(); }); try { - await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, disposeDataReceiver); + await this.runTestsNew(uri, testIds, uuid, runInstance, debugBool, deferredTillEOT); await deferredTillEOT.promise; disposeDataReceiver(this.testServer); } catch (error) { @@ -67,7 +67,7 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { uuid: string, runInstance?: TestRun, debugBool?: boolean, - disposeDataReceiver?: (testServer: ITestServer) => void, + deferredTillEOT?: Deferred, ): Promise { const settings = this.configSettings.getSettings(uri); const { unittestArgs } = settings.testing; @@ -84,15 +84,12 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { testIds, outChannel: this.outputChannel, }; - - const deferred = createDeferred(); traceLog(`Running UNITTEST execution for the following test ids: ${testIds}`); const runTestIdsPort = await startTestIdServer(testIds); await this.testServer.sendCommand(options, runTestIdsPort.toString(), runInstance, testIds, () => { - deferred.resolve(); - disposeDataReceiver?.(this.testServer); + deferredTillEOT?.resolve(); }); // placeholder until after the rewrite is adopted // TODO: remove after adoption. diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index 9cc428ab0a4c..cee14bd279d3 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as assert from 'assert'; -import { TestRun, Uri } from 'vscode'; +import { CancellationToken, CancellationTokenSource, TestRun, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as sinon from 'sinon'; import * as path from 'path'; @@ -22,6 +22,7 @@ import * as util from '../../../../client/testing/testController/common/utils'; import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; import { MockChildProcess } from '../../../mocks/mockChildProcess'; import { traceInfo } from '../../../../client/logging'; +import { UnittestTestExecutionAdapter } from '../../../../client/testing/testController/unittest/testExecutionAdapter'; suite('pytest test execution adapter', () => { let testServer: typeMoq.IMock; @@ -311,4 +312,130 @@ suite('pytest test execution adapter', () => { ); testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); }); + test('PYTEST cancelation token called mid-run resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + execServiceMock + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + cancellationToken.cancel(); + return { + proc: mockProc, + out: typeMoq.Mock.ofType>>().object, + dispose: () => { + /* no-body */ + }, + }; + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + adapter = new PytestTestExecutionAdapter( + testServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await adapter.runTests( + Uri.file(myTestPath), + [], + false, + testRunMock.object, + execFactoryMock.object, + debugLauncher.object, + ); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); + test('UNITTEST cancelation token called mid-run resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + execServiceMock + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + cancellationToken.cancel(); + return { + proc: mockProc, + out: typeMoq.Mock.ofType>>().object, + dispose: () => { + /* no-body */ + }, + }; + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + adapter = new UnittestTestExecutionAdapter( + testServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await adapter.runTests( + Uri.file(myTestPath), + [], + false, + testRunMock.object, + execFactoryMock.object, + debugLauncher.object, + ); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); }); diff --git a/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts new file mode 100644 index 000000000000..d3a4b768da80 --- /dev/null +++ b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts @@ -0,0 +1,224 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { CancellationTokenSource, TestRun, Uri } from 'vscode'; +import * as typeMoq from 'typemoq'; +import * as sinon from 'sinon'; +import * as path from 'path'; +import { Observable } from 'rxjs'; +import { IPythonExecutionFactory, IPythonExecutionService, Output } from '../../../client/common/process/types'; +import { IConfigurationService, ITestOutputChannel } from '../../../client/common/types'; +import { Deferred, createDeferred } from '../../../client/common/utils/async'; +import { EXTENSION_ROOT_DIR } from '../../../client/constants'; +import { ITestDebugLauncher } from '../../../client/testing/common/types'; +import { ITestServer } from '../../../client/testing/testController/common/types'; +import { PytestTestExecutionAdapter } from '../../../client/testing/testController/pytest/pytestExecutionAdapter'; +import { UnittestTestExecutionAdapter } from '../../../client/testing/testController/unittest/testExecutionAdapter'; +import { MockChildProcess } from '../../mocks/mockChildProcess'; +import * as util from '../../../client/testing/testController/common/utils'; + +suite('Execution Flow Run Adapters', () => { + let testServer: typeMoq.IMock; + let configService: IConfigurationService; + let execFactory = typeMoq.Mock.ofType(); + let adapter: PytestTestExecutionAdapter; + let execService: typeMoq.IMock; + let deferred: Deferred; + let debugLauncher: typeMoq.IMock; + (global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR; + let myTestPath: string; + let mockProc: MockChildProcess; + let utilsStartServerStub: sinon.SinonStub; + setup(() => { + testServer = typeMoq.Mock.ofType(); + testServer.setup((t) => t.getPort()).returns(() => 12345); + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + configService = ({ + getSettings: () => ({ + testing: { pytestArgs: ['.'], unittestArgs: ['-v', '-s', '.', '-p', 'test*'] }, + }), + isTestExecution: () => false, + } as unknown) as IConfigurationService; + + // mock out the result resolver + + // set up exec service with child process + mockProc = new MockChildProcess('', ['']); + const output = new Observable>(() => { + /* no op */ + }); + execService = typeMoq.Mock.ofType(); + execService + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + proc: mockProc, + out: output, + dispose: () => { + /* no-body */ + }, + })); + execFactory = typeMoq.Mock.ofType(); + utilsStartServerStub = sinon.stub(util, 'startTestIdServer'); + debugLauncher = typeMoq.Mock.ofType(); + execFactory + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execService.object)); + deferred = createDeferred(); + execService + .setup((x) => x.exec(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + deferred.resolve(); + return Promise.resolve({ stdout: '{}' }); + }); + execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + myTestPath = path.join('/', 'my', 'test', 'path', '/'); + }); + teardown(() => { + sinon.restore(); + }); + test('PYTEST cancelation token called mid-run resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + execServiceMock + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + cancellationToken.cancel(); + return { + proc: mockProc, + out: typeMoq.Mock.ofType>>().object, + dispose: () => { + /* no-body */ + }, + }; + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + adapter = new PytestTestExecutionAdapter( + testServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await adapter.runTests( + Uri.file(myTestPath), + [], + false, + testRunMock.object, + execFactoryMock.object, + debugLauncher.object, + ); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); + test('UNITTEST cancelation token called mid-run resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + + // Stub send command to then have token canceled + const stubTestServer = typeMoq.Mock.ofType(); + stubTestServer + .setup((t) => + t.sendCommand( + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + ), + ) + .returns(() => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + + stubTestServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + stubTestServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + execServiceMock + .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => { + cancellationToken.cancel(); + return { + proc: mockProc, + out: typeMoq.Mock.ofType>>().object, + dispose: () => { + /* no-body */ + }, + }; + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + const unittestAdapter = new UnittestTestExecutionAdapter( + stubTestServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await unittestAdapter.runTests(Uri.file(myTestPath), [], false, testRunMock.object); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + stubTestServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); +}); From 3512173d2d0a9b112e22567c9fbb578b1f690d82 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 14 Sep 2023 08:53:25 -0700 Subject: [PATCH 43/55] fix linting --- .../pytestExecutionAdapter.unit.test.ts | 129 +----------------- 1 file changed, 1 insertion(+), 128 deletions(-) diff --git a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts index cee14bd279d3..9cc428ab0a4c 100644 --- a/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts +++ b/src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts @@ -2,7 +2,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. import * as assert from 'assert'; -import { CancellationToken, CancellationTokenSource, TestRun, Uri } from 'vscode'; +import { TestRun, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as sinon from 'sinon'; import * as path from 'path'; @@ -22,7 +22,6 @@ import * as util from '../../../../client/testing/testController/common/utils'; import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; import { MockChildProcess } from '../../../mocks/mockChildProcess'; import { traceInfo } from '../../../../client/logging'; -import { UnittestTestExecutionAdapter } from '../../../../client/testing/testController/unittest/testExecutionAdapter'; suite('pytest test execution adapter', () => { let testServer: typeMoq.IMock; @@ -312,130 +311,4 @@ suite('pytest test execution adapter', () => { ); testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); }); - test('PYTEST cancelation token called mid-run resolves correctly', async () => { - // mock test run and cancelation token - const testRunMock = typeMoq.Mock.ofType(); - const cancellationToken = new CancellationTokenSource(); - const { token } = cancellationToken; - testRunMock.setup((t) => t.token).returns(() => token); - // mock exec service and exec factory - const execServiceMock = typeMoq.Mock.ofType(); - execServiceMock - .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns(() => { - cancellationToken.cancel(); - return { - proc: mockProc, - out: typeMoq.Mock.ofType>>().object, - dispose: () => { - /* no-body */ - }, - }; - }); - const execFactoryMock = typeMoq.Mock.ofType(); - execFactoryMock - .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - .returns(() => Promise.resolve(execServiceMock.object)); - execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); - execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); - - const deferredStartServer = createDeferred(); - utilsStartServerStub.callsFake(() => { - deferredStartServer.resolve(); - return Promise.resolve(54321); - }); - // mock EOT token - const deferredEOT = createDeferred(); - const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); - utilsCreateEOTStub.callsFake(() => deferredEOT); - // set up test server - testServer - .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns(() => ({ - dispose: () => { - /* no-body */ - }, - })); - testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); - adapter = new PytestTestExecutionAdapter( - testServer.object, - configService, - typeMoq.Mock.ofType().object, - ); - await adapter.runTests( - Uri.file(myTestPath), - [], - false, - testRunMock.object, - execFactoryMock.object, - debugLauncher.object, - ); - // wait for server to start to keep test from failing - await deferredStartServer.promise; - - testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); - }); - test('UNITTEST cancelation token called mid-run resolves correctly', async () => { - // mock test run and cancelation token - const testRunMock = typeMoq.Mock.ofType(); - const cancellationToken = new CancellationTokenSource(); - const { token } = cancellationToken; - testRunMock.setup((t) => t.token).returns(() => token); - // mock exec service and exec factory - const execServiceMock = typeMoq.Mock.ofType(); - execServiceMock - .setup((x) => x.execObservable(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns(() => { - cancellationToken.cancel(); - return { - proc: mockProc, - out: typeMoq.Mock.ofType>>().object, - dispose: () => { - /* no-body */ - }, - }; - }); - const execFactoryMock = typeMoq.Mock.ofType(); - execFactoryMock - .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) - .returns(() => Promise.resolve(execServiceMock.object)); - execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); - execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); - - const deferredStartServer = createDeferred(); - utilsStartServerStub.callsFake(() => { - deferredStartServer.resolve(); - return Promise.resolve(54321); - }); - // mock EOT token - const deferredEOT = createDeferred(); - const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); - utilsCreateEOTStub.callsFake(() => deferredEOT); - // set up test server - testServer - .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) - .returns(() => ({ - dispose: () => { - /* no-body */ - }, - })); - testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); - adapter = new UnittestTestExecutionAdapter( - testServer.object, - configService, - typeMoq.Mock.ofType().object, - ); - await adapter.runTests( - Uri.file(myTestPath), - [], - false, - testRunMock.object, - execFactoryMock.object, - debugLauncher.object, - ); - // wait for server to start to keep test from failing - await deferredStartServer.promise; - - testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); - }); }); From c1079a19b7809b32d117ad49f6e29569b80bb523 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 14 Sep 2023 09:02:36 -0700 Subject: [PATCH 44/55] cancellation for debug --- .../testCancellationRunAdapters.unit.test.ts | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts index d3a4b768da80..2aaffdda41df 100644 --- a/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts +++ b/src/test/testing/testController/testCancellationRunAdapters.unit.test.ts @@ -29,6 +29,7 @@ suite('Execution Flow Run Adapters', () => { let myTestPath: string; let mockProc: MockChildProcess; let utilsStartServerStub: sinon.SinonStub; + setup(() => { testServer = typeMoq.Mock.ofType(); testServer.setup((t) => t.getPort()).returns(() => 12345); @@ -147,6 +148,63 @@ suite('Execution Flow Run Adapters', () => { testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); }); + test('PYTEST cancelation token called mid-debug resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + debugLauncher + .setup((dl) => dl.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(async () => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + testServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + testServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + adapter = new PytestTestExecutionAdapter( + testServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await adapter.runTests( + Uri.file(myTestPath), + [], + true, + testRunMock.object, + execFactoryMock.object, + debugLauncher.object, + ); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + + testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); test('UNITTEST cancelation token called mid-run resolves correctly', async () => { // mock test run and cancelation token const testRunMock = typeMoq.Mock.ofType(); @@ -219,6 +277,74 @@ suite('Execution Flow Run Adapters', () => { // wait for server to start to keep test from failing await deferredStartServer.promise; + stubTestServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); + }); + test('UNITTEST cancelation token called mid-debug resolves correctly', async () => { + // mock test run and cancelation token + const testRunMock = typeMoq.Mock.ofType(); + const cancellationToken = new CancellationTokenSource(); + const { token } = cancellationToken; + testRunMock.setup((t) => t.token).returns(() => token); + + // Stub send command to then have token canceled + const stubTestServer = typeMoq.Mock.ofType(); + stubTestServer + .setup((t) => + t.sendCommand( + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + typeMoq.It.isAny(), + ), + ) + .returns(() => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + + stubTestServer.setup((t) => t.createUUID(typeMoq.It.isAny())).returns(() => 'uuid123'); + stubTestServer + .setup((t) => t.onRunDataReceived(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(() => ({ + dispose: () => { + /* no-body */ + }, + })); + // mock exec service and exec factory + const execServiceMock = typeMoq.Mock.ofType(); + debugLauncher + .setup((dl) => dl.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny())) + .returns(async () => { + cancellationToken.cancel(); + return Promise.resolve(); + }); + const execFactoryMock = typeMoq.Mock.ofType(); + execFactoryMock + .setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny())) + .returns(() => Promise.resolve(execServiceMock.object)); + execFactoryMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + execServiceMock.setup((p) => ((p as unknown) as any).then).returns(() => undefined); + + const deferredStartServer = createDeferred(); + utilsStartServerStub.callsFake(() => { + deferredStartServer.resolve(); + return Promise.resolve(54321); + }); + // mock EOT token + const deferredEOT = createDeferred(); + const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred'); + utilsCreateEOTStub.callsFake(() => deferredEOT); + // set up test server + const unittestAdapter = new UnittestTestExecutionAdapter( + stubTestServer.object, + configService, + typeMoq.Mock.ofType().object, + ); + await unittestAdapter.runTests(Uri.file(myTestPath), [], false, testRunMock.object); + // wait for server to start to keep test from failing + await deferredStartServer.promise; + stubTestServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once()); }); }); From aeabdc33a751a11175a83a8f224d71c40219b418 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 14 Sep 2023 09:14:05 -0700 Subject: [PATCH 45/55] formatting --- pythonFiles/vscode_pytest/run_pytest_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pythonFiles/vscode_pytest/run_pytest_script.py b/pythonFiles/vscode_pytest/run_pytest_script.py index d10a1ace9d01..0fca8208a406 100644 --- a/pythonFiles/vscode_pytest/run_pytest_script.py +++ b/pythonFiles/vscode_pytest/run_pytest_script.py @@ -53,7 +53,7 @@ buffer = b"" # Process the JSON data - print(f"Received JSON data in run script") + print("Received JSON data in run script") break except json.JSONDecodeError: # JSON decoding error, the complete JSON object is not yet received From 0c51954c8043cd3342c44a0c9b9daf5d01f92c73 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 14 Sep 2023 10:48:17 -0700 Subject: [PATCH 46/55] fix seg_fault up test count --- .vscode/settings.json | 11 +- .../testing/common/testingAdapter.test.ts | 135 +++++++++++++----- .../test_parameterized_subtest.py | 4 +- 3 files changed, 110 insertions(+), 40 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 174a850c901e..06011b3d13cd 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -65,5 +65,14 @@ "--max-line-length=88" ], "typescript.preferences.importModuleSpecifier": "relative", - "debug.javascript.usePreview": false + "debug.javascript.usePreview": false, + // Branch name suggestion. + "git.branchRandomName.enable": true, + "git.branchProtection": [ + "main", + "release/*" + ], + "git.pullBeforeCheckout": true, + // Open merge editor for resolving conflicts. + "git.mergeEditor": true, } diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 47263c7638a5..a185a0f166e1 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -196,7 +196,6 @@ suite('End to End Tests: test adapters', () => { // set workspace to test workspace folder workspaceUri = Uri.parse(rootPathSmallWorkspace); - await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { // verification after discovery is complete @@ -253,12 +252,19 @@ suite('End to End Tests: test adapters', () => { // result resolver and saved data for assertions resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; + let failureOccurred = false; + let failureMsg: string | undefined = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; // the payloads that get to the _resolveExecution are all data and should be successful. - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); - assert.ok(payload.result, 'Expected results to be present'); + try { + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err?.toString(); + failureOccurred = true; + } return Promise.resolve(); }; @@ -286,18 +292,30 @@ suite('End to End Tests: test adapters', () => { .finally(() => { // verify that the _resolveExecution was called once per test assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('unittest execution adapter large workspace', async () => { // result resolver and saved data for assertions resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; + let failureOccurred = false; + let failureMsg: string | undefined = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; // the payloads that get to the _resolveExecution are all data and should be successful. - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); - assert.ok(payload.result, 'Expected results to be present'); + try { + const validStatuses = ['subtest-success', 'subtest-failure']; + assert.ok( + validStatuses.includes(payload.status), + `Expected status to be one of ${validStatuses.join(', ')}`, + ); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err?.toString(); + failureOccurred = true; + } return Promise.resolve(); }; @@ -325,19 +343,27 @@ suite('End to End Tests: test adapters', () => { .runTests(workspaceUri, ['test_parameterized_subtest.NumbersTest.test_even'], false, testRun.object) .then(() => { // verify that the _resolveExecution was called once per test - assert.strictEqual(callCount, 3, 'Expected _resolveExecution to be called once'); + assert.strictEqual(callCount, 2000, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('pytest execution adapter small workspace', async () => { // result resolver and saved data for assertions resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; + let failureOccurred = false; + let failureMsg: string | undefined = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; // the payloads that get to the _resolveExecution are all data and should be successful. - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); - assert.ok(payload.result, 'Expected results to be present'); + try { + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err?.toString(); + failureOccurred = true; + } return Promise.resolve(); }; // set workspace to test workspace folder @@ -370,18 +396,26 @@ suite('End to End Tests: test adapters', () => { .then(() => { // verify that the _resolveExecution was called once per test assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('pytest execution adapter large workspace', async () => { // result resolver and saved data for assertions resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; + let failureOccurred = false; + let failureMsg: string | undefined = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; // the payloads that get to the _resolveExecution are all data and should be successful. - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); - assert.ok(payload.result, 'Expected results to be present'); + try { + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err?.toString(); + failureOccurred = true; + } return Promise.resolve(); }; @@ -390,7 +424,7 @@ suite('End to End Tests: test adapters', () => { // generate list of test_ids const testIds: string[] = []; - for (let i = 0; i < 3; i = i + 1) { + for (let i = 0; i < 2000; i = i + 1) { const testId = `${rootPathLargeWorkspace}/test_parameterized_subtest.py::test_odd_even[${i}]`; testIds.push(testId); } @@ -413,27 +447,45 @@ suite('End to End Tests: test adapters', () => { ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).then(() => { // verify that the _resolveExecution was called once per test - assert.strictEqual(callCount, 3, 'Expected _resolveExecution to be called once'); + assert.strictEqual(callCount, 2000, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); - test('UNITTEST execution adapter seg fault error handling', async () => { + test('unittest execution adapter seg fault error handling', async () => { resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; + let failureOccurred = false; + let failureMsg: string | undefined = ''; + resultResolver._resolveExecution = async (payload, _token?) => { + traceLog(`resolveDiscovery ${payload}`); + callCount = callCount + 1; + // the payloads that get to the _resolveExecution are all data and should be successful. + try { + assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.ok(payload.result, 'Expected results to be present'); + } catch (err) { + failureMsg = err?.toString(); + failureOccurred = true; + } + return Promise.resolve(); + }; resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results - assert.notDeepEqual( - JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'), - -1, - 'Expected testId to be present', - ); callCount = callCount + 1; + try { + // 1. Check the status is "success" + assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); + // 2. Confirm no errors + assert.ok(data.error, "Expected errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(data.result, 'Expected results to be present'); + // 4. make sure the testID is found in the results + const indexOfTest = JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'); + assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); + } catch (err) { + failureMsg = err?.toString(); + failureOccurred = true; + } return Promise.resolve(); }; @@ -461,26 +513,34 @@ suite('End to End Tests: test adapters', () => { ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object).finally(() => { assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); test('pytest execution adapter seg fault error handling', async () => { resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); let callCount = 0; + let failureOccurred = false; + let failureMsg: string | undefined = ''; resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. - // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found - assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results - assert.notDeepEqual( - JSON.stringify(data).search('test_seg_fault.py::TestSegmentationFault::test_segfault'), - -1, - 'Expected testId to be present', - ); callCount = callCount + 1; + try { + // 1. Check the status is "success" + assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); + // 2. Confirm no errors + assert.ok(data.error, "Expected errors in 'error' field"); + // 3. Confirm tests are found + assert.ok(data.result, 'Expected results to be present'); + // 4. make sure the testID is found in the results + assert.notDeepEqual( + JSON.stringify(data).search('test_seg_fault.py::TestSegmentationFault::test_segfault'), + -1, + 'Expected testId to be present', + ); + } catch (err) { + failureMsg = err?.toString(); + failureOccurred = true; + } return Promise.resolve(); }; @@ -508,6 +568,7 @@ suite('End to End Tests: test adapters', () => { ); await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => { assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once'); + assert.strictEqual(failureOccurred, false, failureMsg); }); }); }); diff --git a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py index ebbfa7393cb7..a76856ebb929 100644 --- a/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py +++ b/src/testTestingRootWkspc/largeWorkspace/test_parameterized_subtest.py @@ -4,13 +4,13 @@ import unittest -@pytest.mark.parametrize("num", range(0, 3)) +@pytest.mark.parametrize("num", range(0, 2000)) def test_odd_even(num): assert num % 2 == 0 class NumbersTest(unittest.TestCase): def test_even(self): - for i in range(0, 3): + for i in range(0, 2000): with self.subTest(i=i): self.assertEqual(i % 2, 0) From 9712c091e5e6e83e6140e89e079adec017b6e207 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 14 Sep 2023 11:47:08 -0700 Subject: [PATCH 47/55] types --- .../testing/common/testingAdapter.test.ts | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index a185a0f166e1..b605ca16aaf8 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -253,7 +253,7 @@ suite('End to End Tests: test adapters', () => { resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; let failureOccurred = false; - let failureMsg: string | undefined = ''; + let failureMsg = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; @@ -262,7 +262,7 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { - failureMsg = err?.toString(); + failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; } return Promise.resolve(); @@ -300,7 +300,7 @@ suite('End to End Tests: test adapters', () => { resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; let failureOccurred = false; - let failureMsg: string | undefined = ''; + let failureMsg = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; @@ -313,7 +313,7 @@ suite('End to End Tests: test adapters', () => { ); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { - failureMsg = err?.toString(); + failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; } return Promise.resolve(); @@ -352,7 +352,7 @@ suite('End to End Tests: test adapters', () => { resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; let failureOccurred = false; - let failureMsg: string | undefined = ''; + let failureMsg = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; @@ -361,7 +361,7 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { - failureMsg = err?.toString(); + failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; } return Promise.resolve(); @@ -404,7 +404,7 @@ suite('End to End Tests: test adapters', () => { resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; let failureOccurred = false; - let failureMsg: string | undefined = ''; + let failureMsg = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; @@ -413,7 +413,7 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { - failureMsg = err?.toString(); + failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; } return Promise.resolve(); @@ -455,7 +455,7 @@ suite('End to End Tests: test adapters', () => { resultResolver = new PythonResultResolver(testController, unittestProvider, workspaceUri); let callCount = 0; let failureOccurred = false; - let failureMsg: string | undefined = ''; + let failureMsg = ''; resultResolver._resolveExecution = async (payload, _token?) => { traceLog(`resolveDiscovery ${payload}`); callCount = callCount + 1; @@ -464,7 +464,7 @@ suite('End to End Tests: test adapters', () => { assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { - failureMsg = err?.toString(); + failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; } return Promise.resolve(); @@ -483,7 +483,7 @@ suite('End to End Tests: test adapters', () => { const indexOfTest = JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'); assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); } catch (err) { - failureMsg = err?.toString(); + failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; } return Promise.resolve(); @@ -520,7 +520,7 @@ suite('End to End Tests: test adapters', () => { resultResolver = new PythonResultResolver(testController, pytestProvider, workspaceUri); let callCount = 0; let failureOccurred = false; - let failureMsg: string | undefined = ''; + let failureMsg = ''; resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. callCount = callCount + 1; @@ -538,7 +538,7 @@ suite('End to End Tests: test adapters', () => { 'Expected testId to be present', ); } catch (err) { - failureMsg = err?.toString(); + failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; } return Promise.resolve(); From 855d23c0b6081dae330a6523a3c078a789f9300e Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 14 Sep 2023 12:15:09 -0700 Subject: [PATCH 48/55] tests --- .../testing/common/testingAdapter.test.ts | 73 ++++++++++++------- 1 file changed, 48 insertions(+), 25 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index b605ca16aaf8..c0671e2913cb 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -121,7 +121,11 @@ suite('End to End Tests: test adapters', () => { // verification after discovery is complete // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); // 3. Confirm tests are found @@ -161,7 +165,11 @@ suite('End to End Tests: test adapters', () => { await discoveryAdapter.discoverTests(workspaceUri).finally(() => { // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors assert.strictEqual(actualData.error, undefined, "Expected no errors in 'error' field"); // 3. Confirm tests are found @@ -200,8 +208,11 @@ suite('End to End Tests: test adapters', () => { // verification after discovery is complete // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors assert.strictEqual(actualData.error?.length, 0, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); @@ -239,8 +250,11 @@ suite('End to End Tests: test adapters', () => { await discoveryAdapter.discoverTests(workspaceUri, pythonExecFactory).finally(() => { // verification after discovery is complete // 1. Check the status is "success" - assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'"); - // 2. Confirm no errors + assert.strictEqual( + actualData.status, + 'success', + `Expected status to be 'success' instead status is ${actualData.status}`, + ); // 2. Confirm no errors assert.strictEqual(actualData.error?.length, 0, "Expected no errors in 'error' field"); // 3. Confirm tests are found assert.ok(actualData.tests, 'Expected tests to be present'); @@ -259,7 +273,11 @@ suite('End to End Tests: test adapters', () => { callCount = callCount + 1; // the payloads that get to the _resolveExecution are all data and should be successful. try { - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.strictEqual( + payload.status, + 'success', + `Expected status to be 'success', instead status is ${payload.status}`, + ); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { failureMsg = err ? (err as Error).toString() : ''; @@ -309,7 +327,7 @@ suite('End to End Tests: test adapters', () => { const validStatuses = ['subtest-success', 'subtest-failure']; assert.ok( validStatuses.includes(payload.status), - `Expected status to be one of ${validStatuses.join(', ')}`, + `Expected status to be one of ${validStatuses.join(', ')}, but instead status is ${payload.status}`, ); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { @@ -358,7 +376,11 @@ suite('End to End Tests: test adapters', () => { callCount = callCount + 1; // the payloads that get to the _resolveExecution are all data and should be successful. try { - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.strictEqual( + payload.status, + 'success', + `Expected status to be 'success', instead status is ${payload.status}`, + ); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { failureMsg = err ? (err as Error).toString() : ''; @@ -410,7 +432,11 @@ suite('End to End Tests: test adapters', () => { callCount = callCount + 1; // the payloads that get to the _resolveExecution are all data and should be successful. try { - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); + assert.strictEqual( + payload.status, + 'success', + `Expected status to be 'success', instead status is ${payload.status}`, + ); assert.ok(payload.result, 'Expected results to be present'); } catch (err) { failureMsg = err ? (err as Error).toString() : ''; @@ -456,25 +482,17 @@ suite('End to End Tests: test adapters', () => { let callCount = 0; let failureOccurred = false; let failureMsg = ''; - resultResolver._resolveExecution = async (payload, _token?) => { - traceLog(`resolveDiscovery ${payload}`); - callCount = callCount + 1; - // the payloads that get to the _resolveExecution are all data and should be successful. - try { - assert.strictEqual(payload.status, 'success', "Expected status to be 'success'"); - assert.ok(payload.result, 'Expected results to be present'); - } catch (err) { - failureMsg = err ? (err as Error).toString() : ''; - failureOccurred = true; - } - return Promise.resolve(); - }; resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. callCount = callCount + 1; + console.log(`unittest execution adapter seg fault error handling \n ${data}`); try { // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); + assert.strictEqual( + data.status, + 'error', + `Expected status to be 'error', instead status is ${data.status}`, + ); // 2. Confirm no errors assert.ok(data.error, "Expected errors in 'error' field"); // 3. Confirm tests are found @@ -523,10 +541,15 @@ suite('End to End Tests: test adapters', () => { let failureMsg = ''; resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. + console.log(`unittest execution adapter seg fault error handling \n ${data}`); callCount = callCount + 1; try { // 1. Check the status is "success" - assert.strictEqual(data.status, 'error', "Expected status to be 'error'"); + assert.strictEqual( + data.status, + 'error', + `Expected status to be 'error', instead status is ${data.status}`, + ); // 2. Confirm no errors assert.ok(data.error, "Expected errors in 'error' field"); // 3. Confirm tests are found From 5481fa523a99315d5c65876d434bafd2c595cfe6 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Thu, 14 Sep 2023 12:29:57 -0700 Subject: [PATCH 49/55] another --- src/test/testing/common/testingAdapter.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index c0671e2913cb..a4b4d73132ea 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -485,7 +485,7 @@ suite('End to End Tests: test adapters', () => { resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. callCount = callCount + 1; - console.log(`unittest execution adapter seg fault error handling \n ${data}`); + console.log(`unittest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); try { // 1. Check the status is "success" assert.strictEqual( @@ -541,7 +541,7 @@ suite('End to End Tests: test adapters', () => { let failureMsg = ''; resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. - console.log(`unittest execution adapter seg fault error handling \n ${data}`); + console.log(`unittest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); callCount = callCount + 1; try { // 1. Check the status is "success" From a4e559c34997e4e7842f9aa969effaa56064dbac Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 15 Sep 2023 08:39:11 -0700 Subject: [PATCH 50/55] add additional info logging --- .../testController/common/resultResolver.ts | 2 +- .../testing/testController/common/server.ts | 14 +++++++++----- .../pytest/pytestDiscoveryAdapter.ts | 14 +++++++++----- .../pytest/pytestExecutionAdapter.ts | 17 +++++++++++------ .../unittest/testExecutionAdapter.ts | 8 ++++++-- 5 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/client/testing/testController/common/resultResolver.ts b/src/client/testing/testController/common/resultResolver.ts index c4ad53c226cc..5ef6695ca280 100644 --- a/src/client/testing/testController/common/resultResolver.ts +++ b/src/client/testing/testController/common/resultResolver.ts @@ -109,7 +109,7 @@ export class PythonResultResolver implements ITestResultResolver { ): Promise { if (payload !== undefined && 'eot' in payload) { // the payload is an EOT payload, so resolve the deferred promise. - traceLog('ResultResolver EOT received for discovery.'); + traceLog('ResultResolver EOT received for execution.'); const eotPayload = payload as EOTTestPayload; if (eotPayload.eot === true) { deferredTillEOT.resolve(); diff --git a/src/client/testing/testController/common/server.ts b/src/client/testing/testController/common/server.ts index 80d93586d17a..f59c486f7a85 100644 --- a/src/client/testing/testController/common/server.ts +++ b/src/client/testing/testController/common/server.ts @@ -47,7 +47,7 @@ export class PythonTestServer implements ITestServer, Disposable { } buffer = remainingBuffer; } catch (ex) { - traceError(`Error processing test server request: ${ex} observe`); + traceError(`Error reading data from buffer: ${ex} observed.`); buffer = Buffer.alloc(0); this._onDataReceived.fire({ uuid: '', data: '' }); } @@ -81,6 +81,7 @@ export class PythonTestServer implements ITestServer, Disposable { // what payload is so small it doesn't include the whole UUID think got this if (extractedJsonPayload.uuid !== undefined && extractedJsonPayload.cleanedJsonData !== undefined) { // if a full json was found in the buffer, fire the data received event then keep cycling with the remaining raw data. + traceInfo(`Firing data received event, ${extractedJsonPayload.cleanedJsonData}`); this._fireDataReceived(extractedJsonPayload.uuid, extractedJsonPayload.cleanedJsonData); } buffer = Buffer.from(extractedJsonPayload.remainingRawData); @@ -89,7 +90,7 @@ export class PythonTestServer implements ITestServer, Disposable { buffer = Buffer.alloc(0); } } catch (ex) { - traceError(`Error:: ${ex}`); + traceError(`Error attempting to resolve data: ${ex}`); this._onDataReceived.fire({ uuid: '', data: '' }); } return buffer; @@ -108,7 +109,7 @@ export class PythonTestServer implements ITestServer, Disposable { data: extractedJSON, }); } else { - traceLog(`Error processing test server request: request is not recognized as discovery or run.`); + traceError(`Error processing test server request: request is not recognized as discovery or run.`); this._onDataReceived.fire({ uuid: '', data: '' }); } } @@ -214,10 +215,9 @@ export class PythonTestServer implements ITestServer, Disposable { traceLog(`Discovering unittest tests with arguments: ${args}\r\n`); } const deferred = createDeferred>(); - const result = execService.execObservable(args, spawnOptions); - runInstance?.token.onCancellationRequested(() => { + traceInfo('Test run cancelled, killing unittest subprocess.'); result?.proc?.kill(); }); @@ -232,6 +232,9 @@ export class PythonTestServer implements ITestServer, Disposable { result?.proc?.on('exit', (code, signal) => { // if the child has testIds then this is a run request if (code !== 0 && testIds) { + traceError( + `Subprocess exited unsuccessfully with exit code ${code} and signal ${signal}. Creating and sending error execution payload`, + ); // if the child process exited with a non-zero exit code, then we need to send the error payload. this._onRunDataReceived.fire({ uuid, @@ -248,6 +251,7 @@ export class PythonTestServer implements ITestServer, Disposable { await deferred.promise; } } catch (ex) { + traceError(`Error while server attempting to run unittest command: ${ex}`); this.uuids = this.uuids.filter((u) => u !== uuid); this._onDataReceived.fire({ uuid, diff --git a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts index 9c7c5fbb4400..bafa91847b42 100644 --- a/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts +++ b/src/client/testing/testController/pytest/pytestDiscoveryAdapter.ts @@ -11,7 +11,7 @@ import { import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { Deferred, createDeferred } from '../../../common/utils/async'; import { EXTENSION_ROOT_DIR } from '../../../constants'; -import { traceVerbose } from '../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../logging'; import { DataReceivedEvent, DiscoveredTestPayload, @@ -32,15 +32,13 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { ) {} async discoverTests(uri: Uri, executionFactory?: IPythonExecutionFactory): Promise { - const settings = this.configSettings.getSettings(uri); const uuid = this.testServer.createUUID(uri.fsPath); - const { pytestArgs } = settings.testing; - traceVerbose(pytestArgs); const deferredTillEOT: Deferred = createDeferred(); const dataReceivedDisposable = this.testServer.onDiscoveryDataReceived(async (e: DataReceivedEvent) => { this.resultResolver?.resolveDiscovery(JSON.parse(e.data), deferredTillEOT); }); const disposeDataReceiver = function (testServer: ITestServer) { + traceInfo(`Disposing data receiver for ${uri.fsPath} and deleting UUID; pytest discovery.`); testServer.deleteUUID(uuid); dataReceivedDisposable.dispose(); }; @@ -86,6 +84,7 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { // delete UUID following entire discovery finishing. const deferredExec = createDeferred>(); const execArgs = ['-m', 'pytest', '-p', 'vscode_pytest', '--collect-only'].concat(pytestArgs); + traceVerbose(`Running pytest discovery with command: ${execArgs.join(' ')}`); const result = execService?.execObservable(execArgs, spawnOptions); // Take all output from the subprocess and add it to the test output channel. This will be the pytest output. @@ -96,7 +95,12 @@ export class PytestTestDiscoveryAdapter implements ITestDiscoveryAdapter { result?.proc?.stderr?.on('data', (data) => { spawnOptions.outputChannel?.append(data.toString()); }); - result?.proc?.on('exit', () => { + result?.proc?.on('exit', (code, signal) => { + if (code !== 0) { + traceError( + `Subprocess exited unsuccessfully with exit code ${code} and signal ${signal}. Creating and sending error discovery payload`, + ); + } deferredExec.resolve({ stdout: '', stderr: '' }); deferred.resolve(); }); diff --git a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts index 1f8936f772e9..085af40375d4 100644 --- a/src/client/testing/testController/pytest/pytestExecutionAdapter.ts +++ b/src/client/testing/testController/pytest/pytestExecutionAdapter.ts @@ -5,7 +5,7 @@ import { TestRun, Uri } from 'vscode'; import * as path from 'path'; import { IConfigurationService, ITestOutputChannel } from '../../../common/types'; import { Deferred, createDeferred } from '../../../common/utils/async'; -import { traceError, traceInfo, traceVerbose } from '../../../logging'; +import { traceError, traceInfo } from '../../../logging'; import { DataReceivedEvent, ExecutionTestPayload, @@ -42,19 +42,22 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { debugLauncher?: ITestDebugLauncher, ): Promise { const uuid = this.testServer.createUUID(uri.fsPath); - traceVerbose(uri, testIds, debugBool); const deferredTillEOT: Deferred = utils.createEOTDeferred(); const dataReceivedDisposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { if (runInstance) { const eParsed = JSON.parse(e.data); this.resultResolver?.resolveExecution(eParsed, runInstance, deferredTillEOT); + } else { + traceError('No run instance found, cannot resolve execution.'); } }); const disposeDataReceiver = function (testServer: ITestServer) { + traceInfo(`Disposing data receiver for ${uri.fsPath} and deleting UUID; pytest execution.`); testServer.deleteUUID(uuid); dataReceivedDisposable.dispose(); }; runInstance?.token.onCancellationRequested(() => { + traceInfo("Test run cancelled, resolving 'till EOT' deferred."); deferredTillEOT.resolve(); }); try { @@ -95,14 +98,12 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { ): Promise { const relativePathToPytest = 'pythonFiles'; const fullPluginPath = path.join(EXTENSION_ROOT_DIR, relativePathToPytest); - this.configSettings.isTestExecution(); const settings = this.configSettings.getSettings(uri); const { pytestArgs } = settings.testing; const cwd = settings.testing.cwd && settings.testing.cwd.length > 0 ? settings.testing.cwd : uri.fsPath; const pythonPathParts: string[] = process.env.PYTHONPATH?.split(path.delimiter) ?? []; const pythonPathCommand = [fullPluginPath, ...pythonPathParts].join(path.delimiter); - const spawnOptions: SpawnOptions = { cwd, throwOnStdErr: true, @@ -161,12 +162,13 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { // combine path to run script with run args const scriptPath = path.join(fullPluginPath, 'vscode_pytest', 'run_pytest_script.py'); const runArgs = [scriptPath, ...testArgs]; - traceInfo(`Running pytests with arguments: ${runArgs.join(' ')}\r\n`); + traceInfo(`Running pytest with arguments: ${runArgs.join(' ')}\r\n`); const deferredExec = createDeferred>(); const result = execService?.execObservable(runArgs, spawnOptions); runInstance?.token.onCancellationRequested(() => { + traceInfo('Test run cancelled, killing pytest subprocess.'); result?.proc?.kill(); }); @@ -180,8 +182,12 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { }); result?.proc?.on('exit', (code, signal) => { + traceInfo('Test run finished, subprocess exited.'); // if the child has testIds then this is a run request if (code !== 0 && testIds) { + traceError( + `Subprocess exited unsuccessfully with exit code ${code} and signal ${signal}. Creating and sending error execution payload`, + ); // if the child process exited with a non-zero exit code, then we need to send the error payload. this.testServer.triggerRunDataReceivedEvent({ uuid, @@ -194,7 +200,6 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter { }); } deferredExec.resolve({ stdout: '', stderr: '' }); - // disposeDataReceiver?.(this.testServer); }); await deferredExec.promise; } diff --git a/src/client/testing/testController/unittest/testExecutionAdapter.ts b/src/client/testing/testController/unittest/testExecutionAdapter.ts index 373dc626fbcc..9da0872ef601 100644 --- a/src/client/testing/testController/unittest/testExecutionAdapter.ts +++ b/src/client/testing/testController/unittest/testExecutionAdapter.ts @@ -15,7 +15,7 @@ import { TestCommandOptions, TestExecutionCommand, } from '../common/types'; -import { traceLog } from '../../../logging'; +import { traceError, traceInfo, traceLog } from '../../../logging'; import { startTestIdServer } from '../common/utils'; /** @@ -41,13 +41,17 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { const disposedDataReceived = this.testServer.onRunDataReceived((e: DataReceivedEvent) => { if (runInstance) { this.resultResolver?.resolveExecution(JSON.parse(e.data), runInstance, deferredTillEOT); + } else { + traceError('No run instance found, cannot resolve execution.'); } }); const disposeDataReceiver = function (testServer: ITestServer) { + traceInfo(`Disposing data receiver for ${uri.fsPath} and deleting UUID; unittest execution.`); testServer.deleteUUID(uuid); disposedDataReceived.dispose(); }; runInstance?.token.onCancellationRequested(() => { + traceInfo("Test run cancelled, resolving 'till EOT' deferred."); deferredTillEOT.resolve(); }); try { @@ -55,7 +59,7 @@ export class UnittestTestExecutionAdapter implements ITestExecutionAdapter { await deferredTillEOT.promise; disposeDataReceiver(this.testServer); } catch (error) { - traceLog(error); + traceError(`Error in running unittest tests: ${error}`); } const executionPayload: ExecutionTestPayload = { cwd: uri.fsPath, status: 'success', error: '' }; return executionPayload; From 5b48927db7688e51b4025574d55cf055b5d2d6cb Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 15 Sep 2023 11:29:42 -0700 Subject: [PATCH 51/55] fix seg fault on diff machines --- .../testing/common/testingAdapter.test.ts | 51 +++++++++---------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index a4b4d73132ea..8a315616f6f3 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -487,17 +487,18 @@ suite('End to End Tests: test adapters', () => { callCount = callCount + 1; console.log(`unittest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); try { - // 1. Check the status is "success" - assert.strictEqual( - data.status, - 'error', - `Expected status to be 'error', instead status is ${data.status}`, - ); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found + if (data.status === 'error') { + assert.ok(data.error, "Expected errors in 'error' field"); + } else { + const indexOfTest = JSON.stringify(data.result).search('error'); + assert.notDeepEqual( + indexOfTest, + -1, + 'If payload status is not error then the individual tests should be marked as errors. This should occur on windows machines.', + ); + } assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results + // make sure the testID is found in the results const indexOfTest = JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'); assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); } catch (err) { @@ -541,25 +542,23 @@ suite('End to End Tests: test adapters', () => { let failureMsg = ''; resultResolver._resolveExecution = async (data, _token?) => { // do the following asserts for each time resolveExecution is called, should be called once per test. - console.log(`unittest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); + console.log(`pytest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); callCount = callCount + 1; try { - // 1. Check the status is "success" - assert.strictEqual( - data.status, - 'error', - `Expected status to be 'error', instead status is ${data.status}`, - ); - // 2. Confirm no errors - assert.ok(data.error, "Expected errors in 'error' field"); - // 3. Confirm tests are found + if (data.status === 'error') { + assert.ok(data.error, "Expected errors in 'error' field"); + } else { + const indexOfTest = JSON.stringify(data.result).search('error'); + assert.notDeepEqual( + indexOfTest, + -1, + 'If payload status is not error then the individual tests should be marked as errors. This should occur on windows machines.', + ); + } assert.ok(data.result, 'Expected results to be present'); - // 4. make sure the testID is found in the results - assert.notDeepEqual( - JSON.stringify(data).search('test_seg_fault.py::TestSegmentationFault::test_segfault'), - -1, - 'Expected testId to be present', - ); + // make sure the testID is found in the results + const indexOfTest = JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'); + assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); } catch (err) { failureMsg = err ? (err as Error).toString() : ''; failureOccurred = true; From 2dd5d05bdd82931787991a67fca2fab7bf20ca5f Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 15 Sep 2023 11:43:47 -0700 Subject: [PATCH 52/55] fix testid --- src/test/testing/common/testingAdapter.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 8a315616f6f3..d7c900152df0 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -5,6 +5,7 @@ import { TestController, TestRun, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; +import * as os from 'os'; import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; import { ITestController, ITestResultResolver } from '../../../client/testing/testController/common/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; @@ -557,7 +558,7 @@ suite('End to End Tests: test adapters', () => { } assert.ok(data.result, 'Expected results to be present'); // make sure the testID is found in the results - const indexOfTest = JSON.stringify(data).search('test_seg_fault.TestSegmentationFault.test_segfault'); + const indexOfTest = JSON.stringify(data).search('test_seg_fault::TestSegmentationFault::test_segfault'); assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); } catch (err) { failureMsg = err ? (err as Error).toString() : ''; From e290587cfb35a93259893724f6f7c474be76cd75 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 15 Sep 2023 11:49:05 -0700 Subject: [PATCH 53/55] delete unused import --- src/test/testing/common/testingAdapter.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index d7c900152df0..95e7e4979437 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -5,7 +5,6 @@ import { TestController, TestRun, Uri } from 'vscode'; import * as typeMoq from 'typemoq'; import * as path from 'path'; import * as assert from 'assert'; -import * as os from 'os'; import { PytestTestDiscoveryAdapter } from '../../../client/testing/testController/pytest/pytestDiscoveryAdapter'; import { ITestController, ITestResultResolver } from '../../../client/testing/testController/common/types'; import { PythonTestServer } from '../../../client/testing/testController/common/server'; From f0d086037e0378929fcfd5de3ef05bf033152fce Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 15 Sep 2023 11:55:26 -0700 Subject: [PATCH 54/55] fix name testid --- src/test/testing/common/testingAdapter.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index 95e7e4979437..ee7d316e9ec9 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -557,7 +557,9 @@ suite('End to End Tests: test adapters', () => { } assert.ok(data.result, 'Expected results to be present'); // make sure the testID is found in the results - const indexOfTest = JSON.stringify(data).search('test_seg_fault::TestSegmentationFault::test_segfault'); + const indexOfTest = JSON.stringify(data).search( + 'test_seg_fault.py::TestSegmentationFault::test_segfault', + ); assert.notDeepEqual(indexOfTest, -1, 'Expected testId to be present'); } catch (err) { failureMsg = err ? (err as Error).toString() : ''; From 8e72673c6081c22794b2337c554efde8b9f6bbf0 Mon Sep 17 00:00:00 2001 From: eleanorjboyd Date: Fri, 15 Sep 2023 12:17:46 -0700 Subject: [PATCH 55/55] fix search --- src/test/testing/common/testingAdapter.test.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/testing/common/testingAdapter.test.ts b/src/test/testing/common/testingAdapter.test.ts index ee7d316e9ec9..b445772d6958 100644 --- a/src/test/testing/common/testingAdapter.test.ts +++ b/src/test/testing/common/testingAdapter.test.ts @@ -488,7 +488,13 @@ suite('End to End Tests: test adapters', () => { console.log(`unittest execution adapter seg fault error handling \n ${JSON.stringify(data)}`); try { if (data.status === 'error') { - assert.ok(data.error, "Expected errors in 'error' field"); + if (data.error === undefined) { + // Dereference a NULL pointer + const indexOfTest = JSON.stringify(data).search('Dereference a NULL pointer'); + assert.notDeepEqual(indexOfTest, -1, 'Expected test to have a null pointer'); + } else { + assert.ok(data.error, "Expected errors in 'error' field"); + } } else { const indexOfTest = JSON.stringify(data.result).search('error'); assert.notDeepEqual(