From 8047973337747e73c7e30238fe33304a32c8b8ed Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 19 Jan 2023 17:02:20 +0530 Subject: [PATCH 1/6] added fail fast in localstack start --- .github/workflows/main.yml | 6 +- README.md | 2 +- conftest.py | 2 +- pyproject.toml | 2 +- tests/conftest.py | 185 ------------------------------------- 5 files changed, 6 insertions(+), 191 deletions(-) delete mode 100644 tests/conftest.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0d745f9..2b478f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,9 +13,9 @@ on: default: 'localstack/localstack:latest' description: localstack docker image name to test against # disabled pipeline for push events, as we want to run the pipeline only on a schedule or manually -# push: -# branches: -# - pytest-plugin + push: + branches: + - pytest-plugin name: Terraform Tests jobs: diff --git a/README.md b/README.md index 39f9b8e..42857de 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Purpose of this project is to externalize the test cases from the localstack rep ## How to run? 1. Run `python -m terraform_pytest.main patch` to apply the patch to the terraform provider aws 2. Run `python -m terraform_pytest.main build -s s3` to build testing binary for the golang module -3Now you are ready to use `python -m pytest` commands to list and run test cases from golang +3. Now you are ready to use `python -m pytest` commands to list and run test cases from golang ## How to run test cases? - To list down all the test case from a specific service, run `python -m pytest terraform-provider-aws/internal/service/ --collect-only -q` diff --git a/conftest.py b/conftest.py index 4e479a6..dce16ab 100644 --- a/conftest.py +++ b/conftest.py @@ -123,7 +123,7 @@ def _docker_service_health(client): def _start_docker_container(client, config, localstack_image): - env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf"] + env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf", "FAIL_FAST=1"] port_mappings = { "53/tcp": ("127.0.0.1", 53), "53/udp": ("127.0.0.1", 53), diff --git a/pyproject.toml b/pyproject.toml index e9451ca..a9b5ae4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,4 +14,4 @@ line_length = 100 testpaths = [ "terraform-provider-aws/internal/service/", ] -#confcutdir = "tests/conftest.py" \ No newline at end of file +filterwarnings =["ignore::DeprecationWarning"] diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 0c71c01..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,185 +0,0 @@ -import re -import pytest -import docker -import requests -from requests.adapters import HTTPAdapter, Retry -from pathlib import Path -import os -from os.path import realpath, relpath, dirname -from utils import execute_command, build_test_bin - - -def pytest_addoption(parser): - parser.addoption( - '--ls-image', action='store', default='localstack/localstack:latest', help='Base URL for the API tests' - ) - parser.addoption( - '--ls-start', action='store_true', default=False, help='Start localstack service' - ) - - -def pytest_collect_file(parent, file_path): - if file_path.suffix == '.go' and file_path.name.endswith('_test.go'): - return GoFile.from_parent(parent, path=file_path) - - -class GoFile(pytest.File): - def collect(self): - raw = self.path.open().read() - fa = re.findall(r'^(func (TestAcc.*))\(.*\).*', raw, re.MULTILINE) - for _, name in fa: - yield GoItem.from_parent(self, name=name) - - -class GoItem(pytest.Item): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def runtest(self): - tf_root_path = realpath(relpath(self.path).split(os.sep)[0]) - service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) - service = service_path.split(os.sep)[-1] - - env = dict(os.environ) - env.update({ - 'TF_ACC': '1', - 'AWS_ACCESS_KEY_ID': 'test', - 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-west-1', - 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_REGION': 'us-east-2', - 'AWS_THIRD_REGION': 'eu-west-1', - }) - - cmd = [ - f'./test-bin/{service}.test', - '-test.v', - '-test.parallel=1', - '-test.count=1', - '-test.timeout=60m', - f'-test.run={self.name}', - ] - return_code, stdout = execute_command(cmd, env, tf_root_path) - if return_code != 0: - raise GoException(returncode=return_code, stderr=stdout) - - def repr_failure(self, excinfo, **kwargs): - if isinstance(excinfo.value, GoException): - return '\n'.join( - [ - f'Execution failed with return code: {excinfo.value.returncode}', - f'Failure Reason:\n{excinfo.value.stderr}', - ] - ) - - def reportinfo(self): - return self.path, 0, f'Test Case: {self.name}' - - -class ReprCrash: - def __init__(self, message): - self.message = message - - -class LongRepr: - def __init__(self, message, reason): - self.reprcrash = ReprCrash(message) - self.reason = reason - - def __str__(self): - return self.reason - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - report = outcome.get_result() - if report.failed: - splits = report.longrepr.split('\n', 1) - longrepr = LongRepr(splits[0], splits[1]) - delattr(report, 'longrepr') - setattr(report, 'longrepr', longrepr) - - -class GoException(Exception): - def __init__(self, returncode, stderr): - self.returncode = returncode - self.stderr = stderr - - -def _docker_service_health(client): - if not client.ping(): - print('\nPlease start docker daemon and try again') - raise Exception('Docker is not running') - - -def _start_docker_container(client, config, localstack_image): - env_vars = ['DEBUG=1', 'PROVIDER_OVERRIDE_S3=asf'] - port_mappings = { - '53/tcp': ('127.0.0.1', 53), - '53/udp': ('127.0.0.1', 53), - '443': ('127.0.0.1', 443), - '4566': ('127.0.0.1', 4566), - '4571': ('127.0.0.1', 4571), - } - volumes = ['/var/run/docker.sock:/var/run/docker.sock'] - localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, - name='localstack_main', volumes=volumes, auto_remove=True, - environment=env_vars) - setattr(config, 'localstack_container_id', localstack_container.id) - - -def _stop_docker_container(client, config): - client.containers.get(getattr(config, 'localstack_container_id')).stop() - print('LocalStack is stopped') - - -def _localstack_health_check(): - localstack_health_url = 'http://localhost:4566/health' - session = requests.Session() - retry = Retry(connect=3, backoff_factor=2) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - session.get(localstack_health_url) - session.close() - - -def _pull_docker_image(client, localstack_image): - docker_image_list = client.images.list(name=localstack_image) - if len(docker_image_list) == 0: - print(f'Pulling image {localstack_image}') - client.images.pull(localstack_image) - docker_image_list = client.images.list(name=localstack_image) - print(f'Using LocalStack image: {docker_image_list[0].id}') - - -def pytest_configure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - localstack_image = config.getoption(name='--ls-image') - - if not is_collect_only and is_localstack_start: - print('\nStarting LocalStack...') - - client = docker.from_env() - _docker_service_health(client) - _pull_docker_image(client, localstack_image) - _start_docker_container(client, config, localstack_image) - _localstack_health_check() - client.close() - - print('LocalStack is ready...') - - -def pytest_unconfigure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - - if not is_collect_only and is_localstack_start: - print('\nStopping LocalStack...') - client = docker.from_env() - _stop_docker_container(client, config) - client.close() From 2434651c363abda21222253fab81d58689adade1 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 19 Jan 2023 20:36:47 +0530 Subject: [PATCH 2/6] updated: running tests ec2 in parrallel --- .github/workflows/main.yml | 9 +++++++++ .gitignore | 3 ++- Makefile | 2 +- docker-compose.yml | 18 ++++++++++++++++++ pyproject.toml | 3 ++- terraform_pytest/utils.py | 12 +++++++++++- 6 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 docker-compose.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b478f3..961e3eb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,9 +77,18 @@ jobs: ls -la terraform-provider-aws/test-bin - name: Run ${{ matrix.service }} Tests + if: ${{ matrix.service != 'ec2' }} run: | python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} + - name: Run ${{ matrix.service }} Tests + if: ${{ matrix.service == 'ec2' }} + run: | + IMAGE_TAG=${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} docker-compose up -d + sleep 20 + python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v -n 2 + + - name: Publish ${{ matrix.service }} Test Results uses: EnricoMi/publish-unit-test-result-action@v2 if: always() diff --git a/.gitignore b/.gitignore index da78a2b..77c853a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__ target **/*.test -report.xml \ No newline at end of file +report.xml +volume diff --git a/Makefile b/Makefile index 4d7c068..70f2fc1 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,4 @@ install: $(VENV_RUN); $(PIP_CMD) install -r requirements.txt format: - $(VENV_RUN); python -m isort .; python -m black . \ No newline at end of file + $(VENV_RUN); python -m isort .; python -m black . diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4688d35 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +version: "3.8" + +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" + image: ${IMAGE_TAG-localstack/localstack:latest} + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + environment: + - DEBUG=1 + - PROVIDER_OVERRIDE_S3=asf + - FAIL_FAST=1 + - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} + - DOCKER_HOST=unix:///var/run/docker.sock + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/pyproject.toml b/pyproject.toml index a9b5ae4..10888b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,11 +2,12 @@ [tool.black] line_length = 100 -include = '(terraform_pytest\/.*\.py$|tests\/.*\.py$)' +include = '(^terraform_pytest\/.*\.py$|^conftest.py$)' #extend_exclude = '()' [tool.isort] profile = 'black' +skip_glob = ["terraform-provider-aws"] #extend_skip = [] line_length = 100 diff --git a/terraform_pytest/utils.py b/terraform_pytest/utils.py index 4226d9e..0d007a7 100644 --- a/terraform_pytest/utils.py +++ b/terraform_pytest/utils.py @@ -143,6 +143,16 @@ def build_test_bin(service, tf_root_path): if exists(_test_bin_abs_path): return + cmd = ["go", "mod", "tidy"] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") + + cmd = ["go", "mod", "vendor"] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") + cmd = [ "go", "test", @@ -153,7 +163,7 @@ def build_test_bin(service, tf_root_path): ] return_code, stdout = execute_command(cmd, cwd=tf_root_path) if return_code != 0: - raise Exception(f"Error while building test binary for {service}") + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") if exists(_test_bin_abs_path): chmod(_test_bin_abs_path, 0o755) From 85165c58a34a24fd3fe62657bc1c49948c3a497d Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 19 Jan 2023 20:38:09 +0530 Subject: [PATCH 3/6] updated: concurrency, added ec2 for testing --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 961e3eb..cec2a24 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,14 +27,14 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python -m terraform_pytest.get-services ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python -m terraform_pytest.get-services ${{ github.event.inputs.services || 'ec2' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} test_service: needs: prepare_list strategy: - max-parallel: 10 +# max-parallel: 10 fail-fast: false matrix: service: ${{ fromJson(needs.prepare_list.outputs.matrix) }} From 63e6ae20f039aaf65b51d0464913fb74b88a614c Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 20 Jan 2023 12:29:06 +0530 Subject: [PATCH 4/6] updated parrellel run --- conftest.py | 46 +++++++++++++++++++++++----------------------- requirements.txt | 3 ++- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/conftest.py b/conftest.py index dce16ab..048970d 100644 --- a/conftest.py +++ b/conftest.py @@ -85,29 +85,29 @@ def reportinfo(self): return self.path, 0, f"Test Case: {self.name}" -class ReprCrash: - def __init__(self, message): - self.message = message - - -class LongRepr: - def __init__(self, message, reason): - self.reprcrash = ReprCrash(message) - self.reason = reason - - def __str__(self): - return self.reason - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - report = outcome.get_result() - if report.failed: - splits = report.longrepr.split("\n", 1) - longrepr = LongRepr(splits[0], splits[1]) - delattr(report, "longrepr") - setattr(report, "longrepr", longrepr) +# class ReprCrash: +# def __init__(self, message): +# self.message = message +# +# +# class LongRepr: +# def __init__(self, message, reason): +# self.reprcrash = ReprCrash(message) +# self.reason = reason +# +# def __str__(self): +# return self.reason +# +# +# @pytest.hookimpl(tryfirst=True, hookwrapper=True) +# def pytest_runtest_makereport(item, call): +# outcome = yield +# report = outcome.get_result() +# if report.failed: +# splits = report.longrepr.split("\n", 1) +# longrepr = LongRepr(splits[0], splits[1]) +# delattr(report, "longrepr") +# setattr(report, "longrepr", longrepr) class GoException(Exception): diff --git a/requirements.txt b/requirements.txt index ab85810..1a0a00a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ pytest==7.2.0 docker==6.0.1 requests==2.28.2 black>=22.1 -isort>=5.10 \ No newline at end of file +isort>=5.10 +pytest-xdist \ No newline at end of file From 70b251d3f568697464376c01a2c3384a9212b93c Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 20 Jan 2023 13:40:52 +0530 Subject: [PATCH 5/6] updated: changes for upstream --- .github/workflows/main.yml | 10 +++++----- README.md | 11 +++++++++-- conftest.py | 25 ------------------------- 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cec2a24..ca3205b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,9 +13,9 @@ on: default: 'localstack/localstack:latest' description: localstack docker image name to test against # disabled pipeline for push events, as we want to run the pipeline only on a schedule or manually - push: - branches: - - pytest-plugin +# push: +# branches: +# - pytest-plugin name: Terraform Tests jobs: @@ -27,14 +27,14 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python -m terraform_pytest.get-services ${{ github.event.inputs.services || 'ec2' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python -m terraform_pytest.get-services ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} test_service: needs: prepare_list strategy: -# max-parallel: 10 + max-parallel: 10 fail-fast: false matrix: service: ${{ fromJson(needs.prepare_list.outputs.matrix) }} diff --git a/README.md b/README.md index 42857de..0903225 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ This is a test runner for localstack and terraform. It will run a test cases fro Purpose of this project is to externalize the test cases from the localstack repo and run them against localstack to gather parity metrics. ## Installation -1. Clone the repository with submodules` +1. Clone the repository with submodules + - `git clone git@github.com:localstack/localstack-terraform-test.git --recurse-submodules` + - Make sure you have the latest version of the submodules after switching to a different branch using `git submodule update --init --recursive` 2. Run `make venv` to create a virtual environment 3. Run `make install` to install the dependencies @@ -19,7 +21,7 @@ Purpose of this project is to externalize the test cases from the localstack rep - To run a specific test case, run `python -m pytest terraform-provider-aws/internal/service// -k --ls-start` or `python -m pytest terraform-provider-aws/internal/service//:: --ls-start` - Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/internal/service//:: --ls-start` -## Default environment variables +## Default environment variables for Terraform Tests - **TF_ACC**: `1` - **AWS_ACCESS_KEY_ID**: `test` - **AWS_SECRET_ACCESS_KEY**: `test` @@ -30,6 +32,11 @@ Purpose of this project is to externalize the test cases from the localstack rep - **AWS_ALTERNATE_REGION**: `us-east-2` - **AWS_THIRD_REGION**: `eu-west-1` +## Environment variables for Localstack +- **DEBUG**: `1` +- **PROVIDER_OVERRIDE_S3**: `asf` +- **FAIL_FAST**: `1` + ## Options - `--ls-start`: Start localstack instance before running the test cases - `--ls-image`: Specify the localstack image to use, default is `localstack/localstack:latest` \ No newline at end of file diff --git a/conftest.py b/conftest.py index 048970d..c5dad09 100644 --- a/conftest.py +++ b/conftest.py @@ -85,31 +85,6 @@ def reportinfo(self): return self.path, 0, f"Test Case: {self.name}" -# class ReprCrash: -# def __init__(self, message): -# self.message = message -# -# -# class LongRepr: -# def __init__(self, message, reason): -# self.reprcrash = ReprCrash(message) -# self.reason = reason -# -# def __str__(self): -# return self.reason -# -# -# @pytest.hookimpl(tryfirst=True, hookwrapper=True) -# def pytest_runtest_makereport(item, call): -# outcome = yield -# report = outcome.get_result() -# if report.failed: -# splits = report.longrepr.split("\n", 1) -# longrepr = LongRepr(splits[0], splits[1]) -# delattr(report, "longrepr") -# setattr(report, "longrepr", longrepr) - - class GoException(Exception): def __init__(self, returncode, stderr): self.returncode = returncode From 56edc960f00c8d344260dfd33e9ca6c184739a66 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 20 Jan 2023 13:42:47 +0530 Subject: [PATCH 6/6] updated: dependancies version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1a0a00a..f294c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ docker==6.0.1 requests==2.28.2 black>=22.1 isort>=5.10 -pytest-xdist \ No newline at end of file +pytest-xdist>=3.1.0