diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f5f0541..0d745f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ on: # disabled pipeline for push events, as we want to run the pipeline only on a schedule or manually # push: # branches: -# - pytest-plugin-fork +# - pytest-plugin name: Terraform Tests jobs: @@ -27,7 +27,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python -m terraform_pytest.get-services ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -69,16 +69,16 @@ jobs: run: | cd terraform-provider-aws && go mod vendor cd ../ - python3 main.py patch + python -m terraform_pytest.main patch - name: Build ${{ matrix.service }} Binary run: | - python main.py build -s ${{ matrix.service }} + python -m terraform_pytest.main build -s ${{ matrix.service }} ls -la terraform-provider-aws/test-bin - name: Run ${{ matrix.service }} Tests run: | - pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} + python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} - name: Publish ${{ matrix.service }} Test Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4d7c068 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +#!/bin/bash + +VENV_BIN ?= python3 -m venv +VENV_DIR ?= .venv +PIP_CMD ?= pip3 + +ifeq ($(OS), Windows_NT) + VENV_ACTIVATE = $(VENV_DIR)/Scripts/activate +else + VENV_ACTIVATE = $(VENV_DIR)/bin/activate +endif + +$(VENV_ACTIVATE): + test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR) + $(VENV_RUN); $(PIP_CMD) install --upgrade pip setuptools wheel plux + touch $(VENV_ACTIVATE) + +VENV_RUN = . $(VENV_ACTIVATE) + +venv: $(VENV_ACTIVATE) ## Create a new (empty) virtual environment + +install: + $(VENV_RUN); $(PIP_CMD) install -r requirements.txt + +format: + $(VENV_RUN); python -m isort .; python -m black . \ No newline at end of file diff --git a/README.md b/README.md index dde1a3f..39f9b8e 100644 --- a/README.md +++ b/README.md @@ -5,26 +5,30 @@ This is a test runner for localstack and terraform. It will run a test cases fro Purpose of this project is to externalize the test cases from the localstack repo and run them against localstack to gather parity metrics. ## Installation -1. Clone the repository -2. Run `python -m virtualenv venv` to create a virtual environment -3. Run `source venv/bin/activate` to activate the virtual environment -4. Run `pip install -r requirements.txt` to install the dependencies +1. Clone the repository with submodules` +2. Run `make venv` to create a virtual environment +3. Run `make install` to install the dependencies ## How to run? -1. Run `python main.py patch` to apply the patch to the terraform provider aws -2. Now you are ready to use `pytest` commands to list and run test cases from golang +1. Run `python -m terraform_pytest.main patch` to apply the patch to the terraform provider aws +2. Run `python -m terraform_pytest.main build -s s3` to build testing binary for the golang module +3Now you are ready to use `python -m pytest` commands to list and run test cases from golang ## How to run test cases? -- To list down all the test case from a specific service, run `pytest terraform-provider-aws/internal/service/ --collect-only -q` -- To run a specific test case, run `pytest terraform-provider-aws/internal/service// -k --ls-start` or `pytest terraform-provider-aws/internal/service//:: --ls-start` -- Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' pytest terraform-provider-aws/internal/service//:: --ls-start` +- To list down all the test case from a specific service, run `python -m pytest terraform-provider-aws/internal/service/ --collect-only -q` +- To run a specific test case, run `python -m pytest terraform-provider-aws/internal/service// -k --ls-start` or `python -m pytest terraform-provider-aws/internal/service//:: --ls-start` +- Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/internal/service//:: --ls-start` ## Default environment variables -- **TF_LOG**: ``debug``, -- **TF_ACC**: ``1``, -- **AWS_ACCESS_KEY_ID**: ``test``, -- **AWS_SECRET_ACCESS_KEY**: ``test``, -- **AWS_DEFAULT_REGION**: ``'us-east-1``' +- **TF_ACC**: `1` +- **AWS_ACCESS_KEY_ID**: `test` +- **AWS_SECRET_ACCESS_KEY**: `test` +- **AWS_DEFAULT_REGION**: `us-west-1` +- **AWS_ALTERNATE_ACCESS_KEY_ID**: `test` +- **AWS_ALTERNATE_SECRET_ACCESS_KEY**: `test` +- **AWS_ALTERNATE_SECRET_ACCESS_KEY**: `test` +- **AWS_ALTERNATE_REGION**: `us-east-2` +- **AWS_THIRD_REGION**: `eu-west-1` ## Options - `--ls-start`: Start localstack instance before running the test cases diff --git a/conftest.py b/conftest.py index 462a27c..4e479a6 100644 --- a/conftest.py +++ b/conftest.py @@ -1,32 +1,37 @@ +import os import re -import pytest +from os.path import dirname, realpath, relpath +from pathlib import Path + import docker +import pytest import requests from requests.adapters import HTTPAdapter, Retry -from pathlib import Path -import os -from os.path import realpath, relpath, dirname -from utils import execute_command, build_test_bin + +from terraform_pytest.utils import execute_command def pytest_addoption(parser): parser.addoption( - '--ls-image', action='store', default='localstack/localstack:latest', help='Base URL for the API tests' + "--ls-image", + action="store", + default="localstack/localstack:latest", + help="Base URL for the API tests", ) parser.addoption( - '--ls-start', action='store_true', default=False, help='Start localstack service' + "--ls-start", action="store_true", default=False, help="Start localstack service" ) def pytest_collect_file(parent, file_path): - if file_path.suffix == '.go' and file_path.name.endswith('_test.go'): + if file_path.suffix == ".go" and file_path.name.endswith("_test.go"): return GoFile.from_parent(parent, path=file_path) class GoFile(pytest.File): def collect(self): raw = self.path.open().read() - fa = re.findall(r'^(func (TestAcc.*))\(.*\).*', raw, re.MULTILINE) + fa = re.findall(r"^(func (TestAcc.*))\(.*\).*", raw, re.MULTILINE) for _, name in fa: yield GoItem.from_parent(self, name=name) @@ -41,25 +46,27 @@ def runtest(self): service = service_path.split(os.sep)[-1] env = dict(os.environ) - env.update({ - 'TF_ACC': '1', - 'AWS_ACCESS_KEY_ID': 'test', - 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-west-2', - 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_REGION': 'us-east-2', - 'AWS_THIRD_REGION': 'eu-west-1', - }) + env.update( + { + "TF_ACC": "1", + "AWS_ACCESS_KEY_ID": "test", + "AWS_SECRET_ACCESS_KEY": "test", + "AWS_DEFAULT_REGION": "us-west-1", + "AWS_ALTERNATE_ACCESS_KEY_ID": "test", + "AWS_ALTERNATE_SECRET_ACCESS_KEY": "test", + "AWS_ALTERNATE_SECRET_ACCESS_KEY": "test", + "AWS_ALTERNATE_REGION": "us-east-2", + "AWS_THIRD_REGION": "eu-west-1", + } + ) cmd = [ - f'./test-bin/{service}.test', - '-test.v', - '-test.parallel=1', - '-test.count=1', - '-test.timeout=60m', - f'-test.run={self.name}', + f"./test-bin/{service}.test", + "-test.v", + "-test.parallel=1", + "-test.count=1", + "-test.timeout=60m", + f"-test.run={self.name}", ] return_code, stdout = execute_command(cmd, env, tf_root_path) if return_code != 0: @@ -67,15 +74,15 @@ def runtest(self): def repr_failure(self, excinfo, **kwargs): if isinstance(excinfo.value, GoException): - return '\n'.join( + return "\n".join( [ - f'Execution failed with return code: {excinfo.value.returncode}', - f'Failure Reason:\n{excinfo.value.stderr}', + f"Execution failed with return code: {excinfo.value.returncode}", + f"Failure Reason:\n{excinfo.value.stderr}", ] ) def reportinfo(self): - return self.path, 0, f'Test Case: {self.name}' + return self.path, 0, f"Test Case: {self.name}" class ReprCrash: @@ -97,10 +104,10 @@ def pytest_runtest_makereport(item, call): outcome = yield report = outcome.get_result() if report.failed: - splits = report.longrepr.split('\n', 1) + splits = report.longrepr.split("\n", 1) longrepr = LongRepr(splits[0], splits[1]) - delattr(report, 'longrepr') - setattr(report, 'longrepr', longrepr) + delattr(report, "longrepr") + setattr(report, "longrepr", longrepr) class GoException(Exception): @@ -111,38 +118,44 @@ def __init__(self, returncode, stderr): def _docker_service_health(client): if not client.ping(): - print('\nPlease start docker daemon and try again') - raise Exception('Docker is not running') + print("\nPlease start docker daemon and try again") + raise Exception("Docker is not running") def _start_docker_container(client, config, localstack_image): - env_vars = ['DEBUG=1', 'PROVIDER_OVERRIDE_S3=asf'] + env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf"] port_mappings = { - '53/tcp': ('127.0.0.1', 53), - '53/udp': ('127.0.0.1', 53), - '443': ('127.0.0.1', 443), - '4566': ('127.0.0.1', 4566), - '4571': ('127.0.0.1', 4571), + "53/tcp": ("127.0.0.1", 53), + "53/udp": ("127.0.0.1", 53), + "443": ("127.0.0.1", 443), + "4566": ("127.0.0.1", 4566), + "4571": ("127.0.0.1", 4571), } - volumes = ['/var/run/docker.sock:/var/run/docker.sock'] - localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, - name='localstack_main', volumes=volumes, auto_remove=True, - environment=env_vars) - setattr(config, 'localstack_container_id', localstack_container.id) + volumes = ["/var/run/docker.sock:/var/run/docker.sock"] + localstack_container = client.containers.run( + image=localstack_image, + detach=True, + ports=port_mappings, + name="localstack_main", + volumes=volumes, + auto_remove=True, + environment=env_vars, + ) + setattr(config, "localstack_container_id", localstack_container.id) def _stop_docker_container(client, config): - client.containers.get(getattr(config, 'localstack_container_id')).stop() - print('LocalStack is stopped') + client.containers.get(getattr(config, "localstack_container_id")).stop() + print("LocalStack is stopped") def _localstack_health_check(): - localstack_health_url = 'http://localhost:4566/health' + localstack_health_url = "http://localhost:4566/health" session = requests.Session() retry = Retry(connect=3, backoff_factor=2) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) session.get(localstack_health_url) session.close() @@ -150,19 +163,19 @@ def _localstack_health_check(): def _pull_docker_image(client, localstack_image): docker_image_list = client.images.list(name=localstack_image) if len(docker_image_list) == 0: - print(f'Pulling image {localstack_image}') + print(f"Pulling image {localstack_image}") client.images.pull(localstack_image) docker_image_list = client.images.list(name=localstack_image) - print(f'Using LocalStack image: {docker_image_list[0].id}') + print(f"Using LocalStack image: {docker_image_list[0].id}") def pytest_configure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - localstack_image = config.getoption(name='--ls-image') + is_collect_only = config.getoption(name="--collect-only") + is_localstack_start = config.getoption(name="--ls-start") + localstack_image = config.getoption(name="--ls-image") if not is_collect_only and is_localstack_start: - print('\nStarting LocalStack...') + print("\nStarting LocalStack...") client = docker.from_env() _docker_service_health(client) @@ -171,15 +184,15 @@ def pytest_configure(config): _localstack_health_check() client.close() - print('LocalStack is ready...') + print("LocalStack is ready...") def pytest_unconfigure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') + is_collect_only = config.getoption(name="--collect-only") + is_localstack_start = config.getoption(name="--ls-start") if not is_collect_only and is_localstack_start: - print('\nStopping LocalStack...') + print("\nStopping LocalStack...") client = docker.from_env() _stop_docker_container(client, config) client.close() diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index afe7983..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.8" - -services: - localstack: - container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" - image: localstack/localstack - ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - - "127.0.0.1:4510-4559:4510-4559" # external services port range - environment: - - DEBUG=${DEBUG-} - - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} - - DOCKER_HOST=unix:///var/run/docker.sock - volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/main.py b/main.py deleted file mode 100644 index 38b3d27..0000000 --- a/main.py +++ /dev/null @@ -1,46 +0,0 @@ -import click -from timeit import default_timer as timer -from utils import build_test_bin, TF_REPO_NAME, get_services -from os.path import realpath - - -@click.group(name='pytest-golang', help='Golang Test Runner for localstack') -def cli(): - pass - - -@click.command(name='patch', help='Patch the golang test runner') -def patch(): - from utils import patch_repo - patch_repo() - - -@click.command(name='build', help='Build binary for testing') -@click.option('--service', '-s', default=None, help='''Service to build; use "ls-all", "ls-community", "ls-pro" to build all services, example: ---service=ls-all; --service=ec2; --service=ec2,iam''') -def build(service): - """Build binary for testing""" - if not service: - print('No service provided') - print('use --service or -s to specify services to build; for more help try --help to see more options') - return - - services = get_services(service) - - for service in services: - print(f'Building {service}...') - try: - start = timer() - build_test_bin(service=service, tf_root_path=realpath(TF_REPO_NAME)) - end = timer() - print(f'Build {service} in {end - start} seconds') - except KeyboardInterrupt: - print('Interrupted') - return - except Exception as e: - print(f'Failed to build binary for {service}: {e}') - - -cli.add_command(build) -cli.add_command(patch) -cli() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e9451ca --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +# project configuration + +[tool.black] +line_length = 100 +include = '(terraform_pytest\/.*\.py$|tests\/.*\.py$)' +#extend_exclude = '()' + +[tool.isort] +profile = 'black' +#extend_skip = [] +line_length = 100 + +[tool.pytest.ini_options] +testpaths = [ + "terraform-provider-aws/internal/service/", +] +#confcutdir = "tests/conftest.py" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 12240bb..ab85810 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ click==8.1.3 pytest==7.2.0 docker==6.0.1 requests==2.28.2 +black>=22.1 +isort>=5.10 \ No newline at end of file diff --git a/get-services.py b/terraform_pytest/get-services.py similarity index 69% rename from get-services.py rename to terraform_pytest/get-services.py index 3c87155..a702011 100644 --- a/get-services.py +++ b/terraform_pytest/get-services.py @@ -1,7 +1,7 @@ import json import sys -from utils import get_services +from terraform_pytest.utils import get_services services = [] @@ -11,6 +11,5 @@ print(json.dumps(services)) exit(0) else: - print('No service provided') + print("No service provided") exit(1) - diff --git a/terraform_pytest/main.py b/terraform_pytest/main.py new file mode 100644 index 0000000..57cc802 --- /dev/null +++ b/terraform_pytest/main.py @@ -0,0 +1,55 @@ +from os.path import realpath +from timeit import default_timer as timer + +import click + +from terraform_pytest.utils import TF_REPO_NAME, build_test_bin, get_services, patch_repo + + +@click.group(name="pytest-golang", help="Golang Test Runner for localstack") +def cli(): + pass + + +@click.command(name="patch", help="Patch the golang test runner") +def patch(): + patch_repo() + + +@click.command(name="build", help="Build binary for testing") +@click.option( + "--service", + "-s", + default=None, + help="""Service to build; use "ls-all", "ls-community", "ls-pro" to build all services, example: +--service=ls-all; --service=ec2; --service=ec2,iam""", +) +def build(service): + """Build binary for testing""" + if not service: + print("No service provided") + print( + "use --service or -s to specify services to build; for more help try --help to see more options" + ) + return + + services = get_services(service) + + for service in services: + print(f"Building {service}...") + try: + start = timer() + build_test_bin(service=service, tf_root_path=realpath(TF_REPO_NAME)) + end = timer() + print(f"Build {service} in {end - start} seconds") + except KeyboardInterrupt: + print("Interrupted") + return + except Exception as e: + print(f"Failed to build binary for {service}: {e}") + + +if __name__ == "__main__": + cli.add_command(build) + cli.add_command(patch) + cli() diff --git a/terraform_pytest/utils.py b/terraform_pytest/utils.py new file mode 100644 index 0000000..4226d9e --- /dev/null +++ b/terraform_pytest/utils.py @@ -0,0 +1,201 @@ +import signal +from os import chdir, chmod, getcwd, listdir, system +from os.path import exists, realpath +from uuid import uuid4 + +TF_REPO_NAME = "terraform-provider-aws" +TF_REPO_PATH = f"{realpath(TF_REPO_NAME)}" + +TF_REPO_PATCH_FILES = ["etc/001-hardcode-endpoint.patch"] + +TF_TEST_BINARY_FOLDER = "test-bin" +TF_REPO_SERVICE_FOLDER = "./internal/service" + +BLACKLISTED_SERVICES = ["controltower", "greengrass"] +LS_COMMUNITY_SERVICES = [ + "acm", + "apigateway", + "lambda", + "cloudformation", + "cloudwatch", + "configservice", + "dynamodb", + "ec2", + "elasticsearch", + "events", + "firehose", + "iam", + "kinesis", + "kms", + "logs", + "opensearch", + "redshift", + "resourcegroups", + "resourcegroupstaggingapi", + "route53", + "route53resolver", + "s3", + "s3control", + "secretsmanager", + "ses", + "sns", + "sqs", + "ssm", + "sts", + "swf", + "transcribe", +] +LS_PRO_SERVICES = [ + "amplify", + "apigateway", + "apigatewayv2", + "appconfig", + "appautoscaling", + "appsync", + "athena", + "autoscaling", + "backup", + "batch", + "cloudformation", + "cloudfront", + "cloudtrail", + "codecommit", + "cognitoidp", + "cognitoidentity", + "docdb", + "dynamodb", + "ec2", + "ecr", + "ecs", + "efs", + "eks", + "elasticache", + "elasticbeanstalk", + "elb", + "elbv2", + "emr", + "events", + "fis", + "glacier", + "glue", + "iam", + "iot", + "iotanalytics", + "kafka", + "kinesisanalytics", + "kms", + "lakeformation", + "lambda", + "logs", + "mediastore", + "mq", + "mwaa", + "neptune", + "organizations", + "qldb", + "rds", + "redshift", + "route53", + "s3", + "sagemaker", + "secretsmanager", + "serverlessrepo", + "ses", + "sns", + "sqs", + "ssm", + "sts", +] + + +def _get_test_bin_abs_path(service): + return f"{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test" + + +def execute_command(cmd, env=None, cwd=None): + """ + Execute a command and return the return code. + """ + _lwd = getcwd() + if isinstance(cmd, list): + cmd = " ".join(cmd) + else: + raise Exception("Please provide command as list(str)") + if cwd: + chdir(cwd) + if env: + _env = " ".join([f'{k}="{str(v)}"' for k, v in env.items()]) + cmd = f"{_env} {cmd}" + log_file: str = "/tmp/%s" % uuid4().hex + _err = system(f"{cmd} > {log_file} 2>&1") + if _err == signal.SIGINT: + print("SIGNINT is caught") + raise KeyboardInterrupt + _out = open(log_file, "r").read() + chdir(_lwd) + return _err, _out + + +def build_test_bin(service, tf_root_path): + _test_bin_abs_path = _get_test_bin_abs_path(service) + _tf_repo_service_folder = f"{TF_REPO_SERVICE_FOLDER}/{service}" + + if exists(_test_bin_abs_path): + return + + cmd = [ + "go", + "test", + "-c", + _tf_repo_service_folder, + "-o", + _test_bin_abs_path, + ] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}") + + if exists(_test_bin_abs_path): + chmod(_test_bin_abs_path, 0o755) + + return return_code, stdout + + +def get_all_services(): + services = [] + for service in listdir(f"{TF_REPO_PATH}/{TF_REPO_SERVICE_FOLDER}"): + if service not in BLACKLISTED_SERVICES: + services.append(service) + return sorted(services) + + +def get_services(service): + if service == "ls-community": + services = LS_COMMUNITY_SERVICES + elif service == "ls-pro": + services = LS_PRO_SERVICES + elif service == "ls-all": + services = LS_COMMUNITY_SERVICES + LS_PRO_SERVICES + else: + if "," in service: + services = service.split(",") + services = [s for s in services if s] + else: + services = [service] + services = [s for s in services if s not in BLACKLISTED_SERVICES] + return list(set(services)) + + +def patch_repo(): + print(f"Patching {TF_REPO_NAME}...") + for patch_file in TF_REPO_PATCH_FILES: + cmd = [ + "git", + "apply", + f"{realpath(patch_file)}", + ] + return_code, stdout = execute_command(cmd, cwd=realpath(TF_REPO_NAME)) + if return_code != 0: + print("----- error while patching repo -----") + if stdout: + print(f"stdout: {stdout}") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0c71c01 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,185 @@ +import re +import pytest +import docker +import requests +from requests.adapters import HTTPAdapter, Retry +from pathlib import Path +import os +from os.path import realpath, relpath, dirname +from utils import execute_command, build_test_bin + + +def pytest_addoption(parser): + parser.addoption( + '--ls-image', action='store', default='localstack/localstack:latest', help='Base URL for the API tests' + ) + parser.addoption( + '--ls-start', action='store_true', default=False, help='Start localstack service' + ) + + +def pytest_collect_file(parent, file_path): + if file_path.suffix == '.go' and file_path.name.endswith('_test.go'): + return GoFile.from_parent(parent, path=file_path) + + +class GoFile(pytest.File): + def collect(self): + raw = self.path.open().read() + fa = re.findall(r'^(func (TestAcc.*))\(.*\).*', raw, re.MULTILINE) + for _, name in fa: + yield GoItem.from_parent(self, name=name) + + +class GoItem(pytest.Item): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def runtest(self): + tf_root_path = realpath(relpath(self.path).split(os.sep)[0]) + service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) + service = service_path.split(os.sep)[-1] + + env = dict(os.environ) + env.update({ + 'TF_ACC': '1', + 'AWS_ACCESS_KEY_ID': 'test', + 'AWS_SECRET_ACCESS_KEY': 'test', + 'AWS_DEFAULT_REGION': 'us-west-1', + 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', + 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', + 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', + 'AWS_ALTERNATE_REGION': 'us-east-2', + 'AWS_THIRD_REGION': 'eu-west-1', + }) + + cmd = [ + f'./test-bin/{service}.test', + '-test.v', + '-test.parallel=1', + '-test.count=1', + '-test.timeout=60m', + f'-test.run={self.name}', + ] + return_code, stdout = execute_command(cmd, env, tf_root_path) + if return_code != 0: + raise GoException(returncode=return_code, stderr=stdout) + + def repr_failure(self, excinfo, **kwargs): + if isinstance(excinfo.value, GoException): + return '\n'.join( + [ + f'Execution failed with return code: {excinfo.value.returncode}', + f'Failure Reason:\n{excinfo.value.stderr}', + ] + ) + + def reportinfo(self): + return self.path, 0, f'Test Case: {self.name}' + + +class ReprCrash: + def __init__(self, message): + self.message = message + + +class LongRepr: + def __init__(self, message, reason): + self.reprcrash = ReprCrash(message) + self.reason = reason + + def __str__(self): + return self.reason + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + if report.failed: + splits = report.longrepr.split('\n', 1) + longrepr = LongRepr(splits[0], splits[1]) + delattr(report, 'longrepr') + setattr(report, 'longrepr', longrepr) + + +class GoException(Exception): + def __init__(self, returncode, stderr): + self.returncode = returncode + self.stderr = stderr + + +def _docker_service_health(client): + if not client.ping(): + print('\nPlease start docker daemon and try again') + raise Exception('Docker is not running') + + +def _start_docker_container(client, config, localstack_image): + env_vars = ['DEBUG=1', 'PROVIDER_OVERRIDE_S3=asf'] + port_mappings = { + '53/tcp': ('127.0.0.1', 53), + '53/udp': ('127.0.0.1', 53), + '443': ('127.0.0.1', 443), + '4566': ('127.0.0.1', 4566), + '4571': ('127.0.0.1', 4571), + } + volumes = ['/var/run/docker.sock:/var/run/docker.sock'] + localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, + name='localstack_main', volumes=volumes, auto_remove=True, + environment=env_vars) + setattr(config, 'localstack_container_id', localstack_container.id) + + +def _stop_docker_container(client, config): + client.containers.get(getattr(config, 'localstack_container_id')).stop() + print('LocalStack is stopped') + + +def _localstack_health_check(): + localstack_health_url = 'http://localhost:4566/health' + session = requests.Session() + retry = Retry(connect=3, backoff_factor=2) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + session.get(localstack_health_url) + session.close() + + +def _pull_docker_image(client, localstack_image): + docker_image_list = client.images.list(name=localstack_image) + if len(docker_image_list) == 0: + print(f'Pulling image {localstack_image}') + client.images.pull(localstack_image) + docker_image_list = client.images.list(name=localstack_image) + print(f'Using LocalStack image: {docker_image_list[0].id}') + + +def pytest_configure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + localstack_image = config.getoption(name='--ls-image') + + if not is_collect_only and is_localstack_start: + print('\nStarting LocalStack...') + + client = docker.from_env() + _docker_service_health(client) + _pull_docker_image(client, localstack_image) + _start_docker_container(client, config, localstack_image) + _localstack_health_check() + client.close() + + print('LocalStack is ready...') + + +def pytest_unconfigure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + + if not is_collect_only and is_localstack_start: + print('\nStopping LocalStack...') + client = docker.from_env() + _stop_docker_container(client, config) + client.close() diff --git a/utils.py b/utils.py deleted file mode 100644 index a7b1496..0000000 --- a/utils.py +++ /dev/null @@ -1,122 +0,0 @@ -import signal -from os import system, getcwd, chdir, chmod, listdir -from os.path import exists, realpath -from uuid import uuid4 - - -TF_REPO_NAME = 'terraform-provider-aws' -TF_REPO_PATH = f'{realpath(TF_REPO_NAME)}' - -TF_REPO_PATCH_FILES = ['etc/001-hardcode-endpoint.patch'] - -TF_TEST_BINARY_FOLDER = 'test-bin' -TF_REPO_SERVICE_FOLDER = './internal/service' - -BLACKLISTED_SERVICES = ['controltower', 'greengrass'] -LS_COMMUNITY_SERVICES = [ - "acm", "apigateway", "lambda", "cloudformation", "cloudwatch", "configservice", "dynamodb", "ec2", "elasticsearch", - "events", "firehose", "iam", "kinesis", "kms", "logs", "opensearch", "redshift", "resourcegroups", - "resourcegroupstaggingapi", "route53", "route53resolver", "s3", "s3control", "secretsmanager", "ses", "sns", "sqs", - "ssm", "sts", "swf", "transcribe" -] -LS_PRO_SERVICES = [ - "amplify", "apigateway", "apigatewayv2", "appconfig", "appautoscaling", "appsync", "athena", "autoscaling", - "backup", "batch", "cloudformation", "cloudfront", "cloudtrail", "codecommit", "cognitoidp", "cognitoidentity", - "docdb", "dynamodb", "ec2", "ecr", "ecs", "efs", "eks", "elasticache", "elasticbeanstalk", "elb", "elbv2", "emr", - "events", "fis", "glacier", "glue", "iam", "iot", "iotanalytics", "kafka", "kinesisanalytics", "kms", - "lakeformation", "lambda", "logs", "mediastore", "mq", "mwaa", "neptune", "organizations", "qldb", "rds", - "redshift", "route53", "s3", "sagemaker", "secretsmanager", "serverlessrepo", "ses", "sns", "sqs", "ssm", "sts" -] - - -def _get_test_bin_abs_path(service): - return f'{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test' - - -def execute_command(cmd, env=None, cwd=None): - """ - Execute a command and return the return code. - """ - _lwd = getcwd() - if isinstance(cmd, list): - cmd = ' '.join(cmd) - else: - raise Exception("Please provide command as list(str)") - if cwd: - chdir(cwd) - if env: - _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) - cmd = f'{_env} {cmd}' - log_file: str = '/tmp/%s' % uuid4().hex - _err = system(f'{cmd} > {log_file} 2>&1') - if _err == signal.SIGINT: - print("SIGNINT is caught") - raise KeyboardInterrupt - _out = open(log_file, 'r').read() - chdir(_lwd) - return _err, _out - - -def build_test_bin(service, tf_root_path): - _test_bin_abs_path = _get_test_bin_abs_path(service) - _tf_repo_service_folder = f'{TF_REPO_SERVICE_FOLDER}/{service}' - - if exists(_test_bin_abs_path): - return - - cmd = [ - "go", - "test", - "-c", - _tf_repo_service_folder, - "-o", - _test_bin_abs_path, - ] - return_code, stdout = execute_command(cmd, cwd=tf_root_path) - if return_code != 0: - raise Exception(f"Error while building test binary for {service}") - - if exists(_test_bin_abs_path): - chmod(_test_bin_abs_path, 0o755) - - return return_code, stdout - - -def get_all_services(): - services = [] - for service in listdir(f'{TF_REPO_PATH}/{TF_REPO_SERVICE_FOLDER}'): - if service not in BLACKLISTED_SERVICES: - services.append(service) - return sorted(services) - - -def get_services(service): - if service == 'ls-community': - services = LS_COMMUNITY_SERVICES - elif service == 'ls-pro': - services = LS_PRO_SERVICES - elif service == 'ls-all': - services = LS_COMMUNITY_SERVICES + LS_PRO_SERVICES - else: - if ',' in service: - services = service.split(',') - services = [s for s in services if s] - else: - services = [service] - services = [s for s in services if s not in BLACKLISTED_SERVICES] - return list(set(services)) - - -def patch_repo(): - print(f'Patching {TF_REPO_NAME}...') - for patch_file in TF_REPO_PATCH_FILES: - cmd = [ - 'git', - 'apply', - f'{realpath(patch_file)}', - ] - return_code, stdout = execute_command(cmd, cwd=realpath(TF_REPO_NAME)) - if return_code != 0: - print("----- error while patching repo -----") - if stdout: - print(f'stdout: {stdout}')