From d2aa925267e0895c27124549d90f033a47d3e51f Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 17:48:10 +0530 Subject: [PATCH 01/94] added python plugin --- .circleci/config.yml | 66 +---- .gitignore | 13 +- .gitmodules | 6 - README.md | 78 ++---- bin/create-report | 259 ------------------ bin/create-report-cli | 55 ---- bin/create-report-html | 130 --------- bin/install-aws-test | 38 --- bin/list-tests | 84 ------ bin/publish-metrics | 34 --- bin/run-tests | 248 ----------------- bin/setup-on-ec2 | 46 ---- build.py | 47 ++++ conftest.py | 192 +++++++++++++ ...oded-configuration-for-running-tests.patch | 74 ----- etc/0002-route53-reduce-sync-time.patch | 27 -- etc/001-hardcode-endpoint.patch | 27 ++ localstack | 1 - localstack-tests.excl.txt | 6 - localstack-tests.incl.txt | 32 --- main.py | 41 +++ moto | 1 - patch.py | 21 ++ requirements.txt | 4 + 24 files changed, 369 insertions(+), 1161 deletions(-) delete mode 100755 bin/create-report delete mode 100755 bin/create-report-cli delete mode 100755 bin/create-report-html delete mode 100755 bin/install-aws-test delete mode 100755 bin/list-tests delete mode 100755 bin/publish-metrics delete mode 100755 bin/run-tests delete mode 100755 bin/setup-on-ec2 create mode 100644 build.py create mode 100644 conftest.py delete mode 100644 etc/0001-add-simple-hardcoded-configuration-for-running-tests.patch delete mode 100644 etc/0002-route53-reduce-sync-time.patch create mode 100644 etc/001-hardcode-endpoint.patch delete mode 160000 localstack delete mode 100644 localstack-tests.excl.txt delete mode 100644 localstack-tests.incl.txt create mode 100644 main.py delete mode 160000 moto create mode 100644 patch.py create mode 100644 requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index ddd51ef..d100fa5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,78 +5,29 @@ orbs: jobs: build: - # machine: - # image: ubuntu-2004:202101-01 executor: localstack/default - environment: - DOWNLOAD_TEST_BIN: 1 - KINESIS_PROVIDER: kinesalite steps: - checkout - # start LocalStack asynchronously - - localstack/start - - run: git submodule update --init --recursive - run: name: "Installing prerequisites" command: | sudo apt-get update sudo apt install -y python3.8 libsasl2-dev + pip install -r requirements.txt - # download/install and cache aws.test and localstack pip - - restore_cache: - keys: - - localstack-cache - - # uncomment this line to force re-downloading of aws.test binary - - run: rm -f /home/circleci/.cache/localstack/aws.test - - - run: bin/install-aws-test - # - run: - # name: "Installing LocalStack" - # command: | - # cd localstack - # virtualenv --python=`which python3.8` .venv - # make install - # cd .. - - save_cache: - key: localstack-cache - paths: - - /home/circleci/.cache/localstack/ - - /home/circleci/.cache/pip/ - - # wait for LocalStack to become available - - localstack/wait - - # main test suite - run: - name: "Run test suite" - # command: bin/run-tests -i localstack-tests.incl.txt - command: bin/run-tests -t localstack-tests.incl.txt + name: "Install go mod dependancies" + command: | + python main.py patch + cd terraform-provider-aws + go mod vendor - # save build reports as artifacts - run: - name: "Create reports" - when: always + name: "Run test suite" command: | - pip3 install junit2html - bin/create-report || true - bin/create-report-html || true - mkdir -p /tmp/report/tests - mkdir -p /tmp/results/ - mv build/report.html /tmp/report || true - mv build/tests/*.html /tmp/report/tests || true - # temporarily also uploading XML files, for debugging - cp build/tests/*.xml /tmp/report/tests || true - mv build/tests/*.xml /tmp/results || true - - - store_test_results: - path: /tmp/results - - store_artifacts: - path: /tmp/report - - store_artifacts: - path: /tmp/report/tests + pytest terraform-provider-aws/internal/service/s3/bucket_object_test.go -k ignore -s -v --ls-start workflows: main: @@ -84,5 +35,6 @@ workflows: or: - equal: [ build, << pipeline.git.branch >> ] - equal: [ build-new, << pipeline.git.branch >> ] + - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - build diff --git a/.gitignore b/.gitignore index f38249a..a9dccf3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,5 @@ -*~ -*.swp -*.log -*.bak - -.vscode +.DS_Store .idea -*.iml - -build/ +.venv +.pytest_cache +__pycache__ \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 5613b14..b558fd7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,3 @@ -[submodule "localstack"] - path = localstack - url = https://github.com/localstack/localstack.git [submodule "terraform-provider-aws"] path = terraform-provider-aws url = https://github.com/hashicorp/terraform-provider-aws.git -[submodule "moto"] - path = moto - url = git@github.com:localstack/moto diff --git a/README.md b/README.md index 12b48ca..dde1a3f 100644 --- a/README.md +++ b/README.md @@ -1,61 +1,31 @@ -Terraform vs LocalStack -======================= +# Localstack Terraform Test Runner -This repository contains scripts and CI configurations to to run the Terraform Acceptance test suite of the AWS provider against LocalStack +This is a test runner for localstack and terraform. It will run a test cases from the hashicrop [terraform provider aws](https://github.com/hashicorp/terraform-provider-aws.git) against Localstack Instance. -## Utilities +Purpose of this project is to externalize the test cases from the localstack repo and run them against localstack to gather parity metrics. -Some utilities for local development: +## Installation +1. Clone the repository +2. Run `python -m virtualenv venv` to create a virtual environment +3. Run `source venv/bin/activate` to activate the virtual environment +4. Run `pip install -r requirements.txt` to install the dependencies -* `bin/list-tests [--all]`: list the available tests by parsing the go test files. -* `bin/install-aws-test` creates the binary for running the test suite (and installs it into `$HOME/.cache/localstack/aws.test`. requires go 1.16 -* `bin/run-tests [test]` run a specific test. this installs and runs localstack in a background process. add the flag `-t` to test against an already running localstack instance. +## How to run? +1. Run `python main.py patch` to apply the patch to the terraform provider aws +2. Now you are ready to use `pytest` commands to list and run test cases from golang -## Finding and running tests +## How to run test cases? +- To list down all the test case from a specific service, run `pytest terraform-provider-aws/internal/service/ --collect-only -q` +- To run a specific test case, run `pytest terraform-provider-aws/internal/service// -k --ls-start` or `pytest terraform-provider-aws/internal/service//:: --ls-start` +- Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' pytest terraform-provider-aws/internal/service//:: --ls-start` -After running `bin/install-aws-test`, use `bin/run-tests [OPTIONS...] [TESTS...]` to run individual tests or entire test suites. +## Default environment variables +- **TF_LOG**: ``debug``, +- **TF_ACC**: ``1``, +- **AWS_ACCESS_KEY_ID**: ``test``, +- **AWS_SECRET_ACCESS_KEY**: ``test``, +- **AWS_DEFAULT_REGION**: ``'us-east-1``' -Here are some examples: - -* `bin/run-tests TestAccAWSAPIGatewayResource` -* `bin/run-tests -t TestAccAWSAPIGatewayResource`: same as above, but does not start localstack -* `bin/run-tests TestAccAWSAPIGateway`: runs all tests that match `TestAccAWSAPIGateway` (run `bin/list-tests TestAccAWSAPIGateway` to see which ones will be executed) -* `bin/run-tests -e TestAccAWSAPIGatewayV2 TestAccAWSAPIGateway`: same as above, but excludes all tests that match `TestAccAWSAPIGatewayV2`. -* `bin/run-tests -i localstack-tests.incl.txt`: runs all tests listed in the text file - -You can use `bin/list-tests` with the same parameters to see which tests will be executed, -or to find specific tests based on patterns. - -For example: - -``` - % bin/list-tests Queue -TestAccAWSBatchJobQueue -TestAccAWSGameliftGameSessionQueue -TestAccAWSMediaConvertQueue -TestAccAWSSQSQueue -TestAccAWSSQSQueuePolicy -TestAccDataSourceAwsBatchJobQueue -TestAccDataSourceAwsSqsQueue -``` - -or - -``` - % bin/list-tests "Data.*Queue" -TestAccDataSourceAwsBatchJobQueue -TestAccDataSourceAwsSqsQueue -``` - -## Generating the test reports - -Test logs are aggregated into `build/tests/*.log`, the command `bin/create-report` will create junit-like xml reports. -These can then be rendered into html using `bin/create-report-html`, which also creates a summary page in `build/report.html`. -For rendering html, you need `junit2html`. - -## Travis config - -### Build cache - -The Travis-CI worker caches the built `aws.test` binary across builds. -The first build may therefore take a while. +## Options +- `--ls-start`: Start localstack instance before running the test cases +- `--ls-image`: Specify the localstack image to use, default is `localstack/localstack:latest` \ No newline at end of file diff --git a/bin/create-report b/bin/create-report deleted file mode 100755 index dec6c14..0000000 --- a/bin/create-report +++ /dev/null @@ -1,259 +0,0 @@ -#!/usr/bin/python3 - -import glob -import os - -from collections import defaultdict - -prefixes = { - 'go': '[ltt.gotest]', - 'lst': '[ltt.localstack]', - 'runner': '[ltt.runner]' -} - -def strip_special_chars(line): - # remove special characters, convert to str - if not isinstance(line, bytes): - line = line.replace('\x1b', '').encode('utf-8') - return line.decode('utf-8', 'ignore') - - -class TestCase: - def __init__(self, suite, test): - self.suite = suite - self.test = test - self.lines = list() - self.duration = 0 - self.result = None - - -class RunnerLogParser: - def __init__(self, suite): - self.suite = suite - self.tests = dict() - self.active_test = None - self.collecting = False - - def parse_duration(self, fragment): - seconds = fragment[1:-2] # (0.00s) -> 0.00 - return float(seconds) - - def add_line(self, src, line): - tests = self.tests - suite = self.suite - - line = strip_special_chars(line) - - output = line[len(prefixes[src]) + 1:] - - if src == 'runner': - if output.startswith('starting'): - if output.strip().split()[-1] == suite: - self.collecting = True - - if output.startswith('completed'): - if output.strip().split()[-1] == suite: - self.collecting = False - - if not self.collecting: - return - - if src == 'go': - if output.startswith('=== RUN'): - _ ,_ , test = output.split() - if test not in tests: - tests[test] = TestCase(suite, test) - - self.active_test = test - return - - elif output.startswith('=== PAUSE'): - _ ,_ , test = output.split() - self.active_test = None - return - - elif output.startswith('=== CONT'): - _ ,_ , test = output.split() - self.active_test = test - return - - elif output.strip().startswith('--- PASS'): - _ ,_ , test, duration = output.split() - self.active_test = None - tests[test].result = 'passed' - tests[test].duration = self.parse_duration(duration) - return - - elif output.strip().startswith('--- SKIP'): - _ ,_ , test, duration = output.split() - self.active_test = None - tests[test].result = 'skipped' - tests[test].duration = self.parse_duration(duration) - return - - elif output.strip().startswith('--- FAIL'): - _ ,_ , test, duration = output.split() - self.active_test = None - tests[test].result = 'failed' - tests[test].duration = self.parse_duration(duration) - return - - if self.active_test: - tests[self.active_test].lines.append(line) - - -def parse(suite, lines): - parser = RunnerLogParser(suite) - - for line in lines: - for src, prefix in prefixes.items(): - if line.startswith(prefix): - parser.add_line(src, line) - - for test in parser.tests.values(): - if test.result is None: - test.result = 'errored' - test.duration = 0. - - return parser - - -def parser_to_xml(parser): - - testsuite = { - 'name': parser.suite, - 'time': 0, - 'errors': 0, - 'failures': 0, - 'skipped': 0, - 'tests': 0, - } - - def find_fail_message(testcase: TestCase): - for line in testcase.lines: - if not line.startswith('[ltt.gotest]'): - continue - - if 'Step' in line and '.go' in line and 'error' in line: - return line.strip() - - return 'failure' - - - def test_to_junit_dict(testcase: TestCase): - d = { - 'name': testcase.test, - 'classname': testcase.test.split('/')[0], - 'time': "%.2f" % testcase.duration, - } - - testsuite['tests'] += 1 - testsuite['time'] += testcase.duration - - if testcase.result == 'errored': - d['error'] = { - 'type': 'error', - '__CDATA__': ''.join(testcase.lines) - } - testsuite['errors'] += 1 - - if testcase.result == 'failed': - d['failure'] = { - 'type': 'failure', - 'message': escape(find_fail_message(testcase)), - '__CDATA__': ''.join(testcase.lines) - } - testsuite['failures'] += 1 - - - if testcase.result == 'skipped': - d['skipped'] = {} - testsuite['skipped'] += 1 - - - return d - - testcases = [dict2xml(test_to_junit_dict(t), 'testcase') for t in parser.tests.values()] - - testsuite['properties'] = [{ - 'property': {'name': 'test-runner', 'value': 'localstack-terraform-test'} - }] - - testsuite['__XML__'] = testcases - - return dict2xml(testsuite, 'testsuite') - - -def escape(s): - s = s.replace("&", "&") - s = s.replace("<", "<") - s = s.replace(">", ">") - s = s.replace("\"", """) - s = s.replace("'", "'") - return s - - -def dict2xml(d, root_node=None): - wrap = False if None == root_node or isinstance(d, list) else True - root = 'objects' if None == root_node else root_node - root_singular = root[:-1] if 's' == root[-1] and None == root_node else root - xml = '' - children = [] - - if isinstance(d, dict): - for key, value in dict.items(d): - if key == '__CDATA__': - children.append('') - elif key.startswith('__XML__'): - if isinstance(value, list): - children.append('\n'.join(value)) - else: - children.append(value) - elif isinstance(value, dict): - children.append(dict2xml(value, key)) - elif isinstance(value, list): - children.append(dict2xml(value, key)) - else: - xml = xml + ' ' + key + '="' + str(value) + '"' - else: - for value in d: - children.append(dict2xml(value, root_singular)) - - end_tag = '>' if 0 < len(children) else '/>' - - if wrap or isinstance(d, dict): - xml = '<' + root + xml + end_tag - - if 0 < len(children): - for child in children: - xml = xml + child - - if wrap or isinstance(d, dict): - xml = xml + '' - - return xml - - - -def main(): - for f in glob.glob('build/tests/*.log'): - with open(f, 'rb') as fd: - lines = fd.readlines() - - lines = [strip_special_chars(line) for line in lines] - suite = os.path.basename(f)[:-4] # strip `.log` - parser = parse(suite, lines) - - f_xml = f[:-4] + '.xml' - d = os.path.dirname(f_xml) - f = os.path.basename(f_xml) - - f_xml = os.path.join(d, 'TEST-' + f) - - with open(f_xml, 'w') as fd: - print(f_xml) - fd.writelines(parser_to_xml(parser)) - - -if __name__ == '__main__': - main() diff --git a/bin/create-report-cli b/bin/create-report-cli deleted file mode 100755 index c92f30a..0000000 --- a/bin/create-report-cli +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python3 - -import glob -import xml.etree.ElementTree as ET - - -def get_summary(reports): - passed = failures = errors = skipped = total = 0 - for f in reports: - try: - for event, elem in ET.iterparse(f, events=('start',)): - if elem.tag == 'testsuite': - d = dict(elem.attrib) - d['time'] = float(d['time']) - d['tests'] = float(d['tests']) - d['errors'] = float(d['errors']) - d['failures'] = float(d['failures']) - d['skipped'] = float(d['skipped']) - d['passed'] = d['tests'] - (d['errors'] + d['failures'] + d['skipped']) - print(d) - # - total += d["tests"] - passed += d["passed"] - failures += d["failures"] - errors += d["errors"] - except Exception as e: - print(e) - - print('===========================================') - print("Total cases: {}".format(total)) - for name, nr in [("Passed", passed), ("Failures", failures), ("Errors", errors), ("Skipped", skipped)]: - pct = round((nr / total) * 100, 2) - print("{}: {} ({}%)".format(name, nr, pct)) - - failed_tests = failures + errors - if failed_tests > 0: - raise Exception("{} Terraform tests failed!".format(failed_tests)) - - -def main(): - reports = glob.glob('build/tests/*.xml') - if not reports: - print('no reports, run bin/create-report first') - exit(1) - - reports.sort() - - print('===========================================') - get_summary(reports) - print('===========================================') - print() - - -if __name__ == '__main__': - main() diff --git a/bin/create-report-html b/bin/create-report-html deleted file mode 100755 index 3142a85..0000000 --- a/bin/create-report-html +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/python3 - -import glob -import os -import xml.etree.ElementTree as ET -from shutil import which - - -def render(src, dest): - os.system(f'junit2html {src} {dest}') - - -def create_test_reports(reports): - files = [] - for f in reports: - - f_html = f[:-4] + '.html' - - if os.path.exists(f_html): - print('skipping', f) - continue - - print(f) - render(f, f_html) - files.append(f_html) - - return files - - -def parse_testsuite_meta(source): - for event, elem in ET.iterparse(source, events=('start', )): - if elem.tag == 'testsuite': - - d = dict(elem.attrib) - d['time'] = float(d['time']) - d['tests'] = int(d['tests']) - d['errors'] = int(d['errors']) - d['failures'] = int(d['failures']) - d['skipped'] = int(d['skipped']) - d['passed'] = d['tests'] - (d['errors'] + d['failures'] + d['skipped']) - - return d - - return None - - -def create_summary(reports): - testsuites = [] - - for f in reports: - with open(f, 'r') as fd: - try: - ts = parse_testsuite_meta(fd) - if ts: - ts['report'] = os.path.basename(f[:-4] + '.html') - testsuites.append(ts) - except Exception as e: - print('Unable to parse test suite metadata from report %s: %s' % (f, e)) - - summary = {'tests': 0, 'skipped': 0, 'failures': 0, 'errors': 0, 'passed': 0, 'time': 0} - - for ts in testsuites: - for k in summary.keys(): - summary[k] += ts.get(k, 0) - - summary['time'] = round(summary['time'], 2) - - for k in ('skipped', 'failures', 'errors', 'passed'): - if summary['tests'] == 0: - summary[k + '%'] = 0 - else: - summary[k + '%'] = round((summary[k] / summary['tests']) * 100, 2) - - return testsuites, summary - - -def render_summary(testsuites, summary): - # TODO: use jinja templates and render a proper summary page - - html = 'Summary' - - html += '' - html += '' - for k in ('passed', 'failures', 'errors', 'skipped'): - html += f'' - - html += f'' - - html += '
ResultsNumber%
{k}{summary[k]}{summary[k+"%"]}
Total{summary["tests"]}100%
' - - html += '' - for ts in testsuites: - html += ''.format(**ts) - - html += '
TestTestsPassedFailedErroredSkipped
{name}{tests}{passed}{failures}{errors}{skipped}
' - - html += '' - return html - - -def main(): - reports = glob.glob('build/tests/*.xml') - if not reports: - print('no reports, run bin/create-report first') - exit(1) - - if not which('junit2html'): - print('junit2html not found in path, please install it with: pip install junit2html') - exit(1) - - reports.sort() - - create_test_reports(reports) - testsuites, summary = create_summary(reports) - - with open('build/report.html', 'w') as fd: - fd.write(render_summary(testsuites, summary)) - - print('===========================================') - print('report created') - print() - - if which('open'): - print('open build/report.html') - elif which('xdg-open'): - print('xdg-open build/report.html') - - -if __name__ == '__main__': - main() diff --git a/bin/install-aws-test b/bin/install-aws-test deleted file mode 100755 index ef80b06..0000000 --- a/bin/install-aws-test +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -pwd=`pwd` - -TEST_BIN=$HOME/.cache/localstack/aws.test - -TEST_BIN_URL=${TEST_BIN_URL:-"https://localstack-terraform-test.s3.eu-central-1.amazonaws.com/aws.test"} - -[ -f ${TEST_BIN} ] && { echo "aws.test already exists at ${TEST_BIN} skipping"; exit 0; } - -BIN_DIR=$(dirname ${TEST_BIN}) -mkdir -p ${BIN_DIR} - -if [ ! -z "${DOWNLOAD_TEST_BIN}" ] && [ ${DOWNLOAD_TEST_BIN} == 1 ]; then - echo "downloading test_binary from ${TEST_BIN_URL}" - if ! curl ${TEST_BIN_URL} --output ${TEST_BIN}; then - exit 1; - fi - chmod +x ${TEST_BIN} - ls -la ${TEST_BIN} - exit 0 -fi - -PATCH="etc/0001-add-simple-hardcoded-configuration-for-running-tests.patch" - -cd terraform-provider-aws -git apply $pwd/${PATCH} - -PATCH="etc/0002-route53-reduce-sync-time.patch" - -git apply $pwd/${PATCH} - -go get -u ./ - -echo "building ${TEST_BIN} with go test" -go test -c ./aws - -mv aws.test ${TEST_BIN} diff --git a/bin/list-tests b/bin/list-tests deleted file mode 100755 index 661d558..0000000 --- a/bin/list-tests +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -function usage() { - echo "USAGE" - echo " list-tests [OPTION...] [INCLUDE...]" - echo "" - echo "DESCRIPTION" - echo " list available acceptance tests in the terraform test suite and prepare arguments for the go test command" - echo "" - echo "OPTIONS" - echo " -h, --help" - echo " show this message" - echo "" - echo " -a, --all" - echo " list individual test cases instead of test groups" - echo "" - echo " -e PATTERN|FILE, --exclude PATTERN|FILE" - echo " repeatable option to exclude certain tests or test groups" - echo "" - echo " -i PATTERN|FILE, --include PATTERN|FILE" - echo " repeatable option to include certain tests or test groups" - echo "" - echo " -p, --prepare" - echo " prepare the list of tests for 'go test -run' command" -} - -function list_testacc_all() { - grep "^func TestAcc" terraform-provider-aws/aws/**_test.go \ - | cut -d':' -f2 | cut -d '(' -f1 | cut -c 6- \ - | sort -u | grep -v '^$' -} - -function list_testacc() { - list_testacc_all | cut -d'_' -f1 | sort -u | grep -v '^$' -} - -# some logic to invoke the correct command -function main() { - excluded=() - included=() - - cmd=list_testacc - - # parse options - while [[ "$#" -gt 0 ]]; do - case $1 in - -h|--help) usage; exit 0; ;; - -a|--all) cmd=list_testacc_all; ;; - -e|--exclude) excluded+=("$2"); shift; ;; - -i|--include) included+=("$2"); shift; ;; - -p|--prepare) prepare=0; ;; - *) included+=("$1"); ;; - esac - shift - done - - # build grep filters - filters="" - for excl in "${excluded[@]}"; do - if [ -f $excl ]; then - filters="${filters}|grep -v -f "$excl"" - else - filters="${filters}|grep -v '"$excl"'" - fi - done - for incl in "${included[@]}"; do - if [ -f $incl ]; then - filters="${filters}|grep -f "$incl"" - else - filters="${filters}|grep '"$incl"'" - fi - done - - cmd=$(echo "${cmd} ${filters}") - - if [ ! -z $prepare ]; then - echo "($(eval $cmd | tr '\n' '|' | sed 's/|$/_/' | sed 's/|/_|/g'))" - else - eval $cmd - fi - -} - -main "$@" diff --git a/bin/publish-metrics b/bin/publish-metrics deleted file mode 100755 index 51a8694..0000000 --- a/bin/publish-metrics +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -export PROJECT_ROOT=$(pwd) # FIXME - -function publish_metrics() { - export AWS_DEFAULT_REGION=us-east-2 - # check if AWS credentials are configured - aws cloudwatch list-metrics --namespace test123 > /dev/null || return - echo "Publishing test results to CloudWatch Metrics ..." - - # generate report - bin/create-report - bin/create-report-html - - # prepare data - now=$(date +"%Y-%m-%dT%H:%M:%S") - reportFile=$PROJECT_ROOT/build/report.html - passed=$(sed -r 's|.*passed([^<]*).*|\1|' $reportFile) - failures=$(sed -r 's|.*failures([^<]*).*|\1|' $reportFile) - errors=$(sed -r 's|.*errors([^<]*).*|\1|' $reportFile) - skipped=$(sed -r 's|.*skipped([^<]*).*|\1|' $reportFile) - total=$(sed -r 's|.*Total([^<]*).*|\1|' $reportFile) - passedPercent=$(awk "BEGIN { print $passed / $total }") - - # publish data to CloudWatch - names=(testsPassed testsPassedPercent testsFailures testsErrors testsSkipped testsTotal) - values=($passed $passedPercent $failures $errors $skipped $total) - echo ${values[@]} - for (( i=0 ; i < ${#names[@]} ; i++ )) { - aws cloudwatch put-metric-data --namespace ls-tf-tests --metric-name ${names[i]} --value ${values[i]} --timestamp $now - } -} - -publish_metrics diff --git a/bin/run-tests b/bin/run-tests deleted file mode 100755 index aa4532b..0000000 --- a/bin/run-tests +++ /dev/null @@ -1,248 +0,0 @@ -#!/bin/bash - -function usage() { - echo "USAGE" - echo " run-tests [OPTIONS...] [TESTS...]" - echo "" - echo "DESCRIPTION" - echo " runs the Terraform test suite" - echo "" - echo "OPTIONS" - echo " -h, --help" - echo " show this message" - echo "" - echo " -e PATTERN|FILE, --exclude PATTERN|FILE" - echo " repeatable option to exclude certain tests or test groups" - echo "" - echo " -i PATTERN|FILE, --include PATTERN|FILE" - echo " repeatable option to include certain tests or test groups" - echo "" - echo " -t, --tests-only" - echo " run the tests against an already running instance of localstack" -} - -export PROJECT_ROOT=$(pwd) # FIXME - -export LST_DIR=${PROJECT_ROOT}/localstack -export BUILD_DIR=${PROJECT_ROOT}/build - -export LST_LOG=${BUILD_DIR}/localstack.log -export TEST_LOG=${BUILD_DIR}/test.log - -export AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-us-east-1} -export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-test} -export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-test} - -export INSTALL_LOCAL_MOTO=${INSTALL_LOCAL_MOTO:-true} - -export TF_ACC=1 -export TEST_PARALLEL=${TEST_PARALLEL:-1} - -export TIMEOUT_SECS=600 - -SUPPRESS_BIN_LOGS=${SUPPRESS_BIN_LOGS:-0} - -TEST_BIN=$HOME/.cache/localstack/aws.test -[ ! -f $TEST_BIN ] && { echo "aws.test not installed. please run bin/install-aws-test"; exit 1; } - -function run_localstack() { - cd ${LST_DIR} - source .venv/bin/activate - - if [ ${INSTALL_LOCAL_MOTO} == true ]; then - pip install -e ../moto - fi - - exec bin/localstack start --host -} - -function run_watchdog() { - log="${BUILD_DIR}/tests/$1.log" - - i=0 - while true; do - sleep 2 - i=$((i + 1)) - if [ $i -gt $TIMEOUT_SECS ]; then - # TODO remove duplicate code - pkill -f "aws.test" # FIXME - echo "[ltt.runner] terminated (timeout) $1" | tee -a ${log} - break - fi - if cat ${log} | strings | grep --max-count=1 -q "attempt 1/25" &> /dev/null; then - pkill -f "aws.test" # FIXME - echo "[ltt.runner] terminated $1" | tee -a ${log} - break - fi - done -} - -function forward_lst_log() { - trap 'kill $(jobs -p) 2> /dev/null' EXIT - log=$1 - tail -f -n0 ${LST_LOG} 2> /dev/null | stdbuf -oL awk '{print "[ltt.localstack] " $0}' | tee -a $log > /dev/null -} - -function forward_test_log() { - trap 'kill $(jobs -p) 2> /dev/null' EXIT - log=$1 - tail -f -n0 ${TEST_LOG} 2> /dev/null | stdbuf -oL awk '{print "[ltt.gotest] " $0}' | tee -a $log > /dev/null -} - -function run_test() { - cd terraform-provider-aws/aws - echo "running ${TEST_BIN} -test.v -test.parallel=${TEST_PARALLEL} -test.run $@" - if [ "$SUPPRESS_BIN_LOGS" = "1" ]; then - # can be useful for debugging, but seems to break certain API calls, e.g., large S3 file PUTs - ${TEST_BIN} -test.v -test.parallel=${TEST_PARALLEL} -test.run "$@" 2>&1 | tr -cd "[:print:][\011\012]" - else - ${TEST_BIN} -test.v -test.parallel=${TEST_PARALLEL} -test.run "$@" - fi - result=$? - echo - echo Test terminated with exit code $? - # clean up tmp dirs - rm -rf /tmp/plugintest* - return $result -} - -function list_tests() { - # without this function we could not run individual tests, but it's kinda hacky - cmd=$PROJECT_ROOT/bin/list-tests - args_all="" - args="" - - # parse options - while [[ "$#" -gt 0 ]]; do - if [[ $2 =~ "_" ]]; then - args_all="$args_all $1 $2" - else - args="$args $1 $2" - fi - shift - shift - done - - [ -z "$args" ] || $cmd $args - [ -z "$args_all" ] || $cmd --all $args_all -} - -function run_tests() { - tests=$(list_tests "$@") - - if [ -z "${tests}" ]; then - echo "no matching tests" - exit 1 - fi - - # kill jobs once process exits - trap 'kill $(jobs -p) 2> /dev/null' EXIT - - echo "" > ${TEST_LOG} - - tail -F ${TEST_LOG} 2> /dev/null | stdbuf -oL strings | egrep "(=== RUN|=== CONT|=== PAUSE|FAIL|PASS|SKIP)" & - - export TF_LOG=debug - - for t in ${tests}; do - # truncate test log - echo "" > ${TEST_LOG} - - # touch log for test run - log="${BUILD_DIR}/tests/${t}.log" - echo "" > ${log} - - echo "[ltt.runner] starting $t" | tee -a ${log} - - run_watchdog $t & - pid_watchdog=$! - - forward_lst_log $log & - pid_forward_lst_log=$! - - forward_test_log $log & - pid_forward_test_log=$! - - sleep 1 - - if [[ $t =~ "_" ]]; then - arg=$t - else - arg="${t}_" - fi - - run_test "${arg}" 2>&1 | stdbuf -oL tee -a ${TEST_LOG} > /dev/null - - kill $pid_watchdog &> /dev/null - kill $pid_forward_lst_log &> /dev/null - kill $pid_forward_test_log &> /dev/null - - wait $pid_watchdog $pid_forward_test_log $pid_forward_lst_log - echo "[ltt.runner] completed $t" | tee -a ${log} - sleep 1 - - done -} - -function run_lst_and_tests() { - rm -f ${LST_LOG} - # start localstack in the background - DEBUG=1 run_localstack &> ${LST_LOG} & - export lst_pid=$! - - # TODO: subprocesses will stay open if interrupted - - # wait for localstack to be ready - echo "[ltt.runner] waiting on localstack to start on process ${lst_pid}" - - while true; do - sleep 1 - - if `grep --max-count=1 -q "Ready\." ${LST_LOG}`; then - break - fi - if ! ps -p ${lst_pid} > /dev/null; then - echo "[ltt.runner] localstack terminated while waiting" - exit 1 - fi - done - - run_tests "$@" - ret=$? - - # kill the running localstack instance - echo "[ltt.runner] killing localstack ${lst_pid}" - kill ${lst_pid} - echo "[ltt.runner] waiting on localstack to end" - wait ${lst_pid} - - return ${ret} -} - -function main() { - list_test_args="" # test filters (handed to list-tests) - start_lst=true - - while [[ "$#" -gt 0 ]]; do - case $1 in - -h|--help) usage; exit 0 ;; - -i|--include) list_test_args="$list_test_args -i $2"; shift ;; - -e|--exclude) list_test_args="$list_test_args -e $2"; shift ;; - -t|--tests-only) start_lst=false; ;; - *) list_test_args="$list_test_args -i $1" ;; - esac - shift - done - - mkdir -p ${BUILD_DIR}/tests - - if [ $start_lst == false ]; then - run_tests $list_test_args - else - run_lst_and_tests $list_test_args - fi - - exit $? -} - -main "$@" diff --git a/bin/setup-on-ec2 b/bin/setup-on-ec2 deleted file mode 100755 index 818420e..0000000 --- a/bin/setup-on-ec2 +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/bash - -# ONLY RUN THESE COMMANDS ON FRESH EC2 INSTANCES, NOT IN YOUR OWN SYSTEM! - -sudo apt -y update -sudo apt -y install docker.io python3-pip npm -echo 'PATH=$PATH:~/.local/bin' >> ~/.profile - -sudo usermod -aG docker ubuntu - -pip install localstack junit2html awscli awscli-local -sudo npm install -g serve - -# enable swap memory -sudo fallocate -l 2G /swapfile -sudo chmod 600 /swapfile -sudo mkswap /swapfile -sudo swapon /swapfile - -test -e localstack-terraform-test || ( - git clone https://github.com/localstack/localstack-terraform-test - git submodule update --init terraform-provider-aws -) -( - cd localstack-terraform-test - DOWNLOAD_TEST_BIN=1 bin/install-aws-test -) - -# may need to logout, re-login to the machine for groups to become effective - -# STEP 1: -# TODO: set API key below -# export LOCALSTACK_API_KEY=test -# DOCKER_FLAGS=-d localstack start - -# STEP 2: -# start in screen session: -# bin/run-tests -t -i localstack-tests.incl.txt -# to create reports: -# bin/create-report -# bin/create-report-html - -# STEP 3: -# start in separate screen session: -# cd localstack-terraform-test/build -# sudo serve -l tcp://0.0.0.0:80 diff --git a/build.py b/build.py new file mode 100644 index 0000000..f129bbd --- /dev/null +++ b/build.py @@ -0,0 +1,47 @@ +import os.path +from subprocess import Popen, PIPE +from constants import BASE_PATH, SERVICE_BASE_PATH, BIN_PATH +from os.path import realpath, exists + + +def get_all_services(): + services = [] + for service in os.listdir(f'{BASE_PATH}/{SERVICE_BASE_PATH}'): + services.append(service) + return sorted(services) + + +def build_bin(service=None, force=False): + service_path = f'{SERVICE_BASE_PATH}/{service}' + bin_path = f'./{BIN_PATH}/{service}.test' + + if exists(bin_path) and not force: + print(f'Binary already exists for {service}') + return + print(f'Building {service}...') + + cmd = [ + 'go', + 'test', + '-c', + service_path, + '-o', + bin_path, + ] + + proc = Popen( + cmd, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + cwd=realpath(BASE_PATH) + ) + stdout, stderr = proc.communicate() + proc.terminate() + if stdout: + print(f'stdout: {stdout}') + if stderr: + print(f'stderr: {stderr}') + + if exists(bin_path): + os.chmod(bin_path, 0o755) diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..324a7e9 --- /dev/null +++ b/conftest.py @@ -0,0 +1,192 @@ +import pytest +import re +from subprocess import PIPE, Popen +from pathlib import Path +import os +from os.path import realpath, relpath, dirname +import docker +import requests +from requests.adapters import HTTPAdapter, Retry +from os.path import exists + + +def pytest_addoption(parser): + parser.addoption( + '--ls-image', action='store', default='localstack/localstack:latest', help='Base URL for the API tests' + ) + parser.addoption( + '--ls-start', action='store_true', default=False, help='Start localstack service' + ) + + +def pytest_collect_file(parent, file_path): + if file_path.suffix == ".go" and file_path.name.endswith("_test.go"): + return GoFile.from_parent(parent, path=file_path) + + +class GoFile(pytest.File): + def collect(self): + raw = self.path.open().read() + fa = re.findall(r"^(func (TestAcc.*))\(.*\).*", raw, re.MULTILINE) + for _, name in fa: + yield GoItem.from_parent(self, name=name) + + +class GoItem(pytest.Item): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def runtest(self): + tf_root_path = realpath(relpath(self.path).split(os.sep)[0]) + service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) + service = service_path.split(os.sep)[-1] + + _build_test_bin(service=service, tf_root_path=tf_root_path, service_path=service_path) + + env = dict(os.environ) + env.update({ + 'TF_ACC': '1', + 'AWS_ACCESS_KEY_ID': 'test', + 'AWS_SECRET_ACCESS_KEY': 'test', + 'AWS_DEFAULT_REGION': 'us-east-1' + }) + + cmd = [ + f"./test-bin/{service}.test", + "-test.v", + "-test.parallel=1", + "-test.count=1", + "-test.timeout=60m", + "-test.run", f"{self.name}" + ] + + proc = Popen( + cmd, stdout=PIPE, stderr=PIPE, + env=env, bufsize=1, universal_newlines=True, + cwd=tf_root_path + ) + stdout, stderr = proc.communicate() + proc.terminate() + if proc.returncode != 0: + raise GoException(proc.returncode, stdout, stderr) + return proc.returncode + + def repr_failure(self, excinfo, **kwargs): + """Called when self.runtest() raises an exception.""" + if isinstance(excinfo.value, GoException): + return "\n".join( + [ + f'\nExecution failed with return code: {excinfo.value.returncode}', + f'\nFailure Reason:\n{excinfo.value}', + ] + ) + + def reportinfo(self): + return self.path, 0, f"Test Case: {self.name}" + + +class GoException(Exception): + def __init__(self, returncode, stdout, stderr): + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + + +def _build_test_bin(service, tf_root_path, service_path): + bin_path = f'./test-bin/{service}.test' + + if exists(f"{tf_root_path}/{bin_path}"): + return + cmd = [ + "go", + "test", + "-c", + f"./{service_path}", + "-o", + bin_path, + ] + proc = Popen( + cmd, stdout=PIPE, stderr=PIPE, + bufsize=1, universal_newlines=True, + cwd=tf_root_path + ) + stdout, stderr = proc.communicate() + proc.terminate() + if proc.returncode != 0: + raise GoException(proc.returncode, stdout, stderr) + return + + +def _docker_service_health(client): + if not client.ping(): + print("\nPlease start docker daemon and try again") + raise Exception("Docker is not running") + + +def _start_docker_container(client, config, localstack_image): + env_vars = ["DEBUG=1"] + port_mappings = { + '53/tcp': ('127.0.0.1', 53), + '53/udp': ('127.0.0.1', 53), + '443': ('127.0.0.1', 443), + '4566': ('127.0.0.1', 4566), + '4571': ('127.0.0.1', 4571), + } + volumes = ["/var/run/docker.sock:/var/run/docker.sock"] + localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, + name='localstack_main', volumes=volumes, auto_remove=True, + environment=env_vars) + setattr(config, "localstack_container_id", localstack_container.id) + + +def _stop_docker_container(client, config): + client.containers.get(getattr(config, "localstack_container_id")).stop() + print("LocalStack is stopped") + + +def _localstack_health_check(): + localstack_health_url = "http://localhost:4566/health" + session = requests.Session() + retry = Retry(connect=3, backoff_factor=2) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + session.get(localstack_health_url) + + +def _pull_docker_image(client, localstack_image): + docker_image_list = client.images.list(name=localstack_image) + if len(docker_image_list) == 0: + print(f"Pulling image {localstack_image}") + client.images.pull(localstack_image) + print(f"Using LocalStack image: {docker_image_list[0].id}") + + +def pytest_configure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + localstack_image = config.getoption(name='--ls-image') + + if not is_collect_only and is_localstack_start: + + print("\nStarting LocalStack...") + + client = docker.from_env() + _docker_service_health(client) + _pull_docker_image(client, localstack_image) + _start_docker_container(client, config, localstack_image) + _localstack_health_check() + client.close() + + print("LocalStack is ready...") + + +def pytest_unconfigure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + + if not is_collect_only and is_localstack_start: + print("\nStopping LocalStack...") + client = docker.from_env() + _stop_docker_container(client, config) + client.close() \ No newline at end of file diff --git a/etc/0001-add-simple-hardcoded-configuration-for-running-tests.patch b/etc/0001-add-simple-hardcoded-configuration-for-running-tests.patch deleted file mode 100644 index a23c1c1..0000000 --- a/etc/0001-add-simple-hardcoded-configuration-for-running-tests.patch +++ /dev/null @@ -1,74 +0,0 @@ -From 0111e5997250b4d1ac56a218147de086c8456c08 Mon Sep 17 00:00:00 2001 -From: Waldemar Hummer -Date: Sun, 9 May 2021 15:34:37 +0200 -Subject: [PATCH] add simple/hardcoded configuration for running tests against - LocalStack - ---- - aws/config.go | 4 ++++ - aws/provider.go | 17 ++++++++++++++++- - 2 files changed, 20 insertions(+), 1 deletion(-) - -diff --git a/aws/config.go b/aws/config.go -index 20e04deb9..fd1857aa3 100644 ---- a/aws/config.go -+++ b/aws/config.go -@@ -456,6 +456,10 @@ func (c *Config) Client() (interface{}, error) { - dnsSuffix = p.DNSSuffix() - } - -+ // XXX: added by whummer -+ // insert custom endpoints -+ c.Endpoints = localEndpoints -+ - client := &AWSClient{ - accessanalyzerconn: accessanalyzer.New(sess.Copy(&aws.Config{Endpoint: aws.String(c.Endpoints["accessanalyzer"])})), - accountid: accountID, -diff --git a/aws/provider.go b/aws/provider.go -index 366fc2178..ee5654ead 100644 ---- a/aws/provider.go -+++ b/aws/provider.go -@@ -1169,6 +1169,8 @@ func Provider() *schema.Provider { - - var descriptions map[string]string - var endpointServiceNames []string -+const localEndpoint = "http://localhost:4566" -+var localEndpoints map[string]string - - func init() { - descriptions = map[string]string{ -@@ -1376,9 +1378,21 @@ func init() { - "workspaces", - "xray", - } -+ -+ // XXX: added by whummer -+ localEndpoints = map[string]string{} -+ for _, name := range endpointServiceNames { -+ if name == "s3" { -+ localEndpoints[name] = "http://s3.localhost.localstack.cloud:4566" -+ } else { -+ localEndpoints[name] = localEndpoint -+ } -+ } -+ - } - - func providerConfigure(d *schema.ResourceData, terraformVersion string) (interface{}, error) { -+ - config := Config{ - AccessKey: d.Get("access_key").(string), - SecretKey: d.Get("secret_key").(string), -@@ -1387,7 +1401,8 @@ func providerConfigure(d *schema.ResourceData, terraformVersion string) (interfa - Region: d.Get("region").(string), - CredsFilename: d.Get("shared_credentials_file").(string), - DefaultTagsConfig: expandProviderDefaultTags(d.Get("default_tags").([]interface{})), -- Endpoints: make(map[string]string), -+ // Endpoints: make(map[string]string), -+ Endpoints: localEndpoints, - MaxRetries: d.Get("max_retries").(int), - IgnoreTagsConfig: expandProviderIgnoreTags(d.Get("ignore_tags").([]interface{})), - Insecure: d.Get("insecure").(bool), --- -2.25.1 - diff --git a/etc/0002-route53-reduce-sync-time.patch b/etc/0002-route53-reduce-sync-time.patch deleted file mode 100644 index 1df26a1..0000000 --- a/etc/0002-route53-reduce-sync-time.patch +++ /dev/null @@ -1,27 +0,0 @@ -diff --git a/aws/resource_aws_route53_record.go b/aws/resource_aws_route53_record.go -index eeca299609..9fb7deb45e 100644 ---- a/aws/resource_aws_route53_record.go -+++ b/aws/resource_aws_route53_record.go -@@ -469,7 +469,7 @@ func resourceAwsRoute53RecordCreate(d *schema.ResourceData, meta interface{}) er - - func changeRoute53RecordSet(conn *route53.Route53, input *route53.ChangeResourceRecordSetsInput) (interface{}, error) { - var out *route53.ChangeResourceRecordSetsOutput -- err := resource.Retry(1*time.Minute, func() *resource.RetryError { -+ err := resource.Retry(5*time.Second, func() *resource.RetryError { - var err error - out, err = conn.ChangeResourceRecordSets(input) - if isAWSErr(err, route53.ErrCodeNoSuchHostedZone, "") { -@@ -494,10 +494,10 @@ func waitForRoute53RecordSetToSync(conn *route53.Route53, requestId string) erro - wait := resource.StateChangeConf{ - Pending: []string{route53.ChangeStatusPending}, - Target: []string{route53.ChangeStatusInsync}, -- Delay: time.Duration(rand.Int63n(20)+10) * time.Second, -+ Delay: time.Duration(rand.Int63n(2)+1) * time.Second, - MinTimeout: 5 * time.Second, -- PollInterval: 20 * time.Second, -- Timeout: 30 * time.Minute, -+ PollInterval: 2 * time.Second, -+ Timeout: 3 * time.Minute, - Refresh: func() (result interface{}, state string, err error) { - changeRequest := &route53.GetChangeInput{ - Id: aws.String(requestId), diff --git a/etc/001-hardcode-endpoint.patch b/etc/001-hardcode-endpoint.patch new file mode 100644 index 0000000..dd3f9ed --- /dev/null +++ b/etc/001-hardcode-endpoint.patch @@ -0,0 +1,27 @@ +diff --git a/internal/conns/config.go b/internal/conns/config.go +index 12240109bb..3940e4ce73 100644 +--- a/internal/conns/config.go ++++ b/internal/conns/config.go +@@ -77,8 +77,22 @@ type Config struct { + UseFIPSEndpoint bool + } + ++func GetLocalEndpoints() map[string]string { ++ const localEndpoint = "http://localhost:4566" ++ var localEndpoints = map[string]string{} ++ for _, name := range names.Aliases() { ++ if name == "s3" { ++ localEndpoints[name] = "http://s3.localhost.localstack.cloud:4566" ++ } else { ++ localEndpoints[name] = localEndpoint ++ } ++ } ++ return localEndpoints ++} ++ + // ConfigureProvider configures the provided provider Meta (instance data). + func (c *Config) ConfigureProvider(ctx context.Context, client *AWSClient) (*AWSClient, diag.Diagnostics) { ++ c.Endpoints = GetLocalEndpoints() + awsbaseConfig := awsbase.Config{ + AccessKey: c.AccessKey, + APNInfo: StdUserAgentProducts(c.TerraformVersion), diff --git a/localstack b/localstack deleted file mode 160000 index 3a32202..0000000 --- a/localstack +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3a3220268b962a97eeeebff44a3bb5383b1dbfc7 diff --git a/localstack-tests.excl.txt b/localstack-tests.excl.txt deleted file mode 100644 index 2ed9db4..0000000 --- a/localstack-tests.excl.txt +++ /dev/null @@ -1,6 +0,0 @@ -TestAccAWSEc2Fleet -TestAccAWSEc2TransitGateway -TestAccAWSEc2TransitGatewayVpcAttachment -TestAccAWSKinesisAnalytics -TestAccAWSSSMMaintenanceWindow -TestAccAWSSSMMaintenanceWindowTask diff --git a/localstack-tests.incl.txt b/localstack-tests.incl.txt deleted file mode 100644 index 5035061..0000000 --- a/localstack-tests.incl.txt +++ /dev/null @@ -1,32 +0,0 @@ -TestAccAWSAcm -TestAccAWSAmplify -# TestAccAWSAPIGateway -TestAccAWSAppsync -TestAccAWSCloudformation -TestAccAWSCloudTrail -TestAccAwsCloudWatch -TestAccAwsCloudWatch -TestAccAWSCloudWatch -TestAccAWSCloudwatch -TestAccAWSCognito -TestAccAWSDBCluster -TestAccAWSDynamoDb -TestAccAWSEc2 -# TestAccAWSGlue -# TestAccAWSIAM -TestAccAWSKinesis -TestAccAWSKms -TestAccAWSLambda -TestAccAWSRedshift -TestAccAWSResourceGroup -TestAccAWSRoute53 -TestAccAWSS3 -TestAccAwsSecrets -TestAccAWSSES -TestAccAWSSns -TestAccAWSSNS -TestAccAWSSQS -TestAccAWSSSM -TestAccAWSSsm -TestAccAWSStepFunctions -TestAccAWSSwfDomain diff --git a/main.py b/main.py new file mode 100644 index 0000000..35908b2 --- /dev/null +++ b/main.py @@ -0,0 +1,41 @@ +import click + + +@click.group(name='pytest-golang', help='Golang Test Runner for localstack') +def cli(): + pass + + +@click.command(name='patch', help='Patch the golang test runner') +def patch(): + from patch import patch_repo + patch_repo() + + +@click.command(name='build', help='Build binary for testing') +@click.option('--service', '-s', default=None, help='''Service to build; use "all" to build all services, example: +--service=all; --service=ec2; --service=ec2,iam''') +@click.option('--force', '-f', default=False, is_flag=True, help='Force build') +def build(service, force): + """Build binary for testing""" + if not service: + print('No service provided') + print('use --service or -s to specify services to build; for more help try --help to see more options') + return + if service == 'all': + from build import get_all_services + services = get_all_services() + else: + if ',' in service: + services = service.split(',') + else: + services = [service] + + for service in services: + from build import build_bin + build_bin(service=service, force=force) + + +cli.add_command(build) +cli.add_command(patch) +cli() diff --git a/moto b/moto deleted file mode 160000 index 2a44d6e..0000000 --- a/moto +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2a44d6e97ce0f0d199cb54a6e967776e14e85069 diff --git a/patch.py b/patch.py new file mode 100644 index 0000000..67acaa3 --- /dev/null +++ b/patch.py @@ -0,0 +1,21 @@ +from subprocess import Popen, PIPE +from constants import BASE_PATH, PATCH_PATH, PATCH_FILES +from os.path import realpath + +def patch_repo(): + print(f'Patching {BASE_PATH}...') + for patch_file in PATCH_FILES: + cmd = [ + 'git', + 'apply', + f'{realpath(PATCH_PATH)}/{patch_file}', + ] + proc = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, cwd=realpath(BASE_PATH)) + stdout, stderr = proc.communicate() + proc.terminate() + if stdout: + print(f'stdout: {stdout}') + if stderr: + print('----- error while patching repo -----') + print(stderr) + print('Note: This usually happens when the patch has already been applied') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7d3a311 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +click~=8.1.3 +pytest~=7.2.0 +docker~=6.0.1 +requests~=2.28.1 \ No newline at end of file From 72729c2a42faa6f2382d91856d9938ed01eceb00 Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Wed, 4 Jan 2023 17:51:31 +0530 Subject: [PATCH 02/94] Updated config.yml --- .circleci/config.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d100fa5..8491d21 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,12 @@ version: 2.1 -orbs: - localstack: localstack/platform@dev:alpha +# orbs: +# localstack: localstack/platform@dev:alpha jobs: build: - executor: localstack/default + machine: + image: ubuntu-2004:202101-01 steps: - checkout From 2505ba1e94cdaaefc7971d7e1c67cfb28d2c9ab9 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 17:56:58 +0530 Subject: [PATCH 03/94] updated deps --- requirements.txt | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7d3a311..3531328 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,20 @@ -click~=8.1.3 -pytest~=7.2.0 -docker~=6.0.1 -requests~=2.28.1 \ No newline at end of file +attrs==22.2.0 +certifi==2022.12.7 +charset-normalizer==2.1.1 +click==8.1.3 +docker==6.0.1 +exceptiongroup==1.1.0 +idna==3.4 +iniconfig==1.1.1 +Jinja2==3.1.2 +junit2html==30.1.3 +MarkupSafe==2.1.1 +packaging==22.0 +pluggy==1.0.0 +pytest==7.2.0 +pytest-parallel==0.1.1 +requests==2.28.1 +tblib==1.7.0 +tomli==2.0.1 +urllib3==1.26.13 +websocket-client==1.4.2 From 50734c4c5c935c88de6d1df60e3f7ac6892c75c1 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 17:59:40 +0530 Subject: [PATCH 04/94] removed lib versions --- requirements.txt | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3531328..a6b3a70 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,4 @@ -attrs==22.2.0 -certifi==2022.12.7 -charset-normalizer==2.1.1 -click==8.1.3 -docker==6.0.1 -exceptiongroup==1.1.0 -idna==3.4 -iniconfig==1.1.1 -Jinja2==3.1.2 -junit2html==30.1.3 -MarkupSafe==2.1.1 -packaging==22.0 -pluggy==1.0.0 -pytest==7.2.0 -pytest-parallel==0.1.1 -requests==2.28.1 -tblib==1.7.0 -tomli==2.0.1 -urllib3==1.26.13 -websocket-client==1.4.2 +click +pytest +docker +requests \ No newline at end of file From c0aa68a5c820bf31d667db205f8a324a4d41c1df Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 18:03:15 +0530 Subject: [PATCH 05/94] added missing file --- .circleci/config.yml | 2 +- constants.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 constants.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 8491d21..22aafb2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -19,7 +19,7 @@ jobs: pip install -r requirements.txt - run: - name: "Install go mod dependancies" + name: "Configuring repos for tests" command: | python main.py patch cd terraform-provider-aws diff --git a/constants.py b/constants.py new file mode 100644 index 0000000..26090e1 --- /dev/null +++ b/constants.py @@ -0,0 +1,7 @@ +BASE_PATH = 'terraform-provider-aws' +SERVICE_BASE_PATH = './internal/service' + +BIN_PATH = 'test-bin' + +PATCH_PATH = 'etc' +PATCH_FILES = ['001-hardcode-endpoint.patch'] From d4fe6d567f0a60cd9b4ab12a9f141270a828f07c Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 18:07:13 +0530 Subject: [PATCH 06/94] updated python version --- .circleci/config.yml | 2 +- patch.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 22aafb2..458bbbc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: name: "Installing prerequisites" command: | sudo apt-get update - sudo apt install -y python3.8 libsasl2-dev + sudo apt install -y python3 libsasl2-dev pip install -r requirements.txt - run: diff --git a/patch.py b/patch.py index 67acaa3..73e10df 100644 --- a/patch.py +++ b/patch.py @@ -2,6 +2,7 @@ from constants import BASE_PATH, PATCH_PATH, PATCH_FILES from os.path import realpath + def patch_repo(): print(f'Patching {BASE_PATH}...') for patch_file in PATCH_FILES: From b5c7e8ab204ab5696e82bf5184bbb941a2baa93d Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 18:14:45 +0530 Subject: [PATCH 07/94] updated python version --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 458bbbc..358f4e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,13 +15,13 @@ jobs: name: "Installing prerequisites" command: | sudo apt-get update - sudo apt install -y python3 libsasl2-dev - pip install -r requirements.txt + sudo apt install -y libsasl2-dev + pip3 install -r requirements.txt - run: name: "Configuring repos for tests" command: | - python main.py patch + python3 main.py patch cd terraform-provider-aws go mod vendor From 4f96808465cb730f8b20fa4f1b52250386fe05be Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 18:19:57 +0530 Subject: [PATCH 08/94] changed ordering --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 358f4e3..7169bc9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -21,9 +21,10 @@ jobs: - run: name: "Configuring repos for tests" command: | - python3 main.py patch cd terraform-provider-aws go mod vendor + cd ../ + python3 main.py patch - run: name: "Run test suite" From 86ba55d514ce1dc64f552ac8057dfbb5709e2ee7 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 18:58:20 +0530 Subject: [PATCH 09/94] added golang/python org --- .circleci/config.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7169bc9..21dd805 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,8 @@ version: 2.1 -# orbs: +orbs: + go: circleci/go@1.8 + python: circleci/python@1.2 # localstack: localstack/platform@dev:alpha jobs: @@ -14,6 +16,8 @@ jobs: - run: name: "Installing prerequisites" command: | + python --version + go version sudo apt-get update sudo apt install -y libsasl2-dev pip3 install -r requirements.txt @@ -21,8 +25,7 @@ jobs: - run: name: "Configuring repos for tests" command: | - cd terraform-provider-aws - go mod vendor + cd terraform-provider-aws && go mod vendor cd ../ python3 main.py patch From 00250efda6fee5d47dac67c57d6948d659ac62d2 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 18:59:23 +0530 Subject: [PATCH 10/94] added versioninign --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21dd805..a4ae252 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,7 +1,7 @@ version: 2.1 orbs: - go: circleci/go@1.8 + go: circleci/go@1.7.1 python: circleci/python@1.2 # localstack: localstack/platform@dev:alpha From 5701d0ebfe9285c5f3e8198800882327c0e45325 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 20:06:38 +0530 Subject: [PATCH 11/94] updated golang version --- .circleci/config.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a4ae252..0899e17 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,6 @@ version: 2.1 orbs: go: circleci/go@1.7.1 - python: circleci/python@1.2 # localstack: localstack/platform@dev:alpha jobs: @@ -11,13 +10,12 @@ jobs: image: ubuntu-2004:202101-01 steps: - checkout - + - go/install: + version: 1.18 - run: git submodule update --init --recursive - run: name: "Installing prerequisites" command: | - python --version - go version sudo apt-get update sudo apt install -y libsasl2-dev pip3 install -r requirements.txt From 3d0254ca88781a7ec9b9af7e0354fee180382f67 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 20:08:38 +0530 Subject: [PATCH 12/94] golang version update --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0899e17..d7f1baa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ jobs: steps: - checkout - go/install: - version: 1.18 + version: 1.19 - run: git submodule update --init --recursive - run: name: "Installing prerequisites" From 0ed141b318da28225004bfa37372963271bd70ca Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 20:09:14 +0530 Subject: [PATCH 13/94] go versionchanged to string --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d7f1baa..3dbd843 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ jobs: steps: - checkout - go/install: - version: 1.19 + version: '1.19' - run: git submodule update --init --recursive - run: name: "Installing prerequisites" From b576bd1f6372d930a4b89e6dd6876d3046a59682 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 20:19:25 +0530 Subject: [PATCH 14/94] reloaded go images --- conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conftest.py b/conftest.py index 324a7e9..a3eae0a 100644 --- a/conftest.py +++ b/conftest.py @@ -159,6 +159,7 @@ def _pull_docker_image(client, localstack_image): if len(docker_image_list) == 0: print(f"Pulling image {localstack_image}") client.images.pull(localstack_image) + docker_image_list = client.images.list(name=localstack_image) print(f"Using LocalStack image: {docker_image_list[0].id}") From 0090d9707cd59337878a2c396eeba97a0e251f89 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 21:30:21 +0530 Subject: [PATCH 15/94] resource class added --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3dbd843..c4725e8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,4 +40,6 @@ workflows: - equal: [ build-new, << pipeline.git.branch >> ] - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - - build + - build: + name: build + resource_class: medium From 4fe804998b0f0c242c07cf5ad95fe99e59089232 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 4 Jan 2023 21:31:39 +0530 Subject: [PATCH 16/94] resource class added --- .circleci/config.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4725e8..396f201 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,6 +8,7 @@ jobs: build: machine: image: ubuntu-2004:202101-01 + resource_class: medium steps: - checkout - go/install: @@ -40,6 +41,4 @@ workflows: - equal: [ build-new, << pipeline.git.branch >> ] - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - - build: - name: build - resource_class: medium + - build \ No newline at end of file From 7fb9c718cf9c7887de772d5e21d2dc1990ef1903 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 12:22:06 +0530 Subject: [PATCH 17/94] updated build --- .circleci/config.yml | 1 + .gitignore | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 396f201..8c716b9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,6 +27,7 @@ jobs: cd terraform-provider-aws && go mod vendor cd ../ python3 main.py patch + python3 main.py build -s s3 - run: name: "Run test suite" diff --git a/.gitignore b/.gitignore index a9dccf3..f1cb84e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ .idea .venv .pytest_cache -__pycache__ \ No newline at end of file +__pycache__ +**/*.test \ No newline at end of file From ddbfbe81b79a586da1f2d1bda6b52c612d678260 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 12:46:16 +0530 Subject: [PATCH 18/94] changed bin building --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8c716b9..e9f5c39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,7 +27,8 @@ jobs: cd terraform-provider-aws && go mod vendor cd ../ python3 main.py patch - python3 main.py build -s s3 + cd terraform-provider-aws && go test -c ./internal/service/s3 -o ./test-bin/s3.test +# python3 main.py build -s s3 - run: name: "Run test suite" From 3c81b9807d3415be0218c5964e86a2f848bbcc65 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 13:27:53 +0530 Subject: [PATCH 19/94] updated testing mechanism --- .circleci/config.yml | 2 +- conftest.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e9f5c39..1aefdcf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,7 +27,7 @@ jobs: cd terraform-provider-aws && go mod vendor cd ../ python3 main.py patch - cd terraform-provider-aws && go test -c ./internal/service/s3 -o ./test-bin/s3.test +# cd terraform-provider-aws && go test -c ./internal/service/s3 -o ./test-bin/s3.test # python3 main.py build -s s3 - run: diff --git a/conftest.py b/conftest.py index a3eae0a..a0ca65c 100644 --- a/conftest.py +++ b/conftest.py @@ -41,7 +41,7 @@ def runtest(self): service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) service = service_path.split(os.sep)[-1] - _build_test_bin(service=service, tf_root_path=tf_root_path, service_path=service_path) + # _build_test_bin(service=service, tf_root_path=tf_root_path, service_path=service_path) env = dict(os.environ) env.update({ @@ -51,13 +51,17 @@ def runtest(self): 'AWS_DEFAULT_REGION': 'us-east-1' }) + # cmd = [ + # f"./test-bin/{service}.test", + # "-test.v", + # "-test.parallel=1", + # "-test.count=1", + # "-test.timeout=60m", + # "-test.run", f"{self.name}" + # ] + cmd = [ - f"./test-bin/{service}.test", - "-test.v", - "-test.parallel=1", - "-test.count=1", - "-test.timeout=60m", - "-test.run", f"{self.name}" + "go", "test", f"{service_path}", "-count=1", "-v", "-timeout=60", f"-run {self.name}" ] proc = Popen( From 68cbd9d07a201702d758b4f672d6cfa8a56d633e Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 13:59:28 +0530 Subject: [PATCH 20/94] updated value --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index a0ca65c..e285fbc 100644 --- a/conftest.py +++ b/conftest.py @@ -61,7 +61,7 @@ def runtest(self): # ] cmd = [ - "go", "test", f"{service_path}", "-count=1", "-v", "-timeout=60", f"-run {self.name}" + "go", "test", f"{service_path}", "-count=1", "-v", "-timeout=60m", f"-run {self.name}" ] proc = Popen( From 0257868c239db1ee5d5af643bf46919958395ad5 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 14:44:18 +0530 Subject: [PATCH 21/94] updated testing mechanism --- .circleci/config.yml | 2 +- conftest.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1aefdcf..fcf8f60 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,7 +33,7 @@ jobs: - run: name: "Run test suite" command: | - pytest terraform-provider-aws/internal/service/s3/bucket_object_test.go -k ignore -s -v --ls-start + pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start workflows: main: diff --git a/conftest.py b/conftest.py index e285fbc..4d4e69b 100644 --- a/conftest.py +++ b/conftest.py @@ -61,8 +61,9 @@ def runtest(self): # ] cmd = [ - "go", "test", f"{service_path}", "-count=1", "-v", "-timeout=60m", f"-run {self.name}" + "go", "test", f"./{service_path}", "-test.count=1", "-test.v", f"-test.run {self.name}" ] + # print("command: ", cmd) proc = Popen( cmd, stdout=PIPE, stderr=PIPE, From 637d0c43049deefb274000b561ad20a6a9210eb8 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 15:01:59 +0530 Subject: [PATCH 22/94] updated timeout --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index fcf8f60..4b995d3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,7 @@ jobs: - run: name: "Run test suite" + no_output_timeout: 30m command: | pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start From f6cfa35df1d634a6e728e3ef7d638d3fb6fc4004 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 15:08:27 +0530 Subject: [PATCH 23/94] using go commands --- .circleci/config.yml | 6 +++++- docker-compose.yml | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 docker-compose.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b995d3..558a59f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,7 +34,11 @@ jobs: name: "Run test suite" no_output_timeout: 30m command: | - pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start + cd terraform-provider-aws + docker-compose up -d + sleep 20 + TF_ACC=1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION=us-east-1 go test -v ./internal/service/s3 +# pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start workflows: main: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..afe7983 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.8" + +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" + image: localstack/localstack + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + environment: + - DEBUG=${DEBUG-} + - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} + - DOCKER_HOST=unix:///var/run/docker.sock + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" From 8541d1dbc76067f978ca60d1e491e4dd3cd3d739 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 15:10:17 +0530 Subject: [PATCH 24/94] updated env for buff output --- .circleci/config.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 558a59f..9c1fab2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,10 +35,11 @@ jobs: no_output_timeout: 30m command: | cd terraform-provider-aws - docker-compose up -d - sleep 20 - TF_ACC=1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION=us-east-1 go test -v ./internal/service/s3 -# pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start + export PYTHONUNBUFFERED=1 + pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start +# docker-compose up -d +# sleep 20 +# TF_ACC=1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION=us-east-1 go test -v ./internal/service/s3 workflows: main: From 651cdb4e2fadedb3241aa025fa21e0093554643a Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 15:12:59 +0530 Subject: [PATCH 25/94] removed cd --- .circleci/config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9c1fab2..a2589f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,12 +34,8 @@ jobs: name: "Run test suite" no_output_timeout: 30m command: | - cd terraform-provider-aws export PYTHONUNBUFFERED=1 pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start -# docker-compose up -d -# sleep 20 -# TF_ACC=1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test AWS_DEFAULT_REGION=us-east-1 go test -v ./internal/service/s3 workflows: main: From 05f8dc891a492e06749628f6f6fafbcd11c5fad7 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 16:14:30 +0530 Subject: [PATCH 26/94] updated test command --- conftest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/conftest.py b/conftest.py index 4d4e69b..b9a867e 100644 --- a/conftest.py +++ b/conftest.py @@ -48,9 +48,13 @@ def runtest(self): 'TF_ACC': '1', 'AWS_ACCESS_KEY_ID': 'test', 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-east-1' + 'AWS_DEFAULT_REGION': 'us-east-1', + 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', + 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', + 'AWS_ALTERNATE_REGION': 'us-east-2', }) + # cmd = [ # f"./test-bin/{service}.test", # "-test.v", @@ -59,11 +63,11 @@ def runtest(self): # "-test.timeout=60m", # "-test.run", f"{self.name}" # ] - + # go test ./internal/service/$1 -test.count 1 -test.v -test.timeout 60m -parallel $VALUE -run $2 cmd = [ - "go", "test", f"./{service_path}", "-test.count=1", "-test.v", f"-test.run {self.name}" + "go", "test", f"./{service_path}", "-test.count=1", "-test.v", f"-test.run={self.name}" ] - # print("command: ", cmd) + # print("-------> command: ", cmd) proc = Popen( cmd, stdout=PIPE, stderr=PIPE, @@ -129,7 +133,7 @@ def _docker_service_health(client): def _start_docker_container(client, config, localstack_image): - env_vars = ["DEBUG=1"] + env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf"] port_mappings = { '53/tcp': ('127.0.0.1', 53), '53/udp': ('127.0.0.1', 53), From 7a34cde04e73fbb8a9d52b0708d747e55901aa4c Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 17:12:35 +0530 Subject: [PATCH 27/94] using raw golang --- .circleci/config.yml | 17 +++++++++++++++-- conftest.py | 5 ++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a2589f2..39b4513 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -34,8 +34,21 @@ jobs: name: "Run test suite" no_output_timeout: 30m command: | - export PYTHONUNBUFFERED=1 - pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start + docker-compose up -d + export TF_ACC=1 + export TF_LOG=DEBUG + export TF_LOG_CORE=DEBUG + export TF_LOG_PROVIDER=DEBUG + + export AWS_ALTERNATE_ACCESS_KEY_ID=test + export AWS_ALTERNATE_SECRET_ACCESS_KEY=test + export AWS_ALTERNATE_REGION=us-east-2 + export AWS_DEFAULT_REGION=eu-west-1 + export AWS_ACCESS_KEY_ID=test + export AWS_SECRET_ACCESS_KEY=test + go test ./internal/service/s3 -test.count 1 -test.v -test.timeout 60m -parallel 4 +# export PYTHONUNBUFFERED=1 +# pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start workflows: main: diff --git a/conftest.py b/conftest.py index b9a867e..04ccd5c 100644 --- a/conftest.py +++ b/conftest.py @@ -54,7 +54,6 @@ def runtest(self): 'AWS_ALTERNATE_REGION': 'us-east-2', }) - # cmd = [ # f"./test-bin/{service}.test", # "-test.v", @@ -67,7 +66,6 @@ def runtest(self): cmd = [ "go", "test", f"./{service_path}", "-test.count=1", "-test.v", f"-test.run={self.name}" ] - # print("-------> command: ", cmd) proc = Popen( cmd, stdout=PIPE, stderr=PIPE, @@ -161,6 +159,7 @@ def _localstack_health_check(): session.mount('http://', adapter) session.mount('https://', adapter) session.get(localstack_health_url) + session.close() def _pull_docker_image(client, localstack_image): @@ -199,4 +198,4 @@ def pytest_unconfigure(config): print("\nStopping LocalStack...") client = docker.from_env() _stop_docker_container(client, config) - client.close() \ No newline at end of file + client.close() From 4d588f8fa25d4b790616b998bee3abd539c66213 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 5 Jan 2023 17:18:27 +0530 Subject: [PATCH 28/94] using raw golang --- .circleci/config.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 39b4513..0e6fd39 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,17 +35,14 @@ jobs: no_output_timeout: 30m command: | docker-compose up -d - export TF_ACC=1 - export TF_LOG=DEBUG - export TF_LOG_CORE=DEBUG - export TF_LOG_PROVIDER=DEBUG - + export TF_ACC=1 export AWS_ALTERNATE_ACCESS_KEY_ID=test export AWS_ALTERNATE_SECRET_ACCESS_KEY=test export AWS_ALTERNATE_REGION=us-east-2 export AWS_DEFAULT_REGION=eu-west-1 export AWS_ACCESS_KEY_ID=test export AWS_SECRET_ACCESS_KEY=test + cd terraform-provider-aws go test ./internal/service/s3 -test.count 1 -test.v -test.timeout 60m -parallel 4 # export PYTHONUNBUFFERED=1 # pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start From c32f981bd36767560378af4b385938c32f5c4c56 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 10:39:01 +0530 Subject: [PATCH 29/94] updated testing mechansim --- .circleci/config.yml | 35 ++++++++++++++++-- conftest.py | 88 +++++++++++++++++++++++++++++++------------- 2 files changed, 93 insertions(+), 30 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e6fd39..f7184f5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,9 +2,38 @@ version: 2.1 orbs: go: circleci/go@1.7.1 -# localstack: localstack/platform@dev:alpha jobs: + test: + machine: + image: ubuntu-2004:202101-01 + resource_class: xlarge + steps: + - checkout + - go/install: + version: '1.19' + - run: git submodule update --init --recursive + - run: + name: "Installing prerequisites" + command: | + sudo apt-get update + sudo apt install -y libsasl2-dev + pip3 install -r requirements.txt + + - run: + name: "Configuring repos for tests" + command: | + cd terraform-provider-aws && go mod vendor + cd ../ + python3 main.py patch + + - run: + name: "Run test suite" + no_output_timeout: 10m + command: | + cd terraform-provider-aws + go test -c ./internal/service/s3 + build: machine: image: ubuntu-2004:202101-01 @@ -27,8 +56,6 @@ jobs: cd terraform-provider-aws && go mod vendor cd ../ python3 main.py patch -# cd terraform-provider-aws && go test -c ./internal/service/s3 -o ./test-bin/s3.test -# python3 main.py build -s s3 - run: name: "Run test suite" @@ -55,4 +82,4 @@ workflows: - equal: [ build-new, << pipeline.git.branch >> ] - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - - build \ No newline at end of file + - test \ No newline at end of file diff --git a/conftest.py b/conftest.py index 04ccd5c..1b93ff0 100644 --- a/conftest.py +++ b/conftest.py @@ -1,13 +1,14 @@ -import pytest import re -from subprocess import PIPE, Popen -from pathlib import Path -import os -from os.path import realpath, relpath, dirname +import pytest import docker import requests from requests.adapters import HTTPAdapter, Retry -from os.path import exists +from subprocess import PIPE, Popen, run +from pathlib import Path +import os +from os import system, getcwd, chdir +from os.path import realpath, relpath, dirname, exists +from tempfile import NamedTemporaryFile def pytest_addoption(parser): @@ -54,29 +55,41 @@ def runtest(self): 'AWS_ALTERNATE_REGION': 'us-east-2', }) - # cmd = [ - # f"./test-bin/{service}.test", - # "-test.v", - # "-test.parallel=1", - # "-test.count=1", - # "-test.timeout=60m", - # "-test.run", f"{self.name}" - # ] - # go test ./internal/service/$1 -test.count 1 -test.v -test.timeout 60m -parallel $VALUE -run $2 cmd = [ - "go", "test", f"./{service_path}", "-test.count=1", "-test.v", f"-test.run={self.name}" + f"./test-bin/{service}.test", + "-test.v", + "-test.parallel=1", + "-test.count=1", + "-test.timeout=60m", + "-test.run", f"{self.name}" ] - proc = Popen( - cmd, stdout=PIPE, stderr=PIPE, - env=env, bufsize=1, universal_newlines=True, - cwd=tf_root_path - ) - stdout, stderr = proc.communicate() - proc.terminate() - if proc.returncode != 0: - raise GoException(proc.returncode, stdout, stderr) - return proc.returncode + return_code, stdout = _execute_command(cmd, env, tf_root_path) + if return_code != 0: + raise GoException(returncode=return_code, stdout=stdout) + return return_code + + # go test ./internal/service/$1 -test.count 1 -test.v -test.timeout 60m -parallel $VALUE -run $2 + # cmd = [ + # "go", "test", f"./{service_path}", "-test.count=1", "-test.v", f"-test.run={self.name}" + # ] + + # cmd_str = " ".join(str(c) for c in cmd) + # cmd_str = f"cd {tf_root_path} && TF_ACC=1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test {cmd_str}" + # return_code = os.system(cmd_str) + # proc = run(cmd, env=env, cwd=tf_root_path, check=True) + # print(f"proc: {proc}") + # return proc.returncode + # proc = Popen( + # cmd, stdout=PIPE, stderr=PIPE, + # env=env, bufsize=1, universal_newlines=True, + # cwd=tf_root_path + # ) + # stdout, stderr = proc.communicate() + # proc.terminate() + # if proc.returncode != 0: + # raise GoException(proc.returncode, stdout, stderr) + # return proc.returncode def repr_failure(self, excinfo, **kwargs): """Called when self.runtest() raises an exception.""" @@ -171,6 +184,29 @@ def _pull_docker_image(client, localstack_image): print(f"Using LocalStack image: {docker_image_list[0].id}") +def _execute_command(cmd, env=None, cwd=None): + """ + Execute a command and return the return code. + """ + _lwd = getcwd() + if isinstance(cmd, list): + cmd = ' '.join(cmd) + else: + raise Exception("Please provide command as list(str)") + if cwd: + chdir(cwd) + if env: + _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) + cmd = f'{_env} {cmd}' + log_file = NamedTemporaryFile() + _err = system(f'{cmd} &> {log_file.name}') + _log_file = open(log_file.name, 'r') + _out = _log_file.read() + _log_file.close() + chdir(_lwd) + return _err, _out + + def pytest_configure(config): is_collect_only = config.getoption(name='--collect-only') is_localstack_start = config.getoption(name='--ls-start') From ef91855410f1796e412139dafc39804497f5e642 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 10:40:02 +0530 Subject: [PATCH 30/94] updated resource class to large --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7184f5..529efc0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,7 +7,7 @@ jobs: test: machine: image: ubuntu-2004:202101-01 - resource_class: xlarge + resource_class: large steps: - checkout - go/install: From 596807f2a3e09f4d05c9f3ed7c45e62905be0d87 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 11:19:44 +0530 Subject: [PATCH 31/94] testing workflow behaviour --- .circleci/config.yml | 2 +- .circleci/test.yml | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 .circleci/test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 529efc0..40de3c6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,6 +80,6 @@ workflows: or: - equal: [ build, << pipeline.git.branch >> ] - equal: [ build-new, << pipeline.git.branch >> ] - - equal: [ pytest-plugin, << pipeline.git.branch >> ] + - equal: [ pytest-pluginn, << pipeline.git.branch >> ] jobs: - test \ No newline at end of file diff --git a/.circleci/test.yml b/.circleci/test.yml new file mode 100644 index 0000000..e94b579 --- /dev/null +++ b/.circleci/test.yml @@ -0,0 +1,28 @@ +version: 2.1 + +orbs: + go: circleci/go@1.7.1 + +jobs: + generate: + machine: + image: ubuntu-2004:202101-01 + steps: + - checkout + - run: touch testfile.txt + - run: ls + + test: + machine: + image: ubuntu-2004:202101-01 + steps: + - run: ls + +workflows: + main: + when: + or: + - equal: [ pytest-plugin, << pipeline.git.branch >> ] + jobs: + - generate + - test From a82cc5f0e0ffe32d463e8627832ee26d6f4469d4 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 11:20:41 +0530 Subject: [PATCH 32/94] invoking wf --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 40de3c6..0b388d7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,4 +82,4 @@ workflows: - equal: [ build-new, << pipeline.git.branch >> ] - equal: [ pytest-pluginn, << pipeline.git.branch >> ] jobs: - - test \ No newline at end of file + - test From 1e3be8a048f91abd1f177e0e9c6b2c4907b70ea4 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 11:23:42 +0530 Subject: [PATCH 33/94] updated wf --- .circleci/config.yml | 23 ++++++++++++++++++++--- .circleci/test.yml | 28 ---------------------------- 2 files changed, 20 insertions(+), 31 deletions(-) delete mode 100644 .circleci/test.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b388d7..172ea4d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,7 +4,7 @@ orbs: go: circleci/go@1.7.1 jobs: - test: + generate-test-binary: machine: image: ubuntu-2004:202101-01 resource_class: large @@ -34,7 +34,7 @@ jobs: cd terraform-provider-aws go test -c ./internal/service/s3 - build: + run-tests: machine: image: ubuntu-2004:202101-01 resource_class: medium @@ -74,12 +74,29 @@ jobs: # export PYTHONUNBUFFERED=1 # pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start + generate: + machine: + image: ubuntu-2004:202101-01 + steps: + - checkout + - run: touch testfile.txt + - run: ls + + test: + machine: + image: ubuntu-2004:202101-01 + steps: + - run: ls + + workflows: main: when: or: - equal: [ build, << pipeline.git.branch >> ] - equal: [ build-new, << pipeline.git.branch >> ] - - equal: [ pytest-pluginn, << pipeline.git.branch >> ] + - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: + - generate - test + diff --git a/.circleci/test.yml b/.circleci/test.yml deleted file mode 100644 index e94b579..0000000 --- a/.circleci/test.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: 2.1 - -orbs: - go: circleci/go@1.7.1 - -jobs: - generate: - machine: - image: ubuntu-2004:202101-01 - steps: - - checkout - - run: touch testfile.txt - - run: ls - - test: - machine: - image: ubuntu-2004:202101-01 - steps: - - run: ls - -workflows: - main: - when: - or: - - equal: [ pytest-plugin, << pipeline.git.branch >> ] - jobs: - - generate - - test From 1e1491c587d1f746e0ecb2881b3c19c0689f8cc8 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 11:26:14 +0530 Subject: [PATCH 34/94] wf dependancies --- .circleci/config.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 172ea4d..078555a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,5 +98,7 @@ workflows: - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - generate - - test + - test: + requires: + - generate From 0848af05128d774f6da5336cdc0482c68b25f681 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 11:43:18 +0530 Subject: [PATCH 35/94] workspace attach test --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 078555a..3b1c2a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,15 +77,23 @@ jobs: generate: machine: image: ubuntu-2004:202101-01 + working_directory: /tmp steps: + - run: mkdir workspace && cd workspace - checkout - run: touch testfile.txt - run: ls + - persist_to_workspace: + root: . + paths: + - . test: machine: image: ubuntu-2004:202101-01 steps: + - attach_workspace: + at: . - run: ls From 3a467b5e7c22f159009bc9f09c2217ee427767f3 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 11:45:01 +0530 Subject: [PATCH 36/94] removed working dir --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3b1c2a4..a0d7444 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -77,7 +77,6 @@ jobs: generate: machine: image: ubuntu-2004:202101-01 - working_directory: /tmp steps: - run: mkdir workspace && cd workspace - checkout From 43857aaa07cc67f53766c2dd1f3073e9cd924303 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 11:46:01 +0530 Subject: [PATCH 37/94] removed dir creation --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a0d7444..388b17d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -78,7 +78,6 @@ jobs: machine: image: ubuntu-2004:202101-01 steps: - - run: mkdir workspace && cd workspace - checkout - run: touch testfile.txt - run: ls From c7d2ea4bb9779960ae0fbe43926a731b23dd9571 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 12:50:11 +0530 Subject: [PATCH 38/94] added build persistence --- .circleci/config.yml | 37 +++++---------------- build.py | 47 -------------------------- conftest.py | 58 +++----------------------------- constants.py | 7 ---- main.py | 15 ++++++--- patch.py | 22 ------------ utils.py | 79 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 161 deletions(-) delete mode 100644 build.py delete mode 100644 constants.py delete mode 100644 patch.py create mode 100644 utils.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 388b17d..49507a4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,21 +28,19 @@ jobs: python3 main.py patch - run: - name: "Run test suite" - no_output_timeout: 10m + name: "Generate Testing Binary" + no_output_timeout: 20m command: | - cd terraform-provider-aws - go test -c ./internal/service/s3 + python3 main.py build -s s3 run-tests: machine: image: ubuntu-2004:202101-01 resource_class: medium steps: - - checkout - go/install: version: '1.19' - - run: git submodule update --init --recursive + - run: name: "Installing prerequisites" command: | @@ -50,29 +48,12 @@ jobs: sudo apt install -y libsasl2-dev pip3 install -r requirements.txt - - run: - name: "Configuring repos for tests" - command: | - cd terraform-provider-aws && go mod vendor - cd ../ - python3 main.py patch - - run: name: "Run test suite" no_output_timeout: 30m command: | - docker-compose up -d - export TF_ACC=1 - export AWS_ALTERNATE_ACCESS_KEY_ID=test - export AWS_ALTERNATE_SECRET_ACCESS_KEY=test - export AWS_ALTERNATE_REGION=us-east-2 - export AWS_DEFAULT_REGION=eu-west-1 - export AWS_ACCESS_KEY_ID=test - export AWS_SECRET_ACCESS_KEY=test - cd terraform-provider-aws - go test ./internal/service/s3 -test.count 1 -test.v -test.timeout 60m -parallel 4 -# export PYTHONUNBUFFERED=1 -# pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start + export PYTHONUNBUFFERED=1 + pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start generate: machine: @@ -103,8 +84,8 @@ workflows: - equal: [ build-new, << pipeline.git.branch >> ] - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - - generate - - test: + - generate-test-binary + - run-tests: requires: - - generate + - generate-test-binary diff --git a/build.py b/build.py deleted file mode 100644 index f129bbd..0000000 --- a/build.py +++ /dev/null @@ -1,47 +0,0 @@ -import os.path -from subprocess import Popen, PIPE -from constants import BASE_PATH, SERVICE_BASE_PATH, BIN_PATH -from os.path import realpath, exists - - -def get_all_services(): - services = [] - for service in os.listdir(f'{BASE_PATH}/{SERVICE_BASE_PATH}'): - services.append(service) - return sorted(services) - - -def build_bin(service=None, force=False): - service_path = f'{SERVICE_BASE_PATH}/{service}' - bin_path = f'./{BIN_PATH}/{service}.test' - - if exists(bin_path) and not force: - print(f'Binary already exists for {service}') - return - print(f'Building {service}...') - - cmd = [ - 'go', - 'test', - '-c', - service_path, - '-o', - bin_path, - ] - - proc = Popen( - cmd, - stdout=PIPE, - stderr=PIPE, - universal_newlines=True, - cwd=realpath(BASE_PATH) - ) - stdout, stderr = proc.communicate() - proc.terminate() - if stdout: - print(f'stdout: {stdout}') - if stderr: - print(f'stderr: {stderr}') - - if exists(bin_path): - os.chmod(bin_path, 0o755) diff --git a/conftest.py b/conftest.py index 1b93ff0..982cee1 100644 --- a/conftest.py +++ b/conftest.py @@ -3,12 +3,10 @@ import docker import requests from requests.adapters import HTTPAdapter, Retry -from subprocess import PIPE, Popen, run from pathlib import Path import os -from os import system, getcwd, chdir from os.path import realpath, relpath, dirname, exists -from tempfile import NamedTemporaryFile +from utils import execute_command, build_test_bin def pytest_addoption(parser): @@ -42,7 +40,9 @@ def runtest(self): service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) service = service_path.split(os.sep)[-1] - # _build_test_bin(service=service, tf_root_path=tf_root_path, service_path=service_path) + return_code, stdout = build_test_bin(service=service, tf_root_path=tf_root_path) + if return_code != 0: + raise GoException(returncode=return_code, stdout=stdout) env = dict(os.environ) env.update({ @@ -64,7 +64,7 @@ def runtest(self): "-test.run", f"{self.name}" ] - return_code, stdout = _execute_command(cmd, env, tf_root_path) + return_code, stdout = execute_command(cmd, env, tf_root_path) if return_code != 0: raise GoException(returncode=return_code, stdout=stdout) return return_code @@ -112,31 +112,6 @@ def __init__(self, returncode, stdout, stderr): self.stderr = stderr -def _build_test_bin(service, tf_root_path, service_path): - bin_path = f'./test-bin/{service}.test' - - if exists(f"{tf_root_path}/{bin_path}"): - return - cmd = [ - "go", - "test", - "-c", - f"./{service_path}", - "-o", - bin_path, - ] - proc = Popen( - cmd, stdout=PIPE, stderr=PIPE, - bufsize=1, universal_newlines=True, - cwd=tf_root_path - ) - stdout, stderr = proc.communicate() - proc.terminate() - if proc.returncode != 0: - raise GoException(proc.returncode, stdout, stderr) - return - - def _docker_service_health(client): if not client.ping(): print("\nPlease start docker daemon and try again") @@ -184,29 +159,6 @@ def _pull_docker_image(client, localstack_image): print(f"Using LocalStack image: {docker_image_list[0].id}") -def _execute_command(cmd, env=None, cwd=None): - """ - Execute a command and return the return code. - """ - _lwd = getcwd() - if isinstance(cmd, list): - cmd = ' '.join(cmd) - else: - raise Exception("Please provide command as list(str)") - if cwd: - chdir(cwd) - if env: - _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) - cmd = f'{_env} {cmd}' - log_file = NamedTemporaryFile() - _err = system(f'{cmd} &> {log_file.name}') - _log_file = open(log_file.name, 'r') - _out = _log_file.read() - _log_file.close() - chdir(_lwd) - return _err, _out - - def pytest_configure(config): is_collect_only = config.getoption(name='--collect-only') is_localstack_start = config.getoption(name='--ls-start') diff --git a/constants.py b/constants.py deleted file mode 100644 index 26090e1..0000000 --- a/constants.py +++ /dev/null @@ -1,7 +0,0 @@ -BASE_PATH = 'terraform-provider-aws' -SERVICE_BASE_PATH = './internal/service' - -BIN_PATH = 'test-bin' - -PATCH_PATH = 'etc' -PATCH_FILES = ['001-hardcode-endpoint.patch'] diff --git a/main.py b/main.py index 35908b2..543b894 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,7 @@ def cli(): @click.command(name='patch', help='Patch the golang test runner') def patch(): - from patch import patch_repo + from utils import patch_repo patch_repo() @@ -23,7 +23,7 @@ def build(service, force): print('use --service or -s to specify services to build; for more help try --help to see more options') return if service == 'all': - from build import get_all_services + from utils import get_all_services services = get_all_services() else: if ',' in service: @@ -32,8 +32,15 @@ def build(service, force): services = [service] for service in services: - from build import build_bin - build_bin(service=service, force=force) + from utils import build_test_bin + from utils import BASE_PATH + from os.path import realpath + print(f'Building {service}...') + try: + build_test_bin(service=service, tf_root_path=realpath(BASE_PATH)) + except KeyboardInterrupt: + print('Interrupted') + return cli.add_command(build) diff --git a/patch.py b/patch.py deleted file mode 100644 index 73e10df..0000000 --- a/patch.py +++ /dev/null @@ -1,22 +0,0 @@ -from subprocess import Popen, PIPE -from constants import BASE_PATH, PATCH_PATH, PATCH_FILES -from os.path import realpath - - -def patch_repo(): - print(f'Patching {BASE_PATH}...') - for patch_file in PATCH_FILES: - cmd = [ - 'git', - 'apply', - f'{realpath(PATCH_PATH)}/{patch_file}', - ] - proc = Popen(cmd, stdout=PIPE, stderr=PIPE, universal_newlines=True, cwd=realpath(BASE_PATH)) - stdout, stderr = proc.communicate() - proc.terminate() - if stdout: - print(f'stdout: {stdout}') - if stderr: - print('----- error while patching repo -----') - print(stderr) - print('Note: This usually happens when the patch has already been applied') diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..514dbf6 --- /dev/null +++ b/utils.py @@ -0,0 +1,79 @@ +from os import system, getcwd, chdir, chmod, listdir +from os.path import exists, realpath +from tempfile import NamedTemporaryFile + +BASE_PATH = 'terraform-provider-aws' +SERVICE_BASE_PATH = './internal/service' + +BIN_PATH = 'test-bin' + +PATCH_PATH = 'etc' +PATCH_FILES = ['001-hardcode-endpoint.patch'] + + +def execute_command(cmd, env=None, cwd=None): + """ + Execute a command and return the return code. + """ + _lwd = getcwd() + if isinstance(cmd, list): + cmd = ' '.join(cmd) + else: + raise Exception("Please provide command as list(str)") + if cwd: + chdir(cwd) + if env: + _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) + cmd = f'{_env} {cmd}' + log_file = NamedTemporaryFile() + _err = system(f'{cmd} &> {log_file.name}') + _log_file = open(log_file.name, 'r') + _out = _log_file.read() + _log_file.close() + chdir(_lwd) + return _err, _out + + +def build_test_bin(service, tf_root_path): + bin_path = f'./{BIN_PATH}/{service}.test' + + if exists(f"{realpath(BASE_PATH)}/{bin_path}"): + return + + cmd = [ + "go", + "test", + "-c", + f"{SERVICE_BASE_PATH}/{service}", + "-o", + bin_path, + ] + + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + + if exists(bin_path): + chmod(bin_path, 0o755) + + return return_code, stdout + + +def get_all_services(): + services = [] + for service in listdir(f'{BASE_PATH}/{SERVICE_BASE_PATH}'): + services.append(service) + return sorted(services) + + +def patch_repo(): + print(f'Patching {BASE_PATH}...') + for patch_file in PATCH_FILES: + cmd = [ + 'git', + 'apply', + f'{realpath(PATCH_PATH)}/{patch_file}', + ] + return_code, stdout = execute_command(cmd, cwd=realpath(BASE_PATH)) + if return_code != 0: + print("----- error while patching repo -----") + if stdout: + print(f'stdout: {stdout}') From d06082e1cfee8f6144f0ffb13c1cbc5bbd17c088 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 12:50:55 +0530 Subject: [PATCH 39/94] added persistence --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 49507a4..2e65cf3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -33,11 +33,19 @@ jobs: command: | python3 main.py build -s s3 + - persist_to_workspace: + root: . + paths: + - . + run-tests: machine: image: ubuntu-2004:202101-01 resource_class: medium steps: + - attach_workspace: + at: . + - go/install: version: '1.19' From 4372cd47bcc3cf8faee4bbe33b9e9b16c9933c49 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 13:29:10 +0530 Subject: [PATCH 40/94] debugging prints --- .circleci/config.yml | 1 + conftest.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e65cf3..b16b171 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,6 +32,7 @@ jobs: no_output_timeout: 20m command: | python3 main.py build -s s3 + ls terraform-provider-aws/test-bin - persist_to_workspace: root: . diff --git a/conftest.py b/conftest.py index 982cee1..a86f78a 100644 --- a/conftest.py +++ b/conftest.py @@ -40,10 +40,13 @@ def runtest(self): service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) service = service_path.split(os.sep)[-1] - return_code, stdout = build_test_bin(service=service, tf_root_path=tf_root_path) - if return_code != 0: - raise GoException(returncode=return_code, stdout=stdout) - + # return_code, stdout = build_test_bin(service=service, tf_root_path=tf_root_path) + # if return_code != 0: + # raise GoException(returncode=return_code, stdout=stdout) + return_code, stdout = execute_command(["ls -al"], cwd=tf_root_path) + print("---> return_code", return_code, stdout) + return_code, stdout = execute_command(["pwd"], cwd=tf_root_path) + print("---> pwd", return_code, stdout) env = dict(os.environ) env.update({ 'TF_ACC': '1', From bed75d965574881edb4643679a04c2c2b019db18 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 13:32:06 +0530 Subject: [PATCH 41/94] restarting pipeline --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b16b171..50dc41b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,4 +97,3 @@ workflows: - run-tests: requires: - generate-test-binary - From 210ab1ae853b5f6fef9eb97fa5e3b8955fd6cb89 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 13:56:17 +0530 Subject: [PATCH 42/94] updated config --- .circleci/config.yml | 2 +- utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 50dc41b..dd4945b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: no_output_timeout: 20m command: | python3 main.py build -s s3 - ls terraform-provider-aws/test-bin + ls - persist_to_workspace: root: . diff --git a/utils.py b/utils.py index 514dbf6..82cdb58 100644 --- a/utils.py +++ b/utils.py @@ -51,8 +51,8 @@ def build_test_bin(service, tf_root_path): return_code, stdout = execute_command(cmd, cwd=tf_root_path) - if exists(bin_path): - chmod(bin_path, 0o755) + if exists(realpath(bin_path)): + chmod(realpath(bin_path), 0o755) return return_code, stdout From 2a5e505cedc479de29878040477adc3e9f148a2f Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 14:00:13 +0530 Subject: [PATCH 43/94] added list in terraform folder --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd4945b..7c03d45 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,7 +32,7 @@ jobs: no_output_timeout: 20m command: | python3 main.py build -s s3 - ls + ls terraform-provider-aws - persist_to_workspace: root: . From 388f131a1e42fd737198ae42af914f87cee11112 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 21:13:30 +0530 Subject: [PATCH 44/94] added exception --- utils.py | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/utils.py b/utils.py index 82cdb58..4b74441 100644 --- a/utils.py +++ b/utils.py @@ -2,14 +2,22 @@ from os.path import exists, realpath from tempfile import NamedTemporaryFile -BASE_PATH = 'terraform-provider-aws' -SERVICE_BASE_PATH = './internal/service' - -BIN_PATH = 'test-bin' PATCH_PATH = 'etc' PATCH_FILES = ['001-hardcode-endpoint.patch'] +TF_REPO_NAME = 'terraform-provider-aws' +TF_REPO_PATH = f'{realpath(TF_REPO_NAME)}' + +TF_REPO_PATCH_FILES = ['etc/001-hardcode-endpoint.patch'] + +TF_TEST_BINARY_FOLDER = 'test-bin' +TF_REPO_SERVICE_FOLDER = './internal/service' + + +def _get_test_bin_abs_path(service): + return f'{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test' + def execute_command(cmd, env=None, cwd=None): """ @@ -35,44 +43,47 @@ def execute_command(cmd, env=None, cwd=None): def build_test_bin(service, tf_root_path): - bin_path = f'./{BIN_PATH}/{service}.test' + _test_bin_abs_path = _get_test_bin_abs_path(service) + _tf_repo_service_folder = f'{TF_REPO_SERVICE_FOLDER}/{service}' - if exists(f"{realpath(BASE_PATH)}/{bin_path}"): + if exists(_test_bin_abs_path): return cmd = [ "go", "test", "-c", - f"{SERVICE_BASE_PATH}/{service}", + _tf_repo_service_folder, "-o", - bin_path, + _test_bin_abs_path, ] return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}") - if exists(realpath(bin_path)): - chmod(realpath(bin_path), 0o755) + if exists(_test_bin_abs_path): + chmod(_test_bin_abs_path, 0o755) return return_code, stdout def get_all_services(): services = [] - for service in listdir(f'{BASE_PATH}/{SERVICE_BASE_PATH}'): + for service in listdir(f'{TF_REPO_PATH}/{TF_REPO_SERVICE_FOLDER}'): services.append(service) return sorted(services) def patch_repo(): - print(f'Patching {BASE_PATH}...') + print(f'Patching {TF_REPO_NAME}...') for patch_file in PATCH_FILES: cmd = [ 'git', 'apply', - f'{realpath(PATCH_PATH)}/{patch_file}', + f'{realpath(patch_file)}', ] - return_code, stdout = execute_command(cmd, cwd=realpath(BASE_PATH)) + return_code, stdout = execute_command(cmd, cwd=realpath(TF_REPO_NAME)) if return_code != 0: print("----- error while patching repo -----") if stdout: From e51efafa414ec8738c78b047e05a9f05c5305a25 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 21:14:06 +0530 Subject: [PATCH 45/94] commented workflow --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7c03d45..c6e0102 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -94,6 +94,6 @@ workflows: - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - generate-test-binary - - run-tests: - requires: - - generate-test-binary +# - run-tests: +# requires: +# - generate-test-binary From 9a861e878809918bf46ff06bcf23761579016c07 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 21:20:50 +0530 Subject: [PATCH 46/94] updated tf path --- main.py | 4 ++-- utils.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 543b894..aa08768 100644 --- a/main.py +++ b/main.py @@ -33,11 +33,11 @@ def build(service, force): for service in services: from utils import build_test_bin - from utils import BASE_PATH + from utils import TF_REPO_NAME from os.path import realpath print(f'Building {service}...') try: - build_test_bin(service=service, tf_root_path=realpath(BASE_PATH)) + build_test_bin(service=service, tf_root_path=realpath(TF_REPO_NAME)) except KeyboardInterrupt: print('Interrupted') return diff --git a/utils.py b/utils.py index 4b74441..8008e5c 100644 --- a/utils.py +++ b/utils.py @@ -3,9 +3,6 @@ from tempfile import NamedTemporaryFile -PATCH_PATH = 'etc' -PATCH_FILES = ['001-hardcode-endpoint.patch'] - TF_REPO_NAME = 'terraform-provider-aws' TF_REPO_PATH = f'{realpath(TF_REPO_NAME)}' @@ -77,7 +74,7 @@ def get_all_services(): def patch_repo(): print(f'Patching {TF_REPO_NAME}...') - for patch_file in PATCH_FILES: + for patch_file in TF_REPO_PATCH_FILES: cmd = [ 'git', 'apply', From 41a5ddbd6f6f0ac457317825f4538dc75bc45f6b Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 21:23:34 +0530 Subject: [PATCH 47/94] added print --- utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 8008e5c..349ad30 100644 --- a/utils.py +++ b/utils.py @@ -54,7 +54,7 @@ def build_test_bin(service, tf_root_path): "-o", _test_bin_abs_path, ] - + print(f'----> cmd: {cmd}') return_code, stdout = execute_command(cmd, cwd=tf_root_path) if return_code != 0: raise Exception(f"Error while building test binary for {service}") From f7b1dea898cffe4bd8cab414fb0b0ce93bfefc95 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 21:27:25 +0530 Subject: [PATCH 48/94] added more logs --- utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils.py b/utils.py index 349ad30..c595f5d 100644 --- a/utils.py +++ b/utils.py @@ -59,6 +59,9 @@ def build_test_bin(service, tf_root_path): if return_code != 0: raise Exception(f"Error while building test binary for {service}") + print(f"----> return_code: {return_code}") + print(f"----> stdout: {stdout}") + if exists(_test_bin_abs_path): chmod(_test_bin_abs_path, 0o755) From 9288e19243e384f45514ef640ba3c7bc7321014a Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 21:47:42 +0530 Subject: [PATCH 49/94] updated circleci config --- .circleci/config.yml | 4 +++- main.py | 2 +- utils.py | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c6e0102..d9382a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,7 +31,9 @@ jobs: name: "Generate Testing Binary" no_output_timeout: 20m command: | - python3 main.py build -s s3 +# python3 main.py build -s s3 + cd terraform-provider-aws + go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test ls terraform-provider-aws - persist_to_workspace: diff --git a/main.py b/main.py index aa08768..7f03eb7 100644 --- a/main.py +++ b/main.py @@ -16,7 +16,7 @@ def patch(): @click.option('--service', '-s', default=None, help='''Service to build; use "all" to build all services, example: --service=all; --service=ec2; --service=ec2,iam''') @click.option('--force', '-f', default=False, is_flag=True, help='Force build') -def build(service, force): +def build(service): """Build binary for testing""" if not service: print('No service provided') diff --git a/utils.py b/utils.py index c595f5d..5addc9d 100644 --- a/utils.py +++ b/utils.py @@ -55,6 +55,7 @@ def build_test_bin(service, tf_root_path): _test_bin_abs_path, ] print(f'----> cmd: {cmd}') + print(f'----> tf_root_path: {tf_root_path}') return_code, stdout = execute_command(cmd, cwd=tf_root_path) if return_code != 0: raise Exception(f"Error while building test binary for {service}") From ecc5438eb0249a749b5ee0d5a2fff49a6ed3c928 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 6 Jan 2023 21:48:30 +0530 Subject: [PATCH 50/94] updated wf --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d9382a1..621f770 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,10 +31,10 @@ jobs: name: "Generate Testing Binary" no_output_timeout: 20m command: | -# python3 main.py build -s s3 cd terraform-provider-aws go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test ls terraform-provider-aws +# python3 main.py build -s s3 - persist_to_workspace: root: . From 563157808331bb7d4ca3b07cee1655b510562ed5 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 12:01:50 +0530 Subject: [PATCH 51/94] python build command --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 621f770..63c99ea 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,9 +32,9 @@ jobs: no_output_timeout: 20m command: | cd terraform-provider-aws - go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test +# go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test + python3 main.py build -s s3 ls terraform-provider-aws -# python3 main.py build -s s3 - persist_to_workspace: root: . From 0c13fbf19a10ee7fe362968aaa9abddc85aae1f5 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 12:03:09 +0530 Subject: [PATCH 52/94] fixed syntax --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 63c99ea..ae0029c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -32,9 +32,11 @@ jobs: no_output_timeout: 20m command: | cd terraform-provider-aws -# go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test - python3 main.py build -s s3 - ls terraform-provider-aws + go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test + ls + ls test-bin +# python3 main.py build -s s3 +# ls terraform-provider-aws - persist_to_workspace: root: . From 8a4d194ff85289e68094ca97130c8c2b828e6787 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 12:15:40 +0530 Subject: [PATCH 53/94] adding python command --- .circleci/config.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae0029c..f444c7b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,11 +31,11 @@ jobs: name: "Generate Testing Binary" no_output_timeout: 20m command: | - cd terraform-provider-aws - go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test - ls - ls test-bin -# python3 main.py build -s s3 + python3 main.python3 build -s s3 + ls terraform-provider-aws + ls terraform-provider-aws/test-bin +# cd terraform-provider-aws +# go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test # ls terraform-provider-aws - persist_to_workspace: From 38f291e2f617bd50b799cb4487e4922d170e7513 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 12:18:42 +0530 Subject: [PATCH 54/94] updated filename --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f444c7b..d47a909 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -31,7 +31,7 @@ jobs: name: "Generate Testing Binary" no_output_timeout: 20m command: | - python3 main.python3 build -s s3 + python3 main.py build -s s3 ls terraform-provider-aws ls terraform-provider-aws/test-bin # cd terraform-provider-aws From 7591bec1f15e9a1c0fba39ec87108b21509f5c5a Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 12:22:23 +0530 Subject: [PATCH 55/94] removed unused args --- main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/main.py b/main.py index 7f03eb7..84f4e07 100644 --- a/main.py +++ b/main.py @@ -15,7 +15,6 @@ def patch(): @click.command(name='build', help='Build binary for testing') @click.option('--service', '-s', default=None, help='''Service to build; use "all" to build all services, example: --service=all; --service=ec2; --service=ec2,iam''') -@click.option('--force', '-f', default=False, is_flag=True, help='Force build') def build(service): """Build binary for testing""" if not service: From cfb7b7f219e5414cf4501aaa25df7d8f1298d562 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 12:26:37 +0530 Subject: [PATCH 56/94] added support for bin generation --- utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils.py b/utils.py index 5addc9d..7a85daa 100644 --- a/utils.py +++ b/utils.py @@ -40,6 +40,7 @@ def execute_command(cmd, env=None, cwd=None): def build_test_bin(service, tf_root_path): + _test_bin_rel_path = f'./{TF_TEST_BINARY_FOLDER}/{service}.test' _test_bin_abs_path = _get_test_bin_abs_path(service) _tf_repo_service_folder = f'{TF_REPO_SERVICE_FOLDER}/{service}' @@ -52,7 +53,7 @@ def build_test_bin(service, tf_root_path): "-c", _tf_repo_service_folder, "-o", - _test_bin_abs_path, + _test_bin_rel_path, ] print(f'----> cmd: {cmd}') print(f'----> tf_root_path: {tf_root_path}') From 8976c3b80d39629453c653a80f13fdea7956d447 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 12:29:33 +0530 Subject: [PATCH 57/94] added ls for debugging --- utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils.py b/utils.py index 7a85daa..478ea07 100644 --- a/utils.py +++ b/utils.py @@ -47,6 +47,9 @@ def build_test_bin(service, tf_root_path): if exists(_test_bin_abs_path): return + cmd = ["ls"] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + print(f'-----> ls: {stdout}') cmd = [ "go", "test", From 59f2e49164042beef02345fdcf35dd3d8280b7a9 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 13:19:47 +0530 Subject: [PATCH 58/94] removed tempfile usage --- utils.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/utils.py b/utils.py index 478ea07..7a00407 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,7 @@ from os import system, getcwd, chdir, chmod, listdir from os.path import exists, realpath from tempfile import NamedTemporaryFile +from uuid import uuid4 TF_REPO_NAME = 'terraform-provider-aws' @@ -30,9 +31,10 @@ def execute_command(cmd, env=None, cwd=None): if env: _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) cmd = f'{_env} {cmd}' - log_file = NamedTemporaryFile() - _err = system(f'{cmd} &> {log_file.name}') - _log_file = open(log_file.name, 'r') + # log_file = NamedTemporaryFile() + log_file: str = '/tmp/%s' % uuid4().hex + _err = system(f'{cmd} &> {log_file}') + _log_file = open(log_file, 'r') _out = _log_file.read() _log_file.close() chdir(_lwd) @@ -47,9 +49,6 @@ def build_test_bin(service, tf_root_path): if exists(_test_bin_abs_path): return - cmd = ["ls"] - return_code, stdout = execute_command(cmd, cwd=tf_root_path) - print(f'-----> ls: {stdout}') cmd = [ "go", "test", From aca4841c91169ee410d7f2b9e775c80fedffc352 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 13:22:18 +0530 Subject: [PATCH 59/94] removed tempfile usage --- utils.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/utils.py b/utils.py index 7a00407..9d6f876 100644 --- a/utils.py +++ b/utils.py @@ -32,12 +32,14 @@ def execute_command(cmd, env=None, cwd=None): _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) cmd = f'{_env} {cmd}' # log_file = NamedTemporaryFile() - log_file: str = '/tmp/%s' % uuid4().hex - _err = system(f'{cmd} &> {log_file}') - _log_file = open(log_file, 'r') - _out = _log_file.read() - _log_file.close() + # log_file: str = '/tmp/%s' % uuid4().hex + # _err = system(f'{cmd} &> {log_file}') + _err = system(f'{cmd}') + # _log_file = open(log_file, 'r') + # _out = _log_file.read() + # _log_file.close() chdir(_lwd) + _out = "" return _err, _out From 16d28d5717aa8be4383b5d2b2a685ba4f50010f6 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 13:40:58 +0530 Subject: [PATCH 60/94] added new stage --- .circleci/config.yml | 6 +++--- utils.py | 9 ++------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d47a909..467b76d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -98,6 +98,6 @@ workflows: - equal: [ pytest-plugin, << pipeline.git.branch >> ] jobs: - generate-test-binary -# - run-tests: -# requires: -# - generate-test-binary + - run-tests: + requires: + - generate-test-binary diff --git a/utils.py b/utils.py index 9d6f876..32a8454 100644 --- a/utils.py +++ b/utils.py @@ -44,7 +44,7 @@ def execute_command(cmd, env=None, cwd=None): def build_test_bin(service, tf_root_path): - _test_bin_rel_path = f'./{TF_TEST_BINARY_FOLDER}/{service}.test' + # _test_bin_rel_path = f'./{TF_TEST_BINARY_FOLDER}/{service}.test' _test_bin_abs_path = _get_test_bin_abs_path(service) _tf_repo_service_folder = f'{TF_REPO_SERVICE_FOLDER}/{service}' @@ -57,17 +57,12 @@ def build_test_bin(service, tf_root_path): "-c", _tf_repo_service_folder, "-o", - _test_bin_rel_path, + _test_bin_abs_path, ] - print(f'----> cmd: {cmd}') - print(f'----> tf_root_path: {tf_root_path}') return_code, stdout = execute_command(cmd, cwd=tf_root_path) if return_code != 0: raise Exception(f"Error while building test binary for {service}") - print(f"----> return_code: {return_code}") - print(f"----> stdout: {stdout}") - if exists(_test_bin_abs_path): chmod(_test_bin_abs_path, 0o755) From 0bc23586ef2d088628371db34a9129c17e922022 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Mon, 9 Jan 2023 14:09:50 +0530 Subject: [PATCH 61/94] added support for std output --- conftest.py | 5 +---- utils.py | 11 ++++------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/conftest.py b/conftest.py index a86f78a..525c876 100644 --- a/conftest.py +++ b/conftest.py @@ -43,10 +43,7 @@ def runtest(self): # return_code, stdout = build_test_bin(service=service, tf_root_path=tf_root_path) # if return_code != 0: # raise GoException(returncode=return_code, stdout=stdout) - return_code, stdout = execute_command(["ls -al"], cwd=tf_root_path) - print("---> return_code", return_code, stdout) - return_code, stdout = execute_command(["pwd"], cwd=tf_root_path) - print("---> pwd", return_code, stdout) + env = dict(os.environ) env.update({ 'TF_ACC': '1', diff --git a/utils.py b/utils.py index 32a8454..9c9b263 100644 --- a/utils.py +++ b/utils.py @@ -32,14 +32,11 @@ def execute_command(cmd, env=None, cwd=None): _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) cmd = f'{_env} {cmd}' # log_file = NamedTemporaryFile() - # log_file: str = '/tmp/%s' % uuid4().hex - # _err = system(f'{cmd} &> {log_file}') - _err = system(f'{cmd}') - # _log_file = open(log_file, 'r') - # _out = _log_file.read() - # _log_file.close() + log_file: str = '/tmp/%s' % uuid4().hex + _err = system(f'{cmd} > /dev/null 2> {log_file}') + # _err = system(f'{cmd} > ') + _out = open(log_file, 'r').read() chdir(_lwd) - _out = "" return _err, _out From 4a923ac031696318e15537a1e1c681e65893102a Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 14:00:19 +0530 Subject: [PATCH 62/94] updated tests to support parrallel executions --- .circleci/config.yml | 4 +++- conftest.py | 57 ++++++++++++++++++++++---------------------- utils.py | 9 +++---- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 467b76d..6475293 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -65,8 +65,10 @@ jobs: name: "Run test suite" no_output_timeout: 30m command: | + docker-compose up -d + sleep 10 export PYTHONUNBUFFERED=1 - pytest terraform-provider-aws/internal/service/s3 -s -v --ls-start + pytest terraform-provider-aws/internal/service/s3 -s -v -n 4 generate: machine: diff --git a/conftest.py b/conftest.py index 525c876..4ba44cc 100644 --- a/conftest.py +++ b/conftest.py @@ -66,6 +66,7 @@ def runtest(self): return_code, stdout = execute_command(cmd, env, tf_root_path) if return_code != 0: + print(f"error: {stdout}") raise GoException(returncode=return_code, stdout=stdout) return return_code @@ -159,31 +160,31 @@ def _pull_docker_image(client, localstack_image): print(f"Using LocalStack image: {docker_image_list[0].id}") -def pytest_configure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - localstack_image = config.getoption(name='--ls-image') - - if not is_collect_only and is_localstack_start: - - print("\nStarting LocalStack...") - - client = docker.from_env() - _docker_service_health(client) - _pull_docker_image(client, localstack_image) - _start_docker_container(client, config, localstack_image) - _localstack_health_check() - client.close() - - print("LocalStack is ready...") - - -def pytest_unconfigure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - - if not is_collect_only and is_localstack_start: - print("\nStopping LocalStack...") - client = docker.from_env() - _stop_docker_container(client, config) - client.close() +# def pytest_configure(config): +# is_collect_only = config.getoption(name='--collect-only') +# is_localstack_start = config.getoption(name='--ls-start') +# localstack_image = config.getoption(name='--ls-image') +# +# if not is_collect_only and is_localstack_start: +# +# print("\nStarting LocalStack...") +# +# client = docker.from_env() +# _docker_service_health(client) +# _pull_docker_image(client, localstack_image) +# _start_docker_container(client, config, localstack_image) +# _localstack_health_check() +# client.close() +# +# print("LocalStack is ready...") +# +# +# def pytest_unconfigure(config): +# is_collect_only = config.getoption(name='--collect-only') +# is_localstack_start = config.getoption(name='--ls-start') +# +# if not is_collect_only and is_localstack_start: +# print("\nStopping LocalStack...") +# client = docker.from_env() +# _stop_docker_container(client, config) +# client.close() diff --git a/utils.py b/utils.py index 9c9b263..6cfa9a9 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,6 @@ +import signal from os import system, getcwd, chdir, chmod, listdir from os.path import exists, realpath -from tempfile import NamedTemporaryFile from uuid import uuid4 @@ -31,10 +31,11 @@ def execute_command(cmd, env=None, cwd=None): if env: _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) cmd = f'{_env} {cmd}' - # log_file = NamedTemporaryFile() log_file: str = '/tmp/%s' % uuid4().hex - _err = system(f'{cmd} > /dev/null 2> {log_file}') - # _err = system(f'{cmd} > ') + _err = system(f'{cmd} > {log_file} 2>&1') + if _err == signal.SIGINT: + print("SIGNINT is caught") + raise KeyboardInterrupt _out = open(log_file, 'r').read() chdir(_lwd) return _err, _out From 795c0e8ed5cc08f86132bcf53920ad5836a7000a Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 14:17:43 +0530 Subject: [PATCH 63/94] updated requiremnents --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a6b3a70..1715cff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ click pytest docker -requests \ No newline at end of file +requests +pytest-xdist \ No newline at end of file From 32f7e79d59e76666b803c59a0fd7acbc6496a717 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 16:25:26 +0530 Subject: [PATCH 64/94] moved to github actions --- .circleci/config.yml | 2 +- .github/workflows/main.yml | 35 +++++++++++++++++++++++++++++++++++ main.py | 13 ++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/main.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 6475293..4acd793 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,7 +97,7 @@ workflows: or: - equal: [ build, << pipeline.git.branch >> ] - equal: [ build-new, << pipeline.git.branch >> ] - - equal: [ pytest-plugin, << pipeline.git.branch >> ] + - equal: [ pytest-pluginn, << pipeline.git.branch >> ] jobs: - generate-test-binary - run-tests: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..a0dd7cc --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,35 @@ +on: + push: + branches: + - pytest-plugin + +name: Terraform Tests +jobs: + + prepare_list: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: 'true' + - id: set-matrix + run: echo "::set-output name=matrix::$(ls terraform-provider-aws/internal/service)" + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + test_service: + needs: prepare_list + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.prepare_list.outputs.matrix) }} + runs-on: ubuntu-latest + env: + DNS_ADDRESS: 0 + AWS_DEFAULT_REGION: us-east-2 + AWS_ALTERNATE_REGION: eu-west-1 + + steps: + - name: Get list of tests for this service + id: get-list + run: echo "${{ matrix.service }}" diff --git a/main.py b/main.py index 84f4e07..745c287 100644 --- a/main.py +++ b/main.py @@ -1,5 +1,5 @@ import click - +from timeit import default_timer as timer @click.group(name='pytest-golang', help='Golang Test Runner for localstack') def cli(): @@ -17,6 +17,10 @@ def patch(): --service=all; --service=ec2; --service=ec2,iam''') def build(service): """Build binary for testing""" + + ## skips building for the service + skip_services = ["controltower"] + if not service: print('No service provided') print('use --service or -s to specify services to build; for more help try --help to see more options') @@ -31,15 +35,22 @@ def build(service): services = [service] for service in services: + if service in skip_service: + continue from utils import build_test_bin from utils import TF_REPO_NAME from os.path import realpath print(f'Building {service}...') try: + start = timer() build_test_bin(service=service, tf_root_path=realpath(TF_REPO_NAME)) + end = timer() + print(f'Build {service} in {end - start} seconds') except KeyboardInterrupt: print('Interrupted') return + except Exception as e: + print(f'Failed to build binary for {service}: {e}') cli.add_command(build) From a15c9346fc9317e4b7e5dd547dfc6df7744d9784 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 16:32:27 +0530 Subject: [PATCH 65/94] added services --- .github/workflows/main.yml | 4 ++-- get-services.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 get-services.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a0dd7cc..7eaf02d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "::set-output name=matrix::$(ls terraform-provider-aws/internal/service)" + run: echo "::set-output name=matrix::$(python get-services.py)" outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.prepare_list.outputs.matrix) }} + include: ${{ needs.prepare_list.outputs.matrix }} runs-on: ubuntu-latest env: DNS_ADDRESS: 0 diff --git a/get-services.py b/get-services.py new file mode 100644 index 0000000..ccff5ce --- /dev/null +++ b/get-services.py @@ -0,0 +1,7 @@ +import json +import os + +blacklist_services = ["controltower"] + +services = os.listdir('terraform-provider-aws/internal/service') +print(services) From 9edc23db9461e5a3601ba2b4c5ec072875cbcbd1 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 16:35:28 +0530 Subject: [PATCH 66/94] added from json --- .github/workflows/main.yml | 2 +- get-services.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7eaf02d..3fd2874 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ needs.prepare_list.outputs.matrix }} + include: ${{ fromJson(needs.prepare_list.outputs.matrix) }} runs-on: ubuntu-latest env: DNS_ADDRESS: 0 diff --git a/get-services.py b/get-services.py index ccff5ce..c33aa81 100644 --- a/get-services.py +++ b/get-services.py @@ -4,4 +4,17 @@ blacklist_services = ["controltower"] services = os.listdir('terraform-provider-aws/internal/service') -print(services) +print(json.dumps(services)) + + +# with open("tests/terraform/terraform-tests.yaml") as f: +# service_mapping = yaml.load(f, Loader=yaml.FullLoader) +# mapping = [] +# for service, partition_or_tests in service_mapping.items(): +# if isinstance(partition_or_tests, dict): +# partitions = list(partition_or_tests.keys()) +# for partition in partitions: +# mapping.append({"service": service, "partition": str(partition)}) +# else: +# mapping.append({"service": service, "partition": None}) +# print(json.dumps(mapping)) From d32ace74f1eafa4a9aba8d1e1d307ddf01a0e255 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 16:40:54 +0530 Subject: [PATCH 67/94] added serice --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3fd2874..da16629 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "::set-output name=matrix::$(python get-services.py)" + run: echo "matrix=$(python get-services.py)" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJson(needs.prepare_list.outputs.matrix) }} + service: ${{ fromJson(needs.prepare_list.outputs.matrix) }} runs-on: ubuntu-latest env: DNS_ADDRESS: 0 From 4e9ed685029f4c5575f34022c1bee33d62ad7b8a Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 16:49:36 +0530 Subject: [PATCH 68/94] updated action --- .github/workflows/main.yml | 40 +++++++++++++++++++++++++++++++------- get-services.py | 8 +++++++- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index da16629..9e5ff31 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,12 +24,38 @@ jobs: matrix: service: ${{ fromJson(needs.prepare_list.outputs.matrix) }} runs-on: ubuntu-latest - env: - DNS_ADDRESS: 0 - AWS_DEFAULT_REGION: us-east-2 - AWS_ALTERNATE_REGION: eu-west-1 steps: - - name: Get list of tests for this service - id: get-list - run: echo "${{ matrix.service }}" + - uses: actions/checkout@v3 + with: + submodules: 'true' + - uses: actions/setup-go@v3 + with: + go-version: 1.18.x + - name: Set up Python 3.10.5 + uses: actions/setup-python@v2 + with: + python-version: '3.10.5' + - name: Cache Python .venv + id: python-cache + uses: actions/cache@v2 + with: + path: ./.venv/ + key: ${{ runner.os }}-venv-${{ hashFiles('**/setup.cfg') }} + - name: Install system dependencies + run: | + sudo apt update + sudo apt install libsasl2-dev -y + pip install --upgrade pip + pip install -r requirements.txt + + - name: Patch Terraform Provider + run: | + cd terraform-provider-aws && go mod vendor + cd ../ + python3 main.py patch + + - name: Build ${{ matrix.service }} Binary + run: | + python main.py build -s s3 + ls -la terraform-provider-aws/test-bin \ No newline at end of file diff --git a/get-services.py b/get-services.py index c33aa81..1d87fa5 100644 --- a/get-services.py +++ b/get-services.py @@ -2,8 +2,14 @@ import os blacklist_services = ["controltower"] +whitelist_service = ["s3"] -services = os.listdir('terraform-provider-aws/internal/service') +services = [] +for service in os.listdir('terraform-provider-aws/internal/service'): + if service in whitelist_service: + services.append(service) + # if service not in blacklist_services: + # services.append(service) print(json.dumps(services)) From bf8274a4023fa4b11cc1dae7e3000b7039ef89a2 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 16:51:58 +0530 Subject: [PATCH 69/94] minor fix --- get-services.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/get-services.py b/get-services.py index 1d87fa5..383e9f7 100644 --- a/get-services.py +++ b/get-services.py @@ -2,11 +2,11 @@ import os blacklist_services = ["controltower"] -whitelist_service = ["s3"] +whitelist_services = ["s3"] services = [] for service in os.listdir('terraform-provider-aws/internal/service'): - if service in whitelist_service: + if service in whitelist_services: services.append(service) # if service not in blacklist_services: # services.append(service) From 42feab830230e819f049182978e1df943b3cd168 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 16:56:14 +0530 Subject: [PATCH 70/94] minor fix --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 745c287..ce41003 100644 --- a/main.py +++ b/main.py @@ -35,7 +35,7 @@ def build(service): services = [service] for service in services: - if service in skip_service: + if service in skip_services: continue from utils import build_test_bin from utils import TF_REPO_NAME From 025536b6478441f38c86c73ec0d7f9e61844ddee Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 17:17:00 +0530 Subject: [PATCH 71/94] added support for service --- .github/workflows/main.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9e5ff31..0d11442 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,4 +58,11 @@ jobs: - name: Build ${{ matrix.service }} Binary run: | python main.py build -s s3 - ls -la terraform-provider-aws/test-bin \ No newline at end of file + ls -la terraform-provider-aws/test-bin + + - name: Run Localstack + run: docker-compose up -d + + - name: Run ${{ matrix.service }} Tests + run: | + pytest terraform-provider-aws/internal/service/ -s -v \ No newline at end of file From c4e5d660f777be8694259bd206f239e87b4ed434 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 17:24:00 +0530 Subject: [PATCH 72/94] added testing for specific service --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0d11442..9989926 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,4 +65,4 @@ jobs: - name: Run ${{ matrix.service }} Tests run: | - pytest terraform-provider-aws/internal/service/ -s -v \ No newline at end of file + pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v \ No newline at end of file From 43de7ae4a084ab59078ab770cbc1d14b05689944 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Tue, 10 Jan 2023 17:46:56 +0530 Subject: [PATCH 73/94] running tests for all services --- .github/workflows/main.yml | 2 +- get-services.py | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9989926..3037c49 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,7 +57,7 @@ jobs: - name: Build ${{ matrix.service }} Binary run: | - python main.py build -s s3 + python main.py build -s ${{ matrix.service }} ls -la terraform-provider-aws/test-bin - name: Run Localstack diff --git a/get-services.py b/get-services.py index 383e9f7..10c7890 100644 --- a/get-services.py +++ b/get-services.py @@ -2,25 +2,9 @@ import os blacklist_services = ["controltower"] -whitelist_services = ["s3"] services = [] for service in os.listdir('terraform-provider-aws/internal/service'): - if service in whitelist_services: + if service not in blacklist_services: services.append(service) - # if service not in blacklist_services: - # services.append(service) print(json.dumps(services)) - - -# with open("tests/terraform/terraform-tests.yaml") as f: -# service_mapping = yaml.load(f, Loader=yaml.FullLoader) -# mapping = [] -# for service, partition_or_tests in service_mapping.items(): -# if isinstance(partition_or_tests, dict): -# partitions = list(partition_or_tests.keys()) -# for partition in partitions: -# mapping.append({"service": service, "partition": str(partition)}) -# else: -# mapping.append({"service": service, "partition": None}) -# print(json.dumps(mapping)) From 43c3091402c93f6ae7b1de8692e6e75326ca39bb Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 11 Jan 2023 12:53:56 +0530 Subject: [PATCH 74/94] test runs for specific service --- .github/workflows/main.yml | 30 +++++---- conftest.py | 129 ++++++++++++++++--------------------- get-services.py | 26 ++++++-- main.py | 10 +-- requirements.txt | 2 +- utils.py | 2 + 6 files changed, 100 insertions(+), 99 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3037c49..3b7b2ae 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,15 @@ on: + workflow_dispatch: + inputs: + services: + default: 'all' + type: string + description: name of the service to execute tests for + localstack-image: + required: false + type: string + default: 'localstack/localstack:latest' + description: localstack docker image name to test against push: branches: - pytest-plugin @@ -13,7 +24,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py)" >> $GITHUB_OUTPUT + run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -26,25 +37,23 @@ jobs: runs-on: ubuntu-latest steps: + - uses: actions/checkout@v3 with: submodules: 'true' + - uses: actions/setup-go@v3 with: go-version: 1.18.x + - name: Set up Python 3.10.5 uses: actions/setup-python@v2 with: python-version: '3.10.5' - - name: Cache Python .venv - id: python-cache - uses: actions/cache@v2 - with: - path: ./.venv/ - key: ${{ runner.os }}-venv-${{ hashFiles('**/setup.cfg') }} + - name: Install system dependencies run: | - sudo apt update + sudo apt update --fix-missing sudo apt install libsasl2-dev -y pip install --upgrade pip pip install -r requirements.txt @@ -60,9 +69,6 @@ jobs: python main.py build -s ${{ matrix.service }} ls -la terraform-provider-aws/test-bin - - name: Run Localstack - run: docker-compose up -d - - name: Run ${{ matrix.service }} Tests run: | - pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v \ No newline at end of file + pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image }} \ No newline at end of file diff --git a/conftest.py b/conftest.py index 4ba44cc..127b2dc 100644 --- a/conftest.py +++ b/conftest.py @@ -5,7 +5,7 @@ from requests.adapters import HTTPAdapter, Retry from pathlib import Path import os -from os.path import realpath, relpath, dirname, exists +from os.path import realpath, relpath, dirname from utils import execute_command, build_test_bin @@ -19,14 +19,14 @@ def pytest_addoption(parser): def pytest_collect_file(parent, file_path): - if file_path.suffix == ".go" and file_path.name.endswith("_test.go"): + if file_path.suffix == '.go' and file_path.name.endswith('_test.go'): return GoFile.from_parent(parent, path=file_path) class GoFile(pytest.File): def collect(self): raw = self.path.open().read() - fa = re.findall(r"^(func (TestAcc.*))\(.*\).*", raw, re.MULTILINE) + fa = re.findall(r'^(func (TestAcc.*))\(.*\).*', raw, re.MULTILINE) for _, name in fa: yield GoItem.from_parent(self, name=name) @@ -49,53 +49,32 @@ def runtest(self): 'TF_ACC': '1', 'AWS_ACCESS_KEY_ID': 'test', 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-east-1', + 'AWS_DEFAULT_REGION': 'us-west-2', 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', 'AWS_ALTERNATE_REGION': 'us-east-2', }) cmd = [ - f"./test-bin/{service}.test", - "-test.v", - "-test.parallel=1", - "-test.count=1", - "-test.timeout=60m", - "-test.run", f"{self.name}" + f'./test-bin/{service}.test', + '-test.v', + '-test.parallel=1', + '-test.count=1', + '-test.timeout=5m', + 'test.run", f"{self.name}' ] return_code, stdout = execute_command(cmd, env, tf_root_path) + print(f'error: {stdout}') if return_code != 0: - print(f"error: {stdout}") + print(f'error: {stdout}') raise GoException(returncode=return_code, stdout=stdout) return return_code - # go test ./internal/service/$1 -test.count 1 -test.v -test.timeout 60m -parallel $VALUE -run $2 - # cmd = [ - # "go", "test", f"./{service_path}", "-test.count=1", "-test.v", f"-test.run={self.name}" - # ] - - # cmd_str = " ".join(str(c) for c in cmd) - # cmd_str = f"cd {tf_root_path} && TF_ACC=1 AWS_ACCESS_KEY_ID=test AWS_SECRET_ACCESS_KEY=test {cmd_str}" - # return_code = os.system(cmd_str) - # proc = run(cmd, env=env, cwd=tf_root_path, check=True) - # print(f"proc: {proc}") - # return proc.returncode - # proc = Popen( - # cmd, stdout=PIPE, stderr=PIPE, - # env=env, bufsize=1, universal_newlines=True, - # cwd=tf_root_path - # ) - # stdout, stderr = proc.communicate() - # proc.terminate() - # if proc.returncode != 0: - # raise GoException(proc.returncode, stdout, stderr) - # return proc.returncode - def repr_failure(self, excinfo, **kwargs): """Called when self.runtest() raises an exception.""" if isinstance(excinfo.value, GoException): - return "\n".join( + return '\n'.join( [ f'\nExecution failed with return code: {excinfo.value.returncode}', f'\nFailure Reason:\n{excinfo.value}', @@ -103,7 +82,7 @@ def repr_failure(self, excinfo, **kwargs): ) def reportinfo(self): - return self.path, 0, f"Test Case: {self.name}" + return self.path, 0, f'Test Case: {self.name}' class GoException(Exception): @@ -115,12 +94,12 @@ def __init__(self, returncode, stdout, stderr): def _docker_service_health(client): if not client.ping(): - print("\nPlease start docker daemon and try again") - raise Exception("Docker is not running") + print('\nPlease start docker daemon and try again') + raise Exception('Docker is not running') def _start_docker_container(client, config, localstack_image): - env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf"] + env_vars = ['DEBUG=1', 'PROVIDER_OVERRIDE_S3=asf'] port_mappings = { '53/tcp': ('127.0.0.1', 53), '53/udp': ('127.0.0.1', 53), @@ -128,20 +107,20 @@ def _start_docker_container(client, config, localstack_image): '4566': ('127.0.0.1', 4566), '4571': ('127.0.0.1', 4571), } - volumes = ["/var/run/docker.sock:/var/run/docker.sock"] + volumes = ['/var/run/docker.sock:/var/run/docker.sock'] localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, name='localstack_main', volumes=volumes, auto_remove=True, environment=env_vars) - setattr(config, "localstack_container_id", localstack_container.id) + setattr(config, 'localstack_container_id', localstack_container.id) def _stop_docker_container(client, config): - client.containers.get(getattr(config, "localstack_container_id")).stop() - print("LocalStack is stopped") + client.containers.get(getattr(config, 'localstack_container_id')).stop() + print('LocalStack is stopped') def _localstack_health_check(): - localstack_health_url = "http://localhost:4566/health" + localstack_health_url = 'http://localhost:4566/health' session = requests.Session() retry = Retry(connect=3, backoff_factor=2) adapter = HTTPAdapter(max_retries=retry) @@ -154,37 +133,37 @@ def _localstack_health_check(): def _pull_docker_image(client, localstack_image): docker_image_list = client.images.list(name=localstack_image) if len(docker_image_list) == 0: - print(f"Pulling image {localstack_image}") + print(f'Pulling image {localstack_image}') client.images.pull(localstack_image) docker_image_list = client.images.list(name=localstack_image) - print(f"Using LocalStack image: {docker_image_list[0].id}") - - -# def pytest_configure(config): -# is_collect_only = config.getoption(name='--collect-only') -# is_localstack_start = config.getoption(name='--ls-start') -# localstack_image = config.getoption(name='--ls-image') -# -# if not is_collect_only and is_localstack_start: -# -# print("\nStarting LocalStack...") -# -# client = docker.from_env() -# _docker_service_health(client) -# _pull_docker_image(client, localstack_image) -# _start_docker_container(client, config, localstack_image) -# _localstack_health_check() -# client.close() -# -# print("LocalStack is ready...") -# -# -# def pytest_unconfigure(config): -# is_collect_only = config.getoption(name='--collect-only') -# is_localstack_start = config.getoption(name='--ls-start') -# -# if not is_collect_only and is_localstack_start: -# print("\nStopping LocalStack...") -# client = docker.from_env() -# _stop_docker_container(client, config) -# client.close() + print(f'Using LocalStack image: {docker_image_list[0].id}') + + +def pytest_configure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + localstack_image = config.getoption(name='--ls-image') + + if not is_collect_only and is_localstack_start: + + print('\nStarting LocalStack...') + + client = docker.from_env() + _docker_service_health(client) + _pull_docker_image(client, localstack_image) + _start_docker_container(client, config, localstack_image) + _localstack_health_check() + client.close() + + print('LocalStack is ready...') + + +def pytest_unconfigure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + + if not is_collect_only and is_localstack_start: + print('\nStopping LocalStack...') + client = docker.from_env() + _stop_docker_container(client, config) + client.close() diff --git a/get-services.py b/get-services.py index 10c7890..30b631b 100644 --- a/get-services.py +++ b/get-services.py @@ -1,10 +1,24 @@ import json -import os +import sys -blacklist_services = ["controltower"] +import utils services = [] -for service in os.listdir('terraform-provider-aws/internal/service'): - if service not in blacklist_services: - services.append(service) -print(json.dumps(services)) + +if len(sys.argv) > 1: + service = sys.argv[1] + if service == 'all': + from utils import get_all_services + services = get_all_services() + else: + if ',' in service: + services = service.split(',') + services = [s for s in services if s != '' and s not in utils.BLACKLISTED_SERVICES] + else: + services = [service] + print(json.dumps(services)) + exit(0) +else: + print('No service provided') + exit(1) + diff --git a/main.py b/main.py index ce41003..3cb02c3 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,9 @@ import click from timeit import default_timer as timer +import utils + + @click.group(name='pytest-golang', help='Golang Test Runner for localstack') def cli(): pass @@ -18,9 +21,7 @@ def patch(): def build(service): """Build binary for testing""" - ## skips building for the service - skip_services = ["controltower"] - + # skips building for the service if not service: print('No service provided') print('use --service or -s to specify services to build; for more help try --help to see more options') @@ -31,12 +32,11 @@ def build(service): else: if ',' in service: services = service.split(',') + services = [s for s in services if s != '' and s not in utils.BLACKLISTED_SERVICES] else: services = [service] for service in services: - if service in skip_services: - continue from utils import build_test_bin from utils import TF_REPO_NAME from os.path import realpath diff --git a/requirements.txt b/requirements.txt index 1715cff..cbf71bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ click pytest docker requests -pytest-xdist \ No newline at end of file +pytest-xdist diff --git a/utils.py b/utils.py index 6cfa9a9..1735d7a 100644 --- a/utils.py +++ b/utils.py @@ -12,6 +12,8 @@ TF_TEST_BINARY_FOLDER = 'test-bin' TF_REPO_SERVICE_FOLDER = './internal/service' +BLACKLISTED_SERVICES = ['controltower', 'greengrass'] + def _get_test_bin_abs_path(service): return f'{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test' From da90aa6860a9f5b96782ffc4e98f2c9c49209bde Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 11 Jan 2023 12:56:01 +0530 Subject: [PATCH 75/94] fix: workflow to run without dispatch --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3b7b2ae..c449e7d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services | 'all' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -71,4 +71,4 @@ jobs: - name: Run ${{ matrix.service }} Tests run: | - pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image }} \ No newline at end of file + pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image | 'localstack/localstack:latest' }} \ No newline at end of file From 3602799534f37ecdf2354b9d98c2c41dd5babaf9 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Wed, 11 Jan 2023 12:56:40 +0530 Subject: [PATCH 76/94] fix: workflow or syntax --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c449e7d..aa35af3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services | 'all' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'all' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -71,4 +71,4 @@ jobs: - name: Run ${{ matrix.service }} Tests run: | - pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image | 'localstack/localstack:latest' }} \ No newline at end of file + pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} \ No newline at end of file From 12759069fff6be3ffe86b1f4e4113584e8a19423 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Wed, 11 Jan 2023 15:47:14 +0100 Subject: [PATCH 77/94] upgrade actions/setup-python@v4 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aa35af3..e5c1bca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,7 @@ jobs: go-version: 1.18.x - name: Set up Python 3.10.5 - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: '3.10.5' From 207acd8369c3f652adf171d27fe6d4047efd3817 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Wed, 11 Jan 2023 15:58:25 +0100 Subject: [PATCH 78/94] add JUnit test result publishing --- .github/workflows/main.yml | 9 ++++++++- .gitignore | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5c1bca..6c72c95 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -71,4 +71,11 @@ jobs: - name: Run ${{ matrix.service }} Tests run: | - pytest terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} \ No newline at end of file + pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} + + - name: Publish ${{ matrix.service }} Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + if: always() + with: + files: target/reports/*.xml + check_name: ${{ matrix.service }} Terraform Test Results \ No newline at end of file diff --git a/.gitignore b/.gitignore index f1cb84e..ff13e69 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ .venv .pytest_cache __pycache__ +target **/*.test \ No newline at end of file From 4bfeccca6e5ccd8a6a1462919972b936156893f9 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Wed, 11 Jan 2023 16:07:05 +0100 Subject: [PATCH 79/94] upgrade EnricoMi/publish-unit-test-result-action --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c72c95..fb5f895 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -74,7 +74,7 @@ jobs: pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} - name: Publish ${{ matrix.service }} Test Results - uses: EnricoMi/publish-unit-test-result-action@v1 + uses: EnricoMi/publish-unit-test-result-action@v2 if: always() with: files: target/reports/*.xml From c7117fe8e289649a75832bf1a2a216537d6bcf39 Mon Sep 17 00:00:00 2001 From: Alexander Rashed Date: Wed, 11 Jan 2023 17:35:02 +0100 Subject: [PATCH 80/94] fix configuration of publish-unit-test-result-action --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fb5f895..f9853ea 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,5 +77,5 @@ jobs: uses: EnricoMi/publish-unit-test-result-action@v2 if: always() with: - files: target/reports/*.xml + junit_files: target/reports/*.xml check_name: ${{ matrix.service }} Terraform Test Results \ No newline at end of file From 488a2b23464b7a1d6fb55637297eac3f637bc429 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 12 Jan 2023 11:33:00 +0530 Subject: [PATCH 81/94] fix: report generation patched --- .gitignore | 3 ++- conftest.py | 45 ++++++++++++++++++++++++++++++++------------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index ff13e69..da78a2b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ .pytest_cache __pycache__ target -**/*.test \ No newline at end of file +**/*.test +report.xml \ No newline at end of file diff --git a/conftest.py b/conftest.py index 127b2dc..b8b426f 100644 --- a/conftest.py +++ b/conftest.py @@ -49,7 +49,7 @@ def runtest(self): 'TF_ACC': '1', 'AWS_ACCESS_KEY_ID': 'test', 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-west-2', + 'AWS_DEFAULT_REGION': 'us-east-1', 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', 'AWS_ALTERNATE_REGION': 'us-east-2', @@ -60,24 +60,20 @@ def runtest(self): '-test.v', '-test.parallel=1', '-test.count=1', - '-test.timeout=5m', - 'test.run", f"{self.name}' + '-test.timeout=60m', + f'-test.run={self.name}', ] - return_code, stdout = execute_command(cmd, env, tf_root_path) - print(f'error: {stdout}') if return_code != 0: - print(f'error: {stdout}') - raise GoException(returncode=return_code, stdout=stdout) - return return_code + raise GoException(returncode=return_code, stderr=stdout) def repr_failure(self, excinfo, **kwargs): """Called when self.runtest() raises an exception.""" if isinstance(excinfo.value, GoException): return '\n'.join( [ - f'\nExecution failed with return code: {excinfo.value.returncode}', - f'\nFailure Reason:\n{excinfo.value}', + f'Execution failed with return code: {excinfo.value.returncode}', + f'Failure Reason:\n{excinfo.value.stderr}', ] ) @@ -85,10 +81,34 @@ def reportinfo(self): return self.path, 0, f'Test Case: {self.name}' +class ReprCrash: + def __init__(self, message): + self.message = message + + +class LongRepr: + def __init__(self, message, reason): + self.reprcrash = ReprCrash(message) + self.reason = reason + + def __str__(self): + return self.reason + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + if report.failed: + splits = report.longrepr.split('\n', 1) + longrepr = LongRepr(splits[0], splits[1]) + delattr(report, 'longrepr') + setattr(report, 'longrepr', longrepr) + + class GoException(Exception): - def __init__(self, returncode, stdout, stderr): + def __init__(self, returncode, stderr): self.returncode = returncode - self.stdout = stdout self.stderr = stderr @@ -145,7 +165,6 @@ def pytest_configure(config): localstack_image = config.getoption(name='--ls-image') if not is_collect_only and is_localstack_start: - print('\nStarting LocalStack...') client = docker.from_env() From 937ada597f781abc967eb1c7635cc93e5b8b86d4 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 12 Jan 2023 12:45:01 +0530 Subject: [PATCH 82/94] update: added localstack implemented services, reformatted utils, updated wf --- .github/workflows/main.yml | 6 +++--- conftest.py | 7 ++----- get-services.py | 12 ++---------- main.py | 24 ++++++------------------ utils.py | 27 ++++++++++++++++++++++++++- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f9853ea..6afec2e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,9 +2,9 @@ on: workflow_dispatch: inputs: services: - default: 'all' + default: 'ls-community' type: string - description: name of the service to execute tests for + description: name of the service to execute tests for (e.g. "ls-community", "ls-pro", "ls-all", "s3,iam,ec2") localstack-image: required: false type: string @@ -24,7 +24,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'all' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} diff --git a/conftest.py b/conftest.py index b8b426f..8cc058c 100644 --- a/conftest.py +++ b/conftest.py @@ -40,19 +40,16 @@ def runtest(self): service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) service = service_path.split(os.sep)[-1] - # return_code, stdout = build_test_bin(service=service, tf_root_path=tf_root_path) - # if return_code != 0: - # raise GoException(returncode=return_code, stdout=stdout) - env = dict(os.environ) env.update({ 'TF_ACC': '1', 'AWS_ACCESS_KEY_ID': 'test', 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-east-1', + 'AWS_DEFAULT_REGION': 'us-west-2', 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', 'AWS_ALTERNATE_REGION': 'us-east-2', + 'AWS_THIRD_REGION': 'eu-west-1', }) cmd = [ diff --git a/get-services.py b/get-services.py index 30b631b..3c87155 100644 --- a/get-services.py +++ b/get-services.py @@ -1,21 +1,13 @@ import json import sys -import utils +from utils import get_services services = [] if len(sys.argv) > 1: service = sys.argv[1] - if service == 'all': - from utils import get_all_services - services = get_all_services() - else: - if ',' in service: - services = service.split(',') - services = [s for s in services if s != '' and s not in utils.BLACKLISTED_SERVICES] - else: - services = [service] + services = get_services(service) print(json.dumps(services)) exit(0) else: diff --git a/main.py b/main.py index 3cb02c3..38b3d27 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,7 @@ import click from timeit import default_timer as timer - -import utils +from utils import build_test_bin, TF_REPO_NAME, get_services +from os.path import realpath @click.group(name='pytest-golang', help='Golang Test Runner for localstack') @@ -16,30 +16,18 @@ def patch(): @click.command(name='build', help='Build binary for testing') -@click.option('--service', '-s', default=None, help='''Service to build; use "all" to build all services, example: ---service=all; --service=ec2; --service=ec2,iam''') +@click.option('--service', '-s', default=None, help='''Service to build; use "ls-all", "ls-community", "ls-pro" to build all services, example: +--service=ls-all; --service=ec2; --service=ec2,iam''') def build(service): """Build binary for testing""" - - # skips building for the service if not service: print('No service provided') print('use --service or -s to specify services to build; for more help try --help to see more options') return - if service == 'all': - from utils import get_all_services - services = get_all_services() - else: - if ',' in service: - services = service.split(',') - services = [s for s in services if s != '' and s not in utils.BLACKLISTED_SERVICES] - else: - services = [service] + + services = get_services(service) for service in services: - from utils import build_test_bin - from utils import TF_REPO_NAME - from os.path import realpath print(f'Building {service}...') try: start = timer() diff --git a/utils.py b/utils.py index 1735d7a..c35da1a 100644 --- a/utils.py +++ b/utils.py @@ -13,6 +13,13 @@ TF_REPO_SERVICE_FOLDER = './internal/service' BLACKLISTED_SERVICES = ['controltower', 'greengrass'] +LS_COMMUNITY_SERVICES = [ + "acm", "apigateway", "lambda", "cloudformation", "cloudwatch", "configservice", "dynamodb", "ec2", "elasticsearch", + "events", "firehose", "iam", "kinesis", "kms", "logs", "opensearch", "redshift", "resourcegroups", + "resourcegroupstaggingapi", "route53", "route53resolver", "s3", "s3control", "secretsmanager", "ses", "sns", "sqs", + "ssm", "sts", "swf", "transcribe" +] +LS_PRO_SERVICES = [] def _get_test_bin_abs_path(service): @@ -72,10 +79,28 @@ def build_test_bin(service, tf_root_path): def get_all_services(): services = [] for service in listdir(f'{TF_REPO_PATH}/{TF_REPO_SERVICE_FOLDER}'): - services.append(service) + if service not in BLACKLISTED_SERVICES: + services.append(service) return sorted(services) +def get_services(service): + if service == 'ls-community': + services = LS_COMMUNITY_SERVICES + elif service == 'ls-pro': + services = LS_PRO_SERVICES + elif service == 'ls-all': + services = LS_COMMUNITY_SERVICES + LS_PRO_SERVICES + else: + if ',' in service: + services = service.split(',') + services = [s for s in services if s] + else: + services = [service] + services = [s for s in services if s not in BLACKLISTED_SERVICES] + return list(set(services)) + + def patch_repo(): print(f'Patching {TF_REPO_NAME}...') for patch_file in TF_REPO_PATCH_FILES: From 27368e3ca648d4aaa4ae1b94fc566053fa8c90d4 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 12 Jan 2023 13:22:07 +0530 Subject: [PATCH 83/94] added single service --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6afec2e..81d05fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'acm' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -71,6 +71,7 @@ jobs: - name: Run ${{ matrix.service }} Tests run: | + echo "::remove-matcher owner=python::" pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} - name: Publish ${{ matrix.service }} Test Results From fef13bd2355ce16e3cfb1eeca0c1167fbc81b0ec Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 12 Jan 2023 13:45:42 +0530 Subject: [PATCH 84/94] removed annotation --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 81d05fe..eb9ed05 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,4 +79,6 @@ jobs: if: always() with: junit_files: target/reports/*.xml - check_name: ${{ matrix.service }} Terraform Test Results \ No newline at end of file + check_name: ${{ matrix.service }} Terraform Test Results + report_individual_runs: "false" + check_run_annotations: "none" \ No newline at end of file From bd44e0fd955e9befba367a5cc221c9a68831749e Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 12 Jan 2023 14:41:08 +0530 Subject: [PATCH 85/94] added: support to run ls supported services --- .github/workflows/main.yml | 2 -- utils.py | 9 ++++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eb9ed05..1564149 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,5 +80,3 @@ jobs: with: junit_files: target/reports/*.xml check_name: ${{ matrix.service }} Terraform Test Results - report_individual_runs: "false" - check_run_annotations: "none" \ No newline at end of file diff --git a/utils.py b/utils.py index c35da1a..051686b 100644 --- a/utils.py +++ b/utils.py @@ -19,7 +19,14 @@ "resourcegroupstaggingapi", "route53", "route53resolver", "s3", "s3control", "secretsmanager", "ses", "sns", "sqs", "ssm", "sts", "swf", "transcribe" ] -LS_PRO_SERVICES = [] +LS_PRO_SERVICES = [ + "amplify", "apigateway", "apigatewayv2", "appconfig", "appautoscaling", "appsync", "athena", "autoscaling", + "backup", "batch", "cloudformation", "cloudfront", "cloudtrail", "codecommit", "cognitoidp", "cognitoidentity", + "docdb", "dynamodb", "ec2", "ecr", "ecs", "efs", "eks", "elasticache", "elasticbeanstalk", "elb", "elbv2", "emr", + "events", "fis", "glacier", "glue", "iam", "iot", "iotanalytics", "kafka", "kinesisanalytics", "kms", + "lakeformation", "lambda", "logs", "mediastore", "mq", "mwaa", "neptune", "organizations", "qldb", "rds", + "redshift", "route53", "s3", "sagemaker", "secretsmanager", "serverlessrepo", "ses", "sns", "sqs", "ssm", "sts" +] def _get_test_bin_abs_path(service): From 4a8ed4f66baf6e2c6e3f4cbf96405e73863451bb Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Thu, 12 Jan 2023 15:07:44 +0530 Subject: [PATCH 86/94] updated: workflow max-parallel, run for all service --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1564149..21ef7c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,13 +24,14 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'acm' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} test_service: needs: prepare_list strategy: + max-parallel: 10 fail-fast: false matrix: service: ${{ fromJson(needs.prepare_list.outputs.matrix) }} From 416195c44eedce43f021abb80a59332db4be9aff Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Fri, 13 Jan 2023 14:25:33 +0530 Subject: [PATCH 87/94] Pytest plugin fork (#13) * updated: workflow to support caching * fix: branch name * added: schedule for workflow * workflow run: all services * fix: dependancy caching in go * updated: changed the service to check caching * removed: comment from gh workflow * updated: workflow finalization --- .github/workflows/main.yml | 15 ++++++++++----- conftest.py | 1 + requirements.txt | 9 ++++----- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21ef7c3..f5f0541 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,6 @@ on: + schedule: + - cron: '0 19 * * 6' workflow_dispatch: inputs: services: @@ -10,9 +12,10 @@ on: type: string default: 'localstack/localstack:latest' description: localstack docker image name to test against - push: - branches: - - pytest-plugin +# disabled pipeline for push events, as we want to run the pipeline only on a schedule or manually +# push: +# branches: +# - pytest-plugin-fork name: Terraform Tests jobs: @@ -45,12 +48,15 @@ jobs: - uses: actions/setup-go@v3 with: - go-version: 1.18.x + go-version: '1.18.x' + cache: true + cache-dependency-path: terraform-provider-aws/go.sum - name: Set up Python 3.10.5 uses: actions/setup-python@v4 with: python-version: '3.10.5' + cache: 'pip' - name: Install system dependencies run: | @@ -72,7 +78,6 @@ jobs: - name: Run ${{ matrix.service }} Tests run: | - echo "::remove-matcher owner=python::" pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} - name: Publish ${{ matrix.service }} Test Results diff --git a/conftest.py b/conftest.py index 8cc058c..f6fcfe4 100644 --- a/conftest.py +++ b/conftest.py @@ -48,6 +48,7 @@ def runtest(self): 'AWS_DEFAULT_REGION': 'us-west-2', 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', + 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', 'AWS_ALTERNATE_REGION': 'us-east-2', 'AWS_THIRD_REGION': 'eu-west-1', }) diff --git a/requirements.txt b/requirements.txt index cbf71bd..12240bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ -click -pytest -docker -requests -pytest-xdist +click==8.1.3 +pytest==7.2.0 +docker==6.0.1 +requests==2.28.2 From 63bcf722addfe824892d97a30ff17dc500fb2783 Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 13 Jan 2023 15:45:59 +0530 Subject: [PATCH 88/94] removed: circleci workflow --- .circleci/config.yml | 105 ------------------------------------------- conftest.py | 1 - 2 files changed, 106 deletions(-) delete mode 100644 .circleci/config.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 4acd793..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,105 +0,0 @@ -version: 2.1 - -orbs: - go: circleci/go@1.7.1 - -jobs: - generate-test-binary: - machine: - image: ubuntu-2004:202101-01 - resource_class: large - steps: - - checkout - - go/install: - version: '1.19' - - run: git submodule update --init --recursive - - run: - name: "Installing prerequisites" - command: | - sudo apt-get update - sudo apt install -y libsasl2-dev - pip3 install -r requirements.txt - - - run: - name: "Configuring repos for tests" - command: | - cd terraform-provider-aws && go mod vendor - cd ../ - python3 main.py patch - - - run: - name: "Generate Testing Binary" - no_output_timeout: 20m - command: | - python3 main.py build -s s3 - ls terraform-provider-aws - ls terraform-provider-aws/test-bin -# cd terraform-provider-aws -# go test -c ./internal/service/s3 -o /home/circleci/project/terraform-provider-aws/test-bin/s3.test -# ls terraform-provider-aws - - - persist_to_workspace: - root: . - paths: - - . - - run-tests: - machine: - image: ubuntu-2004:202101-01 - resource_class: medium - steps: - - attach_workspace: - at: . - - - go/install: - version: '1.19' - - - run: - name: "Installing prerequisites" - command: | - sudo apt-get update - sudo apt install -y libsasl2-dev - pip3 install -r requirements.txt - - - run: - name: "Run test suite" - no_output_timeout: 30m - command: | - docker-compose up -d - sleep 10 - export PYTHONUNBUFFERED=1 - pytest terraform-provider-aws/internal/service/s3 -s -v -n 4 - - generate: - machine: - image: ubuntu-2004:202101-01 - steps: - - checkout - - run: touch testfile.txt - - run: ls - - persist_to_workspace: - root: . - paths: - - . - - test: - machine: - image: ubuntu-2004:202101-01 - steps: - - attach_workspace: - at: . - - run: ls - - -workflows: - main: - when: - or: - - equal: [ build, << pipeline.git.branch >> ] - - equal: [ build-new, << pipeline.git.branch >> ] - - equal: [ pytest-pluginn, << pipeline.git.branch >> ] - jobs: - - generate-test-binary - - run-tests: - requires: - - generate-test-binary diff --git a/conftest.py b/conftest.py index f6fcfe4..462a27c 100644 --- a/conftest.py +++ b/conftest.py @@ -66,7 +66,6 @@ def runtest(self): raise GoException(returncode=return_code, stderr=stdout) def repr_failure(self, excinfo, **kwargs): - """Called when self.runtest() raises an exception.""" if isinstance(excinfo.value, GoException): return '\n'.join( [ From 8763b0399b555b675ce490f3178ed8dc590f414f Mon Sep 17 00:00:00 2001 From: macnev2013 Date: Fri, 13 Jan 2023 15:48:26 +0530 Subject: [PATCH 89/94] removed: unused code --- utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils.py b/utils.py index 051686b..a7b1496 100644 --- a/utils.py +++ b/utils.py @@ -58,7 +58,6 @@ def execute_command(cmd, env=None, cwd=None): def build_test_bin(service, tf_root_path): - # _test_bin_rel_path = f'./{TF_TEST_BINARY_FOLDER}/{service}.test' _test_bin_abs_path = _get_test_bin_abs_path(service) _tf_repo_service_folder = f'{TF_REPO_SERVICE_FOLDER}/{service}' From 67a5067c6332ff014eeb80c783ce084eda8c1f87 Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Tue, 17 Jan 2023 11:33:10 +0530 Subject: [PATCH 90/94] added formatting, makefile, updated project structure (#15) * comment resolution * minor fix * removed build on push --- .github/workflows/main.yml | 10 +- Makefile | 26 +++ README.md | 32 +-- conftest.py | 135 ++++++------ docker-compose.yml | 16 -- main.py | 46 ---- pyproject.toml | 17 ++ requirements.txt | 2 + .../get-services.py | 5 +- terraform_pytest/main.py | 55 +++++ terraform_pytest/utils.py | 201 ++++++++++++++++++ tests/conftest.py | 185 ++++++++++++++++ utils.py | 122 ----------- 13 files changed, 585 insertions(+), 267 deletions(-) create mode 100644 Makefile delete mode 100644 docker-compose.yml delete mode 100644 main.py create mode 100644 pyproject.toml rename get-services.py => terraform_pytest/get-services.py (69%) create mode 100644 terraform_pytest/main.py create mode 100644 terraform_pytest/utils.py create mode 100644 tests/conftest.py delete mode 100644 utils.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f5f0541..0d745f9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ on: # disabled pipeline for push events, as we want to run the pipeline only on a schedule or manually # push: # branches: -# - pytest-plugin-fork +# - pytest-plugin name: Terraform Tests jobs: @@ -27,7 +27,7 @@ jobs: with: submodules: 'true' - id: set-matrix - run: echo "matrix=$(python get-services.py ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT + run: echo "matrix=$(python -m terraform_pytest.get-services ${{ github.event.inputs.services || 'ls-community' }})" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} @@ -69,16 +69,16 @@ jobs: run: | cd terraform-provider-aws && go mod vendor cd ../ - python3 main.py patch + python -m terraform_pytest.main patch - name: Build ${{ matrix.service }} Binary run: | - python main.py build -s ${{ matrix.service }} + python -m terraform_pytest.main build -s ${{ matrix.service }} ls -la terraform-provider-aws/test-bin - name: Run ${{ matrix.service }} Tests run: | - pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} + python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} - name: Publish ${{ matrix.service }} Test Results uses: EnricoMi/publish-unit-test-result-action@v2 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4d7c068 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +#!/bin/bash + +VENV_BIN ?= python3 -m venv +VENV_DIR ?= .venv +PIP_CMD ?= pip3 + +ifeq ($(OS), Windows_NT) + VENV_ACTIVATE = $(VENV_DIR)/Scripts/activate +else + VENV_ACTIVATE = $(VENV_DIR)/bin/activate +endif + +$(VENV_ACTIVATE): + test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR) + $(VENV_RUN); $(PIP_CMD) install --upgrade pip setuptools wheel plux + touch $(VENV_ACTIVATE) + +VENV_RUN = . $(VENV_ACTIVATE) + +venv: $(VENV_ACTIVATE) ## Create a new (empty) virtual environment + +install: + $(VENV_RUN); $(PIP_CMD) install -r requirements.txt + +format: + $(VENV_RUN); python -m isort .; python -m black . \ No newline at end of file diff --git a/README.md b/README.md index dde1a3f..39f9b8e 100644 --- a/README.md +++ b/README.md @@ -5,26 +5,30 @@ This is a test runner for localstack and terraform. It will run a test cases fro Purpose of this project is to externalize the test cases from the localstack repo and run them against localstack to gather parity metrics. ## Installation -1. Clone the repository -2. Run `python -m virtualenv venv` to create a virtual environment -3. Run `source venv/bin/activate` to activate the virtual environment -4. Run `pip install -r requirements.txt` to install the dependencies +1. Clone the repository with submodules` +2. Run `make venv` to create a virtual environment +3. Run `make install` to install the dependencies ## How to run? -1. Run `python main.py patch` to apply the patch to the terraform provider aws -2. Now you are ready to use `pytest` commands to list and run test cases from golang +1. Run `python -m terraform_pytest.main patch` to apply the patch to the terraform provider aws +2. Run `python -m terraform_pytest.main build -s s3` to build testing binary for the golang module +3Now you are ready to use `python -m pytest` commands to list and run test cases from golang ## How to run test cases? -- To list down all the test case from a specific service, run `pytest terraform-provider-aws/internal/service/ --collect-only -q` -- To run a specific test case, run `pytest terraform-provider-aws/internal/service// -k --ls-start` or `pytest terraform-provider-aws/internal/service//:: --ls-start` -- Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' pytest terraform-provider-aws/internal/service//:: --ls-start` +- To list down all the test case from a specific service, run `python -m pytest terraform-provider-aws/internal/service/ --collect-only -q` +- To run a specific test case, run `python -m pytest terraform-provider-aws/internal/service// -k --ls-start` or `python -m pytest terraform-provider-aws/internal/service//:: --ls-start` +- Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/internal/service//:: --ls-start` ## Default environment variables -- **TF_LOG**: ``debug``, -- **TF_ACC**: ``1``, -- **AWS_ACCESS_KEY_ID**: ``test``, -- **AWS_SECRET_ACCESS_KEY**: ``test``, -- **AWS_DEFAULT_REGION**: ``'us-east-1``' +- **TF_ACC**: `1` +- **AWS_ACCESS_KEY_ID**: `test` +- **AWS_SECRET_ACCESS_KEY**: `test` +- **AWS_DEFAULT_REGION**: `us-west-1` +- **AWS_ALTERNATE_ACCESS_KEY_ID**: `test` +- **AWS_ALTERNATE_SECRET_ACCESS_KEY**: `test` +- **AWS_ALTERNATE_SECRET_ACCESS_KEY**: `test` +- **AWS_ALTERNATE_REGION**: `us-east-2` +- **AWS_THIRD_REGION**: `eu-west-1` ## Options - `--ls-start`: Start localstack instance before running the test cases diff --git a/conftest.py b/conftest.py index 462a27c..4e479a6 100644 --- a/conftest.py +++ b/conftest.py @@ -1,32 +1,37 @@ +import os import re -import pytest +from os.path import dirname, realpath, relpath +from pathlib import Path + import docker +import pytest import requests from requests.adapters import HTTPAdapter, Retry -from pathlib import Path -import os -from os.path import realpath, relpath, dirname -from utils import execute_command, build_test_bin + +from terraform_pytest.utils import execute_command def pytest_addoption(parser): parser.addoption( - '--ls-image', action='store', default='localstack/localstack:latest', help='Base URL for the API tests' + "--ls-image", + action="store", + default="localstack/localstack:latest", + help="Base URL for the API tests", ) parser.addoption( - '--ls-start', action='store_true', default=False, help='Start localstack service' + "--ls-start", action="store_true", default=False, help="Start localstack service" ) def pytest_collect_file(parent, file_path): - if file_path.suffix == '.go' and file_path.name.endswith('_test.go'): + if file_path.suffix == ".go" and file_path.name.endswith("_test.go"): return GoFile.from_parent(parent, path=file_path) class GoFile(pytest.File): def collect(self): raw = self.path.open().read() - fa = re.findall(r'^(func (TestAcc.*))\(.*\).*', raw, re.MULTILINE) + fa = re.findall(r"^(func (TestAcc.*))\(.*\).*", raw, re.MULTILINE) for _, name in fa: yield GoItem.from_parent(self, name=name) @@ -41,25 +46,27 @@ def runtest(self): service = service_path.split(os.sep)[-1] env = dict(os.environ) - env.update({ - 'TF_ACC': '1', - 'AWS_ACCESS_KEY_ID': 'test', - 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-west-2', - 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_REGION': 'us-east-2', - 'AWS_THIRD_REGION': 'eu-west-1', - }) + env.update( + { + "TF_ACC": "1", + "AWS_ACCESS_KEY_ID": "test", + "AWS_SECRET_ACCESS_KEY": "test", + "AWS_DEFAULT_REGION": "us-west-1", + "AWS_ALTERNATE_ACCESS_KEY_ID": "test", + "AWS_ALTERNATE_SECRET_ACCESS_KEY": "test", + "AWS_ALTERNATE_SECRET_ACCESS_KEY": "test", + "AWS_ALTERNATE_REGION": "us-east-2", + "AWS_THIRD_REGION": "eu-west-1", + } + ) cmd = [ - f'./test-bin/{service}.test', - '-test.v', - '-test.parallel=1', - '-test.count=1', - '-test.timeout=60m', - f'-test.run={self.name}', + f"./test-bin/{service}.test", + "-test.v", + "-test.parallel=1", + "-test.count=1", + "-test.timeout=60m", + f"-test.run={self.name}", ] return_code, stdout = execute_command(cmd, env, tf_root_path) if return_code != 0: @@ -67,15 +74,15 @@ def runtest(self): def repr_failure(self, excinfo, **kwargs): if isinstance(excinfo.value, GoException): - return '\n'.join( + return "\n".join( [ - f'Execution failed with return code: {excinfo.value.returncode}', - f'Failure Reason:\n{excinfo.value.stderr}', + f"Execution failed with return code: {excinfo.value.returncode}", + f"Failure Reason:\n{excinfo.value.stderr}", ] ) def reportinfo(self): - return self.path, 0, f'Test Case: {self.name}' + return self.path, 0, f"Test Case: {self.name}" class ReprCrash: @@ -97,10 +104,10 @@ def pytest_runtest_makereport(item, call): outcome = yield report = outcome.get_result() if report.failed: - splits = report.longrepr.split('\n', 1) + splits = report.longrepr.split("\n", 1) longrepr = LongRepr(splits[0], splits[1]) - delattr(report, 'longrepr') - setattr(report, 'longrepr', longrepr) + delattr(report, "longrepr") + setattr(report, "longrepr", longrepr) class GoException(Exception): @@ -111,38 +118,44 @@ def __init__(self, returncode, stderr): def _docker_service_health(client): if not client.ping(): - print('\nPlease start docker daemon and try again') - raise Exception('Docker is not running') + print("\nPlease start docker daemon and try again") + raise Exception("Docker is not running") def _start_docker_container(client, config, localstack_image): - env_vars = ['DEBUG=1', 'PROVIDER_OVERRIDE_S3=asf'] + env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf"] port_mappings = { - '53/tcp': ('127.0.0.1', 53), - '53/udp': ('127.0.0.1', 53), - '443': ('127.0.0.1', 443), - '4566': ('127.0.0.1', 4566), - '4571': ('127.0.0.1', 4571), + "53/tcp": ("127.0.0.1", 53), + "53/udp": ("127.0.0.1", 53), + "443": ("127.0.0.1", 443), + "4566": ("127.0.0.1", 4566), + "4571": ("127.0.0.1", 4571), } - volumes = ['/var/run/docker.sock:/var/run/docker.sock'] - localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, - name='localstack_main', volumes=volumes, auto_remove=True, - environment=env_vars) - setattr(config, 'localstack_container_id', localstack_container.id) + volumes = ["/var/run/docker.sock:/var/run/docker.sock"] + localstack_container = client.containers.run( + image=localstack_image, + detach=True, + ports=port_mappings, + name="localstack_main", + volumes=volumes, + auto_remove=True, + environment=env_vars, + ) + setattr(config, "localstack_container_id", localstack_container.id) def _stop_docker_container(client, config): - client.containers.get(getattr(config, 'localstack_container_id')).stop() - print('LocalStack is stopped') + client.containers.get(getattr(config, "localstack_container_id")).stop() + print("LocalStack is stopped") def _localstack_health_check(): - localstack_health_url = 'http://localhost:4566/health' + localstack_health_url = "http://localhost:4566/health" session = requests.Session() retry = Retry(connect=3, backoff_factor=2) adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) + session.mount("http://", adapter) + session.mount("https://", adapter) session.get(localstack_health_url) session.close() @@ -150,19 +163,19 @@ def _localstack_health_check(): def _pull_docker_image(client, localstack_image): docker_image_list = client.images.list(name=localstack_image) if len(docker_image_list) == 0: - print(f'Pulling image {localstack_image}') + print(f"Pulling image {localstack_image}") client.images.pull(localstack_image) docker_image_list = client.images.list(name=localstack_image) - print(f'Using LocalStack image: {docker_image_list[0].id}') + print(f"Using LocalStack image: {docker_image_list[0].id}") def pytest_configure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - localstack_image = config.getoption(name='--ls-image') + is_collect_only = config.getoption(name="--collect-only") + is_localstack_start = config.getoption(name="--ls-start") + localstack_image = config.getoption(name="--ls-image") if not is_collect_only and is_localstack_start: - print('\nStarting LocalStack...') + print("\nStarting LocalStack...") client = docker.from_env() _docker_service_health(client) @@ -171,15 +184,15 @@ def pytest_configure(config): _localstack_health_check() client.close() - print('LocalStack is ready...') + print("LocalStack is ready...") def pytest_unconfigure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') + is_collect_only = config.getoption(name="--collect-only") + is_localstack_start = config.getoption(name="--ls-start") if not is_collect_only and is_localstack_start: - print('\nStopping LocalStack...') + print("\nStopping LocalStack...") client = docker.from_env() _stop_docker_container(client, config) client.close() diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index afe7983..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "3.8" - -services: - localstack: - container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" - image: localstack/localstack - ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - - "127.0.0.1:4510-4559:4510-4559" # external services port range - environment: - - DEBUG=${DEBUG-} - - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} - - DOCKER_HOST=unix:///var/run/docker.sock - volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/main.py b/main.py deleted file mode 100644 index 38b3d27..0000000 --- a/main.py +++ /dev/null @@ -1,46 +0,0 @@ -import click -from timeit import default_timer as timer -from utils import build_test_bin, TF_REPO_NAME, get_services -from os.path import realpath - - -@click.group(name='pytest-golang', help='Golang Test Runner for localstack') -def cli(): - pass - - -@click.command(name='patch', help='Patch the golang test runner') -def patch(): - from utils import patch_repo - patch_repo() - - -@click.command(name='build', help='Build binary for testing') -@click.option('--service', '-s', default=None, help='''Service to build; use "ls-all", "ls-community", "ls-pro" to build all services, example: ---service=ls-all; --service=ec2; --service=ec2,iam''') -def build(service): - """Build binary for testing""" - if not service: - print('No service provided') - print('use --service or -s to specify services to build; for more help try --help to see more options') - return - - services = get_services(service) - - for service in services: - print(f'Building {service}...') - try: - start = timer() - build_test_bin(service=service, tf_root_path=realpath(TF_REPO_NAME)) - end = timer() - print(f'Build {service} in {end - start} seconds') - except KeyboardInterrupt: - print('Interrupted') - return - except Exception as e: - print(f'Failed to build binary for {service}: {e}') - - -cli.add_command(build) -cli.add_command(patch) -cli() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e9451ca --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +# project configuration + +[tool.black] +line_length = 100 +include = '(terraform_pytest\/.*\.py$|tests\/.*\.py$)' +#extend_exclude = '()' + +[tool.isort] +profile = 'black' +#extend_skip = [] +line_length = 100 + +[tool.pytest.ini_options] +testpaths = [ + "terraform-provider-aws/internal/service/", +] +#confcutdir = "tests/conftest.py" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 12240bb..ab85810 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ click==8.1.3 pytest==7.2.0 docker==6.0.1 requests==2.28.2 +black>=22.1 +isort>=5.10 \ No newline at end of file diff --git a/get-services.py b/terraform_pytest/get-services.py similarity index 69% rename from get-services.py rename to terraform_pytest/get-services.py index 3c87155..a702011 100644 --- a/get-services.py +++ b/terraform_pytest/get-services.py @@ -1,7 +1,7 @@ import json import sys -from utils import get_services +from terraform_pytest.utils import get_services services = [] @@ -11,6 +11,5 @@ print(json.dumps(services)) exit(0) else: - print('No service provided') + print("No service provided") exit(1) - diff --git a/terraform_pytest/main.py b/terraform_pytest/main.py new file mode 100644 index 0000000..57cc802 --- /dev/null +++ b/terraform_pytest/main.py @@ -0,0 +1,55 @@ +from os.path import realpath +from timeit import default_timer as timer + +import click + +from terraform_pytest.utils import TF_REPO_NAME, build_test_bin, get_services, patch_repo + + +@click.group(name="pytest-golang", help="Golang Test Runner for localstack") +def cli(): + pass + + +@click.command(name="patch", help="Patch the golang test runner") +def patch(): + patch_repo() + + +@click.command(name="build", help="Build binary for testing") +@click.option( + "--service", + "-s", + default=None, + help="""Service to build; use "ls-all", "ls-community", "ls-pro" to build all services, example: +--service=ls-all; --service=ec2; --service=ec2,iam""", +) +def build(service): + """Build binary for testing""" + if not service: + print("No service provided") + print( + "use --service or -s to specify services to build; for more help try --help to see more options" + ) + return + + services = get_services(service) + + for service in services: + print(f"Building {service}...") + try: + start = timer() + build_test_bin(service=service, tf_root_path=realpath(TF_REPO_NAME)) + end = timer() + print(f"Build {service} in {end - start} seconds") + except KeyboardInterrupt: + print("Interrupted") + return + except Exception as e: + print(f"Failed to build binary for {service}: {e}") + + +if __name__ == "__main__": + cli.add_command(build) + cli.add_command(patch) + cli() diff --git a/terraform_pytest/utils.py b/terraform_pytest/utils.py new file mode 100644 index 0000000..4226d9e --- /dev/null +++ b/terraform_pytest/utils.py @@ -0,0 +1,201 @@ +import signal +from os import chdir, chmod, getcwd, listdir, system +from os.path import exists, realpath +from uuid import uuid4 + +TF_REPO_NAME = "terraform-provider-aws" +TF_REPO_PATH = f"{realpath(TF_REPO_NAME)}" + +TF_REPO_PATCH_FILES = ["etc/001-hardcode-endpoint.patch"] + +TF_TEST_BINARY_FOLDER = "test-bin" +TF_REPO_SERVICE_FOLDER = "./internal/service" + +BLACKLISTED_SERVICES = ["controltower", "greengrass"] +LS_COMMUNITY_SERVICES = [ + "acm", + "apigateway", + "lambda", + "cloudformation", + "cloudwatch", + "configservice", + "dynamodb", + "ec2", + "elasticsearch", + "events", + "firehose", + "iam", + "kinesis", + "kms", + "logs", + "opensearch", + "redshift", + "resourcegroups", + "resourcegroupstaggingapi", + "route53", + "route53resolver", + "s3", + "s3control", + "secretsmanager", + "ses", + "sns", + "sqs", + "ssm", + "sts", + "swf", + "transcribe", +] +LS_PRO_SERVICES = [ + "amplify", + "apigateway", + "apigatewayv2", + "appconfig", + "appautoscaling", + "appsync", + "athena", + "autoscaling", + "backup", + "batch", + "cloudformation", + "cloudfront", + "cloudtrail", + "codecommit", + "cognitoidp", + "cognitoidentity", + "docdb", + "dynamodb", + "ec2", + "ecr", + "ecs", + "efs", + "eks", + "elasticache", + "elasticbeanstalk", + "elb", + "elbv2", + "emr", + "events", + "fis", + "glacier", + "glue", + "iam", + "iot", + "iotanalytics", + "kafka", + "kinesisanalytics", + "kms", + "lakeformation", + "lambda", + "logs", + "mediastore", + "mq", + "mwaa", + "neptune", + "organizations", + "qldb", + "rds", + "redshift", + "route53", + "s3", + "sagemaker", + "secretsmanager", + "serverlessrepo", + "ses", + "sns", + "sqs", + "ssm", + "sts", +] + + +def _get_test_bin_abs_path(service): + return f"{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test" + + +def execute_command(cmd, env=None, cwd=None): + """ + Execute a command and return the return code. + """ + _lwd = getcwd() + if isinstance(cmd, list): + cmd = " ".join(cmd) + else: + raise Exception("Please provide command as list(str)") + if cwd: + chdir(cwd) + if env: + _env = " ".join([f'{k}="{str(v)}"' for k, v in env.items()]) + cmd = f"{_env} {cmd}" + log_file: str = "/tmp/%s" % uuid4().hex + _err = system(f"{cmd} > {log_file} 2>&1") + if _err == signal.SIGINT: + print("SIGNINT is caught") + raise KeyboardInterrupt + _out = open(log_file, "r").read() + chdir(_lwd) + return _err, _out + + +def build_test_bin(service, tf_root_path): + _test_bin_abs_path = _get_test_bin_abs_path(service) + _tf_repo_service_folder = f"{TF_REPO_SERVICE_FOLDER}/{service}" + + if exists(_test_bin_abs_path): + return + + cmd = [ + "go", + "test", + "-c", + _tf_repo_service_folder, + "-o", + _test_bin_abs_path, + ] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}") + + if exists(_test_bin_abs_path): + chmod(_test_bin_abs_path, 0o755) + + return return_code, stdout + + +def get_all_services(): + services = [] + for service in listdir(f"{TF_REPO_PATH}/{TF_REPO_SERVICE_FOLDER}"): + if service not in BLACKLISTED_SERVICES: + services.append(service) + return sorted(services) + + +def get_services(service): + if service == "ls-community": + services = LS_COMMUNITY_SERVICES + elif service == "ls-pro": + services = LS_PRO_SERVICES + elif service == "ls-all": + services = LS_COMMUNITY_SERVICES + LS_PRO_SERVICES + else: + if "," in service: + services = service.split(",") + services = [s for s in services if s] + else: + services = [service] + services = [s for s in services if s not in BLACKLISTED_SERVICES] + return list(set(services)) + + +def patch_repo(): + print(f"Patching {TF_REPO_NAME}...") + for patch_file in TF_REPO_PATCH_FILES: + cmd = [ + "git", + "apply", + f"{realpath(patch_file)}", + ] + return_code, stdout = execute_command(cmd, cwd=realpath(TF_REPO_NAME)) + if return_code != 0: + print("----- error while patching repo -----") + if stdout: + print(f"stdout: {stdout}") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0c71c01 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,185 @@ +import re +import pytest +import docker +import requests +from requests.adapters import HTTPAdapter, Retry +from pathlib import Path +import os +from os.path import realpath, relpath, dirname +from utils import execute_command, build_test_bin + + +def pytest_addoption(parser): + parser.addoption( + '--ls-image', action='store', default='localstack/localstack:latest', help='Base URL for the API tests' + ) + parser.addoption( + '--ls-start', action='store_true', default=False, help='Start localstack service' + ) + + +def pytest_collect_file(parent, file_path): + if file_path.suffix == '.go' and file_path.name.endswith('_test.go'): + return GoFile.from_parent(parent, path=file_path) + + +class GoFile(pytest.File): + def collect(self): + raw = self.path.open().read() + fa = re.findall(r'^(func (TestAcc.*))\(.*\).*', raw, re.MULTILINE) + for _, name in fa: + yield GoItem.from_parent(self, name=name) + + +class GoItem(pytest.Item): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def runtest(self): + tf_root_path = realpath(relpath(self.path).split(os.sep)[0]) + service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) + service = service_path.split(os.sep)[-1] + + env = dict(os.environ) + env.update({ + 'TF_ACC': '1', + 'AWS_ACCESS_KEY_ID': 'test', + 'AWS_SECRET_ACCESS_KEY': 'test', + 'AWS_DEFAULT_REGION': 'us-west-1', + 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', + 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', + 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', + 'AWS_ALTERNATE_REGION': 'us-east-2', + 'AWS_THIRD_REGION': 'eu-west-1', + }) + + cmd = [ + f'./test-bin/{service}.test', + '-test.v', + '-test.parallel=1', + '-test.count=1', + '-test.timeout=60m', + f'-test.run={self.name}', + ] + return_code, stdout = execute_command(cmd, env, tf_root_path) + if return_code != 0: + raise GoException(returncode=return_code, stderr=stdout) + + def repr_failure(self, excinfo, **kwargs): + if isinstance(excinfo.value, GoException): + return '\n'.join( + [ + f'Execution failed with return code: {excinfo.value.returncode}', + f'Failure Reason:\n{excinfo.value.stderr}', + ] + ) + + def reportinfo(self): + return self.path, 0, f'Test Case: {self.name}' + + +class ReprCrash: + def __init__(self, message): + self.message = message + + +class LongRepr: + def __init__(self, message, reason): + self.reprcrash = ReprCrash(message) + self.reason = reason + + def __str__(self): + return self.reason + + +@pytest.hookimpl(tryfirst=True, hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + if report.failed: + splits = report.longrepr.split('\n', 1) + longrepr = LongRepr(splits[0], splits[1]) + delattr(report, 'longrepr') + setattr(report, 'longrepr', longrepr) + + +class GoException(Exception): + def __init__(self, returncode, stderr): + self.returncode = returncode + self.stderr = stderr + + +def _docker_service_health(client): + if not client.ping(): + print('\nPlease start docker daemon and try again') + raise Exception('Docker is not running') + + +def _start_docker_container(client, config, localstack_image): + env_vars = ['DEBUG=1', 'PROVIDER_OVERRIDE_S3=asf'] + port_mappings = { + '53/tcp': ('127.0.0.1', 53), + '53/udp': ('127.0.0.1', 53), + '443': ('127.0.0.1', 443), + '4566': ('127.0.0.1', 4566), + '4571': ('127.0.0.1', 4571), + } + volumes = ['/var/run/docker.sock:/var/run/docker.sock'] + localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, + name='localstack_main', volumes=volumes, auto_remove=True, + environment=env_vars) + setattr(config, 'localstack_container_id', localstack_container.id) + + +def _stop_docker_container(client, config): + client.containers.get(getattr(config, 'localstack_container_id')).stop() + print('LocalStack is stopped') + + +def _localstack_health_check(): + localstack_health_url = 'http://localhost:4566/health' + session = requests.Session() + retry = Retry(connect=3, backoff_factor=2) + adapter = HTTPAdapter(max_retries=retry) + session.mount('http://', adapter) + session.mount('https://', adapter) + session.get(localstack_health_url) + session.close() + + +def _pull_docker_image(client, localstack_image): + docker_image_list = client.images.list(name=localstack_image) + if len(docker_image_list) == 0: + print(f'Pulling image {localstack_image}') + client.images.pull(localstack_image) + docker_image_list = client.images.list(name=localstack_image) + print(f'Using LocalStack image: {docker_image_list[0].id}') + + +def pytest_configure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + localstack_image = config.getoption(name='--ls-image') + + if not is_collect_only and is_localstack_start: + print('\nStarting LocalStack...') + + client = docker.from_env() + _docker_service_health(client) + _pull_docker_image(client, localstack_image) + _start_docker_container(client, config, localstack_image) + _localstack_health_check() + client.close() + + print('LocalStack is ready...') + + +def pytest_unconfigure(config): + is_collect_only = config.getoption(name='--collect-only') + is_localstack_start = config.getoption(name='--ls-start') + + if not is_collect_only and is_localstack_start: + print('\nStopping LocalStack...') + client = docker.from_env() + _stop_docker_container(client, config) + client.close() diff --git a/utils.py b/utils.py deleted file mode 100644 index a7b1496..0000000 --- a/utils.py +++ /dev/null @@ -1,122 +0,0 @@ -import signal -from os import system, getcwd, chdir, chmod, listdir -from os.path import exists, realpath -from uuid import uuid4 - - -TF_REPO_NAME = 'terraform-provider-aws' -TF_REPO_PATH = f'{realpath(TF_REPO_NAME)}' - -TF_REPO_PATCH_FILES = ['etc/001-hardcode-endpoint.patch'] - -TF_TEST_BINARY_FOLDER = 'test-bin' -TF_REPO_SERVICE_FOLDER = './internal/service' - -BLACKLISTED_SERVICES = ['controltower', 'greengrass'] -LS_COMMUNITY_SERVICES = [ - "acm", "apigateway", "lambda", "cloudformation", "cloudwatch", "configservice", "dynamodb", "ec2", "elasticsearch", - "events", "firehose", "iam", "kinesis", "kms", "logs", "opensearch", "redshift", "resourcegroups", - "resourcegroupstaggingapi", "route53", "route53resolver", "s3", "s3control", "secretsmanager", "ses", "sns", "sqs", - "ssm", "sts", "swf", "transcribe" -] -LS_PRO_SERVICES = [ - "amplify", "apigateway", "apigatewayv2", "appconfig", "appautoscaling", "appsync", "athena", "autoscaling", - "backup", "batch", "cloudformation", "cloudfront", "cloudtrail", "codecommit", "cognitoidp", "cognitoidentity", - "docdb", "dynamodb", "ec2", "ecr", "ecs", "efs", "eks", "elasticache", "elasticbeanstalk", "elb", "elbv2", "emr", - "events", "fis", "glacier", "glue", "iam", "iot", "iotanalytics", "kafka", "kinesisanalytics", "kms", - "lakeformation", "lambda", "logs", "mediastore", "mq", "mwaa", "neptune", "organizations", "qldb", "rds", - "redshift", "route53", "s3", "sagemaker", "secretsmanager", "serverlessrepo", "ses", "sns", "sqs", "ssm", "sts" -] - - -def _get_test_bin_abs_path(service): - return f'{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test' - - -def execute_command(cmd, env=None, cwd=None): - """ - Execute a command and return the return code. - """ - _lwd = getcwd() - if isinstance(cmd, list): - cmd = ' '.join(cmd) - else: - raise Exception("Please provide command as list(str)") - if cwd: - chdir(cwd) - if env: - _env = ' '.join([f'{k}="{str(v)}"' for k, v in env.items()]) - cmd = f'{_env} {cmd}' - log_file: str = '/tmp/%s' % uuid4().hex - _err = system(f'{cmd} > {log_file} 2>&1') - if _err == signal.SIGINT: - print("SIGNINT is caught") - raise KeyboardInterrupt - _out = open(log_file, 'r').read() - chdir(_lwd) - return _err, _out - - -def build_test_bin(service, tf_root_path): - _test_bin_abs_path = _get_test_bin_abs_path(service) - _tf_repo_service_folder = f'{TF_REPO_SERVICE_FOLDER}/{service}' - - if exists(_test_bin_abs_path): - return - - cmd = [ - "go", - "test", - "-c", - _tf_repo_service_folder, - "-o", - _test_bin_abs_path, - ] - return_code, stdout = execute_command(cmd, cwd=tf_root_path) - if return_code != 0: - raise Exception(f"Error while building test binary for {service}") - - if exists(_test_bin_abs_path): - chmod(_test_bin_abs_path, 0o755) - - return return_code, stdout - - -def get_all_services(): - services = [] - for service in listdir(f'{TF_REPO_PATH}/{TF_REPO_SERVICE_FOLDER}'): - if service not in BLACKLISTED_SERVICES: - services.append(service) - return sorted(services) - - -def get_services(service): - if service == 'ls-community': - services = LS_COMMUNITY_SERVICES - elif service == 'ls-pro': - services = LS_PRO_SERVICES - elif service == 'ls-all': - services = LS_COMMUNITY_SERVICES + LS_PRO_SERVICES - else: - if ',' in service: - services = service.split(',') - services = [s for s in services if s] - else: - services = [service] - services = [s for s in services if s not in BLACKLISTED_SERVICES] - return list(set(services)) - - -def patch_repo(): - print(f'Patching {TF_REPO_NAME}...') - for patch_file in TF_REPO_PATCH_FILES: - cmd = [ - 'git', - 'apply', - f'{realpath(patch_file)}', - ] - return_code, stdout = execute_command(cmd, cwd=realpath(TF_REPO_NAME)) - if return_code != 0: - print("----- error while patching repo -----") - if stdout: - print(f'stdout: {stdout}') From b2c1d6fbe7f5f62ea373b670f8dc30ddde817739 Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Fri, 20 Jan 2023 13:44:16 +0530 Subject: [PATCH 91/94] running parallel tests for ec2 (#16) --- .github/workflows/main.yml | 9 ++ .gitignore | 3 +- Makefile | 2 +- README.md | 13 ++- conftest.py | 27 +----- docker-compose.yml | 18 ++++ pyproject.toml | 5 +- requirements.txt | 3 +- terraform_pytest/utils.py | 12 ++- tests/conftest.py | 185 ------------------------------------- 10 files changed, 57 insertions(+), 220 deletions(-) create mode 100644 docker-compose.yml delete mode 100644 tests/conftest.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0d745f9..ca3205b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,9 +77,18 @@ jobs: ls -la terraform-provider-aws/test-bin - name: Run ${{ matrix.service }} Tests + if: ${{ matrix.service != 'ec2' }} run: | python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} + - name: Run ${{ matrix.service }} Tests + if: ${{ matrix.service == 'ec2' }} + run: | + IMAGE_TAG=${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} docker-compose up -d + sleep 20 + python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v -n 2 + + - name: Publish ${{ matrix.service }} Test Results uses: EnricoMi/publish-unit-test-result-action@v2 if: always() diff --git a/.gitignore b/.gitignore index da78a2b..77c853a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__ target **/*.test -report.xml \ No newline at end of file +report.xml +volume diff --git a/Makefile b/Makefile index 4d7c068..70f2fc1 100644 --- a/Makefile +++ b/Makefile @@ -23,4 +23,4 @@ install: $(VENV_RUN); $(PIP_CMD) install -r requirements.txt format: - $(VENV_RUN); python -m isort .; python -m black . \ No newline at end of file + $(VENV_RUN); python -m isort .; python -m black . diff --git a/README.md b/README.md index 39f9b8e..0903225 100644 --- a/README.md +++ b/README.md @@ -5,21 +5,23 @@ This is a test runner for localstack and terraform. It will run a test cases fro Purpose of this project is to externalize the test cases from the localstack repo and run them against localstack to gather parity metrics. ## Installation -1. Clone the repository with submodules` +1. Clone the repository with submodules + - `git clone git@github.com:localstack/localstack-terraform-test.git --recurse-submodules` + - Make sure you have the latest version of the submodules after switching to a different branch using `git submodule update --init --recursive` 2. Run `make venv` to create a virtual environment 3. Run `make install` to install the dependencies ## How to run? 1. Run `python -m terraform_pytest.main patch` to apply the patch to the terraform provider aws 2. Run `python -m terraform_pytest.main build -s s3` to build testing binary for the golang module -3Now you are ready to use `python -m pytest` commands to list and run test cases from golang +3. Now you are ready to use `python -m pytest` commands to list and run test cases from golang ## How to run test cases? - To list down all the test case from a specific service, run `python -m pytest terraform-provider-aws/internal/service/ --collect-only -q` - To run a specific test case, run `python -m pytest terraform-provider-aws/internal/service// -k --ls-start` or `python -m pytest terraform-provider-aws/internal/service//:: --ls-start` - Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/internal/service//:: --ls-start` -## Default environment variables +## Default environment variables for Terraform Tests - **TF_ACC**: `1` - **AWS_ACCESS_KEY_ID**: `test` - **AWS_SECRET_ACCESS_KEY**: `test` @@ -30,6 +32,11 @@ Purpose of this project is to externalize the test cases from the localstack rep - **AWS_ALTERNATE_REGION**: `us-east-2` - **AWS_THIRD_REGION**: `eu-west-1` +## Environment variables for Localstack +- **DEBUG**: `1` +- **PROVIDER_OVERRIDE_S3**: `asf` +- **FAIL_FAST**: `1` + ## Options - `--ls-start`: Start localstack instance before running the test cases - `--ls-image`: Specify the localstack image to use, default is `localstack/localstack:latest` \ No newline at end of file diff --git a/conftest.py b/conftest.py index 4e479a6..c5dad09 100644 --- a/conftest.py +++ b/conftest.py @@ -85,31 +85,6 @@ def reportinfo(self): return self.path, 0, f"Test Case: {self.name}" -class ReprCrash: - def __init__(self, message): - self.message = message - - -class LongRepr: - def __init__(self, message, reason): - self.reprcrash = ReprCrash(message) - self.reason = reason - - def __str__(self): - return self.reason - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - report = outcome.get_result() - if report.failed: - splits = report.longrepr.split("\n", 1) - longrepr = LongRepr(splits[0], splits[1]) - delattr(report, "longrepr") - setattr(report, "longrepr", longrepr) - - class GoException(Exception): def __init__(self, returncode, stderr): self.returncode = returncode @@ -123,7 +98,7 @@ def _docker_service_health(client): def _start_docker_container(client, config, localstack_image): - env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf"] + env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf", "FAIL_FAST=1"] port_mappings = { "53/tcp": ("127.0.0.1", 53), "53/udp": ("127.0.0.1", 53), diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4688d35 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,18 @@ +version: "3.8" + +services: + localstack: + container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" + image: ${IMAGE_TAG-localstack/localstack:latest} + ports: + - "127.0.0.1:4566:4566" # LocalStack Gateway + - "127.0.0.1:4510-4559:4510-4559" # external services port range + environment: + - DEBUG=1 + - PROVIDER_OVERRIDE_S3=asf + - FAIL_FAST=1 + - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} + - DOCKER_HOST=unix:///var/run/docker.sock + volumes: + - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" diff --git a/pyproject.toml b/pyproject.toml index e9451ca..10888b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,11 +2,12 @@ [tool.black] line_length = 100 -include = '(terraform_pytest\/.*\.py$|tests\/.*\.py$)' +include = '(^terraform_pytest\/.*\.py$|^conftest.py$)' #extend_exclude = '()' [tool.isort] profile = 'black' +skip_glob = ["terraform-provider-aws"] #extend_skip = [] line_length = 100 @@ -14,4 +15,4 @@ line_length = 100 testpaths = [ "terraform-provider-aws/internal/service/", ] -#confcutdir = "tests/conftest.py" \ No newline at end of file +filterwarnings =["ignore::DeprecationWarning"] diff --git a/requirements.txt b/requirements.txt index ab85810..f294c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,5 @@ pytest==7.2.0 docker==6.0.1 requests==2.28.2 black>=22.1 -isort>=5.10 \ No newline at end of file +isort>=5.10 +pytest-xdist>=3.1.0 diff --git a/terraform_pytest/utils.py b/terraform_pytest/utils.py index 4226d9e..0d007a7 100644 --- a/terraform_pytest/utils.py +++ b/terraform_pytest/utils.py @@ -143,6 +143,16 @@ def build_test_bin(service, tf_root_path): if exists(_test_bin_abs_path): return + cmd = ["go", "mod", "tidy"] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") + + cmd = ["go", "mod", "vendor"] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") + cmd = [ "go", "test", @@ -153,7 +163,7 @@ def build_test_bin(service, tf_root_path): ] return_code, stdout = execute_command(cmd, cwd=tf_root_path) if return_code != 0: - raise Exception(f"Error while building test binary for {service}") + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") if exists(_test_bin_abs_path): chmod(_test_bin_abs_path, 0o755) diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 0c71c01..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,185 +0,0 @@ -import re -import pytest -import docker -import requests -from requests.adapters import HTTPAdapter, Retry -from pathlib import Path -import os -from os.path import realpath, relpath, dirname -from utils import execute_command, build_test_bin - - -def pytest_addoption(parser): - parser.addoption( - '--ls-image', action='store', default='localstack/localstack:latest', help='Base URL for the API tests' - ) - parser.addoption( - '--ls-start', action='store_true', default=False, help='Start localstack service' - ) - - -def pytest_collect_file(parent, file_path): - if file_path.suffix == '.go' and file_path.name.endswith('_test.go'): - return GoFile.from_parent(parent, path=file_path) - - -class GoFile(pytest.File): - def collect(self): - raw = self.path.open().read() - fa = re.findall(r'^(func (TestAcc.*))\(.*\).*', raw, re.MULTILINE) - for _, name in fa: - yield GoItem.from_parent(self, name=name) - - -class GoItem(pytest.Item): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - def runtest(self): - tf_root_path = realpath(relpath(self.path).split(os.sep)[0]) - service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) - service = service_path.split(os.sep)[-1] - - env = dict(os.environ) - env.update({ - 'TF_ACC': '1', - 'AWS_ACCESS_KEY_ID': 'test', - 'AWS_SECRET_ACCESS_KEY': 'test', - 'AWS_DEFAULT_REGION': 'us-west-1', - 'AWS_ALTERNATE_ACCESS_KEY_ID': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_SECRET_ACCESS_KEY': 'test', - 'AWS_ALTERNATE_REGION': 'us-east-2', - 'AWS_THIRD_REGION': 'eu-west-1', - }) - - cmd = [ - f'./test-bin/{service}.test', - '-test.v', - '-test.parallel=1', - '-test.count=1', - '-test.timeout=60m', - f'-test.run={self.name}', - ] - return_code, stdout = execute_command(cmd, env, tf_root_path) - if return_code != 0: - raise GoException(returncode=return_code, stderr=stdout) - - def repr_failure(self, excinfo, **kwargs): - if isinstance(excinfo.value, GoException): - return '\n'.join( - [ - f'Execution failed with return code: {excinfo.value.returncode}', - f'Failure Reason:\n{excinfo.value.stderr}', - ] - ) - - def reportinfo(self): - return self.path, 0, f'Test Case: {self.name}' - - -class ReprCrash: - def __init__(self, message): - self.message = message - - -class LongRepr: - def __init__(self, message, reason): - self.reprcrash = ReprCrash(message) - self.reason = reason - - def __str__(self): - return self.reason - - -@pytest.hookimpl(tryfirst=True, hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - report = outcome.get_result() - if report.failed: - splits = report.longrepr.split('\n', 1) - longrepr = LongRepr(splits[0], splits[1]) - delattr(report, 'longrepr') - setattr(report, 'longrepr', longrepr) - - -class GoException(Exception): - def __init__(self, returncode, stderr): - self.returncode = returncode - self.stderr = stderr - - -def _docker_service_health(client): - if not client.ping(): - print('\nPlease start docker daemon and try again') - raise Exception('Docker is not running') - - -def _start_docker_container(client, config, localstack_image): - env_vars = ['DEBUG=1', 'PROVIDER_OVERRIDE_S3=asf'] - port_mappings = { - '53/tcp': ('127.0.0.1', 53), - '53/udp': ('127.0.0.1', 53), - '443': ('127.0.0.1', 443), - '4566': ('127.0.0.1', 4566), - '4571': ('127.0.0.1', 4571), - } - volumes = ['/var/run/docker.sock:/var/run/docker.sock'] - localstack_container = client.containers.run(image=localstack_image, detach=True, ports=port_mappings, - name='localstack_main', volumes=volumes, auto_remove=True, - environment=env_vars) - setattr(config, 'localstack_container_id', localstack_container.id) - - -def _stop_docker_container(client, config): - client.containers.get(getattr(config, 'localstack_container_id')).stop() - print('LocalStack is stopped') - - -def _localstack_health_check(): - localstack_health_url = 'http://localhost:4566/health' - session = requests.Session() - retry = Retry(connect=3, backoff_factor=2) - adapter = HTTPAdapter(max_retries=retry) - session.mount('http://', adapter) - session.mount('https://', adapter) - session.get(localstack_health_url) - session.close() - - -def _pull_docker_image(client, localstack_image): - docker_image_list = client.images.list(name=localstack_image) - if len(docker_image_list) == 0: - print(f'Pulling image {localstack_image}') - client.images.pull(localstack_image) - docker_image_list = client.images.list(name=localstack_image) - print(f'Using LocalStack image: {docker_image_list[0].id}') - - -def pytest_configure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - localstack_image = config.getoption(name='--ls-image') - - if not is_collect_only and is_localstack_start: - print('\nStarting LocalStack...') - - client = docker.from_env() - _docker_service_health(client) - _pull_docker_image(client, localstack_image) - _start_docker_container(client, config, localstack_image) - _localstack_health_check() - client.close() - - print('LocalStack is ready...') - - -def pytest_unconfigure(config): - is_collect_only = config.getoption(name='--collect-only') - is_localstack_start = config.getoption(name='--ls-start') - - if not is_collect_only and is_localstack_start: - print('\nStopping LocalStack...') - client = docker.from_env() - _stop_docker_container(client, config) - client.close() From 2d4e3b81f88a0682f52705928dfc75b99be84e69 Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Mon, 23 Jan 2023 13:57:47 +0530 Subject: [PATCH 92/94] review resolution (#17) * added fail fast in localstack start * updated: running tests ec2 in parrallel * updated: concurrency, added ec2 for testing * updated parrellel run * updated: changes for upstream * updated: dependancies version * review resolution * testing pipeline * added comments; added all svc * testing * added all svc * removed hook --- .github/workflows/lint.yml | 20 ++++++++++ .github/workflows/main.yml | 11 ++---- .pre-commit-config.yaml | 19 +++++++++ Makefile | 16 ++++++-- conftest.py | 54 ++++++++++++++++++++----- pyproject.toml | 7 +--- requirements.txt | 9 +++-- terraform_pytest/main.py | 16 +++----- terraform_pytest/utils.py | 80 +++++++++++++++++++++++++++++--------- 9 files changed, 174 insertions(+), 58 deletions(-) create mode 100644 .github/workflows/lint.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..599525d --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,20 @@ +on: [push, pull_request] + +name: Linting +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10.5 + uses: actions/setup-python@v4 + with: + python-version: '3.10.5' + cache: 'pip' + - name: Install system dependencies + run: | + pip install --upgrade pip + pip install -r requirements.txt + - name: Run Linting + run: | + make lint diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca3205b..5b4ecae 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,10 +12,6 @@ on: type: string default: 'localstack/localstack:latest' description: localstack docker image name to test against -# disabled pipeline for push events, as we want to run the pipeline only on a schedule or manually -# push: -# branches: -# - pytest-plugin name: Terraform Tests jobs: @@ -60,8 +56,6 @@ jobs: - name: Install system dependencies run: | - sudo apt update --fix-missing - sudo apt install libsasl2-dev -y pip install --upgrade pip pip install -r requirements.txt @@ -77,9 +71,10 @@ jobs: ls -la terraform-provider-aws/test-bin - name: Run ${{ matrix.service }} Tests - if: ${{ matrix.service != 'ec2' }} + env: + PYTEST_PARALLEL_CONFIG: "${{ matrix.service == 'ec2' && '-n 2' || '' }}" run: | - python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} + python -m pytest $PYTEST_PARALLEL_CONFIG --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} - name: Run ${{ matrix.service }} Tests if: ${{ matrix.service == 'ec2' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..271255b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,19 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + + - repo: https://github.com/pycqa/isort + rev: 5.9.1 + hooks: + - id: isort + name: isort (python) + - id: isort + name: isort (cython) + types: [cython] + - id: isort + name: isort (pyi) + types: [pyi] diff --git a/Makefile b/Makefile index 70f2fc1..6fd2b21 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,9 @@ else VENV_ACTIVATE = $(VENV_DIR)/bin/activate endif +usage: ## Show this help + @fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/:.*##\s*/##/g' | awk -F'##' '{ printf "%-25s %s\n", $$1, $$2 }' + $(VENV_ACTIVATE): test -d $(VENV_DIR) || $(VENV_BIN) $(VENV_DIR) $(VENV_RUN); $(PIP_CMD) install --upgrade pip setuptools wheel plux @@ -17,10 +20,17 @@ $(VENV_ACTIVATE): VENV_RUN = . $(VENV_ACTIVATE) -venv: $(VENV_ACTIVATE) ## Create a new (empty) virtual environment +venv: $(VENV_ACTIVATE) ## Create a new (empty) virtual environment -install: +install: ## Install the package in editable mode $(VENV_RUN); $(PIP_CMD) install -r requirements.txt -format: +init-precommit: ## install te pre-commit hook into your local git repository + ($(VENV_RUN); pre-commit install) + +lint: ## Run linting + @echo "Running black... " + black --check . + +format: ## Run formatting $(VENV_RUN); python -m isort .; python -m black . diff --git a/conftest.py b/conftest.py index c5dad09..0dfc2d6 100644 --- a/conftest.py +++ b/conftest.py @@ -12,6 +12,7 @@ def pytest_addoption(parser): + """Add command line options to pytest""" parser.addoption( "--ls-image", action="store", @@ -24,12 +25,16 @@ def pytest_addoption(parser): def pytest_collect_file(parent, file_path): + """Collect test files from the test directory""" if file_path.suffix == ".go" and file_path.name.endswith("_test.go"): return GoFile.from_parent(parent, path=file_path) class GoFile(pytest.File): + """class for collecting tests from a file.""" + def collect(self): + """Collect test cases from the test file""" raw = self.path.open().read() fa = re.findall(r"^(func (TestAcc.*))\(.*\).*", raw, re.MULTILINE) for _, name in fa: @@ -37,10 +42,14 @@ def collect(self): class GoItem(pytest.Item): + """class for individual test cases.""" + def __init__(self, **kwargs): super().__init__(**kwargs) def runtest(self): + """Run the test case""" + tf_root_path = realpath(relpath(self.path).split(os.sep)[0]) service_path = dirname(Path(*relpath(self.path).split(os.sep)[1:])) service = service_path.split(os.sep)[-1] @@ -73,6 +82,10 @@ def runtest(self): raise GoException(returncode=return_code, stderr=stdout) def repr_failure(self, excinfo, **kwargs): + """Called when self.runtest() raises an exception. + + return: a representation of a collection failure. + """ if isinstance(excinfo.value, GoException): return "\n".join( [ @@ -82,22 +95,33 @@ def repr_failure(self, excinfo, **kwargs): ) def reportinfo(self): + """Get location information for this item for test reports. + + return: a tuple with three elements: + - The path of the test + - The line number of the test + - A name of the test to be shown in reports + """ return self.path, 0, f"Test Case: {self.name}" class GoException(Exception): + """Go test exception - raised when test cases failed""" + def __init__(self, returncode, stderr): self.returncode = returncode self.stderr = stderr def _docker_service_health(client): + """Check if the docker service is healthy""" if not client.ping(): print("\nPlease start docker daemon and try again") raise Exception("Docker is not running") def _start_docker_container(client, config, localstack_image): + """Start the docker container""" env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf", "FAIL_FAST=1"] port_mappings = { "53/tcp": ("127.0.0.1", 53), @@ -120,11 +144,13 @@ def _start_docker_container(client, config, localstack_image): def _stop_docker_container(client, config): + """Stop the docker container""" client.containers.get(getattr(config, "localstack_container_id")).stop() print("LocalStack is stopped") def _localstack_health_check(): + """Check if the localstack service is healthy""" localstack_health_url = "http://localhost:4566/health" session = requests.Session() retry = Retry(connect=3, backoff_factor=2) @@ -136,6 +162,7 @@ def _localstack_health_check(): def _pull_docker_image(client, localstack_image): + """Pull the docker image""" docker_image_list = client.images.list(name=localstack_image) if len(docker_image_list) == 0: print(f"Pulling image {localstack_image}") @@ -144,10 +171,14 @@ def _pull_docker_image(client, localstack_image): print(f"Using LocalStack image: {docker_image_list[0].id}") -def pytest_configure(config): - is_collect_only = config.getoption(name="--collect-only") - is_localstack_start = config.getoption(name="--ls-start") - localstack_image = config.getoption(name="--ls-image") +def pytest_sessionstart(session): + """Called after the Session object has been created and before performing collection and entering the run test loop.""" + is_collect_only = session.config.getoption(name="--collect-only") + is_localstack_start = session.config.getoption(name="--ls-start") + localstack_image = session.config.getoption(name="--ls-image") + + if getattr(session.config, "workerinput", None) is not None: + return if not is_collect_only and is_localstack_start: print("\nStarting LocalStack...") @@ -155,19 +186,24 @@ def pytest_configure(config): client = docker.from_env() _docker_service_health(client) _pull_docker_image(client, localstack_image) - _start_docker_container(client, config, localstack_image) + _start_docker_container(client, session.config, localstack_image) _localstack_health_check() client.close() print("LocalStack is ready...") -def pytest_unconfigure(config): - is_collect_only = config.getoption(name="--collect-only") - is_localstack_start = config.getoption(name="--ls-start") +def pytest_sessionfinish(session, exitstatus): + """Called after whole test run finished, right before returning the exit status to the system.""" + is_collect_only = session.config.getoption(name="--collect-only") + is_localstack_start = session.config.getoption(name="--ls-start") + + # Only run on the master node + if getattr(session.config, "workerinput", None) is not None: + return if not is_collect_only and is_localstack_start: print("\nStopping LocalStack...") client = docker.from_env() - _stop_docker_container(client, config) + _stop_docker_container(client, session.config) client.close() diff --git a/pyproject.toml b/pyproject.toml index 10888b2..968281a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,17 +2,14 @@ [tool.black] line_length = 100 -include = '(^terraform_pytest\/.*\.py$|^conftest.py$)' -#extend_exclude = '()' +extend_exclude = '(terraform-provider-aws)' [tool.isort] profile = 'black' -skip_glob = ["terraform-provider-aws"] -#extend_skip = [] +extend_skip = ["terraform-provider-aws"] line_length = 100 [tool.pytest.ini_options] testpaths = [ "terraform-provider-aws/internal/service/", ] -filterwarnings =["ignore::DeprecationWarning"] diff --git a/requirements.txt b/requirements.txt index f294c38..5de6103 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,8 @@ -click==8.1.3 -pytest==7.2.0 -docker==6.0.1 -requests==2.28.2 +click>=8.1.3 +pytest>=7.2.0 +docker>=6.0.1 +requests>=2.28.2 black>=22.1 isort>=5.10 pytest-xdist>=3.1.0 +pre-commit>=2.21.0 diff --git a/terraform_pytest/main.py b/terraform_pytest/main.py index 57cc802..f67e122 100644 --- a/terraform_pytest/main.py +++ b/terraform_pytest/main.py @@ -21,25 +21,21 @@ def patch(): "--service", "-s", default=None, + required=True, help="""Service to build; use "ls-all", "ls-community", "ls-pro" to build all services, example: --service=ls-all; --service=ec2; --service=ec2,iam""", ) -def build(service): - """Build binary for testing""" - if not service: - print("No service provided") - print( - "use --service or -s to specify services to build; for more help try --help to see more options" - ) - return - +@click.option("--force-build", "-f", is_flag=True, default=False, help="Force rebuilds binary") +def build(service, force_build): services = get_services(service) for service in services: print(f"Building {service}...") try: start = timer() - build_test_bin(service=service, tf_root_path=realpath(TF_REPO_NAME)) + build_test_bin( + service=service, tf_root_path=realpath(TF_REPO_NAME), force_build=force_build + ) end = timer() print(f"Build {service} in {end - start} seconds") except KeyboardInterrupt: diff --git a/terraform_pytest/utils.py b/terraform_pytest/utils.py index 0d007a7..016a835 100644 --- a/terraform_pytest/utils.py +++ b/terraform_pytest/utils.py @@ -4,14 +4,18 @@ from uuid import uuid4 TF_REPO_NAME = "terraform-provider-aws" + +# absolute path to the terraform repo TF_REPO_PATH = f"{realpath(TF_REPO_NAME)}" +# list of patch files to apply to the terraform repo TF_REPO_PATCH_FILES = ["etc/001-hardcode-endpoint.patch"] +# folder name where the testing binaries are stored TF_TEST_BINARY_FOLDER = "test-bin" TF_REPO_SERVICE_FOLDER = "./internal/service" -BLACKLISTED_SERVICES = ["controltower", "greengrass"] +# list of services that are supported by the localstack community edition LS_COMMUNITY_SERVICES = [ "acm", "apigateway", @@ -45,6 +49,7 @@ "swf", "transcribe", ] +# list of services that are supported by the localstack pro edition LS_PRO_SERVICES = [ "amplify", "apigateway", @@ -107,14 +112,19 @@ "sts", ] - -def _get_test_bin_abs_path(service): - return f"{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test" +# list of services that doesn't contain any tests +BLACKLISTED_SERVICES = ["controltower", "greengrass"] def execute_command(cmd, env=None, cwd=None): - """ - Execute a command and return the return code. + """Execute a command and return the return code. + + :param list(str) cmd: + command to execute + :param dict env: + environment variables + :param str cwd: + working directory """ _lwd = getcwd() if isinstance(cmd, list): @@ -136,12 +146,34 @@ def execute_command(cmd, env=None, cwd=None): return _err, _out -def build_test_bin(service, tf_root_path): - _test_bin_abs_path = _get_test_bin_abs_path(service) +def build_test_bin(service, tf_root_path, force_build=False): + """Build the test binary for a given service. + + :param str service: + service name + :param str tf_root_path: + path to the terraform repo + :param bool force_build: + force build the binary + + :return: int, str or None + return code and stdout + """ + _test_bin_abs_path = f"{TF_REPO_PATH}/{TF_TEST_BINARY_FOLDER}/{service}.test" _tf_repo_service_folder = f"{TF_REPO_SERVICE_FOLDER}/{service}" - if exists(_test_bin_abs_path): - return + if exists(_test_bin_abs_path) and not force_build: + return None + + cmd = ["go", "mod", "tidy"] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") + + cmd = ["go", "mod", "vendor"] + return_code, stdout = execute_command(cmd, cwd=tf_root_path) + if return_code != 0: + raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}") cmd = ["go", "mod", "tidy"] return_code, stdout = execute_command(cmd, cwd=tf_root_path) @@ -171,15 +203,17 @@ def build_test_bin(service, tf_root_path): return return_code, stdout -def get_all_services(): - services = [] - for service in listdir(f"{TF_REPO_PATH}/{TF_REPO_SERVICE_FOLDER}"): - if service not in BLACKLISTED_SERVICES: - services.append(service) - return sorted(services) +def get_services(service): + """Get the list of services to test. + :param: str service: + service names in comma separated format + example: ec2,lambda,iam or ls-community or ls-pro or ls-all -def get_services(service): + :return: list: + list of services + """ + result = [] if service == "ls-community": services = LS_COMMUNITY_SERVICES elif service == "ls-pro": @@ -192,11 +226,19 @@ def get_services(service): services = [s for s in services if s] else: services = [service] - services = [s for s in services if s not in BLACKLISTED_SERVICES] - return list(set(services)) + for s in services: + if s in LS_COMMUNITY_SERVICES + LS_PRO_SERVICES and s not in BLACKLISTED_SERVICES: + result.append(s) + else: + print(f"Service {s} is not supported...\nPlease check the service name") + return list(set(result)) def patch_repo(): + """Patches terraform repo. + + return: None + """ print(f"Patching {TF_REPO_NAME}...") for patch_file in TF_REPO_PATCH_FILES: cmd = [ From ce7a583913408de8f052216fa5bdd72365c7f68f Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Mon, 23 Jan 2023 16:43:40 +0530 Subject: [PATCH 93/94] removed docker-compose (#18) * added fail fast in localstack start * updated: running tests ec2 in parrallel * updated: concurrency, added ec2 for testing * updated parrellel run * updated: changes for upstream * updated: dependancies version * review resolution * testing pipeline * added comments; added all svc * testing * added all svc * removed hook * removed: docker-compose * removed: docker-compose --- .github/workflows/main.yml | 8 -------- docker-compose.yml | 18 ------------------ 2 files changed, 26 deletions(-) delete mode 100644 docker-compose.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b4ecae..c07e4c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,14 +76,6 @@ jobs: run: | python -m pytest $PYTEST_PARALLEL_CONFIG --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} - - name: Run ${{ matrix.service }} Tests - if: ${{ matrix.service == 'ec2' }} - run: | - IMAGE_TAG=${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} docker-compose up -d - sleep 20 - python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v -n 2 - - - name: Publish ${{ matrix.service }} Test Results uses: EnricoMi/publish-unit-test-result-action@v2 if: always() diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 4688d35..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3.8" - -services: - localstack: - container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}" - image: ${IMAGE_TAG-localstack/localstack:latest} - ports: - - "127.0.0.1:4566:4566" # LocalStack Gateway - - "127.0.0.1:4510-4559:4510-4559" # external services port range - environment: - - DEBUG=1 - - PROVIDER_OVERRIDE_S3=asf - - FAIL_FAST=1 - - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-} - - DOCKER_HOST=unix:///var/run/docker.sock - volumes: - - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack" - - "/var/run/docker.sock:/var/run/docker.sock" From 26537fbf2a1b33ad8cc14222b2fef12a685b3f80 Mon Sep 17 00:00:00 2001 From: Macwan Nevil Date: Mon, 23 Jan 2023 16:53:27 +0530 Subject: [PATCH 94/94] readme updated (#19) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0903225..77be770 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Purpose of this project is to externalize the test cases from the localstack rep ## How to run? 1. Run `python -m terraform_pytest.main patch` to apply the patch to the terraform provider aws + - **Note: This operation is not idempotent. Please apply the patch only once.** 2. Run `python -m terraform_pytest.main build -s s3` to build testing binary for the golang module 3. Now you are ready to use `python -m pytest` commands to list and run test cases from golang