Skip to content

Commit 1e6167f

Browse files
general-kroll-4-lifegeneral-kroll-4-life
authored
Merge pull request #300 from stackql/dev
testing-capability
2 parents 6c66b08 + 10674cd commit 1e6167f

35 files changed

+8044
-4
lines changed

‎.github/workflows/main.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ jobs:
5757
5858
- name: "[SETUP] get updated providers"
5959
run: |
60-
pip install -q -r scripts/setup/requirements.txt
60+
pip install -q -r requirements.txt
6161
python scripts/setup/get-updated-providers.py
6262
6363
- name: "[SETUP] prepare dist and test dirs"

‎.github/workflows/regression.yml

+310
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,310 @@
1+
name: Integration Testing and Analysis
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
- dev
8+
push:
9+
branches:
10+
- main
11+
- dev
12+
tags:
13+
- robot*
14+
- regression*
15+
- integration*
16+
17+
env:
18+
GO_VERSION: '^1.22'
19+
STACKQL_CORE_REPOSITORY: ${{ vars.STACKQL_CORE_REPOSITORY != '' && vars.STACKQL_CORE_REPOSITORY || 'stackql/stackql' }}
20+
STACKQL_CORE_REF: ${{ vars.STACKQL_CORE_REF != '' && vars.STACKQL_CORE_REF || 'main' }}
21+
STACKQL_ANY_SDK_REPOSITORY: ${{ vars.STACKQL_ANY_SDK_REPOSITORY != '' && vars.STACKQL_ANY_SDK_REPOSITORY || 'stackql/any-sdk' }}
22+
STACKQL_ANY_SDK_REF: ${{ vars.STACKQL_ANY_SDK_REF != '' && vars.STACKQL_ANY_SDK_REF || 'main' }}
23+
24+
jobs:
25+
regression-testing:
26+
name: regression-testing
27+
runs-on: ubuntu-latest
28+
permissions:
29+
id-token: write
30+
contents: read
31+
env:
32+
AWS_DEFAULT_REGION: us-west-1
33+
REG_MAX_VERSIONS: 3
34+
REG_MAX_AGE_MONTHS: 6
35+
REG_WEBSITE_DIR: _deno_website
36+
REG_PROVIDER_PATH: providers/dist
37+
REG_ARTIFACT_REPO_BUCKET: stackql-registry-artifacts
38+
REG_DENO_DEPLOY_ASSET_REPO: deno-deploy-registry
39+
REG_DENO_DEPLOY_API_DEV: stackql-dev-registry
40+
REG_DENO_DEPLOY_API_PROD: stackql-registry
41+
42+
steps:
43+
- uses: actions/[email protected]
44+
name: "[SETUP] checkout repo"
45+
with:
46+
fetch-depth: 0
47+
48+
- name: Set up Go 1.x
49+
uses: actions/[email protected]
50+
with:
51+
go-version: ${{ env.GO_VERSION }}
52+
check-latest: true
53+
cache: true
54+
id: go
55+
56+
- name: Download core
57+
uses: actions/[email protected]
58+
with:
59+
repository: ${{ env.STACKQL_CORE_REPOSITORY }}
60+
ref: ${{ env.STACKQL_CORE_REF }}
61+
token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }}
62+
path: stackql-core
63+
64+
- name: Download any-sdk
65+
uses: actions/[email protected]
66+
with:
67+
repository: ${{ env.STACKQL_ANY_SDK_REPOSITORY }}
68+
ref: ${{ env.STACKQL_ANY_SDK_REF }}
69+
token: ${{ secrets.CI_STACKQL_PACKAGE_DOWNLOAD_TOKEN }}
70+
path: stackql-any-sdk
71+
72+
- name: Setup Python
73+
uses: actions/[email protected]
74+
with:
75+
python-version: '3.11'
76+
77+
- name: Add dependencies
78+
working-directory: stackql-core
79+
run: |
80+
sudo apt-get install -y jq
81+
pip3 install -r cicd/requirements.txt
82+
83+
- name: Build stackql from core source
84+
working-directory: stackql-core
85+
run: |
86+
go get ./...
87+
python3 cicd/python/build.py --build
88+
89+
- name: Build any-sdk cli from source
90+
working-directory: stackql-any-sdk
91+
run: |
92+
93+
go get ./...
94+
95+
go build -x -v \
96+
-o build/anysdk ./cmd/interrogate
97+
98+
- name: Parse tag
99+
id: parse_tag
100+
run: |
101+
if [ "${{ github.ref_type }}" = "tag" ]; then
102+
tag_obj="$(python3 stackql-core/cicd/python/tag_parse.py '${{ github.ref_name }}' --parse-registry-tag)"
103+
echo "tag_obj: $tag_obj"
104+
{
105+
echo "PARSED_TAG_IS_ROBOT=$(echo $tag_obj | jq -r '.is_robot')"
106+
echo "PARSED_TAG_IS_REGRESSION=$(echo $tag_obj | jq -r '.is_regression')"
107+
} | tee -a "$GITHUB_ENV"
108+
else
109+
{
110+
echo "IS_BRANCH=true"
111+
} >> $GITHUB_ENV
112+
fi
113+
114+
115+
- name: Generate rewritten registry for simulations
116+
working-directory: stackql-core
117+
run: |
118+
python3 test/python/registry-rewrite.py
119+
120+
121+
- name: Prepare load balancing materials
122+
working-directory: stackql-core
123+
run: |
124+
sudo cp /etc/hosts /etc/hosts.bak
125+
python3 test/python/tcp_lb.py --generate-hosts-entries | sudo tee -a /etc/hosts
126+
python3 test/python/tcp_lb.py --generate-nginx-lb > test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf
127+
128+
129+
- name: Install and run nginx load balancer
130+
working-directory: stackql-core
131+
run: |
132+
sudo apt-get install -y curl gnupg2 ca-certificates lsb-release ubuntu-keyring
133+
curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor \
134+
| sudo tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null
135+
gpg --dry-run --quiet --no-keyring --import --import-options import-show /usr/share/keyrings/nginx-archive-keyring.gpg
136+
echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \
137+
http://nginx.org/packages/ubuntu $(lsb_release -cs) nginx" \
138+
| sudo tee /etc/apt/sources.list.d/nginx.list
139+
sudo apt-get update
140+
sudo apt-get install nginx
141+
sudo nginx -c "$(pwd)/test/tcp/reverse-proxy/nginx/dynamic-sni-proxy.conf"
142+
143+
- name: Create materials for core tests
144+
working-directory: stackql-core
145+
run: |
146+
openssl req -x509 -keyout test/server/mtls/credentials/pg_server_key.pem -out test/server/mtls/credentials/pg_server_cert.pem -config test/server/mtls/openssl.cnf -days 365
147+
openssl req -x509 -keyout test/server/mtls/credentials/pg_client_key.pem -out test/server/mtls/credentials/pg_client_cert.pem -config test/server/mtls/openssl.cnf -days 365
148+
openssl req -x509 -keyout test/server/mtls/credentials/pg_rubbish_key.pem -out test/server/mtls/credentials/pg_rubbish_cert.pem -config test/server/mtls/openssl.cnf -days 365
149+
150+
151+
- name: Start Core Test Mocks
152+
working-directory: stackql-core
153+
run: |
154+
pgrep -f flask | xargs kill -9 || true
155+
flask --app=./test/python/flask/gcp/app run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 1080 &
156+
flask --app=./test/python/flask/oauth2/token_srv run --cert=./test/server/mtls/credentials/pg_server_cert.pem --key=./test/server/mtls/credentials/pg_server_key.pem --host 0.0.0.0 --port 2091 &
157+
158+
- name: Run any-sdk cli mocked testing
159+
working-directory: stackql-core
160+
run: |
161+
export GCP_SERVICE_ACCOUNT_KEY="$(cat test/assets/credentials/dummy/google/functional-test-dummy-sa-key.json)"
162+
bucketsListIDs="$(${{ github.workspace }}/stackql-any-sdk/build/anysdk query \
163+
--svc-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/services/storage-v1.yaml" \
164+
--tls.allowInsecure \
165+
--prov-file-path="test/registry-mocked/src/googleapis.com/v0.1.2/provider.yaml" \
166+
--resource buckets \
167+
--method list \
168+
--parameters '{ "project": "stackql-demo" }' \
169+
| jq -r '.items[].id')"
170+
matchingBuckets="$(echo "${bucketsListIDs}" | grep "stackql-demo" )"
171+
if [ "${matchingBuckets}" = "" ]; then
172+
echo "Core Test Failed with no matching buckets"
173+
exit 1
174+
else
175+
echo "Core Test passed with matching buckets: $matchingBuckets"
176+
fi
177+
178+
- name: Run core proxied robot functional tests against local registry
179+
if: success()
180+
working-directory: stackql-core
181+
run: |
182+
providerRoot="$(realpath $(pwd)/../providers)"
183+
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
184+
robot \
185+
--variable "${sundryCfg}" \
186+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
187+
--include registry \
188+
-d test/robot/reports \
189+
test/robot/functional || true
190+
191+
- name: Output from core proxied functional tests
192+
if: always()
193+
run: |
194+
cat stackql-core/test/robot/reports/output.xml
195+
python3 scripts/cicd/python/robot-parse.py --robot-output-file stackql-core/test/robot/reports/output.xml > stackql-core/test/robot/reports/proxied_parsed_output.json
196+
197+
- name: Upload core traffic lights
198+
uses: actions/[email protected]
199+
if: success()
200+
with:
201+
name: proxied-core-traffic-lights
202+
path: stackql-core/test/robot/reports/proxied_parsed_output.json
203+
204+
205+
- name: Post core test cleanup
206+
run: |
207+
pgrep -f flask | xargs kill -9 || true
208+
209+
- name: Run local registry mocked robot functional tests
210+
if: success()
211+
run: |
212+
providerRoot="$(realpath $(pwd)/providers)"
213+
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'"}'
214+
robot \
215+
--variable "${sundryCfg}" \
216+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
217+
-d test/robot/reports/mocked \
218+
test/robot/stackql/mocked || true
219+
220+
- name: Output from local registry mocked functional tests
221+
if: always()
222+
run: |
223+
cat test/robot/reports/mocked/output.xml
224+
python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/mocked/output.xml > test/robot/reports/mocked/parsed_output.json
225+
226+
- name: Upload local registry mocked traffic lights
227+
uses: actions/[email protected]
228+
if: success()
229+
with:
230+
name: local-registry-mocked-traffic-lights
231+
path: test/robot/reports/mocked/parsed_output.json
232+
233+
- name: Post registry mocked test cleanup
234+
run: |
235+
pgrep -f flask | xargs kill -9 || true
236+
sudo nginx -s stop || true
237+
sudo cp /etc/hosts.bak /etc/hosts || true
238+
rm -f test/robot/reports/*.xml || true
239+
240+
- name: Run live readonly robot functional tests
241+
if: success()
242+
id: live_integration_tests_readonly
243+
env:
244+
GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RO_SECRET }}
245+
AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RO_AWS_ACCESS_KEY_ID }}
246+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RO_AWS_SECRET_ACCESS_KEY }}
247+
run: |
248+
providerRoot="$(realpath $(pwd)/providers)"
249+
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}'
250+
robot \
251+
--variable "${sundryCfg}" \
252+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
253+
-d test/robot/reports/readonly \
254+
test/robot/stackql/live/readonly || true
255+
256+
- name: Output from live readonly functional tests
257+
if: always()
258+
run: |
259+
cat test/robot/reports/readonly/output.xml
260+
python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/readonly/output.xml > test/robot/reports/readonly/parsed_output.json
261+
262+
- name: Upload readonly traffic lights
263+
uses: actions/[email protected]
264+
if: success()
265+
with:
266+
name: local-registry-readonly-traffic-lights
267+
path: test/robot/reports/readonly/parsed_output.json
268+
269+
- name: Run live readwrite robot functional tests
270+
if: github.ref_type == 'tag'
271+
id: live_integration_tests_readwrite
272+
env:
273+
GOOGLE_CREDENTIALS: ${{ secrets.CI_SCENARIO_GCP_RW_SECRET }}
274+
AWS_ACCESS_KEY_ID: ${{ secrets.CI_SCENARIO_RW_AWS_ACCESS_KEY_ID }}
275+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_SCENARIO_RW_AWS_SECRET_ACCESS_KEY }}
276+
run: |
277+
providerRoot="$(realpath $(pwd)/providers)"
278+
sundryCfg='SUNDRY_CONFIG:{"registry_path": "'"${providerRoot}"'", "GCS_BUCKET_NAME": "stackql-demo-bucket-02", "GCP_PROJECT": "stackql-demo", "AWS_RECORD_SET_ID": "A00000001AAAAAAAAAAAA", "AWS_RECORD_SET_REGION": "us-east-1"}'
279+
robot \
280+
--variable "${sundryCfg}" \
281+
--variable SHOULD_RUN_DOCKER_EXTERNAL_TESTS:true \
282+
-d test/robot/reports/readwrite \
283+
test/robot/stackql/live/readwrite || true
284+
285+
- name: Output from live readwrite functional tests
286+
if: github.ref_type == 'tag'
287+
run: |
288+
cat test/robot/reports/readwrite/output.xml
289+
python3 scripts/cicd/python/robot-parse.py --robot-output-file test/robot/reports/readwrite/output.xml > test/robot/reports/readwrite/parsed_output.json
290+
291+
- name: Upload readonly traffic lights
292+
uses: actions/[email protected]
293+
if: success()
294+
with:
295+
name: local-registry-readwrite-traffic-lights
296+
path: test/robot/reports/readwrite/parsed_output.json
297+
298+
- name: Display traffic lights
299+
run: |
300+
for i in $(ls test/robot/reports/*/parsed_output.json); do
301+
echo "Traffic light for $i"
302+
if [ -f "$i" ]; then
303+
python3 scripts/cicd/python/display-parsed.py --traffic-light-file $i
304+
else
305+
echo "File $i does not exist 🛑"
306+
fi
307+
done
308+
echo "Traffic light for proxied"
309+
python3 scripts/cicd/python/display-parsed.py --traffic-light-file stackql-core/test/robot/reports/proxied_parsed_output.json
310+
echo "traffic lights completed"

‎.gitignore

+5-2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,9 @@ ed25519tool
33
scripts/*/node_modules/
44
scripts/*/package-lock.json
55
stackql*.sh
6-
stackql
6+
/stackql
77
stackql-zip
8-
.stackql/
8+
.stackql/
9+
stackql-core/
10+
stackql-any-sdk/
11+
.venv/

‎providers/src/aws/v00.00.00000/provider.yaml

+9
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,15 @@ providerServices:
290290
title: ce
291291
version: v00.00.00000
292292
description: ce
293+
ce_native:
294+
id: ce:v00.00.00000
295+
name: ce_native
296+
preferred: true
297+
service:
298+
$ref: aws/v00.00.00000/services/ce_native.yaml
299+
title: ce_native
300+
version: v00.00.00000
301+
description: ce_native
293302
certificatemanager:
294303
id: certificatemanager:v00.00.00000
295304
name: certificatemanager

0 commit comments

Comments
 (0)