diff --git a/test/blackbox/annotations.bats b/test/blackbox/annotations.bats index d6b94dfc86..ffa2cc4ae2 100644 --- a/test/blackbox/annotations.bats +++ b/test/blackbox/annotations.bats @@ -32,7 +32,7 @@ function verify_prerequisites { function setup_file() { export COSIGN_PASSWORD="" # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi # Download test data to folder common for the entire suite, not just this file diff --git a/test/blackbox/anonymous_policy.bats b/test/blackbox/anonymous_policy.bats index 385972fdbb..6a6209dc98 100644 --- a/test/blackbox/anonymous_policy.bats +++ b/test/blackbox/anonymous_policy.bats @@ -16,7 +16,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/cloud_only.bats b/test/blackbox/cloud_only.bats index b7a5a59fd9..5990d2c208 100644 --- a/test/blackbox/cloud_only.bats +++ b/test/blackbox/cloud_only.bats @@ -7,7 +7,7 @@ load helpers_wait function setup() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/cve.bats b/test/blackbox/cve.bats index 47e13e5a8f..be389e2d39 100644 --- a/test/blackbox/cve.bats +++ b/test/blackbox/cve.bats @@ -23,7 +23,7 @@ function setup_file() { # Use unique config name based on test file name and test run to avoid conflicts export REGISTRY_NAME=$(basename "${BASH_SOURCE[0]}" .bats)-$(basename "${BATS_FILE_TMPDIR}") # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/detect_manifest_collision.bats b/test/blackbox/detect_manifest_collision.bats index f4d921d55c..428df10fe2 100644 --- a/test/blackbox/detect_manifest_collision.bats +++ b/test/blackbox/detect_manifest_collision.bats @@ -16,7 +16,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/docker_compat.bats b/test/blackbox/docker_compat.bats index f2b5f72ce0..6b1fd8db38 100644 --- a/test/blackbox/docker_compat.bats +++ b/test/blackbox/docker_compat.bats @@ -21,7 +21,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi # Download test data to folder common for the entire suite, not just this file diff --git a/test/blackbox/events_config_decoding.bats b/test/blackbox/events_config_decoding.bats index c55355501f..d56292bd32 100644 --- a/test/blackbox/events_config_decoding.bats +++ b/test/blackbox/events_config_decoding.bats @@ -25,7 +25,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi } diff --git a/test/blackbox/events_http.bats b/test/blackbox/events_http.bats index 8d31b4b935..858587a524 100644 --- a/test/blackbox/events_http.bats +++ b/test/blackbox/events_http.bats @@ -25,7 +25,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/events_http_lint_failure.bats b/test/blackbox/events_http_lint_failure.bats index 5d13ad766c..eac08f2252 100644 --- a/test/blackbox/events_http_lint_failure.bats +++ b/test/blackbox/events_http_lint_failure.bats @@ -30,7 +30,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/events_nats.bats b/test/blackbox/events_nats.bats index 092da7dfd7..b353e86101 100644 --- a/test/blackbox/events_nats.bats +++ b/test/blackbox/events_nats.bats @@ -25,7 +25,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/events_nats_lint_failure.bats b/test/blackbox/events_nats_lint_failure.bats index 3edb0093df..9d265d21a9 100644 --- a/test/blackbox/events_nats_lint_failure.bats +++ b/test/blackbox/events_nats_lint_failure.bats @@ -30,7 +30,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/events_sink_failure.bats b/test/blackbox/events_sink_failure.bats index 8108dc8428..782f7ccda9 100644 --- a/test/blackbox/events_sink_failure.bats +++ b/test/blackbox/events_sink_failure.bats @@ -25,7 +25,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/fips140.bats b/test/blackbox/fips140.bats index f2436db9eb..b963806943 100644 --- a/test/blackbox/fips140.bats +++ b/test/blackbox/fips140.bats @@ -3,392 +3,119 @@ # Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() load helpers_zot +load helpers_pushpull load ../port_helper -function verify_prerequisites { - if [ ! $(command -v curl) ]; then - echo "you need to install curl as a prerequisite to running the tests" >&3 - return 1 - fi - - if [ ! $(command -v jq) ]; then - echo "you need to install jq as a prerequisite to running the tests" >&3 - return 1 - fi - - return 0 -} +PUSHPULL_FIPS_MODE=1 function setup_file() { - # Verify prerequisites are available - if ! $(verify_prerequisites); then - exit 1 - fi - # Download test data to folder common for the entire suite, not just this file - skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 - # Setup zot server - local zot_root_dir=${BATS_FILE_TMPDIR}/zot - local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json - ZOT_LOG_FILE=${zot_root_dir}/zot-log.json - local oci_data_dir=${BATS_FILE_TMPDIR}/oci - mkdir -p ${zot_root_dir} - mkdir -p ${oci_data_dir} - zot_port=$(get_free_port_for_service "zot") - echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port - touch ${ZOT_LOG_FILE} - cat > ${zot_config_file}< config.json - echo "hello world" > artifact.txt - run oras push --plain-http 127.0.0.1:${zot_port}/hello-artifact:v2 \ - --config config.json:application/vnd.acme.rocket.config.v1+json artifact.txt:text/plain -d -v - [ "$status" -eq 0 ] - rm -f artifact.txt - rm -f config.json + helper_push_oras_artifact hello-artifact v2 } @test "pull oras artifact" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run oras pull --plain-http 127.0.0.1:${zot_port}/hello-artifact:v2 -d -v - [ "$status" -eq 0 ] - grep -q "hello world" artifact.txt - rm -f artifact.txt + helper_pull_oras_artifact hello-artifact v2 } @test "attach oras artifacts" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - # attach signature - echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > ${BATS_FILE_TMPDIR}/signature.json - run oras attach --disable-path-validation --plain-http 127.0.0.1:${zot_port}/golang:1.20 --artifact-type 'signature/example' ${BATS_FILE_TMPDIR}/signature.json:application/json - [ "$status" -eq 0 ] - # attach sbom - echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:${zot_port}/golang:1.20'\", \"contents\": \"good\"}" > ${BATS_FILE_TMPDIR}/sbom.json - run oras attach --disable-path-validation --plain-http 127.0.0.1:${zot_port}/golang:1.20 --artifact-type 'sbom/example' ${BATS_FILE_TMPDIR}/sbom.json:application/json - [ "$status" -eq 0 ] + helper_attach_oras_artifacts golang 1.20 } @test "discover oras artifacts" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run oras discover --plain-http --format json 127.0.0.1:${zot_port}/golang:1.20 - [ "$status" -eq 0 ] - [ $(echo "$output" | jq -r ".manifests | length") -eq 2 ] + helper_discover_oras_artifacts golang 1.20 2 } @test "add and list tags using oras" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run skopeo --insecure-policy copy --dest-tls-verify=false \ - oci:${TEST_DATA_DIR}/golang:1.20 \ - docker://127.0.0.1:${zot_port}/oras-tags:1.20 - [ "$status" -eq 0 ] - run oras tag --plain-http 127.0.0.1:${zot_port}/oras-tags:1.20 1 new latest - [ "$status" -eq 0 ] - run oras repo tags --plain-http 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 4 ] - [ "${lines[-1]}" == "new" ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-3]}" == "1.20" ] - [ "${lines[-4]}" == "1" ] - run oras repo tags --plain-http --last new 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ -z $output ] - run oras repo tags --plain-http --last latest 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 1 ] - [ "${lines[-1]}" == "new" ] - run oras repo tags --plain-http --last "1.20" 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 2 ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-1]}" == "new" ] - run oras repo tags --plain-http --last "1" 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 3 ] - [ "${lines[-3]}" == "1.20" ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-1]}" == "new" ] + helper_add_and_list_tags_using_oras } @test "push helm chart" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot -d ${BATS_FILE_TMPDIR} - [ "$status" -eq 0 ] - local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml) - run helm push ${BATS_FILE_TMPDIR}/zot-${chart_version}.tgz oci://localhost:${zot_port}/zot-chart - [ "$status" -eq 0 ] + helper_push_helm_chart } @test "pull helm chart" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml) - run helm pull oci://localhost:${zot_port}/zot-chart/zot --version ${chart_version} -d ${BATS_FILE_TMPDIR} - [ "$status" -eq 0 ] + helper_pull_helm_chart } @test "push image with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl image copy ocidir://${TEST_DATA_DIR}/golang:1.20 localhost:${zot_port}/test-regclient - [ "$status" -eq 0 ] + helper_push_image_with_regclient "ocidir://${TEST_DATA_DIR}/golang:1.20" test-regclient } @test "pull image with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl image copy localhost:${zot_port}/test-regclient ocidir://${TEST_DATA_DIR}/golang:1.20 - [ "$status" -eq 0 ] + helper_pull_image_with_regclient test-regclient "ocidir://${TEST_DATA_DIR}/golang:1.20" } @test "list repositories with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl repo ls localhost:${zot_port} - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - - if [ "$i" = 'test-regclient' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] - - run regctl repo ls --limit 2 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 2 ] - [ "${lines[-2]}" == "busybox" ] - [ "${lines[-1]}" == "golang" ] - - run regctl repo ls --last busybox --limit 1 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 1 ] - [ "${lines[-1]}" == "golang" ] + helper_list_repositories_with_regclient_pagination 2 busybox golang test-regclient "-2:busybox" "-1:golang" } @test "list image tags with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl tag ls localhost:${zot_port}/test-regclient - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - - if [ "$i" = 'latest' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] + helper_list_image_tags_with_regclient test-regclient } @test "push manifest with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - manifest=$(regctl manifest get localhost:${zot_port}/test-regclient --format=raw-body) - run regctl manifest put localhost:${zot_port}/test-regclient:1.0.0 --format oci --content-type application/vnd.oci.image.manifest.v1+json --format oci < Dockerfile < /testfile -EOF - run sh -c 'unset GODEBUG; docker build -f Dockerfile -t localhost:'${zot_port}'/test .' - [ "$status" -eq 0 ] - run docker push localhost:${zot_port}/test - [ "$status" -eq 1 ] - run docker pull localhost:${zot_port}/test - [ "$status" -eq 1 ] +@test "build docker image and verify docker push and pull fail" { + helper_build_docker_image_push_and_pull } diff --git a/test/blackbox/fips140_authn.bats b/test/blackbox/fips140_authn.bats index 0c3e10bd33..b224377619 100644 --- a/test/blackbox/fips140_authn.bats +++ b/test/blackbox/fips140_authn.bats @@ -1,394 +1,105 @@ load helpers_zot +load helpers_pushpull_authn load ../port_helper -function verify_prerequisites { - if [ ! $(command -v curl) ]; then - echo "you need to install curl as a prerequisite to running the tests" >&3 - return 1 - fi - - if [ ! $(command -v jq) ]; then - echo "you need to install jq as a prerequisite to running the tests" >&3 - return 1 - fi - - if [ ! $(command -v htpasswd) ]; then - echo "you need to install htpasswd as a prerequisite to running the tests" >&3 - return 1 - fi - - if [ ! $(command -v mkpasswd) ]; then - echo "you need to install mkpasswd as a prerequisite to running the tests" >&3 - return 1 - fi - - return 0 -} +PUSHPULL_AUTHN_FIPS_MODE=1 function setup_file() { - # Verify prerequisites are available - if ! $(verify_prerequisites); then - exit 1 - fi - - # Download test data to folder common for the entire suite, not just this file - skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/busybox:1.36 oci:${TEST_DATA_DIR}/busybox:1.36 - - # Setup zot server - local zot_root_dir=${BATS_FILE_TMPDIR}/zot - local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json - ZOT_LOG_FILE=${zot_root_dir}/zot-log.json - local zot_htpasswd_file=${BATS_FILE_TMPDIR}/zot_htpasswd - zot_port=$(get_free_port_for_service "zot") - echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port - htpasswd -Bbn ${AUTH_USER} ${AUTH_PASS} >> ${zot_htpasswd_file} # bcrypt - echo "${AUTH_USER2}:$(echo ${AUTH_PASS2} | mkpasswd -s -R 1 -m sha-256)" >> ${zot_htpasswd_file} # sha256 - echo "${AUTH_USER3}:$(echo ${AUTH_PASS3} | mkpasswd -s -R 1 -m sha-512)" >> ${zot_htpasswd_file} # sha512 - echo "${AUTH_USER4}:$(echo ${AUTH_PASS4} | mkpasswd -s -R 0 -m sha-256)" >> ${zot_htpasswd_file} # sha256 zero rounds - echo "${AUTH_USER5}:$(echo ${AUTH_PASS5} | mkpasswd -s -R 0 -m sha-512)" >> ${zot_htpasswd_file} # sha512 zero rounds - - echo ${zot_root_dir} >&3 - - mkdir -p ${zot_root_dir} - - touch ${ZOT_LOG_FILE} - cat > ${zot_config_file}</dev/null || true - fi + authn_teardown } function teardown_file() { - zot_stop_all - unset GODEBUG -} - -# Helper function to verify authentication and image push -# Args: $1=username, $2=password, $3=hash_type, $4=should_succeed (true/false) -function verify_auth_and_push() { - local user="$1" - local pass="$2" - local hash_type="$3" - local should_succeed="$4" - - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - - # Disable TLS for regctl to avoid X25519 issues when regctl runs in FIPS mode - # This must be done before regctl registry login, as login automatically pings the registry - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - - # anonymous authn is set for zot, so all auth is ignored for the /v2/ ping - run regctl registry login localhost:${zot_port} -u ${user} -p ${pass} - [ "$status" -eq 0 ] - - run regctl image copy ocidir://${TEST_DATA_DIR}/busybox:1.36 localhost:${zot_port}/test-${hash_type} - - if [ "$should_succeed" = "true" ]; then - [ "$status" -eq 0 ] - else - [ "$status" -eq 1 ] - log_output | jq 'contains("htpasswd bcrypt failed since fips140 is enabled")' | grep true - fi -} - -@test "push image with regclient - setup registry" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] + authn_teardown_file } @test "push image with bcrypt auth (should fail in FIPS mode)" { - verify_auth_and_push "${AUTH_USER}" "${AUTH_PASS}" "bcrypt" "false" + helper_authn_verify_auth_and_push "${AUTH_USER}" "${AUTH_PASS}" bcrypt false } @test "push image with SHA256 auth (should succeed)" { - verify_auth_and_push "${AUTH_USER2}" "${AUTH_PASS2}" "sha256" "true" + helper_authn_verify_auth_and_push "${AUTH_USER2}" "${AUTH_PASS2}" sha256 true } @test "push image with SHA512 auth (should succeed)" { - verify_auth_and_push "${AUTH_USER3}" "${AUTH_PASS3}" "sha512" "true" + helper_authn_verify_auth_and_push "${AUTH_USER3}" "${AUTH_PASS3}" sha512 true } @test "push image with SHA256 auth with 0 rounds (should succeed)" { - verify_auth_and_push "${AUTH_USER4}" "${AUTH_PASS4}" "sha256-0rounds" "true" + helper_authn_verify_auth_and_push "${AUTH_USER4}" "${AUTH_PASS4}" sha256-0rounds true } @test "push image with SHA512 auth with 0 rounds (should succeed)" { - verify_auth_and_push "${AUTH_USER5}" "${AUTH_PASS5}" "sha512-0rounds" "true" + helper_authn_verify_auth_and_push "${AUTH_USER5}" "${AUTH_PASS5}" sha512-0rounds true } @test "pull image with SHA256 auth" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl registry login localhost:${zot_port} -u ${AUTH_USER2} -p ${AUTH_PASS2} - [ "$status" -eq 0 ] - run regctl image copy localhost:${zot_port}/test-sha256 ocidir://${TEST_DATA_DIR}/busybox:sha256-pulled - [ "$status" -eq 0 ] + helper_authn_pull_image_with_auth "${AUTH_USER2}" "${AUTH_PASS2}" test-sha256 sha256-pulled } @test "pull image with SHA512 auth" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl registry login localhost:${zot_port} -u ${AUTH_USER3} -p ${AUTH_PASS3} - [ "$status" -eq 0 ] - run regctl image copy localhost:${zot_port}/test-sha512 ocidir://${TEST_DATA_DIR}/busybox:sha512-pulled - [ "$status" -eq 0 ] + helper_authn_pull_image_with_auth "${AUTH_USER3}" "${AUTH_PASS3}" test-sha512 sha512-pulled } @test "pull image with SHA256 auth with 0 rounds" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl registry login localhost:${zot_port} -u ${AUTH_USER4} -p ${AUTH_PASS4} - [ "$status" -eq 0 ] - run regctl image copy localhost:${zot_port}/test-sha256-0rounds ocidir://${TEST_DATA_DIR}/busybox:sha256-0rounds-pulled - [ "$status" -eq 0 ] + helper_authn_pull_image_with_auth "${AUTH_USER4}" "${AUTH_PASS4}" test-sha256-0rounds sha256-0rounds-pulled } @test "pull image with SHA512 auth with 0 rounds" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl registry login localhost:${zot_port} -u ${AUTH_USER5} -p ${AUTH_PASS5} - [ "$status" -eq 0 ] - run regctl image copy localhost:${zot_port}/test-sha512-0rounds ocidir://${TEST_DATA_DIR}/busybox:sha512-0rounds-pulled - [ "$status" -eq 0 ] + helper_authn_pull_image_with_auth "${AUTH_USER5}" "${AUTH_PASS5}" test-sha512-0rounds sha512-0rounds-pulled } @test "push OCI artifact with SHA256 auth" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl registry login localhost:${zot_port} -u ${AUTH_USER2} -p ${AUTH_PASS2} - [ "$status" -eq 0 ] - run regctl artifact put localhost:${zot_port}/artifact-sha256:demo < ${BATS_FILE_TMPDIR}/mnist.onnx.check - sha256_out=$(sha256sum ${BATS_FILE_TMPDIR}/mnist.onnx.check | awk '{print $1}') - [ "$sha256_in" = "$sha256_out" ] + helper_authn_ml_artifacts_with_auth } diff --git a/test/blackbox/garbage_collect.bats b/test/blackbox/garbage_collect.bats index 914adad94c..b1115afae8 100644 --- a/test/blackbox/garbage_collect.bats +++ b/test/blackbox/garbage_collect.bats @@ -17,7 +17,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/helpers_pushpull.bash b/test/blackbox/helpers_pushpull.bash new file mode 100644 index 0000000000..68402e71cf --- /dev/null +++ b/test/blackbox/helpers_pushpull.bash @@ -0,0 +1,593 @@ +# Common helper functions and test utilities for blackbox push/pull-style tests. +# Used by pushpull.bats, fips140.bats, and upgrade BATS suites. +# +# pushpull.bats and fips140.bats set PUSHPULL_FIPS_MODE (0 or 1) and call +# pushpull_setup_file / pushpull_teardown / pushpull_teardown_file for lifecycle. + +function verify_prerequisites() { + local tool + + for tool in curl jq git docker; do + if ! command -v "${tool}" >/dev/null; then + echo "you need to install ${tool} as a prerequisite to running the tests" >&3 + return 1 + fi + done + + return 0 +} + +function get_zot_port() { + cat "${BATS_FILE_TMPDIR}/zot.port" +} + +function pushpull_isolate_regctl_config() { + # regctl persists login/TLS settings to config on disk; isolate per BATS file. + export REGCTL_CONFIG="${BATS_FILE_TMPDIR}/regctl.json" +} + +function pushpull_setup_file() { + if ! verify_prerequisites; then + exit 1 + fi + + pushpull_isolate_regctl_config + + skopeo --insecure-policy copy --format=oci \ + docker://ghcr.io/project-zot/golang:1.20 \ + oci:${TEST_DATA_DIR}/golang:1.20 + + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + local log_file=${zot_root_dir}/zot-log.json + + mkdir -p "${zot_root_dir}" "${oci_data_dir}" + touch "${log_file}" + + if [ "${PUSHPULL_FIPS_MODE:-0}" = 1 ]; then + export GODEBUG="fips140=only" + fi + + zot_port=$(get_free_port_for_service "zot") + echo "${zot_port}" >"${BATS_FILE_TMPDIR}/zot.port" + + cat >"${zot_config_file}" < "${work_dir}/config.json" + echo "hello world" > "${work_dir}/artifact.txt" + cat > "${work_dir}/oras-artifact-annotations.json" <<'EOF' +{ + "artifact.txt": { + "org.opencontainers.image.title": "artifact.txt" + } +} +EOF + run bash -c "cd '${work_dir}' && oras push --plain-http '127.0.0.1:${zot_port}/${artifact_name}:${tag}' \ + --annotation-file oras-artifact-annotations.json \ + --config config.json:application/vnd.acme.rocket.config.v1+json \ + artifact.txt:text/plain -d -v" + [ "${status}" -eq 0 ] +} + +# Args: $1 = artifact_name, $2 = tag +function helper_pull_oras_artifact() { + local artifact_name=${1} + local tag=${2} + local zot_port ref digest artifact_file + zot_port=$(get_zot_port) + ref="127.0.0.1:${zot_port}/${artifact_name}:${tag}" + artifact_file=${BATS_TEST_TMPDIR}/artifact.txt + + run oras pull --plain-http -o "${BATS_TEST_TMPDIR}" "${ref}" -d -v + [ "${status}" -eq 0 ] + grep -q "hello world" "${artifact_file}" + + run oras manifest fetch --plain-http "${ref}" + [ "${status}" -eq 0 ] + digest=$(echo "${output}" | jq -r '.layers[] | select(.mediaType == "text/plain") | .digest' | head -1) + [ -n "${digest}" ] + + run curl -fsSL "http://127.0.0.1:${zot_port}/v2/${artifact_name}/blobs/${digest}" + [ "${status}" -eq 0 ] + echo "${output}" | grep -q "hello world" +} + +# Args: $1 = image_name, $2 = tag +function helper_attach_oras_artifacts() { + local image_name=${1} + local tag=${2} + local zot_port + zot_port=$(get_zot_port) + + echo '{"artifact": "", "signature": "pat hancock"}' > "${BATS_FILE_TMPDIR}/signature.json" + run oras attach --disable-path-validation --plain-http "127.0.0.1:${zot_port}/${image_name}:${tag}" \ + --artifact-type 'signature/example' "${BATS_FILE_TMPDIR}/signature.json:application/json" + [ "${status}" -eq 0 ] + + echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:${zot_port}/${image_name}:${tag}'\", \"contents\": \"good\"}" > "${BATS_FILE_TMPDIR}/sbom.json" + run oras attach --disable-path-validation --plain-http "127.0.0.1:${zot_port}/${image_name}:${tag}" \ + --artifact-type 'sbom/example' "${BATS_FILE_TMPDIR}/sbom.json:application/json" + [ "${status}" -eq 0 ] +} + +# Args: $1 = image_name, $2 = tag, $3 = expected artifact count +function helper_discover_oras_artifacts() { + local image_name=${1} + local tag=${2} + local expected_count=${3} + local zot_port + zot_port=$(get_zot_port) + + run oras discover --plain-http --format json "127.0.0.1:${zot_port}/${image_name}:${tag}" + [ "${status}" -eq 0 ] + [ "$(echo "${output}" | jq -r '.manifests | length')" -eq "${expected_count}" ] +} + +function helper_add_and_list_tags_using_oras() { + local zot_port + zot_port=$(get_zot_port) + + run skopeo --insecure-policy copy --dest-tls-verify=false \ + "oci:${TEST_DATA_DIR}/golang:1.20" \ + "docker://127.0.0.1:${zot_port}/oras-tags:1.20" + [ "${status}" -eq 0 ] + + run oras tag --plain-http "127.0.0.1:${zot_port}/oras-tags:1.20" 1 new latest + [ "${status}" -eq 0 ] + + run oras repo tags --plain-http "127.0.0.1:${zot_port}/oras-tags" + [ "${status}" -eq 0 ] + echo "${output}" + [ "$(echo "${output}" | wc -l)" -eq 4 ] + [ "${lines[-1]}" = "new" ] + [ "${lines[-2]}" = "latest" ] + [ "${lines[-3]}" = "1.20" ] + [ "${lines[-4]}" = "1" ] + + run oras repo tags --plain-http --last new "127.0.0.1:${zot_port}/oras-tags" + [ "${status}" -eq 0 ] + echo "${output}" + [ -z "${output}" ] + + run oras repo tags --plain-http --last latest "127.0.0.1:${zot_port}/oras-tags" + [ "${status}" -eq 0 ] + echo "${output}" + [ "$(echo "${output}" | wc -l)" -eq 1 ] + [ "${lines[-1]}" = "new" ] + + run oras repo tags --plain-http --last "1.20" "127.0.0.1:${zot_port}/oras-tags" + [ "${status}" -eq 0 ] + echo "${output}" + [ "$(echo "${output}" | wc -l)" -eq 2 ] + [ "${lines[-2]}" = "latest" ] + [ "${lines[-1]}" = "new" ] + + run oras repo tags --plain-http --last "1" "127.0.0.1:${zot_port}/oras-tags" + [ "${status}" -eq 0 ] + echo "${output}" + [ "$(echo "${output}" | wc -l)" -eq 3 ] + [ "${lines[-3]}" = "1.20" ] + [ "${lines[-2]}" = "latest" ] + [ "${lines[-1]}" = "new" ] +} + +function helper_push_helm_chart() { + local zot_port chart_version + zot_port=$(get_zot_port) + + run helm package "${BATS_FILE_TMPDIR}/helm-charts/charts/zot" -d "${BATS_FILE_TMPDIR}" + [ "${status}" -eq 0 ] + + chart_version=$(awk '/version/{printf $2}' "${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml") + run helm push "${BATS_FILE_TMPDIR}/zot-${chart_version}.tgz" "oci://localhost:${zot_port}/zot-chart" + [ "${status}" -eq 0 ] +} + +function helper_pull_helm_chart() { + local zot_port chart_version + zot_port=$(get_zot_port) + chart_version=$(awk '/version/{printf $2}' "${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml") + + run helm pull "oci://localhost:${zot_port}/zot-chart/zot" --version "${chart_version}" -d "${BATS_FILE_TMPDIR}" + [ "${status}" -eq 0 ] +} + +# Args: $1 = source reference, $2 = destination repository +function helper_push_image_with_regclient() { + local source_ref=${1} + local dest_repo=${2} + local zot_port + zot_port=$(get_zot_port) + + run regctl registry set "localhost:${zot_port}" --tls disabled + [ "${status}" -eq 0 ] + run regctl image copy "${source_ref}" "localhost:${zot_port}/${dest_repo}" + [ "${status}" -eq 0 ] +} + +# Args: $1 = source repository, $2 = destination reference +function helper_pull_image_with_regclient() { + local source_repo=${1} + local dest_ref=${2} + local zot_port + zot_port=$(get_zot_port) + + run regctl image copy "localhost:${zot_port}/${source_repo}" "${dest_ref}" + [ "${status}" -eq 0 ] +} + +# Args: $1 = page limit, $2 = --last cursor repo, $3 = expected next repo, +# $4 = repository expected in full catalog listing, +# $@ = index:repo assertions against the limited result page. +function helper_list_repositories_with_regclient_pagination() { + local limit=${1} + local cursor_repo=${2} + local expected_next_repo=${3} + local expected_catalog_repo=${4} + local zot_port + zot_port=$(get_zot_port) + + run regctl repo ls "localhost:${zot_port}" + [ "${status}" -eq 0 ] + helper_assert_output_contains_line "${expected_catalog_repo}" + + run regctl repo ls --limit "${limit}" "localhost:${zot_port}" + [ "${status}" -eq 0 ] + echo "${output}" + [ "$(echo "${output}" | wc -l)" -eq "${limit}" ] + helper_assert_line_specs "${@:5}" + + run regctl repo ls --last "${cursor_repo}" --limit 1 "localhost:${zot_port}" + [ "${status}" -eq 0 ] + echo "${output}" + [ "$(echo "${output}" | wc -l)" -eq 1 ] + [ "${lines[-1]}" = "${expected_next_repo}" ] +} + +# Args: $1 = repository +function helper_list_image_tags_with_regclient() { + local repo=${1} + local zot_port + zot_port=$(get_zot_port) + + run regctl tag ls "localhost:${zot_port}/${repo}" + [ "${status}" -eq 0 ] + helper_assert_output_contains_line latest +} + +# Args: $1 = repository, $2 = manifest tag +function helper_push_manifest_with_regclient() { + local repo=${1} + local tag=${2} + local zot_port manifest + zot_port=$(get_zot_port) + manifest=$(regctl manifest get "localhost:${zot_port}/${repo}" --format=raw-body) + + run regctl manifest put "localhost:${zot_port}/${repo}:${tag}" \ + --format oci \ + --content-type application/vnd.oci.image.manifest.v1+json < "${dockerfile}" < /testfile +DOCKERFILE + run sh -c "unset GODEBUG; docker build -f '${dockerfile}' -t localhost:${zot_port}/test '${BATS_TEST_TMPDIR}'" + [ "${status}" -eq 0 ] + run docker push "localhost:${zot_port}/test" + [ "${status}" -eq 1 ] + run docker pull "localhost:${zot_port}/test" + [ "${status}" -eq 1 ] +} diff --git a/test/blackbox/helpers_pushpull_authn.bash b/test/blackbox/helpers_pushpull_authn.bash new file mode 100644 index 0000000000..4704f76b00 --- /dev/null +++ b/test/blackbox/helpers_pushpull_authn.bash @@ -0,0 +1,316 @@ +# Common helper functions for authenticated push/pull blackbox tests. +# Used by pushpull_authn.bats and fips140_authn.bats. + +load helpers_pushpull + +function verify_authn_prerequisites() { + if ! verify_prerequisites; then + return 1 + fi + + if ! command -v htpasswd >/dev/null; then + echo "you need to install htpasswd as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ "${PUSHPULL_AUTHN_FIPS_MODE:-0}" = 1 ]; then + if ! command -v mkpasswd >/dev/null; then + echo "you need to install mkpasswd as a prerequisite to running the tests" >&3 + return 1 + fi + fi + + return 0 +} + +function authn_write_htpasswd_file() { + local htpasswd_file=${1} + + htpasswd -Bbn "${AUTH_USER}" "${AUTH_PASS}" >>"${htpasswd_file}" + + if [ "${PUSHPULL_AUTHN_FIPS_MODE:-0}" = 1 ]; then + echo "${AUTH_USER2}:$(echo "${AUTH_PASS2}" | mkpasswd -s -R 1 -m sha-256)" >>"${htpasswd_file}" + echo "${AUTH_USER3}:$(echo "${AUTH_PASS3}" | mkpasswd -s -R 1 -m sha-512)" >>"${htpasswd_file}" + echo "${AUTH_USER4}:$(echo "${AUTH_PASS4}" | mkpasswd -s -R 0 -m sha-256)" >>"${htpasswd_file}" + echo "${AUTH_USER5}:$(echo "${AUTH_PASS5}" | mkpasswd -s -R 0 -m sha-512)" >>"${htpasswd_file}" + fi +} + +function authn_write_zot_config() { + local zot_config_file=${1} + local zot_root_dir=${2} + local zot_port=${3} + local zot_htpasswd_file=${4} + local log_file=${5} + + cat >"${zot_config_file}" <"${BATS_FILE_TMPDIR}/zot.port" + authn_write_htpasswd_file "${zot_htpasswd_file}" + + echo "${zot_root_dir}" >&3 + mkdir -p "${zot_root_dir}" + touch "${log_file}" + + authn_write_zot_config "${zot_config_file}" "${zot_root_dir}" "${zot_port}" \ + "${zot_htpasswd_file}" "${log_file}" + + if [ "${PUSHPULL_AUTHN_FIPS_MODE:-0}" = 1 ]; then + export GODEBUG="fips140=only" + fi + + zot_serve "${ZOT_PATH}" "${zot_config_file}" + wait_zot_reachable "${zot_port}" + + run regctl registry set "localhost:${zot_port}" --tls disabled + [ "${status}" -eq 0 ] + + if [ "${PUSHPULL_AUTHN_FIPS_MODE:-0}" = 1 ]; then + log_output | jq 'contains("fips140 is currently enabled")?' | grep true + fi +} + +function authn_teardown() { + cat "${BATS_FILE_TMPDIR}/zot/zot-log.json" +} + +function authn_teardown_file() { + zot_stop_all + + if [ "${PUSHPULL_AUTHN_FIPS_MODE:-0}" = 1 ]; then + unset GODEBUG + fi +} + +function helper_authn_regctl_login() { + local user=${1} + local pass=${2} + + run regctl registry login "localhost:$(get_zot_port)" -u "${user}" -p "${pass}" + [ "${status}" -eq 0 ] +} + +# Args: $1 = source reference, $2 = destination repository +function helper_authn_push_image_with_regclient() { + local source_ref=${1} + local dest_repo=${2} + + helper_authn_regctl_login "${AUTH_USER}" "${AUTH_PASS}" + run regctl image copy "${source_ref}" "localhost:$(get_zot_port)/${dest_repo}" + [ "${status}" -eq 0 ] +} + +# Args: $1 = source repository, $2 = destination reference +function helper_authn_pull_image_with_regclient() { + local source_repo=${1} + local dest_ref=${2} + + helper_authn_regctl_login "${AUTH_USER}" "${AUTH_PASS}" + run regctl image copy "localhost:$(get_zot_port)/${source_repo}" "${dest_ref}" + [ "${status}" -eq 0 ] +} + +# Args: $1 = artifact reference +function helper_authn_push_oci_artifact_with_regclient() { + local ref=${1} + + helper_authn_regctl_login "${AUTH_USER}" "${AUTH_PASS}" + helper_push_oci_artifact_with_regclient "${ref}" +} + +# Args: $1 = artifact reference, $2 = expected artifact content +function helper_authn_pull_oci_artifact_with_regclient() { + local ref=${1} + local expected_content=${2} + + helper_authn_regctl_login "${AUTH_USER}" "${AUTH_PASS}" + helper_pull_oci_artifact_with_regclient "${ref}" "${expected_content}" +} + +function helper_authn_push_oci_artifact_references_with_regclient() { + helper_authn_regctl_login "${AUTH_USER}" "${AUTH_PASS}" + helper_push_oci_artifact_references_with_regclient 0 +} + +function helper_authn_list_oci_artifact_references_with_regclient() { + helper_authn_regctl_login "${AUTH_USER}" "${AUTH_PASS}" + helper_pull_oci_artifact_references_with_regclient 1 +} + +# Args: $1=username, $2=password +function helper_authn_ml_artifacts() { + local user=${1} + local pass=${2} + local zot_port sha256_in sha256_out + + helper_authn_regctl_login "${user}" "${pass}" + zot_port=$(get_zot_port) + + run curl --fail -L -0 \ + https://github.com/tarilabs/demo20231212/raw/main/v1.nb20231206162408/mnist.onnx \ + -o "${BATS_FILE_TMPDIR}/mnist.onnx" + [ "${status}" -eq 0 ] + + run sha256sum "${BATS_FILE_TMPDIR}/mnist.onnx" + [ "${status}" -eq 0 ] + sha256_in=$(echo "${output}" | awk '{print $1}') + [ -n "${sha256_in}" ] + + run regctl artifact put \ + --annotation description="used for demo purposes" \ + --annotation model_format_name="onnx" \ + --annotation model_format_version="1" \ + --artifact-type "application/vnd.model.type" \ + "localhost:${zot_port}/models/my-model-from-gh:v1" \ + -f "${BATS_FILE_TMPDIR}/mnist.onnx" + [ "${status}" -eq 0 ] + + run regctl artifact list "localhost:${zot_port}/models/my-model-from-gh:v1" \ + --format '{{jsonPretty .}}' + [ "${status}" -eq 0 ] + + run regctl artifact list --filter-artifact-type "application/vnd.model.type" \ + "localhost:${zot_port}/models/my-model-from-gh:v1" \ + --format '{{jsonPretty .}}' + [ "${status}" -eq 0 ] + + run bash -c "regctl artifact get 'localhost:${zot_port}/models/my-model-from-gh:v1' >'${BATS_FILE_TMPDIR}/mnist.onnx.check'" + [ "${status}" -eq 0 ] + + run sha256sum "${BATS_FILE_TMPDIR}/mnist.onnx.check" + [ "${status}" -eq 0 ] + sha256_out=$(echo "${output}" | awk '{print $1}') + [ -n "${sha256_out}" ] + [ "${sha256_in}" = "${sha256_out}" ] +} + +# Args: $1=username, $2=password, $3=hash_type, $4=should_succeed (true/false) +function helper_authn_verify_auth_and_push() { + local user=${1} + local pass=${2} + local hash_type=${3} + local should_succeed=${4} + + helper_authn_regctl_login "${user}" "${pass}" + run regctl image copy "ocidir://${TEST_DATA_DIR}/busybox:1.36" \ + "localhost:$(get_zot_port)/test-${hash_type}" + + if [ "${should_succeed}" = true ]; then + [ "${status}" -eq 0 ] + else + [ "${status}" -eq 1 ] + log_output | jq 'contains("htpasswd bcrypt failed since fips140 is enabled")?' | grep true + fi +} + +# Args: $1=username, $2=password, $3=source_repo, $4=dest_ref +function helper_authn_pull_image_with_auth() { + local user=${1} + local pass=${2} + local source_repo=${3} + local dest_ref=${4} + + helper_authn_regctl_login "${user}" "${pass}" + run regctl image copy "localhost:$(get_zot_port)/${source_repo}" \ + "ocidir://${TEST_DATA_DIR}/busybox:${dest_ref}" + [ "${status}" -eq 0 ] +} + +# Args: $1=username, $2=password, $3=artifact_ref, $4=artifact_body +function helper_authn_push_oci_artifact_with_auth() { + local user=${1} + local pass=${2} + local artifact_ref=${3} + local artifact_body=${4} + + helper_authn_regctl_login "${user}" "${pass}" + run regctl artifact put "localhost:$(get_zot_port)/${artifact_ref}" <&3 - return 1 - fi - - if [ ! $(command -v jq) ]; then - echo "you need to install jq as a prerequisite to running the tests" >&3 - return 1 - fi - - return 0 -} - -# Common teardown function - prints zot log on failure -function teardown() { - # conditionally printing on failure is possible from teardown but not from teardown_file - cat ${BATS_FILE_TMPDIR}/zot.log -} - -# Common teardown_file function - stops all zot instances -function teardown_file() { - zot_stop_all -} - -# ============================================================================== -# COMMON HELPER FUNCTIONS -# These are the core functions used by both release and new test functions -# ============================================================================== - -# Helper: Get the zot port -function get_zot_port() { - cat ${BATS_FILE_TMPDIR}/zot.port -} - -# Helper: Push an image using skopeo -# Args: $1 = image_name, $2 = tag -function helper_push_image() { - local image_name=${1:-golang} - local tag=${2:-1.20} - local zot_port=$(get_zot_port) - run skopeo --insecure-policy copy --dest-tls-verify=false \ - oci:${TEST_DATA_DIR}/${image_name}:${tag} \ - docker://127.0.0.1:${zot_port}/${image_name}:${tag} - [ "$status" -eq 0 ] - run curl http://127.0.0.1:${zot_port}/v2/_catalog - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq --arg name "${image_name}" 'any(.repositories[]; . == $name)') = true ] - run curl http://127.0.0.1:${zot_port}/v2/${image_name}/tags/list - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq --arg tag "${tag}" 'any(.tags[]; . == $tag)') = true ] -} - -# Helper: Pull an image using skopeo -# Args: $1 = image_name, $2 = tag -function helper_pull_image() { - local image_name=${1:-golang} - local tag=${2:-1.20} - local oci_data_dir=${BATS_FILE_TMPDIR}/oci - local zot_port=$(get_zot_port) - run skopeo --insecure-policy copy --src-tls-verify=false \ - docker://127.0.0.1:${zot_port}/${image_name}:${tag} \ - oci:${oci_data_dir}/${image_name}:${tag} - [ "$status" -eq 0 ] - run cat ${oci_data_dir}/${image_name}/index.json - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq --arg tag "${tag}" '.manifests[].annotations."org.opencontainers.image.ref.name" == $tag') = true ] -} - -# Helper: Push an image index (multi-arch) -# Args: $1 = source_image, $2 = dest_image_name, $3 = tag -function helper_push_image_index() { - local source_image=${1:-docker://public.ecr.aws/docker/library/busybox:latest} - local dest_name=${2:-busybox} - local tag=${3:-latest} - local zot_port=$(get_zot_port) - run skopeo --insecure-policy copy --format=oci --dest-tls-verify=false --multi-arch=all \ - ${source_image} \ - docker://127.0.0.1:${zot_port}/${dest_name}:${tag} - [ "$status" -eq 0 ] - run curl http://127.0.0.1:${zot_port}/v2/_catalog - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq --arg name "${dest_name}" 'any(.repositories[]; . == $name)') = true ] - run curl http://127.0.0.1:${zot_port}/v2/${dest_name}/tags/list - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq --arg tag "${tag}" 'any(.tags[]; . == $tag)') = true ] -} - -# Helper: Pull an image index (multi-arch) -# Args: $1 = image_name, $2 = tag -function helper_pull_image_index() { - local image_name=${1:-busybox} - local tag=${2:-latest} - local oci_data_dir=${BATS_FILE_TMPDIR}/oci - local zot_port=$(get_zot_port) - run skopeo --insecure-policy copy --src-tls-verify=false --multi-arch=all \ - docker://127.0.0.1:${zot_port}/${image_name}:${tag} \ - oci:${oci_data_dir}/${image_name}:${tag} - [ "$status" -eq 0 ] - run cat ${oci_data_dir}/${image_name}/index.json - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq --arg tag "${tag}" '.manifests[].annotations."org.opencontainers.image.ref.name" == $tag') = true ] - run skopeo --insecure-policy --override-arch=arm64 --override-os=linux copy --src-tls-verify=false --multi-arch=all \ - docker://127.0.0.1:${zot_port}/${image_name}:${tag} \ - oci:${oci_data_dir}/${image_name}:${tag} - [ "$status" -eq 0 ] - run cat ${oci_data_dir}/${image_name}/index.json - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq --arg tag "${tag}" '.manifests[].annotations."org.opencontainers.image.ref.name" == $tag') = true ] -} - -# Helper: Push an ORAS artifact -# Args: $1 = artifact_name, $2 = tag -function helper_push_oras_artifact() { - local artifact_name=${1:-hello-artifact} - local tag=${2:-v2} - local zot_port=$(get_zot_port) - echo "{\"name\":\"foo\",\"value\":\"bar\"}" > config.json - echo "hello world" > artifact.txt - run oras push --plain-http 127.0.0.1:${zot_port}/${artifact_name}:${tag} \ - --config config.json:application/vnd.acme.rocket.config.v1+json artifact.txt:text/plain -d -v - [ "$status" -eq 0 ] - rm -f artifact.txt - rm -f config.json -} - -# Helper: Pull an ORAS artifact -# Args: $1 = artifact_name, $2 = tag -function helper_pull_oras_artifact() { - local artifact_name=${1:-hello-artifact} - local tag=${2:-v2} - local zot_port=$(get_zot_port) - run oras pull --plain-http 127.0.0.1:${zot_port}/${artifact_name}:${tag} -d -v - [ "$status" -eq 0 ] - grep -q "hello world" artifact.txt - rm -f artifact.txt -} - -# Helper: Attach ORAS artifacts (signature and sbom) to an image -# Args: $1 = image_name, $2 = tag -function helper_attach_oras_artifacts() { - local image_name=${1:-golang} - local tag=${2:-1.20} - local zot_port=$(get_zot_port) - # attach signature - echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > ${BATS_FILE_TMPDIR}/signature.json - run oras attach --disable-path-validation --plain-http 127.0.0.1:${zot_port}/${image_name}:${tag} --artifact-type 'signature/example' ${BATS_FILE_TMPDIR}/signature.json:application/json - [ "$status" -eq 0 ] - # attach sbom - echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:${zot_port}/${image_name}:${tag}'\", \"contents\": \"good\"}" > ${BATS_FILE_TMPDIR}/sbom.json - run oras attach --disable-path-validation --plain-http 127.0.0.1:${zot_port}/${image_name}:${tag} --artifact-type 'sbom/example' ${BATS_FILE_TMPDIR}/sbom.json:application/json - [ "$status" -eq 0 ] -} - -# Helper: Discover ORAS artifacts -# Args: $1 = image_name, $2 = tag, $3 = expected_count -function helper_discover_oras_artifacts() { - local image_name=${1:-golang} - local tag=${2:-1.20} - local expected_count=${3:-2} - local zot_port=$(get_zot_port) - run oras discover --plain-http --format json 127.0.0.1:${zot_port}/${image_name}:${tag} - [ "$status" -eq 0 ] - [ $(echo "$output" | jq -r ".manifests | length") -eq ${expected_count} ] -} - -# Helper: Add and list tags using ORAS -function helper_add_and_list_tags_using_oras() { - local zot_port=$(get_zot_port) - run skopeo --insecure-policy copy --dest-tls-verify=false \ - oci:${TEST_DATA_DIR}/golang:1.20 \ - docker://127.0.0.1:${zot_port}/oras-tags:1.20 - [ "$status" -eq 0 ] - run oras tag --plain-http 127.0.0.1:${zot_port}/oras-tags:1.20 1 new latest - [ "$status" -eq 0 ] - run oras repo tags --plain-http 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 4 ] - [ "${lines[-1]}" == "new" ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-3]}" == "1.20" ] - [ "${lines[-4]}" == "1" ] - run oras repo tags --plain-http --last new 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ -z "$output" ] - run oras repo tags --plain-http --last latest 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 1 ] - [ "${lines[-1]}" == "new" ] - run oras repo tags --plain-http --last "1.20" 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 2 ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-1]}" == "new" ] - run oras repo tags --plain-http --last "1" 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 3 ] - [ "${lines[-3]}" == "1.20" ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-1]}" == "new" ] -} - -# Helper: Push a Helm chart -function helper_push_helm_chart() { - local zot_port=$(get_zot_port) - run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot -d ${BATS_FILE_TMPDIR} - [ "$status" -eq 0 ] - local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml) - run helm push ${BATS_FILE_TMPDIR}/zot-${chart_version}.tgz oci://localhost:${zot_port}/zot-chart - [ "$status" -eq 0 ] -} - -# Helper: Pull a Helm chart -function helper_pull_helm_chart() { - local zot_port=$(get_zot_port) - local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml) - run helm pull oci://localhost:${zot_port}/zot-chart/zot --version ${chart_version} -d ${BATS_FILE_TMPDIR} - [ "$status" -eq 0 ] -} - -# Helper: Push image with regclient -function helper_push_image_with_regclient() { - local zot_port=$(get_zot_port) - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl image copy ocidir://${TEST_DATA_DIR}/golang:1.20 localhost:${zot_port}/test-regclient - [ "$status" -eq 0 ] -} - -# Helper: Pull image with regclient -function helper_pull_image_with_regclient() { - local zot_port=$(get_zot_port) - run regctl image copy localhost:${zot_port}/test-regclient ocidir://${TEST_DATA_DIR}/golang:1.20 - [ "$status" -eq 0 ] -} - -# Helper: List repositories with regclient -# Args: $1 = limit (optional), $2 = expected_first_repos (optional, space separated) -function helper_list_repositories_with_regclient() { - local limit=${1:-2} - local first_repo=${2:-busybox} - local second_repo=${3:-golang} - local zot_port=$(get_zot_port) - run regctl repo ls localhost:${zot_port} - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - if [ "$i" = 'test-regclient' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] - - run regctl repo ls --limit ${limit} localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq ${limit} ] -} - -# Helper: List image tags with regclient -function helper_list_image_tags_with_regclient() { - local zot_port=$(get_zot_port) - run regctl tag ls localhost:${zot_port}/test-regclient - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - if [ "$i" = 'latest' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] -} - -# Helper: Push manifest with regclient -function helper_push_manifest_with_regclient() { - local zot_port=$(get_zot_port) - manifest=$(regctl manifest get localhost:${zot_port}/test-regclient --format=raw-body) - run regctl manifest put localhost:${zot_port}/test-regclient:1.0.0 --format oci --content-type application/vnd.oci.image.manifest.v1+json --format oci < Dockerfile < /testfile -DOCKERFILE - docker build -f Dockerfile . -t localhost:${zot_port}/test - run docker push localhost:${zot_port}/test - [ "$status" -eq 1 ] - run docker pull localhost:${zot_port}/test - [ "$status" -eq 1 ] -} - -# ============================================================================== -# RELEASE TEST FUNCTIONS -# These functions are used to test the released version of zot before upgrade -# ============================================================================== - -function test_release_push_image() { - local zot_port=$(get_zot_port) - run skopeo --insecure-policy copy --dest-tls-verify=false \ - oci:${TEST_DATA_DIR}/golang:1.20 \ - docker://127.0.0.1:${zot_port}/golang:1.20 - [ "$status" -eq 0 ] - run curl http://127.0.0.1:${zot_port}/v2/_catalog - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ] - run curl http://127.0.0.1:${zot_port}/v2/golang/tags/list - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ] -} - -function test_release_pull_image() { - helper_pull_image golang 1.20 -} - -function test_release_push_image_index() { - helper_push_image_index docker://public.ecr.aws/docker/library/busybox:latest busybox latest -} - -function test_release_pull_image_index() { - helper_pull_image_index busybox latest -} - -function test_release_push_oras_artifact() { - helper_push_oras_artifact hello-artifact v2 -} - -function test_release_pull_oras_artifact() { - helper_pull_oras_artifact hello-artifact v2 -} - -function test_release_attach_oras_artifacts() { - helper_attach_oras_artifacts golang 1.20 -} - -function test_release_discover_oras_artifacts() { - helper_discover_oras_artifacts golang 1.20 2 -} - -function test_release_add_and_list_tags_using_oras() { - helper_add_and_list_tags_using_oras -} - -function test_release_push_helm_chart() { - helper_push_helm_chart -} - -function test_release_pull_helm_chart() { - helper_pull_helm_chart -} - -function test_release_push_image_with_regclient() { - helper_push_image_with_regclient -} - -function test_release_pull_image_with_regclient() { - helper_pull_image_with_regclient -} - -function test_release_list_repositories_with_regclient() { - local zot_port=$(get_zot_port) - run regctl repo ls localhost:${zot_port} - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - if [ "$i" = 'test-regclient' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] - - run regctl repo ls --limit 2 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 2 ] - [ "${lines[-2]}" == "busybox" ] - [ "${lines[-1]}" == "golang" ] - - run regctl repo ls --last busybox --limit 1 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 1 ] - [ "${lines[-1]}" == "golang" ] -} - -function test_release_list_image_tags_with_regclient() { - helper_list_image_tags_with_regclient -} - -function test_release_push_manifest_with_regclient() { - helper_push_manifest_with_regclient -} - -function test_release_pull_manifest_with_regclient() { - helper_pull_manifest_with_regclient -} - -function test_release_pull_manifest_with_docker_client() { - helper_pull_manifest_with_docker_client -} - -function test_release_pull_manifest_with_crictl() { - helper_pull_manifest_with_crictl -} - -function test_release_push_oci_artifact_with_regclient() { - helper_push_oci_artifact_with_regclient -} - -function test_release_pull_oci_artifact_with_regclient() { - helper_pull_oci_artifact_with_regclient -} - -function test_release_push_oci_artifact_references_with_regclient() { - helper_push_oci_artifact_references_with_regclient 0 -} - -function test_release_pull_oci_artifact_references_with_regclient() { - helper_pull_oci_artifact_references_with_regclient 1 -} - -function test_release_push_docker_image() { - helper_push_docker_image -} - -# ============================================================================== -# NEW (POST-UPGRADE) TEST FUNCTIONS -# These functions are used to test the new version of zot after upgrade -# ============================================================================== - -function test_new_existing_pull_image() { - helper_pull_image golang 1.20 -} - -function test_new_existing_pull_image_index() { - helper_pull_image_index busybox latest -} - -function test_new_existing_pull_oras_artifact() { - helper_pull_oras_artifact hello-artifact v2 -} - -function test_new_push_image() { - local zot_port=$(get_zot_port) - # first check existing images - run curl http://127.0.0.1:${zot_port}/v2/_catalog - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq 'any(.repositories[]; . == "golang")') = true ] - run skopeo --insecure-policy copy --dest-tls-verify=false \ - oci:${TEST_DATA_DIR}/golang:1.20 \ - docker://127.0.0.1:${zot_port}/golang:1.20 - [ "$status" -eq 0 ] - run skopeo --insecure-policy copy --dest-tls-verify=false \ - docker://ghcr.io/project-zot/test-images/alpine:3.17.3 \ - docker://127.0.0.1:${zot_port}/alpine:3.17.3 - [ "$status" -eq 0 ] - run curl http://127.0.0.1:${zot_port}/v2/_catalog - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq 'any(.repositories[]; . == "golang")') = true ] - [ $(echo "${lines[-1]}" | jq 'any(.repositories[]; . == "alpine")') = true ] - run curl http://127.0.0.1:${zot_port}/v2/golang/tags/list - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ] -} - -function test_new_pull_image() { - helper_pull_image golang 1.20 -} - -function test_new_push_image_index() { - local zot_port=$(get_zot_port) - # --multi-arch below pushes an image index (containing many images) instead - # of an image manifest (single image) - run skopeo --insecure-policy copy --format=oci --dest-tls-verify=false --multi-arch=all \ - docker://public.ecr.aws/docker/library/busybox:latest \ - docker://127.0.0.1:${zot_port}/busybox:latest - [ "$status" -eq 0 ] - run curl http://127.0.0.1:${zot_port}/v2/_catalog - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq 'any(.repositories[]; . == "busybox")') = true ] - run curl http://127.0.0.1:${zot_port}/v2/busybox/tags/list - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.tags[]') = '"latest"' ] -} - -function test_new_pull_image_index() { - local oci_data_dir=${BATS_FILE_TMPDIR}/oci - local zot_port=$(get_zot_port) - run skopeo --insecure-policy copy --src-tls-verify=false --multi-arch=all \ - docker://127.0.0.1:${zot_port}/busybox:latest \ - oci:${oci_data_dir}/busybox:latest - [ "$status" -eq 0 ] - run cat ${BATS_FILE_TMPDIR}/oci/busybox/index.json - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.manifests[].annotations."org.opencontainers.image.ref.name"') = '"latest"' ] - run skopeo --insecure-policy --override-arch=arm64 --override-os=linux copy --src-tls-verify=false --multi-arch=all \ - docker://127.0.0.1:${zot_port}/busybox:latest \ - oci:${oci_data_dir}/busybox:latest - [ "$status" -eq 0 ] - run cat ${BATS_FILE_TMPDIR}/oci/busybox/index.json - [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.manifests[].annotations."org.opencontainers.image.ref.name"') = '"latest"' ] - run curl -X DELETE http://127.0.0.1:${zot_port}/v2/busybox/manifests/latest - [ "$status" -eq 0 ] -} - -function test_new_push_oras_artifact() { - helper_push_oras_artifact hello-artifact v2 -} - -function test_new_pull_oras_artifact() { - helper_pull_oras_artifact hello-artifact v2 -} - -function test_new_attach_oras_artifacts() { - helper_attach_oras_artifacts golang 1.20 -} - -# Note: expected_count parameter allows different counts for full vs minimal zot -function test_new_discover_oras_artifacts() { - local expected_count=${1:-4} - helper_discover_oras_artifacts golang 1.20 ${expected_count} -} - -function test_new_add_and_list_tags_using_oras() { - helper_add_and_list_tags_using_oras -} - -function test_new_push_helm_chart() { - helper_push_helm_chart -} - -function test_new_pull_helm_chart() { - helper_pull_helm_chart -} - -function test_new_push_image_with_regclient() { - helper_push_image_with_regclient -} - -function test_new_pull_image_with_regclient() { - helper_pull_image_with_regclient -} - -function test_new_list_repositories_with_regclient() { - local zot_port=$(get_zot_port) - run regctl repo ls localhost:${zot_port} - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - if [ "$i" = 'test-regclient' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] - - run regctl repo ls --limit 4 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 4 ] - [ "${lines[0]}" == "alpine" ] - [ "${lines[-1]}" == "busybox" ] - - run regctl repo ls --last busybox --limit 1 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 1 ] - [ "${lines[-1]}" == "golang" ] -} - -function test_new_list_image_tags_with_regclient() { - helper_list_image_tags_with_regclient -} - -function test_new_push_manifest_with_regclient() { - helper_push_manifest_with_regclient -} - -function test_new_pull_manifest_with_regclient() { - helper_pull_manifest_with_regclient -} - -function test_new_pull_manifest_with_docker_client() { - helper_pull_manifest_with_docker_client -} - -function test_new_pull_manifest_with_crictl() { - helper_pull_manifest_with_crictl -} - -function test_new_push_oci_artifact_with_regclient() { - helper_push_oci_artifact_with_regclient -} - -function test_new_pull_oci_artifact_with_regclient() { - helper_pull_oci_artifact_with_regclient -} - -function test_new_push_oci_artifact_references_with_regclient() { - helper_push_oci_artifact_references_with_regclient 1 -} - -function test_new_pull_oci_artifact_references_with_regclient() { - helper_pull_oci_artifact_references_with_regclient 1 -} - -function test_new_push_docker_image() { - helper_push_docker_image -} diff --git a/test/blackbox/helpers_wait.bash b/test/blackbox/helpers_wait.bash index 2d24b78c5c..469cec87ac 100644 --- a/test/blackbox/helpers_wait.bash +++ b/test/blackbox/helpers_wait.bash @@ -26,3 +26,22 @@ function wait_file() { until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do sleep 1; done } + +# Args: $1 = max attempts, $2 = delay seconds, $3+ = command +function retry_until_success() { + local attempts=${1} + local delay=${2} + shift 2 + + while [ "${attempts}" -gt 0 ]; do + run "$@" + if [ "${status}" -eq 0 ]; then + return 0 + fi + attempts=$((attempts - 1)) + if [ "${attempts}" -eq 0 ]; then + return "${status}" + fi + sleep "${delay}" + done +} diff --git a/test/blackbox/metadata.bats b/test/blackbox/metadata.bats index 6a6d9eb50f..2262035caa 100644 --- a/test/blackbox/metadata.bats +++ b/test/blackbox/metadata.bats @@ -26,7 +26,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/metrics.bats b/test/blackbox/metrics.bats index 4220d57e30..9b51e2fca3 100644 --- a/test/blackbox/metrics.bats +++ b/test/blackbox/metrics.bats @@ -22,7 +22,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/metrics_minimal.bats b/test/blackbox/metrics_minimal.bats index 6f89340536..e1ff9f118f 100644 --- a/test/blackbox/metrics_minimal.bats +++ b/test/blackbox/metrics_minimal.bats @@ -23,7 +23,7 @@ function verify_prerequisites() { function setup_file() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/multiarch_index.bats b/test/blackbox/multiarch_index.bats index 62446a4f7b..d61c1c2d08 100644 --- a/test/blackbox/multiarch_index.bats +++ b/test/blackbox/multiarch_index.bats @@ -16,7 +16,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi # Setup zot server diff --git a/test/blackbox/openid_claim_mapping.bats b/test/blackbox/openid_claim_mapping.bats index 49f27cc2d5..7002b9b1f1 100755 --- a/test/blackbox/openid_claim_mapping.bats +++ b/test/blackbox/openid_claim_mapping.bats @@ -40,7 +40,7 @@ IDP_PID="" function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/pushpull.bats b/test/blackbox/pushpull.bats index 7d35b2bd93..0716971774 100644 --- a/test/blackbox/pushpull.bats +++ b/test/blackbox/pushpull.bats @@ -3,386 +3,119 @@ # Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() load helpers_zot +load helpers_pushpull load ../port_helper -function verify_prerequisites { - if [ ! $(command -v curl) ]; then - echo "you need to install curl as a prerequisite to running the tests" >&3 - return 1 - fi - - if [ ! $(command -v jq) ]; then - echo "you need to install jq as a prerequisite to running the tests" >&3 - return 1 - fi - - return 0 -} +PUSHPULL_FIPS_MODE=0 function setup_file() { - # Verify prerequisites are available - if ! $(verify_prerequisites); then - exit 1 - fi - # Download test data to folder common for the entire suite, not just this file - skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 - # Setup zot server - local zot_root_dir=${BATS_FILE_TMPDIR}/zot - local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json - local oci_data_dir=${BATS_FILE_TMPDIR}/oci - mkdir -p ${zot_root_dir} - mkdir -p ${oci_data_dir} - zot_port=$(get_free_port_for_service "zot") - echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port - cat > ${zot_config_file}< config.json - echo "hello world" > artifact.txt - run oras push --plain-http 127.0.0.1:${zot_port}/hello-artifact:v2 \ - --config config.json:application/vnd.acme.rocket.config.v1+json artifact.txt:text/plain -d -v - [ "$status" -eq 0 ] - rm -f artifact.txt - rm -f config.json + helper_push_oras_artifact hello-artifact v2 } @test "pull oras artifact" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run oras pull --plain-http 127.0.0.1:${zot_port}/hello-artifact:v2 -d -v - [ "$status" -eq 0 ] - grep -q "hello world" artifact.txt - rm -f artifact.txt + helper_pull_oras_artifact hello-artifact v2 } @test "attach oras artifacts" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - # attach signature - echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > ${BATS_FILE_TMPDIR}/signature.json - run oras attach --disable-path-validation --plain-http 127.0.0.1:${zot_port}/golang:1.20 --artifact-type 'signature/example' ${BATS_FILE_TMPDIR}/signature.json:application/json - [ "$status" -eq 0 ] - # attach sbom - echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:${zot_port}/golang:1.20'\", \"contents\": \"good\"}" > ${BATS_FILE_TMPDIR}/sbom.json - run oras attach --disable-path-validation --plain-http 127.0.0.1:${zot_port}/golang:1.20 --artifact-type 'sbom/example' ${BATS_FILE_TMPDIR}/sbom.json:application/json - [ "$status" -eq 0 ] + helper_attach_oras_artifacts golang 1.20 } @test "discover oras artifacts" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run oras discover --plain-http --format json 127.0.0.1:${zot_port}/golang:1.20 - [ "$status" -eq 0 ] - [ $(echo "$output" | jq -r ".manifests | length") -eq 2 ] + helper_discover_oras_artifacts golang 1.20 2 } @test "add and list tags using oras" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run skopeo --insecure-policy copy --dest-tls-verify=false \ - oci:${TEST_DATA_DIR}/golang:1.20 \ - docker://127.0.0.1:${zot_port}/oras-tags:1.20 - [ "$status" -eq 0 ] - run oras tag --plain-http 127.0.0.1:${zot_port}/oras-tags:1.20 1 new latest - [ "$status" -eq 0 ] - run oras repo tags --plain-http 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 4 ] - [ "${lines[-1]}" == "new" ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-3]}" == "1.20" ] - [ "${lines[-4]}" == "1" ] - run oras repo tags --plain-http --last new 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ -z $output ] - run oras repo tags --plain-http --last latest 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 1 ] - [ "${lines[-1]}" == "new" ] - run oras repo tags --plain-http --last "1.20" 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 2 ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-1]}" == "new" ] - run oras repo tags --plain-http --last "1" 127.0.0.1:${zot_port}/oras-tags - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 3 ] - [ "${lines[-3]}" == "1.20" ] - [ "${lines[-2]}" == "latest" ] - [ "${lines[-1]}" == "new" ] + helper_add_and_list_tags_using_oras } @test "push helm chart" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot -d ${BATS_FILE_TMPDIR} - [ "$status" -eq 0 ] - local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml) - run helm push ${BATS_FILE_TMPDIR}/zot-${chart_version}.tgz oci://localhost:${zot_port}/zot-chart - [ "$status" -eq 0 ] + helper_push_helm_chart } @test "pull helm chart" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml) - run helm pull oci://localhost:${zot_port}/zot-chart/zot --version ${chart_version} -d ${BATS_FILE_TMPDIR} - [ "$status" -eq 0 ] + helper_pull_helm_chart } @test "push image with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl registry set localhost:${zot_port} --tls disabled - [ "$status" -eq 0 ] - run regctl image copy ocidir://${TEST_DATA_DIR}/golang:1.20 localhost:${zot_port}/test-regclient - [ "$status" -eq 0 ] + helper_push_image_with_regclient "ocidir://${TEST_DATA_DIR}/golang:1.20" test-regclient } @test "pull image with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl image copy localhost:${zot_port}/test-regclient ocidir://${TEST_DATA_DIR}/golang:1.20 - [ "$status" -eq 0 ] + helper_pull_image_with_regclient test-regclient "ocidir://${TEST_DATA_DIR}/golang:1.20" } @test "list repositories with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl repo ls localhost:${zot_port} - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - - if [ "$i" = 'test-regclient' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] - - run regctl repo ls --limit 2 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 2 ] - [ "${lines[-2]}" == "busybox" ] - [ "${lines[-1]}" == "golang" ] - - run regctl repo ls --last busybox --limit 1 localhost:${zot_port} - [ "$status" -eq 0 ] - echo "$output" - [ $(echo "$output" | wc -l) -eq 1 ] - [ "${lines[-1]}" == "golang" ] + helper_list_repositories_with_regclient_pagination 2 busybox golang test-regclient "-2:busybox" "-1:golang" } @test "list image tags with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - run regctl tag ls localhost:${zot_port}/test-regclient - [ "$status" -eq 0 ] - - found=0 - for i in "${lines[@]}" - do - - if [ "$i" = 'latest' ]; then - found=1 - fi - done - [ "$found" -eq 1 ] + helper_list_image_tags_with_regclient test-regclient } @test "push manifest with regclient" { - zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port` - manifest=$(regctl manifest get localhost:${zot_port}/test-regclient --format=raw-body) - run regctl manifest put localhost:${zot_port}/test-regclient:1.0.0 --format oci --content-type application/vnd.oci.image.manifest.v1+json --format oci < Dockerfile < /testfile -EOF - docker build -f Dockerfile . -t localhost:${zot_port}/test - run docker push localhost:${zot_port}/test - [ "$status" -eq 1 ] - run docker pull localhost:${zot_port}/test - [ "$status" -eq 1 ] +@test "build docker image and verify docker push and pull fail" { + helper_build_docker_image_push_and_pull } diff --git a/test/blackbox/pushpull_authn.bats b/test/blackbox/pushpull_authn.bats index 63b0ec1c96..e0b2fab173 100644 --- a/test/blackbox/pushpull_authn.bats +++ b/test/blackbox/pushpull_authn.bats @@ -1,198 +1,45 @@ load helpers_zot +load helpers_pushpull_authn load ../port_helper -function verify_prerequisites { - if [ ! $(command -v curl) ]; then - echo "you need to install curl as a prerequisite to running the tests" >&3 - return 1 - fi - - if [ ! $(command -v jq) ]; then - echo "you need to install jq as a prerequisite to running the tests" >&3 - return 1 - fi - - if [ ! $(command -v htpasswd) ]; then - echo "you need to install htpasswd as a prerequisite to running the tests" >&3 - return 1 - fi - - return 0 -} +PUSHPULL_AUTHN_FIPS_MODE=0 function setup_file() { - # Verify prerequisites are available - if ! $(verify_prerequisites); then - exit 1 - fi - - # Download test data to folder common for the entire suite, not just this file - skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/busybox:1.36 oci:${TEST_DATA_DIR}/busybox:1.36 - - # Setup zot server - local zot_root_dir=${BATS_FILE_TMPDIR}/zot - local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json - local zot_htpasswd_file=${BATS_FILE_TMPDIR}/zot_htpasswd - zot_port=$(get_free_port_for_service "zot") - echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port - htpasswd -Bbn ${AUTH_USER} ${AUTH_PASS} >> ${zot_htpasswd_file} - - echo ${zot_root_dir} >&3 - - mkdir -p ${zot_root_dir} - - cat > ${zot_config_file}< ${BATS_FILE_TMPDIR}/mnist.onnx.check - sha256_out=$(sha256sum ${BATS_FILE_TMPDIR}/mnist.onnx.check | awk '{print $1}') - [ "$sha256_in" = "$sha256_out" ] + helper_authn_ml_artifacts "${AUTH_USER}" "${AUTH_PASS}" } diff --git a/test/blackbox/pushpull_running_dedupe.bats b/test/blackbox/pushpull_running_dedupe.bats index 68c42551e0..dd3c48be6d 100644 --- a/test/blackbox/pushpull_running_dedupe.bats +++ b/test/blackbox/pushpull_running_dedupe.bats @@ -21,7 +21,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi # Download test data to folder common for the entire suite, not just this file diff --git a/test/blackbox/quota.bats b/test/blackbox/quota.bats index 8eef8bbba5..49ad34a790 100644 --- a/test/blackbox/quota.bats +++ b/test/blackbox/quota.bats @@ -26,7 +26,7 @@ function verify_prerequisites { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/redis_local.bats b/test/blackbox/redis_local.bats index 4e379ec454..ef009c3347 100644 --- a/test/blackbox/redis_local.bats +++ b/test/blackbox/redis_local.bats @@ -27,7 +27,7 @@ function verify_prerequisites() { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/redis_s3.bats b/test/blackbox/redis_s3.bats index 46d719fc37..671a70bbd0 100644 --- a/test/blackbox/redis_s3.bats +++ b/test/blackbox/redis_s3.bats @@ -18,7 +18,7 @@ function verify_prerequisites() { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/redis_session_store.bats b/test/blackbox/redis_session_store.bats index 0b9abf4fad..d35b39f9ef 100644 --- a/test/blackbox/redis_session_store.bats +++ b/test/blackbox/redis_session_store.bats @@ -31,7 +31,7 @@ REDIS_TEST_CONTAINER_NAME="redis_sessions_server_local" function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/referrers.bats b/test/blackbox/referrers.bats index 56f0141ab4..1d77c090de 100644 --- a/test/blackbox/referrers.bats +++ b/test/blackbox/referrers.bats @@ -21,7 +21,7 @@ function verify_prerequisites() { function setup() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/restore_s3_blobs.bats b/test/blackbox/restore_s3_blobs.bats index f43da6873b..1e43c65c01 100644 --- a/test/blackbox/restore_s3_blobs.bats +++ b/test/blackbox/restore_s3_blobs.bats @@ -8,7 +8,7 @@ load ../port_helper function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/sbom.bats b/test/blackbox/sbom.bats index 7720954d9d..a385d7d929 100644 --- a/test/blackbox/sbom.bats +++ b/test/blackbox/sbom.bats @@ -24,7 +24,7 @@ function verify_prerequisites { } function setup_file() { - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/scrub.bats b/test/blackbox/scrub.bats index 51b71965d7..c83bff9792 100644 --- a/test/blackbox/scrub.bats +++ b/test/blackbox/scrub.bats @@ -16,7 +16,7 @@ function setup_file(){ function setup() { # verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/sync.bats b/test/blackbox/sync.bats index 9d5d339565..78a9b63b3e 100644 --- a/test/blackbox/sync.bats +++ b/test/blackbox/sync.bats @@ -29,7 +29,7 @@ function setup_file() { export COSIGN_PASSWORD="" # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi @@ -321,14 +321,11 @@ EOF @test "sync signatures periodically" { zot_port1=`cat ${BATS_FILE_TMPDIR}/zot.port1` - # wait for signatures to be copied + # wait for signatures to be copied (PollInterval is 10s; allow extra margin on slow CI) run sleep 15s - run notation verify --insecure-registry localhost:${zot_port1}/golang:1.20 - [ "$status" -eq 0 ] - - run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.pub localhost:${zot_port1}/golang:1.20 - [ "$status" -eq 0 ] + retry_until_success 12 5 notation verify --insecure-registry localhost:${zot_port1}/golang:1.20 + retry_until_success 12 5 cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.pub localhost:${zot_port1}/golang:1.20 } @test "sync signatures ondemand" { diff --git a/test/blackbox/sync_cloud.bats b/test/blackbox/sync_cloud.bats index 119b325387..27a6e4be0c 100644 --- a/test/blackbox/sync_cloud.bats +++ b/test/blackbox/sync_cloud.bats @@ -29,7 +29,7 @@ function setup_file() { export COSIGN_PASSWORD="" # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi @@ -347,14 +347,11 @@ EOF @test "sync signatures periodically" { zot_port1=`cat ${BATS_FILE_TMPDIR}/zot.port1` - # wait for signatures to be copied + # wait for signatures to be copied (PollInterval is 10s; allow extra margin on slow CI) run sleep 15s - run notation verify --insecure-registry localhost:${zot_port1}/golang:1.20 - [ "$status" -eq 0 ] - - run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.pub localhost:${zot_port1}/golang:1.20 - [ "$status" -eq 0 ] + retry_until_success 12 5 notation verify --insecure-registry localhost:${zot_port1}/golang:1.20 + retry_until_success 12 5 cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.pub localhost:${zot_port1}/golang:1.20 } @test "sync signatures ondemand" { diff --git a/test/blackbox/sync_docker.bats b/test/blackbox/sync_docker.bats index 43c2119e62..fd07d44c60 100644 --- a/test/blackbox/sync_docker.bats +++ b/test/blackbox/sync_docker.bats @@ -26,7 +26,7 @@ function verify_prerequisites() { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/sync_harness.bats b/test/blackbox/sync_harness.bats index 282592f90b..6eca8b8236 100644 --- a/test/blackbox/sync_harness.bats +++ b/test/blackbox/sync_harness.bats @@ -12,7 +12,7 @@ function verify_prerequisites() { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/sync_replica_cluster.bats b/test/blackbox/sync_replica_cluster.bats index a841ad318a..b59b2fbc18 100644 --- a/test/blackbox/sync_replica_cluster.bats +++ b/test/blackbox/sync_replica_cluster.bats @@ -26,7 +26,7 @@ function verify_prerequisites() { function setup_file() { # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi diff --git a/test/blackbox/upgrade.bats b/test/blackbox/upgrade.bats index 196df773af..26aa690bec 100644 --- a/test/blackbox/upgrade.bats +++ b/test/blackbox/upgrade.bats @@ -3,16 +3,17 @@ # Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites() load helpers_zot -load helpers_upgrade +load helpers_pushpull load ../port_helper function setup_file() { # Use unique config name based on test file name and test run to avoid conflicts export REGISTRY_NAME=$(basename "${BASH_SOURCE[0]}" .bats)-$(basename "${BATS_FILE_TMPDIR}") # Verify prerequisites are available - if ! $(verify_prerequisites); then + if ! verify_prerequisites; then exit 1 fi + pushpull_isolate_regctl_config # Download test data to folder common for the entire suite, not just this file skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 # Setup zot server @@ -21,6 +22,7 @@ function setup_file() { local oci_data_dir=${BATS_FILE_TMPDIR}/oci mkdir -p ${zot_root_dir} mkdir -p ${oci_data_dir} + touch "${zot_root_dir}/zot-log.json" zot_port=$(get_free_port_for_service "zot") echo ${zot_port} > ${BATS_FILE_TMPDIR}/zot.port cat > ${zot_config_file}< ${BATS_FILE_TMPDIR}/zot.port cat > ${zot_config_file}<