From 4cf58aad30ed3d928e1d2fe42b42d50aaae40a87 Mon Sep 17 00:00:00 2001 From: Ken Kania Date: Tue, 23 Sep 2025 09:09:08 -0400 Subject: [PATCH 1/6] merge other --- .github/workflows/perf-test.yaml | 45 + .gitignore | 2 + run_test.sh | 113 +++ tests/maven-0-artifacts/pom.xml | 29 + tests/maven-1-artifact/pom.xml | 37 + tests/maven-100-artifacts/pom.xml | 37 + tests/payment-svc/pom.xml | 1311 +++++++++++++++++++++++++++++ tests/quarkus-starter/pom.xml | 128 +++ 8 files changed, 1702 insertions(+) create mode 100644 .github/workflows/perf-test.yaml create mode 100755 run_test.sh create mode 100644 tests/maven-0-artifacts/pom.xml create mode 100644 tests/maven-1-artifact/pom.xml create mode 100644 tests/maven-100-artifacts/pom.xml create mode 100644 tests/payment-svc/pom.xml create mode 100644 tests/quarkus-starter/pom.xml diff --git a/.github/workflows/perf-test.yaml b/.github/workflows/perf-test.yaml new file mode 100644 index 0000000..b747334 --- /dev/null +++ b/.github/workflows/perf-test.yaml @@ -0,0 +1,45 @@ +name: Perf Test + +on: + pull_request: + +# Cancel an in-progress run if a new run is started for the same PR or commit. +concurrency: + group: "${{ github.workflow }} @ ${{ github.event.pull_request.number }}" + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + input: + - tests/maven-0-artifacts + - tests/quarkus-starter + - tests/payment-svc + + permissions: + contents: read + id-token: write # required for google-github-actions/auth + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Authenticate to Google Cloud + uses: google-github-actions/auth@v2 + with: + workload_identity_provider: projects/620518129011/locations/global/workloadIdentityPools/bilt-rewards-prod/providers/github-provider + service_account: github-actions@bilt-cicd.iam.gserviceaccount.com + create_credentials_file: true + + - name: Set up JDK + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + cache: gradle + + - name: Run Perf Test + run: | + ./run_test.sh ${{ matrix.input }} diff --git a/.gitignore b/.gitignore index 306b1c3..3d95db1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .gradle/ .idea/ +bin/ build/ +target/ local.properties \ No newline at end of file diff --git a/run_test.sh b/run_test.sh new file mode 100755 index 0000000..bf9c0e0 --- /dev/null +++ b/run_test.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +########################################################################### +##### BEGIN CONFIG +########################################################################### + +TEST_DIR=$1 + +REPO_URL=artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central + +# MVN_IMAGE=maven:3.9.11-eclipse-temurin-21-alpine +# 3.8.7-eclipse-temurin-17-alpine +# 3.8.8-eclipse-temurin-21-alpine + +MVN_ACTION=dependency:go-offline + +MVN_FLAGS="" #-Dmaven.artifact.threads=40 -Daether.dependencyCollector.impl=bf" + +RUN_DIR=${RUN_DIR:-$SCRIPT_DIR/target/tests/$(date +%s.%6N)} + +EXTENSIONS_XML=" + + + com.google.cloud.artifactregistry + artifactregistry-maven-wagon + 2.2.6-SNAPSHOT + + +" + +SETTINGS_XML=" + + + + + + custom-repos + + + virtual + $REPO_URL + + + + + virtual + $REPO_URL + + + + + + + custom-repos + + +" + +########################################################################### +##### END CONFIG +########################################################################### + +function run_test() { + run_dir=$1 + repo_dir=$run_dir/_repo + cmd="mvn $MVN_ACTION -B -Dmaven.test.skip=true $MVN_FLAGS" + + touch $run_dir/settings.xml + + if [[ -n "$REPO_URL" ]]; then + echo "Building wagon..." + (cd $SCRIPT_DIR && ./gradlew publishToMavenLocal -Dmaven.repo.local="$repo_dir") + + mkdir -p $run_dir/.mvn + echo "$EXTENSIONS_XML" > $run_dir/.mvn/extensions.xml + + echo "$SETTINGS_XML" > $run_dir/settings.xml + cmd="$cmd -s settings.xml" + fi + + if [[ -n "$MVN_IMAGE" ]]; then + if [[ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then + creds="-v $GOOGLE_APPLICATION_CREDENTIALS:/root/.config/gcloud/application_default_credentials.json:ro" + creds="$creds -e GOOGLE_APPLICATION_CREDENTIALS=/root/.config/gcloud/application_default_credentials.json" + else + creds="-v $HOME/.config:/root/.config:ro" + fi + cmd="docker run --rm \ + -v $run_dir:/src \ + $creds \ + -w /src \ + $MVN_IMAGE \ + $cmd -Dmaven.repo.local=/src/_repo" + else + cd $run_dir + cmd="$cmd -Dmaven.repo.local=$repo_dir" + fi + + echo "RUNNING $cmd" + $cmd | tee -a "$run_dir"/test.log +} + +mkdir -p "$RUN_DIR" +cp -rf $TEST_DIR/. "$RUN_DIR" +run_test "$RUN_DIR" diff --git a/tests/maven-0-artifacts/pom.xml b/tests/maven-0-artifacts/pom.xml new file mode 100644 index 0000000..4aad72f --- /dev/null +++ b/tests/maven-0-artifacts/pom.xml @@ -0,0 +1,29 @@ + + + 4.0.0 + org.acme + slow + 1.0.0-SNAPSHOT + + + 3.14.0 + 21 + UTF-8 + UTF-8 + true + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + + \ No newline at end of file diff --git a/tests/maven-1-artifact/pom.xml b/tests/maven-1-artifact/pom.xml new file mode 100644 index 0000000..8c6cdb5 --- /dev/null +++ b/tests/maven-1-artifact/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + org.acme + slow + 1.0.0-SNAPSHOT + + + 3.14.0 + 21 + UTF-8 + UTF-8 + true + + + + + oro + oro + 2.0.6 + + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + + \ No newline at end of file diff --git a/tests/maven-100-artifacts/pom.xml b/tests/maven-100-artifacts/pom.xml new file mode 100644 index 0000000..7ae5be9 --- /dev/null +++ b/tests/maven-100-artifacts/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + org.acme + slow + 1.0.0-SNAPSHOT + + + 3.14.0 + 21 + UTF-8 + UTF-8 + true + + + + + software.amazon.awssdk + s3 + 2.21.0 + + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + + \ No newline at end of file diff --git a/tests/payment-svc/pom.xml b/tests/payment-svc/pom.xml new file mode 100644 index 0000000..497c6ab --- /dev/null +++ b/tests/payment-svc/pom.xml @@ -0,0 +1,1311 @@ + + + 4.0.0 + com.biltcard + payment-svc + 1.0.0-SNAPSHOT + + + 3.13.0 + true + 11 + 11 + UTF-8 + UTF-8 + 1.12.2.Final + quarkus-universe-bom + io.quarkus + 3.3.1 + + + 1.18.34 + 1.5.5.Final + 1.6.14 + + biltrewards_payment-svc + bilt + https://sonarcloud.io + + **/benefits/enums/*.java, + **/UnleashFeatures.java, + **/payment/util/DiffLogger.java, + **/payment/resource/*.java + + + + 2.43.0 + + + + + + ${quarkus.platform.group-id} + ${quarkus.platform.artifact-id} + ${quarkus.platform.version} + pom + import + + + com.google.cloud + libraries-bom + 21.0.0 + pom + import + + + + + + + io.quarkus + quarkus-resteasy-jackson + + + io.quarkus + quarkus-rest-client-jackson + + + io.quarkus + quarkus-rest-client-mutiny + + + io.quarkus + quarkus-cache + + + io.quarkus + quarkus-smallrye-fault-tolerance + + + io.quarkus + quarkus-jdbc-postgresql + + + org.postgresql + postgresql + + + + + org.postgresql + postgresql + 42.7.3 + + + io.quarkus + quarkus-agroal + + + io.quarkus + quarkus-redis-client + + + io.quarkus + quarkus-container-image-jib + + + com.biltcard + common + ${quarkus.platform.version}-2.5.4-SNAPSHOT + + + com.biltcard.common + bilt-feature-flag + 3.9.0 + + + com.biltcard.common + egress-proxy + 1.1.2 + + + com.bilt.payment-log-svc + db-migration + 1.0.0-SNAPSHOT + test + + + + io.swagger + swagger-annotations + ${swagger-annotations.version} + provided + true + + + + org.mapstruct + mapstruct + ${mapstruct.version} + + + org.mapstruct + mapstruct-processor + ${mapstruct.version} + provided + + + + org.apache.commons + commons-csv + 1.11.0 + + + org.apache.commons + commons-lang3 + 3.15.0 + + + + commons-io + commons-io + 2.16.1 + + + + com.google.cloud + google-cloud-workflow-executions + + 2.0.2 + + + + com.biltcard.residential-property + client + 1.3.0-SNAPSHOT + + + + com.biltcard.user-bank + api + 1.4.10-SNAPSHOT + + + + com.biltcard.yardi-reactive + api + 1.0.5-SNAPSHOT + + + + com.biltcard.mri + api + 2.0.1-SNAPSHOT + + + + com.biltcard.realpage + client + 1.0.0-SNAPSHOT + + + + com.biltrewards.support + api + 1.0.2-SNAPSHOT + + + * + * + + + + + + com.biltcard.common + log-events + 1.4.0 + + + + com.biltrewards.transactis_svc + api + 3.26.0-SNAPSHOT + + + com.opencsv + opencsv + 5.9 + + + + net.java.dev.jna + jna + 5.14.0 + test + + + + io.quarkus + quarkus-junit5 + test + + + io.quarkus + quarkus-junit5-component + 3.4.1 + test + + + io.rest-assured + rest-assured + test + + + org.assertj + assertj-core + test + + + org.awaitility + awaitility + 4.2.0 + test + + + io.quarkus + quarkus-junit5-mockito + test + + + org.mockito + mockito-inline + 4.11.0 + test + + + org.testcontainers + postgresql + 1.20.1 + test + + + org.testcontainers + testcontainers + 1.20.1 + test + + + org.testcontainers + gcloud + 1.20.1 + test + + + com.google.cloud + google-cloud-storage + + + com.github.tomakehurst + wiremock + 2.27.2 + test + + + io.quarkus + quarkus-flyway + test + + + org.projectlombok + lombok + ${lombok.version} + provided + + + com.biltcard + common + tests + test-jar + ${quarkus.platform.version}-2.1.0-SNAPSHOT + test + + + com.biltrewards.realpage-v2 + api + 1.0.0-SNAPSHOT + + + + org.junit-pioneer + junit-pioneer + 2.2.0 + test + + + com.vdurmont + semver4j + 3.1.0 + + + com.biltrewards.payco + api-prejakarta + 1.146.0-SNAPSHOT + + + org.jboss + jandex + 2.4.5.Final + + + org.javers + javers-core + 6.9.1 + + + + + + + io.quarkus + quarkus-maven-plugin + ${quarkus.platform.version} + + + + build + + + + + 52201 + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + ${maven.compiler.source} + ${maven.compiler.target} + + + org.projectlombok + lombok + ${lombok.version} + + + org.mapstruct + mapstruct-processor + ${mapstruct.version} + + + + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + me.fabriciorby + maven-surefire-junit5-tree-reporter + 1.3.0 + + + + plain + + true + + + UNICODE + true + true + true + true + false + true + true + false + + 1 + integration + false + + org.jboss.logmanager.LogManager + + 60 + 1 + + + + integration-tests + integration-test + + test + + + !integration + integration + + + + + + + org.openapitools + openapi-generator-maven-plugin + 4.3.1 + + + payment-svc + + generate + + + ${project.basedir}/src/main/resources/openapi.yml + jaxrs-spec + quarkus + false + com.biltcard.api + com.biltcard.model + + ChargeAllocation=com.biltcard.payment.dto.ChargeAllocation + CheckMailType=com.biltcard.payment.domain.CheckMailType + ChargeType=com.biltcard.payment.client.residential_property_reactive.FeeSchedule.ChargeType + FeeSource=com.biltcard.payment.client.residential_property_reactive.FeeSchedule.FeeSource + PubsubBody=com.biltcard.common.gcp.pubsub.model.PubsubBody + DeleteBiltCardRequest=com.biltcard.model.DeleteBiltCardRequest + CreditCardIssuer=com.biltcard.payment.dto.CreditCardIssuer + PaymentAuthorizationSourceType=com.biltcard.payment.domain.PaymentAuthorizationSourceType + + + src/main/java + true + false + true + java8 + + + + + autopay-svc + + generate + + + ${project.basedir}/src/main/resources/autopay-api.yml + jaxrs-spec + quarkus + false + com.biltcard.api + com.biltcard.model + + LocalDateTime=Date + + + java.time.LocalDateTime=java.util.Date + + + src/main/java + true + false + true + java8 + + + + + + autopay-internal-svc + + generate + + + ${project.basedir}/src/main/resources/autopay-internal-api.yml + jaxrs-spec + quarkus + false + com.biltcard.autopay.internal.api + com.biltcard.autopay.internal.model + + OffsetDateTime=LocalDateTime + + + java.time.OffsetDateTime=java.time.LocalDateTime + PaymentRequest=com.biltcard.model.PaymentRequest + + + src/main/java + true + false + true + java8 + true + + + + + realpage-svc-api + + generate + + + ${project.basedir}/src/main/resources/realpage-svc-api.yml + java + microprofile + com.biltcard.generated.client.realpage.api + com.biltcard.generated.client.realpage.model + false + false + + disableMultipart=true + + + src/main/java + true + false + true + + + + + realpage-svc-model + + generate + + + ${project.basedir}/src/main/resources/realpage-svc-api.yml + java + native + false + false + false + com.biltcard.generated.client.realpage.model + false + + src/main/java + true + + + + + checkbook-svc-api + + generate + + + ${project.basedir}/src/main/resources/checkbook-svc-api.yml + java + microprofile + com.biltcard.generated.check.checkbook.api + com.biltcard.generated.check.checkbook.model + false + false + false + + disableMultipart=true + + + src/main/java + true + false + true + + + + + checkbook-svc-model + + generate + + + ${project.basedir}/src/main/resources/checkbook-svc-api.yml + jaxrs-spec + quarkus + com.biltcard.generated.check.checkbook.model + false + true + false + false + + disableMultipart=true + + + src/main/java + true + false + true + + + + + check-svc + + generate + + + ${project.basedir}/src/main/resources/external-apis/check-svc.yml + jaxrs-spec + quarkus + false + com.biltcard.check.api + com.biltcard.check.model + false + PubsubBody + PubsubBody=com.biltcard.common.gcp.pubsub.model.PubsubBody + + src/main/java + true + false + true + + + + + funds-account-svc-api + + generate + + + ${project.basedir}/src/main/resources/funds-account-svc-api.yml + java + native + false + false + false + com.biltcard.funds_account.model + false + + src/main/java + true + + @io.quarkus.runtime.annotations.RegisterForReflection + + + + + + cardworks-api + + generate + + + ${project.basedir}/src/main/resources/cardworks-api.yml + java + native + false + false + false + com.biltcard.cardworks.model + + src/main/java + true + java8 + + @io.quarkus.runtime.annotations.RegisterForReflection + + + + + + loyalty-v2-svc + + generate + + + ${project.basedir}/src/main/resources/loyalty-v2-svc-api.yml + java + microprofile + com.biltcard.generated.client.loyalty_v2.api + com.biltcard.generated.client.loyalty_v2.model + false + false + + disableMultipart=true + + + src/main/java + true + false + true + + + + + column-bank-svc + + generate + + + ${project.basedir}/src/main/resources/column-bank-svc-api.yml + java + microprofile + com.biltcard.generated.client.column_bank.api + com.biltcard.generated.client.column_bank.model + false + false + + disableMultipart=true + + + LocalDateTime=Date + + + java.time.LocalDateTime=java.util.Date + + + src/main/java + true + true + true + java8 + true + + + + + payment-svc-resume-payment + + generate + + + ${project.basedir}/src/main/resources/internal-api.yml + jaxrs-spec + quarkus + false + com.biltcard.api + com.biltcard.model + + OffsetDateTime=Date + + + java.time.OffsetDateTime=java.util.Date + TransactisPaymentRequest=com.biltrewards.transactis_svc.model.internal.PaymentRequest + TransactisMultiPaymentResponse=com.biltrewards.transactis_svc.model.internal.MultiPaymentResponse + TransactisPaymentResponse=com.biltrewards.transactis_svc.model.internal.PaymentResponse + CheckbookCheckResponse=com.biltcard.generated.check.checkbook.model.CheckbookCheckResponse + CheckbookCheckTrackingResponse=com.biltcard.generated.check.checkbook.model.CheckbookCheckTrackingResponse + PaymentInfoResponse=com.biltcard.model.PaymentInfoResponse + Error=com.biltcard.common.exception.response.Error + Address=com.biltcard.payment.domain.Address + CheckMailType=com.biltcard.payment.domain.CheckMailType + PubsubBody=com.biltcard.common.gcp.pubsub.model.PubsubBody + ThirdPartyCardDetails=com.biltcard.model.ThirdPartyCardDetails + VAccountBlockField=com.biltcard.payment.service.util.VAccountBlockField + CreditCardIssuer=com.biltcard.payment.dto.CreditCardIssuer + + RecurringAuthorizationEvent=com.biltcard.payment.domain.RecurringAuthorizationEvent + + + + src/main/java + true + false + true + java8 + + + + + statement-credit-api + + generate + + + ${project.basedir}/src/main/resources/statement-credit-api.yml + jaxrs-spec + quarkus + false + com.biltcard.api.statementcredit + com.biltcard.model.statementcredit + + src/main/java + true + false + true + java8 + + + + + fiserv-api + + generate + + + ${project.basedir}/src/main/resources/fiserv-api.yml + java + native + + com.biltcard.model + false + false + false + false + + src/main/java + true + false + true + + @io.quarkus.runtime.annotations.RegisterForReflection + + + + + + third-party-card-api + + generate + + + ${project.basedir}/src/main/resources/third-party-card-api.yml + jaxrs-spec + quarkus + false + com.biltcard.api.thirdpartycard + com.biltcard.model + + LocalDateTime=Date + + + java.time.LocalDateTime=java.util.Date + + + src/main/java + true + false + true + java8 + + + + + mastercard-consents-api-models + + generate + + + ${project.basedir}/src/main/resources/mastercard-consents-api-models.yml + java + native + false + false + false + com.biltcard.mastercard.consents + + src/main/java + true + java8 + + @io.quarkus.runtime.annotations.RegisterForReflection + + + + + + check-processing-svc-api + + generate + + + ${project.basedir}/src/main/resources/check-processing-svc-api.yml + java + microprofile + com.biltcard.generated.client.check.api + com.biltcard.generated.client.check.model + false + false + + CheckbookCheckTrackingResponse=com.biltcard.generated.check.checkbook.model.CheckbookCheckTrackingResponse + CheckbookCheckResponse=com.biltcard.generated.check.checkbook.model.CheckbookCheckResponse + ApiException=com.biltcard.generated.check.checkbook.api.ApiException + ApiExceptionMapper=com.biltcard.generated.check.checkbook.api.ApiExceptionMapper + + + disableMultipart=true + + + src/main/java + true + false + true + + + + + refund-api + + generate + + + ${project.basedir}/src/main/resources/refund-api.yml + jaxrs-spec + quarkus + true + false + true + com.biltcard.api.refund + com.biltcard.model.refund + + TransactisAsyncMultiPaymentResponse=com.biltcard.model.TransactisAsyncMultiPaymentResponse + + + true + false + src/main/java + true + java8 + + @io.quarkus.runtime.annotations.RegisterForReflection + + + + + + fraud-assessment-api + + generate + + + ${project.basedir}/src/main/resources/openapi-fraud-assessment.yaml + jaxrs-spec + quarkus + com.biltcard.generated.client.fraudassessment.api + com.biltcard.generated.client.fraudassessment.model + false + false + + disableMultipart=true + + + src/main/java + true + false + true + true + java8 + + @io.quarkus.runtime.annotations.RegisterForReflection + + + + + + fraud-management-svc-api + + generate + + + ${project.basedir}/src/main/resources/fraud-management-svc-api.yml + jaxrs-spec + quarkus + com.biltcard.generated.client.fraudmanagementsvc.api + com.biltcard.generated.client.fraudmanagementsvc.model + false + false + + disableMultipart=true + + + src/main/java + true + false + true + true + java8 + + @io.quarkus.runtime.annotations.RegisterForReflection + + + + FraudAssessmentResponse=com.biltcard.generated.client.fraudassessment.model.FraudAssessmentResponse + + + + + internal-api-no-import-mappings + + generate + + + ${project.basedir}/src/main/resources/internal-api-no-import-mappings.yml + jaxrs-spec + quarkus + false + com.biltcard.api.noimportmappings + com.biltcard.model + + src/main/java + true + java8 + + @io.quarkus.runtime.annotations.RegisterForReflection + + true + false + true + + + + + wells-fargo-service-client-api + + generate + + + ${project.basedir}/src/main/resources/openapi-wells-fargo-svc-client.yaml + jaxrs-spec + false + quarkus + false + false + false + com.biltcard.api.wellsfargo + com.biltcard.model.wellsfargo + + disableMultipart=true + + + true + false + java8 + @io.quarkus.runtime.annotations.RegisterForReflection + true + + + OffsetDateTime=LocalDateTime + + + java.time.OffsetDateTime=java.time.LocalDateTime + + + + + card-svc-client + + generate + + + ${project.basedir}/src/main/resources/card-svc-api.yaml + jaxrs-spec + quarkus + false + com.biltcard.generated.client.cardsvc.api + com.biltcard.generated.client.cardsvc.model + + src/main/java + true + false + true + Api + java8 + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + 3.6.0 + + + add-source + generate-sources + + add-source + + + + ${project.build.directory}/generated-sources/openapi/src/gen/java + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + test-jar + + + + + + org.apache.maven.plugins + maven-source-plugin + 3.3.1 + + + attach-sources + verify + + jar-no-fork + test-jar-no-fork + + + + + + + org.flywaydb + flyway-maven-plugin + 10.17.0 + + payments,public + + filesystem:${basedir}/src/main/resources/db/migration/schema/payments + + + + + org.postgresql + postgresql + 42.7.3 + + + org.flywaydb + flyway-database-postgresql + 10.17.3 + + + + + + org.jacoco + jacoco-maven-plugin + 0.8.12 + + + + prepare-agent + + + + report + test + + report + + + + + + org.jboss.jandex + jandex-maven-plugin + 1.2.3 + + + make-index + + jandex + + + + + + + com.diffplug.spotless + spotless-maven-plugin + ${spotless.version} + + + + check + + + + + + + + src/main/java/**/*.java + src/test/java/**/*.java + + + + + 1.18.1 + + true + + + + + + + + + + native + + + native + + + + + + maven-failsafe-plugin + ${surefire-plugin.version} + + + + integration-test + verify + + + + + ${project.build.directory}/${project.build.finalName}-runner + + org.jboss.logmanager.LogManager + + + + + + + + + + native + + + + com.google.cloud + google-cloud-graalvm-support + 0.7.0 + + + net.java.dev.jna + jna + + + + + flyway-dev + + jdbc:postgresql://localhost:52109/defaultdb + migrations + migrations + + + + flyway-seed + + + filesystem:${basedir}/src/main/resources/db/migration + + + + + \ No newline at end of file diff --git a/tests/quarkus-starter/pom.xml b/tests/quarkus-starter/pom.xml new file mode 100644 index 0000000..024e646 --- /dev/null +++ b/tests/quarkus-starter/pom.xml @@ -0,0 +1,128 @@ + + + 4.0.0 + org.acme + code-with-quarkus + 1.0.0-SNAPSHOT + + + 3.14.0 + 21 + UTF-8 + UTF-8 + quarkus-bom + io.quarkus.platform + 3.26.4 + true + 3.5.3 + + + + + + ${quarkus.platform.group-id} + ${quarkus.platform.artifact-id} + ${quarkus.platform.version} + pom + import + + + + + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-rest + + + io.quarkus + quarkus-junit5 + test + + + io.rest-assured + rest-assured + test + + + + + + + ${quarkus.platform.group-id} + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + + build + generate-code + generate-code-tests + native-image-agent + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + maven-failsafe-plugin + ${surefire-plugin.version} + + + + integration-test + verify + + + + + + + ${project.build.directory}/${project.build.finalName}-runner + org.jboss.logmanager.LogManager + ${maven.home} + + + + + + + + + + native + + + native + + + + false + true + + + + \ No newline at end of file From 7ce6ad99632f869de8c556386bbe9a568c4da9f7 Mon Sep 17 00:00:00 2001 From: Ken Kania Date: Tue, 23 Sep 2025 09:26:03 -0400 Subject: [PATCH 2/6] . --- .github/workflows/perf-test.yaml | 2 +- run_test.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/perf-test.yaml b/.github/workflows/perf-test.yaml index b747334..14f9003 100644 --- a/.github/workflows/perf-test.yaml +++ b/.github/workflows/perf-test.yaml @@ -42,4 +42,4 @@ jobs: - name: Run Perf Test run: | - ./run_test.sh ${{ matrix.input }} + mvn --version diff --git a/run_test.sh b/run_test.sh index bf9c0e0..37afb91 100755 --- a/run_test.sh +++ b/run_test.sh @@ -10,7 +10,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) TEST_DIR=$1 -REPO_URL=artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central +REPO_URL=${REPO_URL:-artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central} # MVN_IMAGE=maven:3.9.11-eclipse-temurin-21-alpine # 3.8.7-eclipse-temurin-17-alpine @@ -45,13 +45,13 @@ SETTINGS_XML=" custom-repos - virtual + custom $REPO_URL - virtual + custom $REPO_URL From d2c168376ff27dc558ecc2df4e37fc766a3ee9ce Mon Sep 17 00:00:00 2001 From: Ken Kania Date: Tue, 23 Sep 2025 09:59:39 -0400 Subject: [PATCH 3/6] . --- .github/workflows/perf-test.yaml | 26 ++++++------- .../wagon/ArtifactRegistryWagon.java | 18 +++++++-- run_test.sh | 39 +++++++++++++++++++ 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/.github/workflows/perf-test.yaml b/.github/workflows/perf-test.yaml index 14f9003..2004014 100644 --- a/.github/workflows/perf-test.yaml +++ b/.github/workflows/perf-test.yaml @@ -26,19 +26,19 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Authenticate to Google Cloud - uses: google-github-actions/auth@v2 - with: - workload_identity_provider: projects/620518129011/locations/global/workloadIdentityPools/bilt-rewards-prod/providers/github-provider - service_account: github-actions@bilt-cicd.iam.gserviceaccount.com - create_credentials_file: true - - - name: Set up JDK - uses: actions/setup-java@v4 - with: - java-version: '21' - distribution: 'temurin' - cache: gradle + # - name: Authenticate to Google Cloud + # uses: google-github-actions/auth@v2 + # with: + # workload_identity_provider: projects/620518129011/locations/global/workloadIdentityPools/bilt-rewards-prod/providers/github-provider + # service_account: github-actions@bilt-cicd.iam.gserviceaccount.com + # create_credentials_file: true + + # - name: Set up JDK + # uses: actions/setup-java@v4 + # with: + # java-version: '21' + # distribution: 'temurin' + # cache: gradle - name: Run Perf Test run: | diff --git a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java index 8c2e72a..fd35e5b 100644 --- a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java +++ b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java @@ -35,6 +35,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + import org.apache.maven.wagon.AbstractWagon; import org.apache.maven.wagon.ConnectionException; import org.apache.maven.wagon.ResourceDoesNotExistException; @@ -240,12 +243,19 @@ private static class GoogleRepository { } GenericUrl constructURL(String artifactPath) { + // Use the new Artifact Registry download API format GenericUrl url = new GenericUrl(); url.setScheme("https"); - url.setHost(repository.getHost()); - url.appendRawPath(repository.getBasedir()); - url.appendRawPath("/"); - url.appendRawPath(artifactPath); + url.setHost("artifactregistry.googleapis.com"); + url.appendRawPath("/download/v1/projects/single-scholar-280421/locations/us/repositories/maven-central-cache/files/"); + try { + url.appendRawPath(URLEncoder.encode(artifactPath, "UTF-8")); + } catch (UnsupportedEncodingException e) { + // UTF-8 is always supported, this should never happen + throw new RuntimeException("UTF-8 encoding not supported", e); + } + url.appendRawPath(":download"); + url.set("alt", "media"); return url; } } diff --git a/run_test.sh b/run_test.sh index 37afb91..4cfa106 100755 --- a/run_test.sh +++ b/run_test.sh @@ -64,6 +64,45 @@ SETTINGS_XML=" " +FALLBACK_SETTINGS_XML=" + + + + + + custom-repos + + + custom + $REPO_URL + + + custom2 + artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache + + + + + custom + $REPO_URL + + + custom2 + artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache + + + + + + + custom-repos + + +" + ########################################################################### ##### END CONFIG ########################################################################### From e55ad3dc97d8fa3541556c267f821f0811c5f318 Mon Sep 17 00:00:00 2001 From: Ken Kania Date: Tue, 23 Sep 2025 11:13:02 -0400 Subject: [PATCH 4/6] . --- README_bilt.md | 59 +++++++++++++ .../wagon/ArtifactRegistryWagon.java | 12 ++- run_test.sh | 86 +++++++++++-------- tests/bilt-1-artifact/pom.xml | 37 ++++++++ 4 files changed, 155 insertions(+), 39 deletions(-) create mode 100644 README_bilt.md create mode 100644 tests/bilt-1-artifact/pom.xml diff --git a/README_bilt.md b/README_bilt.md new file mode 100644 index 0000000..7f07e3d --- /dev/null +++ b/README_bilt.md @@ -0,0 +1,59 @@ +# Bilt AR Wagon Fork + +go/arwagon + +This describes how to use Bilt's GCP Artifact Registry maven wagon fork, which has much improved +performance. GCP promised they are treating poor Java AR performance as critical, but it is unclear +when significant improvements will actually be made. + +Q: Would it be just as fast to use virtual with fallback to maven central in pom.xml? +Q: Does remote API have quota problems unlike remote? +Q: Does remote API not pull from central unlike remote? +Q: how to do jobrunr crap? + + + +## Interface + +On GitHub, we use remote API; everywhere else we just use maven central. This can be overriden by: + +* **-Dcom.bilt.internal.arwagon.bilt-redirect=**, where value is either: + * none + * standard +* **-Dcom.bilt.internal.arwagon.other-redirect=**, where value is either: + * none + * remote + * remote-api + * maven-central + + +## Performance + +On Github ubuntu-latest, with wagon 2.2.5, mvn 3.9.11 (default), mvn go:dependency-offline took: + +* tests/maven-0-artifacts: 19:21 min +* tests/quarkus-starter: 39:04 min +* tests/payment-svc: 179 min + +On local dev machine in VA, maven 3.8.7, tests/maven-0-artifacts (~450 downloads, excluding SHAs): + +* maven central: 12s +* wagon 2.2.5, bilt-maven (mvn will fallback to maven central): 1:27 min +* wagon 2.2.5, bilt-maven, fallback to maven-central-cache: 5:32 min +* wagon 2.2.5, virtual: 13:08 min +* wagon 2.2.5 (edited), virtual, redirect non-bilt to remote AR API: 1:43min + +See here for more numbers: https://docs.google.com/document/d/1z8XYDS-xTj2Y3EzUMFGM-lqa9zGcHnjo3B_KI-aiiBc/edit?tab=t.0#heading=h.hhx1fxwzd219 + +## Future Improvements + +use latest maven with parallelization flags and bf collector (2-10x) + +run builds in GCP network (2-10x) + +prefetch SHAs/JARs (or lazily verify SHAs) + +> Maven fetches pom, then pom.sha1, then eventually jar, then jar.sha1. We can prefetch on +> pom request, and/or compute the SHAs ourself and verify async. + +better connection pooling / http2 (~15% from early experiments) diff --git a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java index fd35e5b..05d8a58 100644 --- a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java +++ b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java @@ -62,6 +62,7 @@ private InputStream getInputStream(Resource resource) throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { try { GenericUrl url = googleRepository.constructURL(resource.getName()); + System.out.println(url); HttpRequest request = requestFactory.buildGetRequest(url); HttpResponse response = request.execute(); return response.getContent(); @@ -243,7 +244,16 @@ private static class GoogleRepository { } GenericUrl constructURL(String artifactPath) { - // Use the new Artifact Registry download API format + if (artifactPath.startsWith("com/bilt/") || artifactPath.startsWith("com/biltrewards/") || artifactPath.startsWith("com/biltcard/")) { + GenericUrl url = new GenericUrl(); + url.setScheme("https"); + url.setHost(repository.getHost()); + url.appendRawPath("/single-scholar-280421/bilt-maven"); + url.appendRawPath("/"); + url.appendRawPath(artifactPath); + return url; + } + GenericUrl url = new GenericUrl(); url.setScheme("https"); url.setHost("artifactregistry.googleapis.com"); diff --git a/run_test.sh b/run_test.sh index 4cfa106..673173f 100755 --- a/run_test.sh +++ b/run_test.sh @@ -33,38 +33,17 @@ EXTENSIONS_XML=" " -SETTINGS_XML=" - - +# SETTINGS_XML=" +# +# - - - custom-repos - - - custom - $REPO_URL - - - - - custom - $REPO_URL - - - - +# +# " - - custom-repos - - -" - -FALLBACK_SETTINGS_XML=" +SETTINGS_XML=" " +# SETTINGS_XML=" +# +# + +# +# +# custom-repos +# +# +# custom +# $REPO_URL +# +# +# custom2 +# artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache +# +# +# +# +# custom +# $REPO_URL +# +# +# custom2 +# artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache +# +# +# +# + +# +# custom-repos +# +# +# " + ########################################################################### ##### END CONFIG ########################################################################### diff --git a/tests/bilt-1-artifact/pom.xml b/tests/bilt-1-artifact/pom.xml new file mode 100644 index 0000000..f871ccc --- /dev/null +++ b/tests/bilt-1-artifact/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + org.acme + slow + 1.0.0-SNAPSHOT + + + 3.14.0 + 21 + UTF-8 + UTF-8 + true + + + + + com.biltcard.common + egress-proxy + 1.1.2 + + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + + \ No newline at end of file From cebc68c65308c3e5f0664c8c2e89df80f5b1a3a3 Mon Sep 17 00:00:00 2001 From: Ken Kania Date: Tue, 23 Sep 2025 13:16:45 -0400 Subject: [PATCH 5/6] . --- run_test.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/run_test.sh b/run_test.sh index 673173f..647d710 100755 --- a/run_test.sh +++ b/run_test.sh @@ -122,19 +122,18 @@ function run_test() { repo_dir=$run_dir/_repo cmd="mvn $MVN_ACTION -B -Dmaven.test.skip=true $MVN_FLAGS" - touch $run_dir/settings.xml + mkdir -p $run_dir/.mvn if [[ -n "$REPO_URL" ]]; then echo "Building wagon..." (cd $SCRIPT_DIR && ./gradlew publishToMavenLocal -Dmaven.repo.local="$repo_dir") - mkdir -p $run_dir/.mvn echo "$EXTENSIONS_XML" > $run_dir/.mvn/extensions.xml - - echo "$SETTINGS_XML" > $run_dir/settings.xml - cmd="$cmd -s settings.xml" fi + echo "$SETTINGS_XML" > $run_dir/.mvn/settings.xml + cmd="$cmd -s settings.xml" + if [[ -n "$MVN_IMAGE" ]]; then if [[ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then creds="-v $GOOGLE_APPLICATION_CREDENTIALS:/root/.config/gcloud/application_default_credentials.json:ro" From d7486d661ccc223fbfe738d2bf4e6d6b14efa0d3 Mon Sep 17 00:00:00 2001 From: Ken Kania Date: Tue, 23 Sep 2025 13:35:39 -0400 Subject: [PATCH 6/6] . --- .../wagon/ArtifactRegistryWagon.java | 163 ++++++++++++++++++ run_test.sh | 2 +- 2 files changed, 164 insertions(+), 1 deletion(-) diff --git a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java index 05d8a58..6f24507 100644 --- a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java +++ b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java @@ -32,11 +32,24 @@ import com.google.cloud.artifactregistry.auth.CredentialProvider; import com.google.cloud.artifactregistry.auth.DefaultCredentialProvider; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.Map; import org.apache.maven.wagon.AbstractWagon; import org.apache.maven.wagon.ConnectionException; @@ -57,6 +70,14 @@ public final class ArtifactRegistryWagon extends AbstractWagon { private HttpTransportFactory httpTransportFactory = NetHttpTransport::new; private CredentialProvider credentialProvider = DefaultCredentialProvider.getInstance(); private Credentials credentials; + private static final Map sha1Cache = new ConcurrentHashMap<>(); + private static final ExecutorService jarPrefetchExecutor = new ThreadPoolExecutor( + 10, // core pool size + 50, // maximum pool size + 60L, TimeUnit.SECONDS, // keep alive time + new LinkedBlockingQueue<>() // work queue + ); + private static final String CACHE_DIR = "/tmp/biltarwagon/"; private InputStream getInputStream(Resource resource) throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { @@ -121,12 +142,66 @@ public boolean resourceExists(String resource) @Override public boolean getIfNewer(String resourceName, File destination, long timestamp) throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { + // Check if this is a request for a .sha1 file + if (resourceName.endsWith(".sha1")) { + String originalResource = resourceName.substring(0, resourceName.length() - 5); // Remove .sha1 suffix + String cachedSha1 = sha1Cache.get(originalResource); + if (cachedSha1 != null) { + // Write the cached SHA1 to the destination file + try (OutputStream out = new java.io.FileOutputStream(destination)) { + out.write(cachedSha1.getBytes("UTF-8")); + System.out.println("CACHE HIT SHA1: " + resourceName); + return true; + } catch (IOException e) { + throw new TransferFailedException("Failed to write SHA1 to destination file.", e); + } + } else { + System.out.println("CACHE MISS SHA1: " + resourceName); + } + } + + // Check if this is a .jar request and we have it cached + if (resourceName.endsWith(".jar")) { + File cachedJarFile = getCachedJarFile(resourceName); + if (cachedJarFile.exists()) { + try { + Files.copy(cachedJarFile.toPath(), destination.toPath(), StandardCopyOption.REPLACE_EXISTING); + System.out.println("CACHE HIT JAR: " + resourceName); + + // Ensure SHA1 is cached for this jar + if (!sha1Cache.containsKey(resourceName)) { + String sha1Hash = computeSha1(destination); + sha1Cache.put(resourceName, sha1Hash); + } + + return true; + } catch (IOException e) { + System.err.println("Failed to copy cached jar file, falling back to remote fetch: " + e.getMessage()); + // Fall through to normal fetch + } + } else { + System.out.println("CACHE MISS JAR: " + resourceName); + } + } + + // If this is a .pom file, prefetch the corresponding .jar file before downloading + if (resourceName.endsWith(".pom")) { + prefetchJarFile(resourceName); + } + Resource resource = new Resource(resourceName); this.fireGetInitiated(resource, destination); try { this.fireGetStarted(resource, destination); InputStream input = getInputStream(resource); this.getTransfer(resource, destination, input); + + // Compute and cache SHA1 for .pom and .jar files + if (resourceName.endsWith(".pom") || resourceName.endsWith(".jar")) { + String sha1Hash = computeSha1(destination); + sha1Cache.put(resourceName, sha1Hash); + } + this.fireGetCompleted(resource, destination); } catch (Exception e) { this.fireTransferError(resource, e, TransferEvent.REQUEST_GET); @@ -142,6 +217,94 @@ public void setHttpTransportFactory(HttpTransportFactory httpTransportFactory) { public void setCredentialProvider(CredentialProvider provider) { this.credentialProvider = provider; } + + private String computeSha1(File file) throws TransferFailedException { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + try (FileInputStream fis = new FileInputStream(file)) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = fis.read(buffer)) != -1) { + digest.update(buffer, 0, bytesRead); + } + } + byte[] hashBytes = digest.digest(); + StringBuilder sb = new StringBuilder(); + for (byte b : hashBytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException | IOException e) { + throw new TransferFailedException("Failed to compute SHA1 hash.", e); + } + } + + private void ensureCacheDirectory() { + try { + Path cacheDir = Paths.get(CACHE_DIR); + if (!Files.exists(cacheDir)) { + Files.createDirectories(cacheDir); + } + } catch (IOException e) { + System.err.println("Failed to create cache directory: " + e.getMessage()); + } + } + + private File getCachedJarFile(String jarResourceName) { + // Replace path separators with underscores to create a flat file structure + String fileName = jarResourceName.replace('/', '_').replace('\\', '_'); + return new File(CACHE_DIR + fileName); + } + + private void prefetchJarFile(String pomResourceName) { + // Convert .pom resource name to .jar resource name + if (!pomResourceName.endsWith(".pom")) { + return; + } + + String jarResourceName = pomResourceName.substring(0, pomResourceName.length() - 4) + ".jar"; + File cachedJarFile = getCachedJarFile(jarResourceName); + + // Skip if already cached + if (cachedJarFile.exists()) { + return; + } + + jarPrefetchExecutor.submit(() -> { + try { + ensureCacheDirectory(); + Resource jarResource = new Resource(jarResourceName); + InputStream input = getInputStream(jarResource); + + // Create temporary file first, then move to final location atomically + File tempFile = new File(cachedJarFile.getAbsolutePath() + ".tmp"); + try (OutputStream out = new java.io.FileOutputStream(tempFile)) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = input.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + + // Atomically move temp file to final location + Files.move(tempFile.toPath(), cachedJarFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // Compute and cache SHA1 for the prefetched jar + String sha1Hash = computeSha1(cachedJarFile); + sha1Cache.put(jarResourceName, sha1Hash); + + System.out.println("PREFETCHED JAR: " + jarResourceName); + + } catch (Exception e) { + System.err.println("Failed to prefetch jar " + jarResourceName + ": " + e.getMessage()); + // Clean up any partial files + File tempFile = new File(cachedJarFile.getAbsolutePath() + ".tmp"); + if (tempFile.exists()) { + tempFile.delete(); + } + } + }); + } private void handlePutRequest(File source, Resource resource, GenericUrl url) throws AuthorizationException, ResourceDoesNotExistException, TransferFailedException { diff --git a/run_test.sh b/run_test.sh index 647d710..6ef3af5 100755 --- a/run_test.sh +++ b/run_test.sh @@ -131,7 +131,7 @@ function run_test() { echo "$EXTENSIONS_XML" > $run_dir/.mvn/extensions.xml fi - echo "$SETTINGS_XML" > $run_dir/.mvn/settings.xml + echo "$SETTINGS_XML" > $run_dir/settings.xml cmd="$cmd -s settings.xml" if [[ -n "$MVN_IMAGE" ]]; then