diff --git a/README_bilt.md b/README_bilt.md new file mode 100644 index 0000000..7f07e3d --- /dev/null +++ b/README_bilt.md @@ -0,0 +1,59 @@ +# Bilt AR Wagon Fork + +go/arwagon + +This describes how to use Bilt's GCP Artifact Registry maven wagon fork, which has much improved +performance. GCP promised they are treating poor Java AR performance as critical, but it is unclear +when significant improvements will actually be made. + +Q: Would it be just as fast to use virtual with fallback to maven central in pom.xml? +Q: Does remote API have quota problems unlike remote? +Q: Does remote API not pull from central unlike remote? +Q: how to do jobrunr crap? + + + +## Interface + +On GitHub, we use remote API; everywhere else we just use maven central. This can be overriden by: + +* **-Dcom.bilt.internal.arwagon.bilt-redirect=**, where value is either: + * none + * standard +* **-Dcom.bilt.internal.arwagon.other-redirect=**, where value is either: + * none + * remote + * remote-api + * maven-central + + +## Performance + +On Github ubuntu-latest, with wagon 2.2.5, mvn 3.9.11 (default), mvn go:dependency-offline took: + +* tests/maven-0-artifacts: 19:21 min +* tests/quarkus-starter: 39:04 min +* tests/payment-svc: 179 min + +On local dev machine in VA, maven 3.8.7, tests/maven-0-artifacts (~450 downloads, excluding SHAs): + +* maven central: 12s +* wagon 2.2.5, bilt-maven (mvn will fallback to maven central): 1:27 min +* wagon 2.2.5, bilt-maven, fallback to maven-central-cache: 5:32 min +* wagon 2.2.5, virtual: 13:08 min +* wagon 2.2.5 (edited), virtual, redirect non-bilt to remote AR API: 1:43min + +See here for more numbers: https://docs.google.com/document/d/1z8XYDS-xTj2Y3EzUMFGM-lqa9zGcHnjo3B_KI-aiiBc/edit?tab=t.0#heading=h.hhx1fxwzd219 + +## Future Improvements + +use latest maven with parallelization flags and bf collector (2-10x) + +run builds in GCP network (2-10x) + +prefetch SHAs/JARs (or lazily verify SHAs) + +> Maven fetches pom, then pom.sha1, then eventually jar, then jar.sha1. We can prefetch on +> pom request, and/or compute the SHAs ourself and verify async. + +better connection pooling / http2 (~15% from early experiments) diff --git a/run_test.sh b/run_test.sh index bf9c0e0..b3502c6 100755 --- a/run_test.sh +++ b/run_test.sh @@ -10,7 +10,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) TEST_DIR=$1 -REPO_URL=artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central +REPO_URL=${REPO_URL:-artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central} # MVN_IMAGE=maven:3.9.11-eclipse-temurin-21-alpine # 3.8.7-eclipse-temurin-17-alpine @@ -45,14 +45,22 @@ SETTINGS_XML=" custom-repos - virtual - $REPO_URL + central2 + artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache + + + bilt-maven + artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-maven - virtual - $REPO_URL + central2 + artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache + + + bilt-maven + artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-maven @@ -73,19 +81,17 @@ function run_test() { repo_dir=$run_dir/_repo cmd="mvn $MVN_ACTION -B -Dmaven.test.skip=true $MVN_FLAGS" - touch $run_dir/settings.xml + mkdir -p $run_dir/.mvn if [[ -n "$REPO_URL" ]]; then echo "Building wagon..." (cd $SCRIPT_DIR && ./gradlew publishToMavenLocal -Dmaven.repo.local="$repo_dir") - - mkdir -p $run_dir/.mvn echo "$EXTENSIONS_XML" > $run_dir/.mvn/extensions.xml - - echo "$SETTINGS_XML" > $run_dir/settings.xml - cmd="$cmd -s settings.xml" fi + echo "$SETTINGS_XML" > $run_dir/settings.xml + cmd="$cmd -s settings.xml" + if [[ -n "$MVN_IMAGE" ]]; then if [[ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]]; then creds="-v $GOOGLE_APPLICATION_CREDENTIALS:/root/.config/gcloud/application_default_credentials.json:ro" diff --git a/tests/bilt-1-artifact/pom.xml b/tests/bilt-1-artifact/pom.xml new file mode 100644 index 0000000..f871ccc --- /dev/null +++ b/tests/bilt-1-artifact/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + org.acme + slow + 1.0.0-SNAPSHOT + + + 3.14.0 + 21 + UTF-8 + UTF-8 + true + + + + + com.biltcard.common + egress-proxy + 1.1.2 + + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + + \ No newline at end of file