diff --git a/README_bilt.md b/README_bilt.md new file mode 100644 index 0000000..7f07e3d --- /dev/null +++ b/README_bilt.md @@ -0,0 +1,59 @@ +# Bilt AR Wagon Fork + +go/arwagon + +This describes how to use Bilt's GCP Artifact Registry maven wagon fork, which has much improved +performance. GCP promised they are treating poor Java AR performance as critical, but it is unclear +when significant improvements will actually be made. + +Q: Would it be just as fast to use virtual with fallback to maven central in pom.xml? +Q: Does remote API have quota problems unlike remote? +Q: Does remote API not pull from central unlike remote? +Q: how to do jobrunr crap? + + + +## Interface + +On GitHub, we use remote API; everywhere else we just use maven central. This can be overriden by: + +* **-Dcom.bilt.internal.arwagon.bilt-redirect=**, where value is either: + * none + * standard +* **-Dcom.bilt.internal.arwagon.other-redirect=**, where value is either: + * none + * remote + * remote-api + * maven-central + + +## Performance + +On Github ubuntu-latest, with wagon 2.2.5, mvn 3.9.11 (default), mvn go:dependency-offline took: + +* tests/maven-0-artifacts: 19:21 min +* tests/quarkus-starter: 39:04 min +* tests/payment-svc: 179 min + +On local dev machine in VA, maven 3.8.7, tests/maven-0-artifacts (~450 downloads, excluding SHAs): + +* maven central: 12s +* wagon 2.2.5, bilt-maven (mvn will fallback to maven central): 1:27 min +* wagon 2.2.5, bilt-maven, fallback to maven-central-cache: 5:32 min +* wagon 2.2.5, virtual: 13:08 min +* wagon 2.2.5 (edited), virtual, redirect non-bilt to remote AR API: 1:43min + +See here for more numbers: https://docs.google.com/document/d/1z8XYDS-xTj2Y3EzUMFGM-lqa9zGcHnjo3B_KI-aiiBc/edit?tab=t.0#heading=h.hhx1fxwzd219 + +## Future Improvements + +use latest maven with parallelization flags and bf collector (2-10x) + +run builds in GCP network (2-10x) + +prefetch SHAs/JARs (or lazily verify SHAs) + +> Maven fetches pom, then pom.sha1, then eventually jar, then jar.sha1. We can prefetch on +> pom request, and/or compute the SHAs ourself and verify async. + +better connection pooling / http2 (~15% from early experiments) diff --git a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java index 8c2e72a..05d8a58 100644 --- a/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java +++ b/artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java @@ -35,6 +35,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; + import org.apache.maven.wagon.AbstractWagon; import org.apache.maven.wagon.ConnectionException; import org.apache.maven.wagon.ResourceDoesNotExistException; @@ -59,6 +62,7 @@ private InputStream getInputStream(Resource resource) throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException { try { GenericUrl url = googleRepository.constructURL(resource.getName()); + System.out.println(url); HttpRequest request = requestFactory.buildGetRequest(url); HttpResponse response = request.execute(); return response.getContent(); @@ -240,12 +244,28 @@ private static class GoogleRepository { } GenericUrl constructURL(String artifactPath) { + if (artifactPath.startsWith("com/bilt/") || artifactPath.startsWith("com/biltrewards/") || artifactPath.startsWith("com/biltcard/")) { + GenericUrl url = new GenericUrl(); + url.setScheme("https"); + url.setHost(repository.getHost()); + url.appendRawPath("/single-scholar-280421/bilt-maven"); + url.appendRawPath("/"); + url.appendRawPath(artifactPath); + return url; + } + GenericUrl url = new GenericUrl(); url.setScheme("https"); - url.setHost(repository.getHost()); - url.appendRawPath(repository.getBasedir()); - url.appendRawPath("/"); - url.appendRawPath(artifactPath); + url.setHost("artifactregistry.googleapis.com"); + url.appendRawPath("/download/v1/projects/single-scholar-280421/locations/us/repositories/maven-central-cache/files/"); + try { + url.appendRawPath(URLEncoder.encode(artifactPath, "UTF-8")); + } catch (UnsupportedEncodingException e) { + // UTF-8 is always supported, this should never happen + throw new RuntimeException("UTF-8 encoding not supported", e); + } + url.appendRawPath(":download"); + url.set("alt", "media"); return url; } } diff --git a/run_test.sh b/run_test.sh index bf9c0e0..673173f 100755 --- a/run_test.sh +++ b/run_test.sh @@ -10,7 +10,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) TEST_DIR=$1 -REPO_URL=artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central +REPO_URL=${REPO_URL:-artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central} # MVN_IMAGE=maven:3.9.11-eclipse-temurin-21-alpine # 3.8.7-eclipse-temurin-17-alpine @@ -33,6 +33,16 @@ EXTENSIONS_XML=" " +# SETTINGS_XML=" +# +# + +# +# " + SETTINGS_XML=" " +# SETTINGS_XML=" +# +# + +# +# +# custom-repos +# +# +# custom +# $REPO_URL +# +# +# custom2 +# artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache +# +# +# +# +# custom +# $REPO_URL +# +# +# custom2 +# artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache +# +# +# +# + +# +# custom-repos +# +# +# " + ########################################################################### ##### END CONFIG ########################################################################### diff --git a/tests/bilt-1-artifact/pom.xml b/tests/bilt-1-artifact/pom.xml new file mode 100644 index 0000000..f871ccc --- /dev/null +++ b/tests/bilt-1-artifact/pom.xml @@ -0,0 +1,37 @@ + + + 4.0.0 + org.acme + slow + 1.0.0-SNAPSHOT + + + 3.14.0 + 21 + UTF-8 + UTF-8 + true + + + + + com.biltcard.common + egress-proxy + 1.1.2 + + + + + + + maven-compiler-plugin + ${compiler-plugin.version} + + true + + + + + \ No newline at end of file