Wagon perf improvements

README_bilt.md

@@ -0,0 +1,59 @@
+# Bilt AR Wagon Fork
+
+go/arwagon
+
+This describes how to use Bilt's GCP Artifact Registry maven wagon fork, which has much improved
+performance. GCP promised they are treating poor Java AR performance as critical, but it is unclear
+when significant improvements will actually be made.
+
+Q: Would it be just as fast to use virtual with fallback to maven central in pom.xml?
+Q: Does remote API have quota problems unlike remote?
+Q: Does remote API not pull from central unlike remote?
+Q: how to do jobrunr crap?
+
+
+
+## Interface
+
+On GitHub, we use remote API; everywhere else we just use maven central. This can be overriden by:
+
+* **-Dcom.bilt.internal.arwagon.bilt-redirect=<value>**, where value is either:
+  * none
+  * standard
+* **-Dcom.bilt.internal.arwagon.other-redirect=<value>**, where value is either:
+  * none
+  * remote
+  * remote-api
+  * maven-central
+
+
+## Performance
+
+On Github ubuntu-latest, with wagon 2.2.5, mvn 3.9.11 (default), mvn go:dependency-offline took:
+
+* tests/maven-0-artifacts: 19:21 min
+* tests/quarkus-starter: 39:04 min
+* tests/payment-svc: 179 min
+
+On local dev machine in VA, maven 3.8.7, tests/maven-0-artifacts (~450 downloads, excluding SHAs):
+
+* maven central: 12s
+* wagon 2.2.5, bilt-maven (mvn will fallback to maven central): 1:27 min
+* wagon 2.2.5, bilt-maven, fallback to maven-central-cache: 5:32 min
+* wagon 2.2.5, virtual: 13:08 min
+* wagon 2.2.5 (edited), virtual, redirect non-bilt to remote AR API: 1:43min
+
+See here for more numbers: https://docs.google.com/document/d/1z8XYDS-xTj2Y3EzUMFGM-lqa9zGcHnjo3B_KI-aiiBc/edit?tab=t.0#heading=h.hhx1fxwzd219
+
+## Future Improvements
+
+use latest maven with parallelization flags and bf collector (2-10x)
+
+run builds in GCP network (2-10x)
+
+prefetch SHAs/JARs (or lazily verify SHAs)
+
+> Maven fetches pom, then pom.sha1, then eventually jar, then jar.sha1. We can prefetch on
+> pom request, and/or compute the SHAs ourself and verify async.
+
+better connection pooling / http2 (~15% from early experiments)

artifactregistry-maven-wagon/src/main/java/com/google/cloud/artifactregistry/wagon/ArtifactRegistryWagon.java

@@ -35,6 +35,9 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+
 import org.apache.maven.wagon.AbstractWagon;
 import org.apache.maven.wagon.ConnectionException;
 import org.apache.maven.wagon.ResourceDoesNotExistException;
@@ -59,6 +62,7 @@ private InputStream getInputStream(Resource resource)
       throws TransferFailedException, ResourceDoesNotExistException, AuthorizationException {
     try {
       GenericUrl url = googleRepository.constructURL(resource.getName());
+      System.out.println(url);
       HttpRequest request = requestFactory.buildGetRequest(url);
       HttpResponse response = request.execute();
       return response.getContent();
@@ -240,12 +244,28 @@ private static class GoogleRepository {
     }
 
     GenericUrl constructURL(String artifactPath) {
+      if (artifactPath.startsWith("com/bilt/") || artifactPath.startsWith("com/biltrewards/") || artifactPath.startsWith("com/biltcard/")) {        
+        GenericUrl url = new GenericUrl();
+        url.setScheme("https");
+        url.setHost(repository.getHost());
+        url.appendRawPath("/single-scholar-280421/bilt-maven");
+        url.appendRawPath("/");
+        url.appendRawPath(artifactPath);
+        return url;
+      }
+
       GenericUrl url = new GenericUrl();
       url.setScheme("https");
-      url.setHost(repository.getHost());
-      url.appendRawPath(repository.getBasedir());
-      url.appendRawPath("/");
-      url.appendRawPath(artifactPath);
+      url.setHost("artifactregistry.googleapis.com");
+      url.appendRawPath("/download/v1/projects/single-scholar-280421/locations/us/repositories/maven-central-cache/files/");
+      try {
+        url.appendRawPath(URLEncoder.encode(artifactPath, "UTF-8"));
+      } catch (UnsupportedEncodingException e) {
+        // UTF-8 is always supported, this should never happen
+        throw new RuntimeException("UTF-8 encoding not supported", e);
+      }
+      url.appendRawPath(":download");
+      url.set("alt", "media");
       return url;
     }
   }

run_test.sh

@@ -10,7 +10,7 @@ SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
 TEST_DIR=$1
 
-REPO_URL=artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central
+REPO_URL=${REPO_URL:-artifactregistry://us-maven.pkg.dev/single-scholar-280421/bilt-private-and-maven-central}
 
 # MVN_IMAGE=maven:3.9.11-eclipse-temurin-21-alpine
 # 3.8.7-eclipse-temurin-17-alpine
@@ -33,6 +33,16 @@ EXTENSIONS_XML="
 </extensions>
 "
 
+# SETTINGS_XML="
+# <?xml version='1.0' encoding='UTF-8'?>
+# <settings xmlns='http://maven.apache.org/SETTINGS/1.0.0'
+#           xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+#           xsi:schemaLocation='http://maven.apache.org/SETTINGS/1.0.0 
+#                               http://maven.apache.org/xsd/settings-1.0.0.xsd'>
+
+# </settings>
+# "
+
 SETTINGS_XML="
 <?xml version='1.0' encoding='UTF-8'?>
 <settings xmlns='http://maven.apache.org/SETTINGS/1.0.0'
@@ -45,13 +55,13 @@ SETTINGS_XML="
       <id>custom-repos</id>
       <repositories>
         <repository>
-          <id>virtual</id>
+          <id>custom</id>
           <url>$REPO_URL</url>
         </repository>
       </repositories>
       <pluginRepositories>
         <pluginRepository>
-          <id>virtual</id>
+          <id>custom</id>
           <url>$REPO_URL</url>
         </pluginRepository>
       </pluginRepositories>
@@ -64,6 +74,45 @@ SETTINGS_XML="
 </settings>
 "
 
+# SETTINGS_XML="
+# <?xml version='1.0' encoding='UTF-8'?>
+# <settings xmlns='http://maven.apache.org/SETTINGS/1.0.0'
+#           xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
+#           xsi:schemaLocation='http://maven.apache.org/SETTINGS/1.0.0 
+#                               http://maven.apache.org/xsd/settings-1.0.0.xsd'>
+
+#   <profiles>
+#     <profile>
+#       <id>custom-repos</id>
+#       <repositories>
+#         <repository>
+#           <id>custom</id>
+#           <url>$REPO_URL</url>
+#         </repository>
+#         <repository>
+#           <id>custom2</id>
+#           <url>artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache</url>
+#         </repository>
+#       </repositories>
+#       <pluginRepositories>
+#         <pluginRepository>
+#           <id>custom</id>
+#           <url>$REPO_URL</url>
+#         </pluginRepository>
+#         <pluginRepository>
+#           <id>custom2</id>
+#           <url>artifactregistry://us-maven.pkg.dev/single-scholar-280421/maven-central-cache</url>
+#         </pluginRepository>
+#       </pluginRepositories>
+#     </profile>
+#   </profiles>
+
+#   <activeProfiles>
+#     <activeProfile>custom-repos</activeProfile>
+#   </activeProfiles>
+# </settings>
+# "
+
 ###########################################################################
 ##### END CONFIG
 ###########################################################################

tests/bilt-1-artifact/pom.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.acme</groupId>
+    <artifactId>slow</artifactId>
+    <version>1.0.0-SNAPSHOT</version>
+
+    <properties>
+        <compiler-plugin.version>3.14.0</compiler-plugin.version>
+        <maven.compiler.release>21</maven.compiler.release>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+        <skipITs>true</skipITs>
+    </properties>
+
+      <dependencies>
+        <dependency>
+            <groupId>com.biltcard.common</groupId>
+            <artifactId>egress-proxy</artifactId>
+            <version>1.1.2</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>${compiler-plugin.version}</version>
+                <configuration>
+                    <parameters>true</parameters>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>