feat: add panamax for mirroring crates.io

Makefile

@@ -58,17 +58,20 @@ endif
 POSTGRES_EXTRA_PATH?=./extras/postgres
 POSTGRES_TAG?=14
 
+PANAMAX_EXTRA_PATH?=./extras/panamax
+PANAMAX_TAG?=1.0.6
+
 RUST_LOG?=debug
 
-DOCKER_COMPOSE_ENV=STACK=$(STACK) BACKEND_TAG=$(BACKEND_TAG) DEPLOYER_TAG=$(DEPLOYER_TAG) PROVISIONER_TAG=$(PROVISIONER_TAG) POSTGRES_TAG=${POSTGRES_TAG} APPS_FQDN=$(APPS_FQDN) DB_FQDN=$(DB_FQDN) POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) RUST_LOG=$(RUST_LOG) CONTAINER_REGISTRY=$(CONTAINER_REGISTRY) MONGO_INITDB_ROOT_USERNAME=$(MONGO_INITDB_ROOT_USERNAME) MONGO_INITDB_ROOT_PASSWORD=$(MONGO_INITDB_ROOT_PASSWORD) DD_ENV=$(DD_ENV) USE_TLS=$(USE_TLS)
+DOCKER_COMPOSE_ENV=STACK=$(STACK) BACKEND_TAG=$(BACKEND_TAG) DEPLOYER_TAG=$(DEPLOYER_TAG) PROVISIONER_TAG=$(PROVISIONER_TAG) POSTGRES_TAG=${POSTGRES_TAG} PANAMAX_TAG=${PANAMAX_TAG} APPS_FQDN=$(APPS_FQDN) DB_FQDN=$(DB_FQDN) POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) RUST_LOG=$(RUST_LOG) CONTAINER_REGISTRY=$(CONTAINER_REGISTRY) MONGO_INITDB_ROOT_USERNAME=$(MONGO_INITDB_ROOT_USERNAME) MONGO_INITDB_ROOT_PASSWORD=$(MONGO_INITDB_ROOT_PASSWORD) DD_ENV=$(DD_ENV) USE_TLS=$(USE_TLS)
 
 .PHONY: images clean src up down deploy shuttle-% postgres docker-compose.rendered.yml test bump-% deploy-examples publish publish-% --validate-version
 
 clean:
 	rm .shuttle-*
 	rm docker-compose.rendered.yml
 
-images: shuttle-provisioner shuttle-deployer shuttle-gateway postgres
+images: shuttle-provisioner shuttle-deployer shuttle-gateway postgres panamax
 
 postgres:
 	docker buildx build \
@@ -78,6 +81,14 @@ postgres:
 	       -f $(POSTGRES_EXTRA_PATH)/Containerfile \
 	       $(POSTGRES_EXTRA_PATH)
 
+panamax:
+	docker buildx build \
+	       --build-arg PANAMAX_TAG=$(PANAMAX_TAG) \
+	       --tag $(CONTAINER_REGISTRY)/panamax:$(PANAMAX_TAG) \
+	       $(BUILDX_FLAGS) \
+	       -f $(PANAMAX_EXTRA_PATH)/Containerfile \
+	       $(PANAMAX_EXTRA_PATH)
+
 docker-compose.rendered.yml: docker-compose.yml docker-compose.dev.yml
 	$(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) $(DOCKER_COMPOSE_FILES) $(DOCKER_COMPOSE_CONFIG_FLAGS) -p $(STACK) config > $@
 

deployer/prepare.sh

@@ -18,3 +18,10 @@ shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' >
 # Prefetch crates.io index
 cd /usr/src/shuttle/service
 cargo fetch
+
+# Make future crates requests to our own mirror
+echo '
+[source.shuttle-crates-io-mirror]
+registry = "http://panamax:8080/git/crates.io-index"
+[source.crates-io]
+replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml

docker-compose.yml

@@ -2,6 +2,8 @@ version: "3.7"
 volumes:
   gateway-vol:
   postgres-vol:
+  panamax-crates-vol:
+  panamax-io-index-vol:
 networks:
   user-net:
     attachable: true
@@ -140,3 +142,15 @@ services:
       placement:
         constraints:
           - node.hostname==controller
+  panamax:
+    image: "${CONTAINER_REGISTRY}/panamax:${PANAMAX_TAG}"
+    restart: always
+    networks:
+      user-net:
+    volumes:
+      - panamax-crates-vol:/mirror/crates
+      - panamax-io-index-vol:/mirror/crates.io-index
+    deploy:
+      placement:
+        constraints:
+          - node.hostname==controller

extras/panamax/Containerfile

@@ -0,0 +1,7 @@
+ARG PANAMAX_TAG=
+
+FROM panamaxrs/panamax:${PANAMAX_TAG}
+
+COPY mirror.toml /mirror/mirror.toml
+
+CMD ["serve", "/mirror"]

extras/panamax/mirror.toml

@@ -0,0 +1,128 @@
+# This is a Panamax mirror. It is a self-contained directory made to be easily copied
+# to an offline network or machine via rsync, USB, or another method.
+
+# When offline, Panamax also includes a "serve" command that can be used to serve
+# rustup and cargo clients from the mirror. This will also give setup instructions
+# on the homepage.
+
+[mirror]
+# Global mirror settings.
+
+
+# Number of download retries before giving up.
+retries = 5
+
+
+# Contact information for the user agent.
+# This is entirely optional, and is not required for the crates.io CDN.
+# You may want to set this if you are mirroring from somewhere else.
+# contact = "[email protected]"
+
+
+[rustup]
+# These are the configuration parameters for the rustup half of the mirror.
+# This will download the rustup-init files, as well as all components needed
+# to run Rust on a machine.
+
+
+# Perform rustup synchronization. Set this to false if you only want to mirror crates.io.
+sync = false
+
+
+# Whether to mirror XZ archives. These archives are more efficiently compressed
+# than the GZ archives, and rustup uses them by default.
+download_xz = true
+# Whether to mirror GZ archives, for further backwards compatibility with rustup.
+download_gz = false
+
+
+# Number of downloads that can be ran in parallel.
+download_threads = 16
+
+
+# Where to download rustup files from.
+source = "https://static.rust-lang.org"
+
+
+# How many historical versions of Rust to keep.
+# Setting these to 1 will keep only the latest version.
+# Setting these to 2 or higher will keep the latest version, as well as historical versions.
+# Setting these to 0 will stop Panamax from downloading the release entirely.
+# Removing the line will keep all release versions.
+keep_latest_stables = 1
+keep_latest_betas = 1
+keep_latest_nightlies = 1
+
+
+# Pinned versions of Rust to download and keep alongside latest stable/beta/nightly
+# Version specifiers should be in the rustup toolchain format:
+#
+# <channel>[-<date>][-<host>]
+#
+# <channel>       = stable|beta|nightly|<major.minor>|<major.minor.patch>
+# <date>          = YYYY-MM-DD
+# <host>          = <target-triple>
+#
+# e.g. valid versions could be "1.42", "1.42.0", and "nightly-2014-12-18"
+# Uncomment the following lines to pin extra rust versions:
+
+#pinned_rust_versions = [
+#    "1.42"
+#]
+
+
+# UNIX platforms to include in the mirror
+# Uncomment the following lines to limit which platforms get downloaded.
+# This affects both rustup-inits and components.
+
+# platforms_unix = [
+#     "arm-unknown-linux-gnueabi",
+#     "x86_64-unknown-linux-gnu",
+#     "x86_64-unknown-linux-musl",
+# ]
+
+
+# Windows platforms to include in the mirror
+# Uncomment the following lines to limit which platforms get downloaded.
+# This affects both rustup-inits and components.
+
+# platforms_windows = [
+#     "x86_64-pc-windows-gnu",
+#     "x86_64-pc-windows-msvc",
+# ]
+
+
+# Whether to download the rustc-dev component.
+# This component isn't always needed, so setting this to false can save lots of space.
+download_dev = false
+
+
+[crates]
+# These are the configuration parameters for the crates.io half of the mirror.
+# This will download the crates.io-index, as well as the crates themselves.
+# Once downloaded, it will then (optionally) rewrite the config.json to point to your mirror.
+
+
+# Perform crates synchronization. Set this to false if you only want to mirror rustup.
+sync = true
+
+
+# Number of downloads that can be ran in parallel.
+download_threads = 64
+
+
+# Where to download the crates from.
+# The default, "https://crates.io/api/v1/crates", will actually instead use the corresponding
+# url at https://static.crates.io in order to avoid a redirect and rate limiting
+source = "https://crates.io/api/v1/crates"
+
+
+# Where to clone the crates.io-index repository from.
+source_index = "https://github.com/rust-lang/crates.io-index"
+
+
+# URL where this mirror's crates directory can be accessed from.
+# Used for rewriting crates.io-index's config.json.
+# Remove this parameter to perform no rewriting.
+# If removed, the `panamax rewrite` command can be used later.
+base_url = "http://panamax:8080/crates"