Add support for "Builder: oci-import" in diff-pr

Also, update other scripts to use `bashbrew fetch` and `gitCache` effectively to remove unnecessary reimplementations of `BASHBREW_CACHE` default value calculation (new in bashbrew v0.1.8).

As a parting gift, add the raw list of total supported architectures to a file so it's more obvious when new ones are added or removed (like `ubuntu` losing `riscv64`).

Author: tianon

.github/workflows/.bashbrew/action.yml

@@ -10,8 +10,8 @@ runs:
 
     # these two version numbers are intentionally as close together as I could possibly get them because no matter what I tried, GitHub will not allow me to DRY them (can't have any useful variables in `uses:` and can't even have YAML references to steal it in `env:` or something)
     - shell: 'bash -Eeuo pipefail -x {0}'
-      run:    echo BASHBREW_VERSION=v0.1.7 >> "$GITHUB_ENV"
-    - uses: docker-library/[email protected].7
+      run:    echo BASHBREW_VERSION=v0.1.8 >> "$GITHUB_ENV"
+    - uses: docker-library/[email protected].8
       if: inputs.build == 'host'
 
     - run: docker build --pull --tag oisupport/bashbrew:base "https://github.com/docker-library/bashbrew.git#$BASHBREW_VERSION"

diff-pr.sh

@@ -91,7 +91,6 @@ if [ "$#" -eq 0 ]; then
 	set -- $images
 fi
 
-export BASHBREW_CACHE="${BASHBREW_CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/bashbrew}"
 export BASHBREW_LIBRARY="$PWD/oi/library"
 
 : "${BASHBREW_ARCH:=amd64}" # TODO something smarter with arches
@@ -103,9 +102,8 @@ template='
 	{{- "\n" -}}
 	{{- range $.Entries -}}
 		{{- $arch := .HasArchitecture arch | ternary arch (.Architectures | first) -}}
-		{{- $froms := $.ArchDockerFroms $arch . -}}
 		{{- $outDir := join "_" $.RepoName (.Tags | last) -}}
-		git -C "$BASHBREW_CACHE/git" archive --format=tar
+		git -C "{{ gitCache }}" archive --format=tar
 		{{- " " -}}
 		{{- "--prefix=" -}}
 		{{- $outDir -}}
@@ -116,12 +114,28 @@ template='
 		{{- $dir := .ArchDirectory $arch -}}
 		{{- (eq $dir ".") | ternary "" $dir -}}
 		{{- "\n" -}}
-		mkdir -p "$tempDir/{{- $outDir -}}" && echo "{{- .ArchFile $arch -}}" > "$tempDir/{{- $outDir -}}/.bashbrew-dockerfile-name"
+		mkdir -p "$tempDir/{{- $outDir -}}" && echo "{{- .ArchBuilder $arch -}}" > "$tempDir/{{- $outDir -}}/.bashbrew-builder" && echo "{{- .ArchFile $arch -}}" > "$tempDir/{{- $outDir -}}/.bashbrew-file"
 		{{- "\n" -}}
 	{{- end -}}
 	tar -cC "$tempDir" . && rm -rf "$tempDir"
 '
 
+_tar-t() {
+	tar -t "$@" \
+		| grep -vE "$uninterestingTarballGrep" \
+		| sed -e 's!^[.]/!!' \
+			-r \
+			-e 's!([/.-]|^)((lib)?(c?python|py)-?)[0-9]+([.][0-9]+)?([/.-]|$)!\1\2XXX\6!g' \
+		| sort
+}
+
+_jq() {
+	if [ "$#" -eq 0 ]; then
+		set -- '.'
+	fi
+	jq --tab -S "$@"
+}
+
 copy-tar() {
 	local src="$1"; shift
 	local dst="$1"; shift
@@ -132,19 +146,73 @@ copy-tar() {
 		return
 	fi
 
-	local d dockerfiles=()
-	for d in "$src"/*/.bashbrew-dockerfile-name; do
+	local d indexes=() dockerfiles=()
+	for d in "$src"/*/.bashbrew-file; do
 		[ -f "$d" ] || continue
 		local bf; bf="$(< "$d")"
 		local dDir; dDir="$(dirname "$d")"
-		dockerfiles+=( "$dDir/$bf" )
-		if [ "$bf" = 'Dockerfile' ]; then
-			# if "Dockerfile.builder" exists, let's check that too (busybox, hello-world)
-			if [ -f "$dDir/$bf.builder" ]; then
-				dockerfiles+=( "$dDir/$bf.builder" )
+		local builder; builder="$(< "$dDir/.bashbrew-builder")"
+		if [ "$builder" = 'oci-import' ]; then
+			indexes+=( "$dDir/$bf" )
+		else
+			dockerfiles+=( "$dDir/$bf" )
+			if [ "$bf" = 'Dockerfile' ]; then
+				# if "Dockerfile.builder" exists, let's check that too (busybox, hello-world)
+				if [ -f "$dDir/$bf.builder" ]; then
+					dockerfiles+=( "$dDir/$bf.builder" )
+				fi
 			fi
 		fi
-		rm "$d" # remove the ".bashbrew-dockerfile-name" file we created
+		rm "$d" "$dDir/.bashbrew-builder" # remove the ".bashbrew-*" files we created
+	done
+
+	# now that we're done with our globbing needs, let's disable globbing so it doesn't give us wrong answers
+	local -
+	set -o noglob
+
+	for i in "${indexes[@]}"; do
+		local iName; iName="$(basename "$i")"
+		local iDir; iDir="$(dirname "$i")"
+		local iDirName; iDirName="$(basename "$iDir")"
+		local iDst="$dst/$iDirName"
+
+		mkdir -p "$iDst"
+
+		_jq . "$i" > "$iDst/$iName"
+
+		local digest
+		digest="$(jq -r --arg name "$iName" '
+			if $name == "index.json" then
+				.manifests[0].digest
+			else
+				.digest
+			end
+		' "$i")"
+
+		local blob="blobs/${digest//://}"
+		local blobDir; blobDir="$(dirname "$blob")"
+		local manifest="$iDir/$blob"
+		mkdir -p "$iDst/$blobDir"
+		_jq . "$manifest" > "$iDst/$blob"
+
+		local configDigest; configDigest="$(jq -r '.config.digest' "$manifest")"
+		local blob="blobs/${configDigest//://}"
+		local blobDir; blobDir="$(dirname "$blob")"
+		local config="$iDir/$blob"
+		mkdir -p "$iDst/$blobDir"
+		_jq . "$config" > "$iDst/$blob"
+
+		local layers
+		layers="$(jq -r '[ .layers[].digest | @sh ] | join(" ")' "$manifest")"
+		eval "layers=( $layers )"
+		local layerDigest
+		for layerDigest in "${layers[@]}"; do
+			local blob="blobs/${layerDigest//://}"
+			local blobDir; blobDir="$(dirname "$blob")"
+			local layer="$iDir/$blob"
+			mkdir -p "$iDst/$blobDir"
+			_tar-t -f "$layer" > "$iDst/$blob  'tar -t'"
+		done
 	done
 
 	for d in "${dockerfiles[@]}"; do
@@ -238,13 +306,7 @@ copy-tar() {
 					case "$g" in
 						*.tar.* | *.tgz)
 							if [ -s "$dstG" ]; then
-								tar -tf "$dstG" \
-									| grep -vE "$uninterestingTarballGrep" \
-									| sed -e 's!^[.]/!!' \
-										-r \
-										-e 's!([/.-]|^)((lib)?(c?python|py)-?)[0-9]+([.][0-9]+)?([/.-]|$)!\1\2XXX\6!g' \
-									| sort \
-									> "$dstG  'tar -t'"
+								_tar-t -f "$dstG" > "$dstG  'tar -t'"
 							fi
 							;;
 					esac
@@ -269,6 +331,8 @@ _metadata-files() {
 	if [ "$#" -gt 0 ]; then
 		bashbrew list "$@" 2>>temp/_bashbrew.err | sort -uV > temp/_bashbrew-list || :
 
+		bashbrew cat --format '{{ range .Entries }}{{ range .Architectures }}{{ . }}{{ "\n" }}{{ end }}{{ end }}' "$@" 2>>temp/_bashbrew.err | sort -u > temp/_bashbrew-arches || :
+
 		"$diffDir/_bashbrew-cat-sorted.sh" "$@" 2>>temp/_bashbrew.err > temp/_bashbrew-cat || :
 
 		bashbrew list --uniq "$@" \
@@ -277,14 +341,15 @@ _metadata-files() {
 			| xargs -r bashbrew cat --format "$templateLastTags" 2>>temp/_bashbrew.err \
 			> temp/_bashbrew-list-build-order || :
 
+		bashbrew fetch --arch-filter "$@"
 		script="$(bashbrew cat --format "$template" "$@")"
 		mkdir tar
 		( eval "$script" | tar -xiC tar )
 		copy-tar tar temp
 		rm -rf tar
 	fi
 
-	if [ -n "$externalPins" ] && command -v crane &> /dev/null && command -v jq &> /dev/null; then
+	if [ -n "$externalPins" ] && command -v crane &> /dev/null; then
 		local file
 		for file in $externalPins; do
 			[ -e "oi/$file" ] || continue
@@ -293,7 +358,7 @@ _metadata-files() {
 			digest="$(< "oi/$file")"
 			dir="temp/$file"
 			mkdir -p "$dir"
-			bashbrew remote arches --json "$pin@$digest" | jq -S . > "$dir/bashbrew.json"
+			bashbrew remote arches --json "$pin@$digest" | _jq > "$dir/bashbrew.json"
 			local manifests manifest
 			manifests="$(jq -r '
 				[ (
@@ -308,10 +373,10 @@ _metadata-files() {
 			' "$dir/bashbrew.json")"
 			eval "manifests=( $manifests )"
 			for manifest in "${manifests[@]}"; do
-				crane manifest "$pin@$manifest" | jq -S . > "$dir/manifest-${manifest//:/_}.json"
+				crane manifest "$pin@$manifest" | _jq > "$dir/manifest-${manifest//:/_}.json"
 				local config
 				config="$(jq -r '.config.digest' "$dir/manifest-${manifest//:/_}.json")"
-				crane blob "$pin@$config" | jq -S . > "$dir/manifest-${manifest//:/_}-config.json"
+				crane blob "$pin@$config" | _jq > "$dir/manifest-${manifest//:/_}-config.json"
 			done
 		done
 	fi

naughty-commits.sh

@@ -3,16 +3,11 @@ set -Eeuo pipefail
 
 fileSizeThresholdMB='2'
 
-: "${BASHBREW_CACHE:=$HOME/.cache/bashbrew}"
-export BASHBREW_CACHE BASHBREW_ARCH=
-
-if [ ! -d "$BASHBREW_CACHE/git" ]; then
-	# initialize the "bashbrew cache"
-	bashbrew --arch amd64 from --uniq --apply-constraints hello-world:linux > /dev/null
-fi
+export BASHBREW_ARCH=
 
+gitCache="$(bashbrew cat --format '{{ gitCache }}' <(echo 'Maintainers: empty hack (@example)'))"
 _git() {
-	git -C "$BASHBREW_CACHE/git" "$@"
+	git -C "$gitCache" "$@"
 }
 
 if [ "$#" -eq 0 ]; then
@@ -21,12 +16,11 @@ fi
 
 imgs="$(bashbrew list --repos "$@" | sort -u)"
 for img in $imgs; do
+	bashbrew fetch --all "$img" # force `git fetch`
 	commits="$(
 		bashbrew cat --format '
 			{{- range $e := .Entries -}}
 				{{- range $a := .Architectures -}}
-					{{- /* force `git fetch` */ -}}
-					{{- $froms := $.ArchDockerFroms $a $e -}}
 					{
 						{{- json "GitRepo" }}:{{ json ($e.ArchGitRepo $a) -}},
 						{{- json "GitFetch" }}:{{ json ($e.ArchGitFetch $a) -}},

naughty-constraints.sh

@@ -1,13 +1,7 @@
 #!/usr/bin/env bash
 set -Eeuo pipefail
 
-: "${BASHBREW_CACHE:=$HOME/.cache/bashbrew}"
-export BASHBREW_CACHE BASHBREW_ARCH=
-
-if [ ! -d "$BASHBREW_CACHE/git" ]; then
-	# initialize the "bashbrew cache"
-	bashbrew --arch amd64 from --uniq --apply-constraints hello-world:linux > /dev/null
-fi
+export BASHBREW_ARCH=
 
 if [ "$#" -eq 0 ]; then
 	set -- '--all'