Use image manifest when exporting layers

A tar archive of a Docker image contains a `mainfest.json` file that
lists the path to each embedded tar file containing the contents of a
layer in the image. This manifest file should be used to identify the
layer files instead of relying on file naming conventions and
assumptions on the directory structure that are not consistent
between container engine implementations.

Fixes gh-34324

Author: scottfrederick

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/Builder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2022 the original author or authors.
+ * Copyright 2012-2023 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@
 package org.springframework.boot.buildpack.platform.build;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.List;
 import java.util.function.Consumer;
 
@@ -32,7 +33,6 @@
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
-import org.springframework.boot.buildpack.platform.io.TarArchive;
 import org.springframework.util.Assert;
 import org.springframework.util.StringUtils;
 
@@ -273,9 +273,8 @@ public Image fetchImage(ImageReference reference, ImageType imageType) throws IO
 		}
 
 		@Override
-		public void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
-				throws IOException {
-			Builder.this.docker.image().exportLayers(reference, exports);
+		public void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
+			Builder.this.docker.image().exportLayerFiles(reference, exports);
 		}
 
 	}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/BuildpackResolverContext.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2022 the original author or authors.
+ * Copyright 2012-2023 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,12 +17,12 @@
 package org.springframework.boot.buildpack.platform.build;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.List;
 
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
-import org.springframework.boot.buildpack.platform.io.TarArchive;
 
 /**
  * Context passed to a {@link BuildpackResolver}.
@@ -52,6 +52,6 @@ interface BuildpackResolverContext {
 	 * during the callback)
 	 * @throws IOException on IO error
 	 */
-	void exportImageLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports) throws IOException;
+	void exportImageLayers(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException;
 
 }

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/ImageBuildpack.java

@@ -17,6 +17,7 @@
 package org.springframework.boot.buildpack.platform.build;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -35,7 +36,6 @@
 import org.springframework.boot.buildpack.platform.docker.type.Layer;
 import org.springframework.boot.buildpack.platform.docker.type.LayerId;
 import org.springframework.boot.buildpack.platform.io.IOConsumer;
-import org.springframework.boot.buildpack.platform.io.TarArchive;
 import org.springframework.util.StreamUtils;
 
 /**
@@ -115,23 +115,16 @@ private static class ExportedLayers {
 
 		ExportedLayers(BuildpackResolverContext context, ImageReference imageReference) throws IOException {
 			List<Path> layerFiles = new ArrayList<>();
-			context.exportImageLayers(imageReference, (name, archive) -> layerFiles.add(copyToTemp(name, archive)));
+			context.exportImageLayers(imageReference, (name, path) -> layerFiles.add(copyToTemp(path)));
 			this.layerFiles = Collections.unmodifiableList(layerFiles);
 		}
 
-		private Path copyToTemp(String name, TarArchive archive) throws IOException {
-			String[] parts = name.split("/");
-			Path path = Files.createTempFile("create-builder-scratch-", parts[0]);
-			try (OutputStream out = Files.newOutputStream(path)) {
-				archive.writeTo(out);
-			}
-			return path;
-		}
-
-		void apply(IOConsumer<Layer> layers) throws IOException {
-			for (Path path : this.layerFiles) {
-				layers.accept(Layer.fromTarArchive((out) -> copyLayerTar(path, out)));
+		private Path copyToTemp(Path path) throws IOException {
+			Path outputPath = Files.createTempFile("create-builder-scratch-", null);
+			try (OutputStream out = Files.newOutputStream(outputPath)) {
+				copyLayerTar(path, out);
 			}
+			return outputPath;
 		}
 
 		private void copyLayerTar(Path path, OutputStream out) throws IOException {
@@ -147,7 +140,16 @@ private void copyLayerTar(Path path, OutputStream out) throws IOException {
 				}
 				tarOut.finish();
 			}
-			Files.delete(path);
+		}
+
+		void apply(IOConsumer<Layer> layers) throws IOException {
+			for (Path path : this.layerFiles) {
+				layers.accept(Layer.fromTarArchive((out) -> {
+					InputStream in = Files.newInputStream(path);
+					StreamUtils.copy(in, out);
+				}));
+				Files.delete(path);
+			}
 		}
 
 	}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/DockerApi.java

@@ -16,13 +16,24 @@
 
 package org.springframework.boot.buildpack.platform.docker;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
 
 import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
@@ -37,6 +48,7 @@
 import org.springframework.boot.buildpack.platform.docker.type.ContainerStatus;
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageArchive;
+import org.springframework.boot.buildpack.platform.docker.type.ImageArchiveManifest;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.docker.type.VolumeName;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
@@ -250,28 +262,57 @@ public void load(ImageArchive archive, UpdateListener<LoadImageUpdateEvent> list
 		}
 
 		/**
-		 * Export the layers of an image.
+		 * Export the layers of an image as {@link TarArchive}s.
 		 * @param reference the reference to export
 		 * @param exports a consumer to receive the layers (contents can only be accessed
 		 * during the callback)
 		 * @throws IOException on IO error
 		 */
 		public void exportLayers(ImageReference reference, IOBiConsumer<String, TarArchive> exports)
 				throws IOException {
+			exportLayerFiles(reference, (name, path) -> {
+				try (InputStream in = Files.newInputStream(path)) {
+					TarArchive archive = (out) -> StreamUtils.copy(in, out);
+					exports.accept(name, archive);
+				}
+			});
+		}
+
+		/**
+		 * Export the layers of an image as paths to layer tar files.
+		 * @param reference the reference to export
+		 * @param exports a consumer to receive the layer tar file paths (file can only be
+		 * accessed during the callback)
+		 * @throws IOException on IO error
+		 */
+		public void exportLayerFiles(ImageReference reference, IOBiConsumer<String, Path> exports) throws IOException {
 			Assert.notNull(reference, "Reference must not be null");
 			Assert.notNull(exports, "Exports must not be null");
 			URI saveUri = buildUrl("/images/" + reference + "/get");
 			Response response = http().get(saveUri);
+			ImageArchiveManifest manifest = null;
+			Map<String, Path> layerFiles = new HashMap<>();
 			try (TarArchiveInputStream tar = new TarArchiveInputStream(response.getContent())) {
 				TarArchiveEntry entry = tar.getNextTarEntry();
 				while (entry != null) {
-					if (entry.getName().endsWith("/layer.tar")) {
-						TarArchive archive = (out) -> StreamUtils.copy(tar, out);
-						exports.accept(entry.getName(), archive);
+					if (entry.getName().equals("manifest.json")) {
+						manifest = readManifest(tar);
+					}
+					if (entry.getName().endsWith(".tar")) {
+						layerFiles.put(entry.getName(), copyToTemp(tar));
 					}
 					entry = tar.getNextTarEntry();
 				}
 			}
+			Assert.notNull(manifest, "Manifest not found in image " + reference);
+			for (Map.Entry<String, Path> entry : layerFiles.entrySet()) {
+				String name = entry.getKey();
+				Path path = entry.getValue();
+				if (manifestContainsLayerEntry(manifest, name)) {
+					exports.accept(name, path);
+				}
+				Files.delete(path);
+			}
 		}
 
 		/**
@@ -308,6 +349,24 @@ public void tag(ImageReference sourceReference, ImageReference targetReference)
 			http().post(uri).close();
 		}
 
+		private ImageArchiveManifest readManifest(TarArchiveInputStream tar) throws IOException {
+			String manifestContent = new BufferedReader(new InputStreamReader(tar, StandardCharsets.UTF_8)).lines()
+				.collect(Collectors.joining());
+			return ImageArchiveManifest.of(new ByteArrayInputStream(manifestContent.getBytes(StandardCharsets.UTF_8)));
+		}
+
+		private Path copyToTemp(TarArchiveInputStream in) throws IOException {
+			Path path = Files.createTempFile("create-builder-scratch-", null);
+			try (OutputStream out = Files.newOutputStream(path)) {
+				StreamUtils.copy(in, out);
+			}
+			return path;
+		}
+
+		private boolean manifestContainsLayerEntry(ImageArchiveManifest manifest, String layerId) {
+			return manifest.getEntries().stream().anyMatch((content) -> content.getLayers().contains(layerId));
+		}
+
 	}
 
 	/**

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/type/ImageArchiveManifest.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2012-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.buildpack.platform.docker.type;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import org.springframework.boot.buildpack.platform.json.MappedObject;
+
+/**
+ * Image archive manifest information.
+ *
+ * @author Scott Frederick
+ * @since 2.7.9
+ */
+public class ImageArchiveManifest extends MappedObject {
+
+	private final List<ManifestEntry> entries = new ArrayList<>();
+
+	protected ImageArchiveManifest(JsonNode node) {
+		super(node, MethodHandles.lookup());
+		getNode().elements().forEachRemaining((element) -> this.entries.add(ManifestEntry.of(element)));
+	}
+
+	/**
+	 * Return the entries contained in the manifest.
+	 * @return the manifest entries
+	 */
+	public List<ManifestEntry> getEntries() {
+		return this.entries;
+	}
+
+	/**
+	 * Create an {@link ImageArchiveManifest} from the provided JSON input stream.
+	 * @param content the JSON input stream
+	 * @return a new {@link ImageArchiveManifest} instance
+	 * @throws IOException on IO error
+	 */
+	public static ImageArchiveManifest of(InputStream content) throws IOException {
+		return of(content, ImageArchiveManifest::new);
+	}
+
+	public static class ManifestEntry extends MappedObject {
+
+		private final List<String> layers;
+
+		protected ManifestEntry(JsonNode node) {
+			super(node, MethodHandles.lookup());
+			this.layers = extractLayers();
+		}
+
+		/**
+		 * Return the collection of layer IDs from a section of the manifest.
+		 * @return a collection of layer IDs
+		 */
+		public List<String> getLayers() {
+			return this.layers;
+		}
+
+		static ManifestEntry of(JsonNode node) {
+			return new ManifestEntry(node);
+		}
+
+		@SuppressWarnings("unchecked")
+		private List<String> extractLayers() {
+			List<String> layers = valueAt("/Layers", List.class);
+			if (layers == null) {
+				return Collections.emptyList();
+			}
+			return layers;
+		}
+
+	}
+
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/build/ImageBuildpackTests.java

@@ -18,7 +18,10 @@
 
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
@@ -33,7 +36,6 @@
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
-import org.springframework.boot.buildpack.platform.io.TarArchive;
 import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -173,20 +175,20 @@ void resolveWhenUnqualifiedReferenceWithInvalidImageReferenceReturnsNull() {
 
 	private Object withMockLayers(InvocationOnMock invocation) {
 		try {
-			IOBiConsumer<String, TarArchive> consumer = invocation.getArgument(1);
-			TarArchive archive = (out) -> {
-				try (TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
-					tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
-					writeTarEntry(tarOut, "/cnb/");
-					writeTarEntry(tarOut, "/cnb/buildpacks/");
-					writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/");
-					writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/");
-					writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/buildpack.toml");
-					writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/" + this.longFilePath);
-					tarOut.finish();
-				}
-			};
-			consumer.accept("test", archive);
+			IOBiConsumer<String, Path> consumer = invocation.getArgument(1);
+			File tarFile = File.createTempFile("create-builder-test-", null);
+			FileOutputStream out = new FileOutputStream(tarFile);
+			try (TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
+				tarOut.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
+				writeTarEntry(tarOut, "/cnb/");
+				writeTarEntry(tarOut, "/cnb/buildpacks/");
+				writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/");
+				writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/");
+				writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/buildpack.toml");
+				writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/" + this.longFilePath);
+				tarOut.finish();
+			}
+			consumer.accept("test", tarFile.toPath());
 		}
 		catch (IOException ex) {
 			fail("Error writing mock layers", ex);