Merge pull request #4604 from ggiguash/optimize_skopeo_copy

USHIFT-5288: Mirroring of images in CI should be performed for the current platform only

Author: openshift-merge-bot[bot]

scripts/mirror-images.sh

@@ -21,17 +21,30 @@ function usage() {
 function skopeo_retry() {
     for attempt in $(seq 3) ; do
         if ! skopeo "$@" ; then
-            echo "WARNING: Failed to run skopeo, retry #${attempt}"
+            echo "WARNING: Failed to run skopeo, retry #${attempt}" >&2
         else
             return 0
         fi
         sleep $(( "${attempt}" * 10 ))
     done
 
-    echo "ERROR: Failed to run skopeo, quitting after 3 tries"
+    echo "ERROR: Failed to run skopeo, quitting after 3 tries" >&2
     return 1
 }
 
+function skopeo_opts_for_image() {
+    local -r img_pull_file="$1"
+    local -r img_src_ref="$2"
+
+    # Cannot copy a single platform for manifest image references
+    local media_type
+    media_type=$(skopeo_retry inspect --raw --authfile "${img_pull_file}" "${img_src_ref}" | jq -r .mediaType)
+    if [ "${media_type}" == "application/vnd.docker.distribution.manifest.list.v2+json" ] ; then
+        echo -n "--all"
+    fi
+    echo -n ""
+}
+
 function mirror_registry() {
     local -r img_pull_file=$1
     local -r img_file_list=$2
@@ -50,25 +63,29 @@ function mirror_registry() {
         # Add the target registry prefix
         dst_img="${dest_registry}/${dst_img}"
         dst_img_no_tag="${dest_registry}/${dst_img_no_tag}"
+        # Determine skopeo options
+        local -r skopeo_opts=$(skopeo_opts_for_image "${img_pull_file}" docker://"${src_img}")
 
         # Run the image mirror and tag command
         echo "Mirroring '${src_img}' to '${dst_img}'"
-        skopeo_retry copy --all --quiet \
+        # shellcheck disable=SC2086
+        skopeo_retry copy ${skopeo_opts} --quiet \
             --preserve-digests \
             --authfile "${img_pull_file}" \
             docker://"${src_img}" docker://"${dst_img}"
 
         echo "Tagging '${dst_img_no_tag}' as 'latest'"
-        skopeo_retry copy --all --quiet \
+        # shellcheck disable=SC2086
+        skopeo_retry copy ${skopeo_opts} --quiet \
             --preserve-digests \
             --authfile "${img_pull_file}" \
             docker://"${dst_img}" docker://"${dst_img_no_tag}:latest"
-
     }
 
     # Export functions for xargs to use
     export -f process_image_copy
     export -f skopeo_retry
+    export -f skopeo_opts_for_image
     # Note that images are passed as one argument by replacing "{}" in xarg input.
     xargs -P 8 -I {} -a "${img_file_list}" \
         bash -c 'process_image_copy "$@"' _ "${img_pull_file}" "${dest_registry}" "{}"
@@ -87,11 +104,14 @@ function registry_to_dir() {
         # Remove the source registry prefix
         local dst_img
         dst_img=$(echo "${src_img}" | cut -d '/' -f 2-)
+        # Determine skopeo options
+        local -r skopeo_opts=$(skopeo_opts_for_image "${img_pull_file}" docker://"${src_img}")
 
         # Run the image download command
         echo "Downloading '${src_img}' to '${local_dir}'"
         mkdir -p "${local_dir}/${dst_img}"
-        skopeo_retry copy --all --quiet \
+        # shellcheck disable=SC2086
+        skopeo_retry copy ${skopeo_opts} --quiet \
             --preserve-digests \
             --authfile "${img_pull_file}" \
             docker://"${src_img}" dir://"${local_dir}/${dst_img}"
@@ -100,6 +120,7 @@ function registry_to_dir() {
     # Export functions for xargs to use
     export -f process_image_copy
     export -f skopeo_retry
+    export -f skopeo_opts_for_image
     # Generate a list for each image and run copy in parallel.
     # Note that the image is passed by replacing "{}" in xarg input.
     xargs -P 8 -I {} -a "${img_file_list}" \
@@ -123,16 +144,20 @@ function dir_to_registry() {
         # Add the target registry prefix and remove SHA
         local -r dst_img="${dest_registry}/${src_img}"
         local -r dst_img_no_tag="${dest_registry}/${src_img%%[@:]*}"
+        # Determine skopeo options
+        local -r skopeo_opts=$(skopeo_opts_for_image "${img_pull_file}" dir://"${src_img}")
 
         # Run the image upload and tag commands
         echo "Uploading '${src_img}' to '${dst_img}'"
-        skopeo_retry copy --all --quiet \
+        # shellcheck disable=SC2086
+        skopeo_retry copy ${skopeo_opts} --quiet \
             --preserve-digests \
             --authfile "${img_pull_file}" \
             dir://"${local_dir}/${src_img}" docker://"${dst_img}"
 
         echo "Tagging '${dst_img}' as '${dst_img_no_tag}:latest'"
-        skopeo_retry copy --all --quiet \
+        # shellcheck disable=SC2086
+        skopeo_retry copy ${skopeo_opts} --quiet \
             --preserve-digests \
             --authfile "${img_pull_file}" \
             docker://"${dst_img}" docker://"${dst_img_no_tag}:latest"
@@ -141,6 +166,7 @@ function dir_to_registry() {
     # Export functions for xargs to use
     export -f process_image_copy
     export -f skopeo_retry
+    export -f skopeo_opts_for_image
 
     # Generate a list with an incremental counter for each image and run copy in parallel.
     # Note that the counter and image pairs are passed as one argument by replacing "{}" in xarg input.

test/bin/pyutils/build_bootc_images.py

@@ -568,7 +568,7 @@ def main():
 
         # Determine versions of RPM packages
         set_rpm_version_info_vars()
-        # Prepare container image lists for mirroring registries
+        # Prepare container images list for mirroring registries
         common.delete_file(CONTAINER_LIST)
         if args.no_extract_images:
             common.print_msg("Skipping container image extraction")
@@ -578,6 +578,9 @@ def main():
             extract_container_images(f"4.{FAKE_NEXT_MINOR_VERSION}.*", NEXT_REPO, CONTAINER_LIST, args.dry_run)
             extract_container_images(PREVIOUS_RELEASE_VERSION, PREVIOUS_RELEASE_REPO, CONTAINER_LIST, args.dry_run)
             extract_container_images(YMINUS2_RELEASE_VERSION, YMINUS2_RELEASE_REPO, CONTAINER_LIST, args.dry_run)
+        # Sort the images list, only leaving unique entries
+        common.sort_uniq_file(CONTAINER_LIST)
+
         # Process package source templates
         ipkgdir = f"{SCRIPTDIR}/../package-sources-bootc"
         for ifile in os.listdir(ipkgdir):

test/bin/pyutils/common.py

@@ -197,6 +197,17 @@ def delete_file(file_path: str):
         pass
 
 
+def sort_uniq_file(file_path: str):
+    """Sort the file by lines and overwrite it with the sorted and unique line output"""
+    # Read, sort, and filter unique lines
+    with open(file_path, 'r') as f:
+        unique_lines = sorted(set(f.readlines()))
+
+    # Write the result back to the file
+    with open(file_path, 'w') as f:
+        f.writelines(unique_lines)
+
+
 def file_has_valid_lines(file_path: str) -> bool:
     """Check if a text file contains at least one non-empty, non-commented line"""
     try: