fix!(storage-resize-images): disable backfill (#2534)

* fix!(storage-resize-images): disable backfill

* fix(storage-resize-image): build

* Update storage-resize-images/CHANGELOG.md

* fix(storage-resize-images): format and getting rid of some backfill comments

* chore(storage-resize-images): format

---------

Co-authored-by: Corie Watson <[email protected]>

Author: cabljac

storage-resize-images/CHANGELOG.md

@@ -1,3 +1,7 @@
+## Version 0.3.0
+
+fix! - remove backfill, due to architectural flaws.
+
 ## Version 0.2.10
 
 feat - added param for adjusting backfill max batch size

storage-resize-images/README.md

@@ -144,7 +144,7 @@ You can find more information about this extension in the following articles:
 * Sizes of resized images: What sizes of images would you like (in pixels)? Enter the sizes as a comma-separated list of WIDTHxHEIGHT values. Learn more about [how this parameter works](https://firebase.google.com/products/extensions/storage-resize-images).
 
 
-* Deletion of original file: Do you want to automatically delete the original file from the Cloud Storage bucket? Warning: these deletions cannot be undone, and if you reconfigure this instance to use different image dimensions, you won't be able to backfill deleted images.
+* Deletion of original file: Do you want to automatically delete the original file from the Cloud Storage bucket? Warning: these deletions cannot be undone.
 
 * Make resized images public: Do you want to make the resized images public automatically? So you can access them by URL. For example: https://storage.googleapis.com/{bucket}/{path}
 
@@ -182,13 +182,6 @@ Leave this field empty if you do not want to store failed images in a separate d
 
 * Cloud Function memory: Memory of the function responsible of resizing images.  Choose how much memory to give to the function that resize images. (For animated GIF => GIF we recommend using a minimum of 2GB).
 
-* Backfill existing images: Should existing, unresized images in the Storage bucket be resized as well?
-
-
-* Backfill batch size: The maximum number of images to resize in a single batch. By default, the function is configured to resize 3 images at a time. This is a conservative default to work with smallest memory option (512 MB).
-WARNING: Ensure your function has enough memory to handle the batch size.
-
-
 * Assign new access token: Should resized images have a new access token assigned to them,  different from the original image?
 
 
@@ -207,8 +200,6 @@ WARNING: Ensure your function has enough memory to handle the batch size.
 
 * **generateResizedImage:** Listens for new images uploaded to your specified Cloud Storage bucket, resizes the images, then stores the resized images in the same bucket. Optionally keeps or deletes the original images.
 
-* **backfillResizedImages:** Handles tasks from startBackfill to resize existing images.
-
 
 
 **APIs Used**:

storage-resize-images/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: storage-resize-images
-version: 0.2.10
+version: 0.3.10
 specVersion: v1beta
 
 displayName: Resize Images
@@ -65,14 +65,15 @@ resources:
       eventTrigger:
         eventType: google.storage.object.finalize
         resource: projects/_/buckets/${param:IMG_BUCKET}
-  - name: backfillResizedImages
-    type: firebaseextensions.v1beta.function
-    description: >-
-      Handles tasks from startBackfill to resize existing images.
-    properties:
-      runtime: nodejs20
-      availableMemoryMb: ${param:FUNCTION_MEMORY}
-      taskQueueTrigger: {}
+  # Backfill feature disabled - commented out to preserve code
+  # - name: backfillResizedImages
+  #   type: firebaseextensions.v1beta.function
+  #   description: >-
+  #     Handles tasks from startBackfill to resize existing images.
+  #   properties:
+  #     runtime: nodejs20
+  #     availableMemoryMb: ${param:FUNCTION_MEMORY}
+  #     taskQueueTrigger: {}
 
 params:
   - param: IMG_BUCKET
@@ -111,9 +112,7 @@ params:
     label: Deletion of original file
     description: >-
       Do you want to automatically delete the original file from the Cloud
-      Storage bucket? Warning: these deletions cannot be undone, and if you
-      reconfigure this instance to use different image dimensions, you won't be
-      able to backfill deleted images.
+      Storage bucket? Warning: these deletions cannot be undone.
     type: select
     options:
       - label: Don't delete
@@ -316,32 +315,33 @@ params:
     required: true
     immutable: false
 
-  - param: DO_BACKFILL
-    label: Backfill existing images
-    description: >
-      Should existing, unresized images in the Storage bucket be resized as
-      well?
-    type: select
-    required: true
-    options:
-      - label: Yes
-        value: true
-      - label: No
-        value: false
-
-  - param: BACKFILL_BATCH_SIZE
-    label: Backfill batch size
-    description: >
-      The maximum number of images to resize in a single batch. By default, the
-      function is configured to resize 3 images at a time. This is a
-      conservative default to work with smallest memory option (512 MB).
-
-      WARNING: Ensure your function has enough memory to handle the batch size.
-    type: string
-    default: 3
-    validationRegex: ^[1-9][0-9]*$
-    validationErrorMessage: Please provide a valid number.
-    required: false
+  # Backfill parameters disabled - commented out to preserve code
+  # - param: DO_BACKFILL
+  #   label: Backfill existing images
+  #   description: >
+  #     Should existing, unresized images in the Storage bucket be resized as
+  #     well?
+  #   type: select
+  #   required: true
+  #   options:
+  #     - label: Yes
+  #       value: true
+  #     - label: No
+  #       value: false
+
+  # - param: BACKFILL_BATCH_SIZE
+  #   label: Backfill batch size
+  #   description: >
+  #     The maximum number of images to resize in a single batch. By default, the
+  #     function is configured to resize 3 images at a time. This is a
+  #     conservative default to work with smallest memory option (512 MB).
+  #
+  #     WARNING: Ensure your function has enough memory to handle the batch size.
+  #   type: string
+  #   default: 3
+  #   validationRegex: ^[1-9][0-9]*$
+  #   validationErrorMessage: Please provide a valid number.
+  #   required: false
 
   - param: REGENERATE_TOKEN
     label: Assign new access token
@@ -426,14 +426,14 @@ events:
     description:
       Occurs when the function is settled. Provides no customized data other
       than the context.
-
-lifecycleEvents:
-  onInstall:
-    function: backfillResizedImages
-    processingMessage: Resizing existing images in ${param:IMG_BUCKET}
-  onUpdate:
-    function: backfillResizedImages
-    processingMessage: Resizing existing images in ${param:IMG_BUCKET}
-  onConfigure:
-    function: backfillResizedImages
-    processingMessage: Resizing existing images in ${param:IMG_BUCKET}
+# Lifecycle events disabled - backfill feature commented out
+# lifecycleEvents:
+#   onInstall:
+#     function: backfillResizedImages
+#     processingMessage: Resizing existing images in ${param:IMG_BUCKET}
+#   onUpdate:
+#     function: backfillResizedImages
+#     processingMessage: Resizing existing images in ${param:IMG_BUCKET}
+#   onConfigure:
+#     function: backfillResizedImages
+#     processingMessage: Resizing existing images in ${param:IMG_BUCKET}

storage-resize-images/functions/__tests__/__mocks__/src/config.ts

@@ -51,7 +51,7 @@
 export const config = {
   bucket: "extensions-testing.appspot.com",
   cacheControlHeader: undefined,
-  doBackfill: false,
+  // doBackfill: false,
   imageSizes: ["200x200"],
   regenerateToken: false,
   makePublic: false,

storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap

@@ -3,13 +3,11 @@
 exports[`extension configuration detected from environment variables 1`] = `
 Object {
   "animated": false,
-  "backfillBatchSize": 3,
   "bucket": "extensions-testing.appspot.com",
   "cacheControlHeader": undefined,
   "contentFilterLevel": null,
   "customFilterPrompt": null,
   "deleteOriginalFile": 0,
-  "doBackfill": false,
   "excludePathList": undefined,
   "failedImagesPath": undefined,
   "imageSizes": Array [

storage-resize-images/functions/__tests__/filters.test.ts

@@ -11,7 +11,7 @@ jest.mock("../src/config", () => {
       imgSizes: ["200x200"],
       resizedImagesPath: undefined,
       deleteOriginalFile: "true",
-      backfillBatchSize: 3,
+      // backfillBatchSize: 3,
     },
   };
 });

storage-resize-images/functions/__tests__/function.test.ts

@@ -27,7 +27,7 @@ jest.mock("../src/config", () => {
       imgSizes: ["200x200"],
       resizedImagesPath: undefined,
       deleteOriginalFile: "true",
-      backfillBatchSize: 3,
+      // backfillBatchSize: 3,
     },
   };
 });

storage-resize-images/functions/src/config.ts

@@ -68,7 +68,8 @@ export const convertHarmBlockThreshold: (
 export const config = {
   bucket: process.env.IMG_BUCKET,
   cacheControlHeader: process.env.CACHE_CONTROL_HEADER,
-  doBackfill: process.env.DO_BACKFILL === "true",
+  // Backfill feature disabled - commented out to preserve code
+  // doBackfill: process.env.DO_BACKFILL === "true",
   imageSizes: process.env.IMG_SIZES.split(","),
   regenerateToken: process.env.REGENERATE_TOKEN == "true",
   makePublic: process.env.MAKE_PUBLIC === "true",
@@ -88,7 +89,7 @@ export const config = {
   ),
   customFilterPrompt: process.env.CUSTOM_FILTER_PROMPT || null,
   placeholderImagePath: process.env.PLACEHOLDER_IMAGE_PATH || null,
-  backfillBatchSize: Number(process.env.BACKFILL_BATCH_SIZE) || 3,
+  // backfillBatchSize: Number(process.env.BACKFILL_BATCH_SIZE) || 3,
 };
 
 export type Config = typeof config;

storage-resize-images/functions/src/index.ts

@@ -156,80 +156,80 @@ export const generateResizedImage = functions.storage
 /**
  *
  */
-export const backfillResizedImages = functions.tasks
-  .taskQueue()
-  .onDispatch(async (data) => {
-    const runtime = getExtensions().runtime();
-    if (!config.doBackfill) {
-      await runtime.setProcessingState(
-        "PROCESSING_COMPLETE",
-        "Existing images were not resized because 'Backfill existing images' was configured to false." +
-          " If you want to resize existing images, reconfigure this instance."
-      );
-      return;
-    }
-    if (data?.nextPageQuery == undefined) {
-      logs.startBackfill();
-    }
-
-    const bucket = admin.storage().bucket(process.env.IMG_BUCKET);
-    const query = data.nextPageQuery || {
-      autoPaginate: false,
-      maxResults: config.backfillBatchSize,
-    };
-    const [files, nextPageQuery] = await bucket.getFiles(query);
-    const filesToResize = files.filter((f: File) => {
-      logs.continueBackfill(f.metadata.name);
-      return shouldResize(convertToObjectMetadata(f.metadata));
-    });
-
-    const filePromises = filesToResize.map((f) => {
-      return generateResizedImageHandler(
-        convertToObjectMetadata(f.metadata),
-        /*verbose=*/ false
-      );
-    });
-    const results = await Promise.allSettled(filePromises);
-
-    const pageErrorsCount = results.filter(
-      (r) => r.status === "rejected"
-    ).length;
-    const pageSuccessCount = results.filter(
-      (r) => r.status === "fulfilled"
-    ).length;
-    const oldErrorsCount = Number(data.errorsCount) || 0;
-    const oldSuccessCount = Number(data.successCount) || 0;
-    const errorsCount = pageErrorsCount + oldErrorsCount;
-    const successCount = pageSuccessCount + oldSuccessCount;
-
-    if (nextPageQuery) {
-      const queue = getFunctions().taskQueue(
-        `locations/${config.location}/functions/backfillResizedImages`,
-        process.env.EXT_INSTANCE_ID
-      );
-      await queue.enqueue({
-        nextPageQuery,
-        errorsCount,
-        successCount,
-      });
-    } else {
-      logs.backfillComplete(successCount, errorsCount);
-      if (errorsCount == 0) {
-        await runtime.setProcessingState(
-          "PROCESSING_COMPLETE",
-          `Successfully resized ${successCount} images.`
-        );
-      } else if (errorsCount > 0 && successCount > 0) {
-        await runtime.setProcessingState(
-          "PROCESSING_WARNING",
-          `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.`
-        );
-      }
-      if (errorsCount > 0 && successCount == 0) {
-        await runtime.setProcessingState(
-          "PROCESSING_FAILED",
-          `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.`
-        );
-      }
-    }
-  });
+// export const backfillResizedImages = functions.tasks
+//   .taskQueue()
+//   .onDispatch(async (data) => {
+//     const runtime = getExtensions().runtime();
+//     if (!config.doBackfill) {
+//       await runtime.setProcessingState(
+//         "PROCESSING_COMPLETE",
+//         "Existing images were not resized because 'Backfill existing images' was configured to false." +
+//           " If you want to resize existing images, reconfigure this instance."
+//       );
+//       return;
+//     }
+//     if (data?.nextPageQuery == undefined) {
+//       logs.startBackfill();
+//     }
+
+//     const bucket = admin.storage().bucket(process.env.IMG_BUCKET);
+//     const query = data.nextPageQuery || {
+//       autoPaginate: false,
+//       maxResults: config.backfillBatchSize,
+//     };
+//     const [files, nextPageQuery] = await bucket.getFiles(query);
+//     const filesToResize = files.filter((f: File) => {
+//       logs.continueBackfill(f.metadata.name);
+//       return shouldResize(convertToObjectMetadata(f.metadata));
+//     });
+
+//     const filePromises = filesToResize.map((f) => {
+//       return generateResizedImageHandler(
+//         convertToObjectMetadata(f.metadata),
+//         /*verbose=*/ false
+//       );
+//     });
+//     const results = await Promise.allSettled(filePromises);
+
+//     const pageErrorsCount = results.filter(
+//       (r) => r.status === "rejected"
+//     ).length;
+//     const pageSuccessCount = results.filter(
+//       (r) => r.status === "fulfilled"
+//     ).length;
+//     const oldErrorsCount = Number(data.errorsCount) || 0;
+//     const oldSuccessCount = Number(data.successCount) || 0;
+//     const errorsCount = pageErrorsCount + oldErrorsCount;
+//     const successCount = pageSuccessCount + oldSuccessCount;
+
+//     if (nextPageQuery) {
+//       const queue = getFunctions().taskQueue(
+//         `locations/${config.location}/functions/backfillResizedImages`,
+//         process.env.EXT_INSTANCE_ID
+//       );
+//       await queue.enqueue({
+//         nextPageQuery,
+//         errorsCount,
+//         successCount,
+//       });
+//     } else {
+//       logs.backfillComplete(successCount, errorsCount);
+//       if (errorsCount == 0) {
+//         await runtime.setProcessingState(
+//           "PROCESSING_COMPLETE",
+//           `Successfully resized ${successCount} images.`
+//         );
+//       } else if (errorsCount > 0 && successCount > 0) {
+//         await runtime.setProcessingState(
+//           "PROCESSING_WARNING",
+//           `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.`
+//         );
+//       }
+//       if (errorsCount > 0 && successCount == 0) {
+//         await runtime.setProcessingState(
+//           "PROCESSING_FAILED",
+//           `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.`
+//         );
+//       }
+//     }
+//   });