feat(nextjs-component, aws-lambda): allow removing old lambda versions (#1884)

Author: dphang

jest-sequencer.js

@@ -0,0 +1,25 @@
+const Sequencer = require("@jest/test-sequencer").default;
+
+class CustomSequencer extends Sequencer {
+  constructor() {
+    super();
+  }
+
+  sort(tests) {
+    // Test structure information
+    // https://github.com/facebook/jest/blob/6b8b1404a1d9254e7d5d90a8934087a9c9899dab/packages/jest-runner/src/types.ts#L17-L21
+    const copyTests = Array.from(tests);
+    return copyTests.sort((testA, testB) => {
+      // FIXME: figure out why this test started failing if run after another test
+      if (testA.path.includes("serverless-trace.test")) {
+        return -1;
+      }
+      if (testB.path.includes("serverless-trace.test")) {
+        return 1;
+      }
+      return testA.path > testB.path ? 1 : -1;
+    });
+  }
+}
+
+module.exports = CustomSequencer;

package.json

@@ -131,7 +131,8 @@
     ],
     "modulePathIgnorePatterns": [
       "/sharp_node_modules/"
-    ]
+    ],
+    "testSequencer": "<rootDir>/jest-sequencer.js"
   },
   "dependencies": {
     "opencollective-postinstall": "^2.0.3",

packages/e2e-tests/next-app/serverless.yml

@@ -1,6 +1,7 @@
 next-app:
   component: "../../serverless-components/nextjs-component"
   inputs:
+    removeOldLambdaVersions: true
     sqs:
       tags:
         foo: bar

packages/libs/lambda-at-edge/package.json

@@ -51,6 +51,7 @@
     "@types/react": "17.0.31",
     "@types/react-dom": "^17.0.10",
     "@types/sharp": "^0.29.2",
+    "@types/uuid": "^8.3.1",
     "fetch-mock-jest": "^1.5.1",
     "klaw": "^3.0.0",
     "rimraf": "^3.0.2",
@@ -62,7 +63,8 @@
     "sharp": "^0.28.3",
     "ts-loader": "^9.2.6",
     "ts-node": "^10.3.0",
-    "typescript": "^4.4.4"
+    "typescript": "^4.4.4",
+    "uuid": "^8.3.2"
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.37.0",

packages/libs/lambda-at-edge/tests/serverless-trace/serverless-trace.test.ts

@@ -5,14 +5,16 @@ import Builder, {
   DEFAULT_LAMBDA_CODE_DIR,
   API_LAMBDA_CODE_DIR
 } from "../../src/build";
+import { jest } from "@jest/globals";
+import { v4 as uuidv4 } from "uuid";
 
 describe("Serverless Trace", () => {
   const fixturePath = path.join(__dirname, "./fixture");
   let outputDir: string;
   let fseRemoveSpy: jest.SpyInstance;
 
   beforeEach(async () => {
-    outputDir = path.join(os.tmpdir(), `${Date.now()}`);
+    outputDir = path.join(os.tmpdir(), `${uuidv4()}`);
 
     fseRemoveSpy = jest.spyOn(fse, "remove").mockImplementation(() => {
       return;

packages/libs/lambda-at-edge/yarn.lock

@@ -1362,31 +1362,12 @@
     picomatch "^2.2.2"
 
 "@sls-next/aws-common@link:../aws-common":
-  version "3.5.0-alpha.7"
-  dependencies:
-    "@aws-sdk/client-s3" "^3.37.0"
-    "@aws-sdk/client-sqs" "^3.37.0"
-    "@sls-next/core" "link:../core"
+  version "0.0.0"
+  uid ""
 
 "@sls-next/core@link:../core":
-  version "3.5.0-alpha.7"
-  dependencies:
-    "@hapi/accept" "^5.0.1"
-    cookie "^0.4.1"
-    execa "^5.1.1"
-    fast-glob "^3.2.7"
-    fresh "^0.5.2"
-    fs-extra "^9.1.0"
-    is-animated "^2.0.1"
-    jsonwebtoken "^8.5.1"
-    next "^11.1.2"
-    node-fetch "2.6.5"
-    normalize-path "^3.0.0"
-    path-to-regexp "^6.1.0"
-    react "^17.0.2"
-    react-dom "^17.0.2"
-    send "^0.17.1"
-    sharp "^0.29.1"
+  version "0.0.0"
+  uid ""
 
 "@tsconfig/node10@^1.0.7":
   version "1.0.8"
@@ -1508,6 +1489,11 @@
   dependencies:
     "@types/node" "*"
 
+"@types/uuid@^8.3.1":
+  version "8.3.1"
+  resolved "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.1.tgz#1a32969cf8f0364b3d8c8af9cc3555b7805df14f"
+  integrity sha512-Y2mHTRAbqfFkpjldbkHGY8JIzRN6XqYRliG8/24FcHm2D2PwW24fl5xMRTVGdrb7iMrwCaIEbLWerGIkXuFWVg==
+
 "@vercel/nft@^0.17.0":
   version "0.17.0"
   resolved "https://registry.npmjs.org/@vercel/nft/-/nft-0.17.0.tgz#28851fefe42fae7a116dc5e23a0a9da29929a18b"

packages/serverless-components/aws-lambda/__mocks__/aws-sdk.mock.ts

@@ -1,3 +1,5 @@
+import { jest } from "@jest/globals";
+
 const promisifyMock = (mockFn) => {
   const promise = jest.fn();
   mockFn.mockImplementation(() => ({
@@ -60,6 +62,12 @@ export const mockTagResource = jest.fn();
 export const mockTagResourcePromise = promisifyMock(mockTagResource);
 export const mockUntagResource = jest.fn();
 export const mockUntagResourcePromise = promisifyMock(mockUntagResource);
+export const mockListVersionsByFunction = jest.fn();
+export const mockListVersionsByFunctionPromise = promisifyMock(
+  mockListVersionsByFunction
+);
+export const mockDeleteFunction = jest.fn();
+export const mockDeleteFunctionPromise = promisifyMock(mockDeleteFunction);
 
 export default {
   SQS: jest.fn(() => ({
@@ -77,6 +85,8 @@ export default {
     updateFunctionConfiguration: mockUpdateFunctionConfiguration,
     listTags: mockListTags,
     tagResource: mockTagResource,
-    untagResource: mockUntagResource
+    untagResource: mockUntagResource,
+    listVersionsByFunction: mockListVersionsByFunction,
+    deleteFunction: mockDeleteFunction
   }))
 };

packages/serverless-components/aws-lambda/__tests__/removeLambdaVersions.test.ts

@@ -0,0 +1,87 @@
+import {
+  mockGetFunctionConfigurationPromise,
+  mockListVersionsByFunctionPromise,
+  mockGetFunctionConfiguration,
+  mockListVersionsByFunction,
+  mockDeleteFunction,
+  mockDeleteFunctionPromise
+} from "../__mocks__/aws-sdk.mock";
+import { removeLambdaVersions } from "../src/removeLambdaVersions";
+import { jest } from "@jest/globals";
+
+jest.mock("aws-sdk", () => require("../__mocks__/aws-sdk.mock"));
+
+describe("publishVersion", () => {
+  it("removes all old lambda versions", async () => {
+    mockGetFunctionConfigurationPromise.mockResolvedValue({
+      FunctionName: "test-function",
+      Version: "4"
+    });
+
+    mockListVersionsByFunctionPromise.mockResolvedValue({
+      Versions: [
+        {
+          FunctionName: "test-function",
+          Version: "1"
+        },
+        {
+          FunctionName: "test-function",
+          Version: "2"
+        },
+        {
+          FunctionName: "test-function",
+          Version: "3"
+        },
+        {
+          FunctionName: "test-function",
+          Version: "4"
+        }
+      ]
+    });
+
+    mockDeleteFunctionPromise.mockResolvedValueOnce(undefined);
+    mockDeleteFunctionPromise.mockResolvedValueOnce(undefined);
+    // Simulate last function couldn't be deleted, but it will not fail the process.
+    mockDeleteFunctionPromise.mockRejectedValueOnce({
+      message: "Mocked error"
+    });
+
+    await removeLambdaVersions(
+      {
+        debug: () => {
+          // intentionally empty
+        }
+      },
+      "test-function",
+      "us-east-1"
+    );
+
+    expect(mockDeleteFunction).toBeCalledWith({
+      FunctionName: "test-function",
+      Qualifier: "1"
+    });
+
+    expect(mockDeleteFunction).toBeCalledWith({
+      FunctionName: "test-function",
+      Qualifier: "2"
+    });
+
+    expect(mockDeleteFunction).toBeCalledWith({
+      FunctionName: "test-function",
+      Qualifier: "3"
+    });
+
+    expect(mockDeleteFunction).toBeCalledTimes(3);
+
+    expect(mockGetFunctionConfiguration).toBeCalledWith({
+      FunctionName: "test-function"
+    });
+    expect(mockGetFunctionConfiguration).toBeCalledTimes(1);
+
+    expect(mockListVersionsByFunction).toBeCalledWith({
+      FunctionName: "test-function",
+      MaxItems: 50
+    });
+    expect(mockListVersionsByFunction).toBeCalledTimes(1);
+  });
+});

packages/serverless-components/aws-lambda/src/removeLambdaVersions.ts

@@ -0,0 +1,75 @@
+// Cleanup Lambda code adapted from https://github.com/davidmenger/cleanup-lambda-versions/blob/master/src/cleanupVersions.js
+import AWS from "aws-sdk";
+import {
+  FunctionConfiguration,
+  ListVersionsByFunctionResponse
+} from "aws-sdk/clients/lambda";
+
+async function listLambdaVersions(
+  lambda: AWS.Lambda,
+  fnName: string
+): Promise<ListVersionsByFunctionResponse> {
+  return await lambda
+    .listVersionsByFunction({
+      FunctionName: fnName,
+      MaxItems: 50
+    })
+    .promise();
+}
+
+async function removeLambdaVersion(
+  lambda: AWS.Lambda,
+  fnName: string,
+  version: string
+): Promise<unknown> {
+  return await lambda
+    .deleteFunction({ FunctionName: fnName, Qualifier: version })
+    .promise();
+}
+
+async function getLambdaFunction(
+  lambda: AWS.Lambda,
+  fnName: string
+): Promise<FunctionConfiguration> {
+  return await lambda
+    .getFunctionConfiguration({ FunctionName: fnName })
+    .promise();
+}
+
+/**
+ * Clean up old lambda versions, up to 50 at a time.
+ * Currently it just removes the version that's not the current version,
+ * but if needed we could add support for preserving the latest X versions.
+ * @param context
+ * @param fnName
+ * @param region
+ */
+export async function removeLambdaVersions(
+  context: any,
+  fnName: string,
+  region: string
+) {
+  const lambda: AWS.Lambda = new AWS.Lambda({ region });
+  const fnConfig = await getLambdaFunction(lambda, fnName);
+
+  const versions = await listLambdaVersions(lambda, fnConfig.FunctionName);
+
+  for (const version of versions.Versions ?? []) {
+    if (version.Version && version.Version !== fnConfig.Version) {
+      try {
+        context.debug(
+          `Removing function: ${fnConfig.FunctionName} - ${version.Version}`
+        );
+        await removeLambdaVersion(
+          lambda,
+          fnConfig.FunctionName,
+          version.Version
+        );
+      } catch (e) {
+        context.debug(
+          `Remove failed (${fnConfig.FunctionName} - ${version.Version}): ${e.message}`
+        );
+      }
+    }
+  }
+}

packages/serverless-components/nextjs-component/__mocks__/@sls-next/aws-lambda/dist/removeLambdaVersions.js

@@ -0,0 +1,8 @@
+import { jest } from "@jest/globals";
+
+const mockRemoveLambdaVersions = jest.fn();
+
+module.exports = {
+  mockRemoveLambdaVersions,
+  removeLambdaVersions: mockRemoveLambdaVersions
+};

packages/serverless-components/nextjs-component/__mocks__/@sls-next/aws-lambda.js renamed to packages/serverless-components/nextjs-component/__mocks__/@sls-next/aws-lambda/index.js

@@ -13,10 +13,10 @@ import obtainDomains from "../src/lib/obtainDomains";
 import {
   DEFAULT_LAMBDA_CODE_DIR,
   API_LAMBDA_CODE_DIR,
-  IMAGE_LAMBDA_CODE_DIR,
-  REGENERATION_LAMBDA_CODE_DIR
+  IMAGE_LAMBDA_CODE_DIR
 } from "../src/constants";
 import { cleanupFixtureDirectory } from "../src/lib/test-utils";
+import { mockRemoveLambdaVersions } from "@sls-next/aws-lambda/dist/removeLambdaVersions";
 
 // unfortunately can't use __mocks__ because aws-sdk is being mocked in other
 // packages in the monorepo
@@ -474,6 +474,28 @@ describe("Custom inputs", () => {
     });
   });
 
+  describe("Old lambda function version removal", () => {
+    let tmpCwd: string;
+    const fixturePath = path.join(__dirname, "./fixtures/generic-fixture");
+
+    beforeEach(async () => {
+      tmpCwd = process.cwd();
+      process.chdir(fixturePath);
+
+      mockServerlessComponentDependencies({ expectedDomain: undefined });
+
+      const component = createNextComponent();
+
+      componentOutputs = await component.default({
+        removeOldLambdaVersions: true
+      });
+    });
+
+    it("removes old versions of lambda functions", () => {
+      expect(mockRemoveLambdaVersions).toBeCalledTimes(3); // 4 if there is regeneration lambda
+    });
+  });
+
   describe.each`
     inputTimeout                            | expectedTimeout
     ${undefined}                            | ${{ defaultTimeout: 10, apiTimeout: 10 }}

packages/serverless-components/nextjs-component/__tests__/custom-inputs.test.ts

@@ -3,6 +3,7 @@ import fse from "fs-extra";
 import { mockS3 } from "@sls-next/aws-s3";
 import { mockCloudFront } from "@sls-next/aws-cloudfront";
 import { mockLambda, mockLambdaPublish } from "@sls-next/aws-lambda";
+import { mockRemoveLambdaVersions } from "@sls-next/aws-lambda/dist/removeLambdaVersions";
 import {
   mockCreateInvalidation,
   mockCheckCloudFrontDistributionReady
@@ -447,6 +448,10 @@ describe.each`
         distributionId: "cloudfrontdistrib"
       });
     });
+
+    it("does not remove old versions of lambda functions by default", () => {
+      expect(mockRemoveLambdaVersions).toBeCalledTimes(0);
+    });
   });
 
   it("uploads static assets to S3 correctly", () => {