feat: optimize Docker builds and parallelize staging CI #2155
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - prod | |
| - staging | |
| pull_request: | |
| concurrency: | |
| group: ${{github.workflow}}-${{github.ref}} | |
| cancel-in-progress: true | |
| jobs: | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Use Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'yarn' | |
| - name: Enable Corepack for Yarn 3 | |
| run: corepack enable | |
| - name: Prepare Yarn version | |
| run: corepack prepare yarn@3.3.1 --activate | |
| - name: Check Yarn version | |
| run: yarn --version | |
| - name: Install dependencies | |
| run: yarn install --immutable | |
| - name: Build | |
| run: yarn build | |
| - name: Run unit tests | |
| shell: bash | |
| run: yarn test:unit | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Lint PR title | |
| uses: amannn/action-semantic-pull-request@v5 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| if: github.event_name == 'pull_request' | |
| - name: Lint commit | |
| uses: wagoid/commitlint-github-action@v5 | |
| - name: Use Node.js | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 20.x | |
| cache: 'yarn' | |
| - name: Enable Corepack for Yarn 3 | |
| run: corepack enable | |
| - name: Prepare Yarn version | |
| run: corepack prepare yarn@3.3.1 --activate | |
| - name: Check Yarn version | |
| run: yarn --version | |
| - name: Install dependencies | |
| run: yarn install --immutable | |
| - name: Lint workspaces | |
| run: yarn lint | |
| # Production deployments - triggered on prod branch | |
| build-and-deploy-prod: | |
| if: github.ref == 'refs/heads/prod' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| environment: | |
| - name: mark | |
| region: ap-northeast-1 | |
| terraform_dir: ./ops/mainnet/mark | |
| - name: mandy | |
| region: eu-south-2 | |
| terraform_dir: ./ops/mainnet/mandy | |
| - name: matoshi | |
| region: ap-southeast-1 | |
| terraform_dir: ./ops/mainnet/matoshi | |
| env: | |
| AWS_REGION: ${{ matrix.environment.region }} | |
| REGISTRY: 679752396206.dkr.ecr.${{ matrix.environment.region }}.amazonaws.com | |
| POLLER_REPOSITORY: mark-poller | |
| POLLER_IMAGE_TAG: mark-poller-${{ github.sha }} | |
| HANDLER_REPOSITORY: mark-handler | |
| HANDLER_IMAGE_TAG: mark-handler-${{ github.sha }} | |
| ADMIN_REPOSITORY: mark-admin | |
| ADMIN_IMAGE_TAG: mark-admin-${{ github.sha }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-region: ${{ matrix.environment.region }} | |
| aws-access-key-id: ${{ secrets.DEPLOYER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOYER_AWS_SECRET_ACCESS_KEY }} | |
| - name: Login to Private ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| with: | |
| mask-password: 'true' | |
| - name: Ensure ECR repositories exist | |
| run: | | |
| aws ecr describe-repositories --repository-names $ADMIN_REPOSITORY --region $AWS_REGION || \ | |
| aws ecr create-repository --repository-name $ADMIN_REPOSITORY --region $AWS_REGION --image-scanning-configuration scanOnPush=true --image-tag-mutability MUTABLE | |
| aws ecr describe-repositories --repository-names $HANDLER_REPOSITORY --region $AWS_REGION || \ | |
| aws ecr create-repository --repository-name $HANDLER_REPOSITORY --region $AWS_REGION --image-scanning-configuration scanOnPush=true --image-tag-mutability MUTABLE | |
| aws ecr describe-repositories --repository-names $POLLER_REPOSITORY --region $AWS_REGION || \ | |
| aws ecr create-repository --repository-name $POLLER_REPOSITORY --region $AWS_REGION --image-scanning-configuration scanOnPush=true --image-tag-mutability MUTABLE | |
| - name: Build and push Admin Docker image | |
| run: | | |
| docker build --provenance=false --sbom=false -f docker/admin/Dockerfile -t $REGISTRY/$ADMIN_REPOSITORY:$ADMIN_IMAGE_TAG . | |
| docker push $REGISTRY/$ADMIN_REPOSITORY:$ADMIN_IMAGE_TAG | |
| - name: Build and push Poller Docker image | |
| run: | | |
| docker build --provenance=false --sbom=false -f docker/poller/Dockerfile -t $REGISTRY/$POLLER_REPOSITORY:$POLLER_IMAGE_TAG . | |
| docker push $REGISTRY/$POLLER_REPOSITORY:$POLLER_IMAGE_TAG | |
| - name: Build and push Invoice Handler Docker image | |
| run: | | |
| docker build -f docker/handler/Dockerfile -t $REGISTRY/$HANDLER_REPOSITORY:$HANDLER_IMAGE_TAG . | |
| docker push $REGISTRY/$HANDLER_REPOSITORY:$HANDLER_IMAGE_TAG | |
| # Remove the main poller Lambda before deploying the invoice handler (prevents duplicate intent creation) | |
| - name: Remove Main Poller Lambda Function | |
| run: | | |
| bash ops/scripts/remove-poller-lambda.sh ${{ matrix.environment.name }} mainnet prod $AWS_REGION | |
| - name: Use Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn install | |
| env: | |
| YARN_ENABLE_IMMUTABLE_INSTALLS: false | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v1 | |
| with: | |
| terraform_version: 1.5.7 | |
| terraform_wrapper: false | |
| - name: Setup Sops | |
| uses: mdgreenwald/mozilla-sops-action@v1.2.0 | |
| with: | |
| version: '3.7.2' | |
| - name: Configure AWS Profile | |
| uses: Fooji/create-aws-profile-action@v1 | |
| with: | |
| profile: aws-deployer-connext | |
| region: ${{ matrix.environment.region }} | |
| key: ${{ secrets.DEPLOYER_AWS_ACCESS_KEY_ID }} | |
| secret: ${{ secrets.DEPLOYER_AWS_SECRET_ACCESS_KEY }} | |
| - name: Terraform Init | |
| working-directory: ${{ matrix.environment.terraform_dir }} | |
| run: terraform init > /dev/null 2>&1 | |
| env: | |
| AWS_PROFILE: aws-deployer-connext | |
| - name: Terraform Apply | |
| working-directory: ${{ matrix.environment.terraform_dir }} | |
| env: | |
| AWS_PROFILE: aws-deployer-connext | |
| run: | | |
| terraform apply \ | |
| -var "handler_image_uri=${REGISTRY}/${HANDLER_REPOSITORY}:${HANDLER_IMAGE_TAG}" \ | |
| -var "image_uri=${REGISTRY}/${POLLER_REPOSITORY}:${POLLER_IMAGE_TAG}" \ | |
| -var "admin_image_uri=${REGISTRY}/${ADMIN_REPOSITORY}:${ADMIN_IMAGE_TAG}" \ | |
| -auto-approve > /dev/null 2>&1 | |
| - name: Show Admin API Endpoint URL | |
| if: success() | |
| working-directory: ${{ matrix.environment.terraform_dir }} | |
| run: | | |
| echo "Admin API Endpoint URL for ${{ matrix.environment.name }}:" | |
| terraform output -raw admin_api_endpoint | |
| - name: Show Invoice Handler URL | |
| if: success() | |
| working-directory: ${{ matrix.environment.terraform_dir }} | |
| run: | | |
| echo "Invoice Handler URL for ${{ matrix.environment.name }}:" | |
| terraform output -raw invoice_handler_url | |
| - name: Post-Deploy Health Check | |
| if: success() | |
| uses: ./.github/actions/post-deploy-health-check | |
| with: | |
| aws-region: ${{ matrix.environment.region }} | |
| ecs-cluster: ${{ matrix.environment.name }}-ecs-mainnet-prod | |
| services: "${{ matrix.environment.name }}-web3signer-mainnet-prod,${{ matrix.environment.name }}-fillservice-web3signer-mainnet-prod,${{ matrix.environment.name }}-handler-mainnet-prod,${{ matrix.environment.name }}-prometheus-mainnet-prod,${{ matrix.environment.name }}-pushgateway-mainnet-prod" | |
| aws-access-key-id: ${{ secrets.DEPLOYER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOYER_AWS_SECRET_ACCESS_KEY }} | |
| # Staging Docker builds - parallel via matrix | |
| build-staging-images: | |
| if: github.ref == 'refs/heads/staging' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - service: admin | |
| dockerfile: docker/admin/Dockerfile | |
| repository: mark-admin | |
| image_tag: mark-admin-${{ github.sha }} | |
| - service: handler | |
| dockerfile: docker/handler/Dockerfile | |
| repository: mark-handler | |
| image_tag: mark-handler-${{ github.sha }} | |
| - service: poller | |
| dockerfile: docker/poller/Dockerfile | |
| repository: mark-poller | |
| image_tag: mark-poller-${{ github.sha }} | |
| env: | |
| AWS_REGION: sa-east-1 | |
| REGISTRY: 679752396206.dkr.ecr.sa-east-1.amazonaws.com | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-region: sa-east-1 | |
| aws-access-key-id: ${{ secrets.DEPLOYER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOYER_AWS_SECRET_ACCESS_KEY }} | |
| - name: Login to Private ECR | |
| id: login-ecr | |
| uses: aws-actions/amazon-ecr-login@v1 | |
| with: | |
| mask-password: 'true' | |
| - name: Ensure ECR repository exists | |
| run: | | |
| aws ecr describe-repositories --repository-names ${{ matrix.repository }} --region $AWS_REGION || \ | |
| aws ecr create-repository --repository-name ${{ matrix.repository }} --region $AWS_REGION --image-scanning-configuration scanOnPush=true --image-tag-mutability MUTABLE | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Build and push ${{ matrix.service }} Docker image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| file: ${{ matrix.dockerfile }} | |
| push: true | |
| tags: ${{ env.REGISTRY }}/${{ matrix.repository }}:${{ matrix.image_tag }} | |
| cache-from: type=gha,scope=staging-${{ matrix.service }} | |
| cache-to: type=gha,scope=staging-${{ matrix.service }},mode=max | |
| provenance: false | |
| sbom: false | |
| # Staging deployment - runs after all images are built | |
| deploy-staging: | |
| if: github.ref == 'refs/heads/staging' | |
| needs: build-staging-images | |
| runs-on: ubuntu-latest | |
| env: | |
| AWS_REGION: sa-east-1 | |
| REGISTRY: 679752396206.dkr.ecr.sa-east-1.amazonaws.com | |
| POLLER_REPOSITORY: mark-poller | |
| POLLER_IMAGE_TAG: mark-poller-${{ github.sha }} | |
| HANDLER_REPOSITORY: mark-handler | |
| HANDLER_IMAGE_TAG: mark-handler-${{ github.sha }} | |
| ADMIN_REPOSITORY: mark-admin | |
| ADMIN_IMAGE_TAG: mark-admin-${{ github.sha }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-region: sa-east-1 | |
| aws-access-key-id: ${{ secrets.DEPLOYER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOYER_AWS_SECRET_ACCESS_KEY }} | |
| # ============================================================================ | |
| # POLLER REMOVAL - TEMPORARY | |
| # ============================================================================ | |
| # Remove only the main poller Lambda function (mark_poller) before deploying the invoice | |
| # handler to prevent duplicate intent creation. Other poller Lambdas remain active. | |
| # | |
| # TODO: Remove this step once poller migration is complete | |
| # ============================================================================ | |
| - name: Remove Main Poller Lambda Function | |
| run: | | |
| bash ops/scripts/remove-poller-lambda.sh mason mainnet staging $AWS_REGION | |
| - name: Use Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn install | |
| env: | |
| YARN_ENABLE_IMMUTABLE_INSTALLS: false | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v1 | |
| with: | |
| terraform_version: 1.5.7 | |
| terraform_wrapper: false | |
| - name: Setup Sops | |
| uses: mdgreenwald/mozilla-sops-action@v1.2.0 | |
| with: | |
| version: '3.7.2' | |
| - name: Configure AWS Profile | |
| uses: Fooji/create-aws-profile-action@v1 | |
| with: | |
| profile: aws-deployer-connext | |
| region: sa-east-1 | |
| key: ${{ secrets.DEPLOYER_AWS_ACCESS_KEY_ID }} | |
| secret: ${{ secrets.DEPLOYER_AWS_SECRET_ACCESS_KEY }} | |
| - name: Terraform Init | |
| working-directory: ./ops/mainnet/mason | |
| run: terraform init > /dev/null 2>&1 | |
| env: | |
| AWS_PROFILE: aws-deployer-connext | |
| - name: Terraform Apply | |
| working-directory: ./ops/mainnet/mason | |
| env: | |
| AWS_PROFILE: aws-deployer-connext | |
| run: | | |
| terraform apply \ | |
| -var "handler_image_uri=${REGISTRY}/${HANDLER_REPOSITORY}:${HANDLER_IMAGE_TAG}" \ | |
| -var "image_uri=${REGISTRY}/${POLLER_REPOSITORY}:${POLLER_IMAGE_TAG}" \ | |
| -var "admin_image_uri=${REGISTRY}/${ADMIN_REPOSITORY}:${ADMIN_IMAGE_TAG}" \ | |
| -auto-approve > /dev/null 2>&1 | |
| - name: Show Admin API Endpoint URL | |
| if: success() | |
| working-directory: ./ops/mainnet/mason | |
| run: | | |
| echo "Admin API Endpoint URL for mason (staging):" | |
| terraform output -raw admin_api_endpoint | |
| - name: Show Invoice Handler URL | |
| if: success() | |
| working-directory: ./ops/mainnet/mason | |
| run: | | |
| echo "Invoice Handler URL for mason (staging):" | |
| terraform output -raw invoice_handler_url | |
| - name: Post-Deploy Health Check | |
| if: success() | |
| uses: ./.github/actions/post-deploy-health-check | |
| with: | |
| aws-region: sa-east-1 | |
| ecs-cluster: mason-ecs-mainnet-staging | |
| services: "mason-web3signer-mainnet-staging,mason-fillservice-web3signer-mainnet-staging,mason-handler-mainnet-staging,mason-prometheus-mainnet-staging,mason-pushgateway-mainnet-staging" | |
| aws-access-key-id: ${{ secrets.DEPLOYER_AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.DEPLOYER_AWS_SECRET_ACCESS_KEY }} |