[DISCO-2169] Delete CircleCI approval task for prod deployment (#250)

* chore: remove unhold-to-deploy-to-prod build step

* docs: update release process documentation

* docs: add clarification for CD Jenkins trigger

* chore: consolidate stage and prod Docker image publication

* fix: remove redundant dependency 'docker-image-build' from 'docker-image-publish' in CircleCI config

* docs: update release process

Co-authored-by: Raphael Pierzina <[email protected]>

* chore: add check-for-deployment step

* chore: add check for docker-image-publish-locust

* fix: spacing in CircleCI config

* fix: schema in docker-image-publish-locust

* docs: add activity diagram of CircleCI main-workflow

---------

Co-authored-by: Raphael Pierzina <[email protected]>

Author: Trinaa

.circleci/config.yml

@@ -28,7 +28,6 @@ workflows:
       - contract-tests:
           <<: *pr-filters
           requires:
-            - checks
             - docker-image-build
       - docs-build:
           <<: *pr-filters
@@ -53,45 +52,27 @@ workflows:
       - contract-tests:
           <<: *main-filters
           requires:
-            - checks
             - docker-image-build
       - docs-build:
           <<: *main-filters
       - docs-publish-github-pages:
           <<: *main-filters
           requires:
             - docs-build
-      - docker-image-publish-stage:
+      - check-for-deployment:
           <<: *main-filters
           requires:
             - checks
-            - unit-tests
-            - integration-tests
             - test-coverage-check
             - contract-tests
-            - docker-image-build
-      - docker-image-publish-merino-locust:
+      - docker-image-publish-locust:
           <<: *main-filters
           requires:
-            - docker-image-publish-stage
-      # The following job will require manual approval in the CircleCI web application.
-      # Once provided, and when all the requirements are fullfilled (e.g. tests)
-      - unhold-to-deploy-to-prod:
+            - check-for-deployment
+      - docker-image-publish:
           <<: *main-filters
-          type: approval
           requires:
-            - checks
-            - unit-tests
-            - integration-tests
-            - test-coverage-check
-            - contract-tests
-            - docker-image-build
-      # On approval of the `unhold-to-deploy-to-prod` job, any successive job that requires it
-      # will run. In this case, it's manually triggering deployment to production.
-      - docker-image-publish-prod:
-          <<: *main-filters
-          requires:
-            - unhold-to-deploy-to-prod
+            - docker-image-publish-locust
 
 jobs:
   checks:
@@ -192,12 +173,19 @@ jobs:
           command: |
             sudo chown 1000:1000 tests/contract/kinto-attachments
             make run-contract-tests
-  docker-image-publish-stage:
+  check-for-deployment:
     docker:
       - image: cimg/base:2022.08
     steps:
       - checkout
       - skip-if-do-not-deploy
+  docker-image-publish:
+    # Pushing a new production Docker image to the Docker Hub registry triggers a
+    # webhook that starts the Jenkins deployment workflow
+    docker:
+      - image: cimg/base:2022.08
+    steps:
+      - checkout
       - attach_workspace:
           at: /tmp/workspace
       - setup_remote_docker
@@ -206,33 +194,48 @@ jobs:
           command: docker load -i /tmp/workspace/merinopy.tar.gz
       - dockerhub-login
       - run:
-          name: Push to Docker Hub # Load tests are triggered though [load test: (abort|warn)] label.
+          name: Push to Docker Hub
+          # The commit tag signals deployment and load test instructions to Jenkins by
+          # modifying the Docker image tag name. The convention looks as follows:
+          #^(?P<environment>stage|prod)(?:-(?P<task>\w+)-(?P<onfailure>warn|abort))?-(?P<commit>[a-z0-9]+)$
           command: |
             if [ "${CIRCLE_BRANCH}" == "main" ]; then
+              PROD_DOCKER_TAG="prod-${CIRCLE_SHA1}"
               if git log -1 "$CIRCLE_SHA1" | grep -q '\[load test: warn\]'; then
                 echo "Load test requested. Slack warning will be output if test fails and deployment workflow for prod will proceed."
-                DOCKER_TAG="stage-loadtest-warn-${CIRCLE_SHA1}"
+                STAGE_DOCKER_TAG="stage-loadtest-warn-${CIRCLE_SHA1}"
               elif git log -1 "$CIRCLE_SHA1" | grep -q '\[load test: abort\]'; then
                 echo "Load test requested. Deployment workflow for prod will abort if load test fails."
-                DOCKER_TAG="stage-loadtest-abort-${CIRCLE_SHA1}"
+                STAGE_DOCKER_TAG="stage-loadtest-abort-${CIRCLE_SHA1}"
               else
-                DOCKER_TAG="stage-${CIRCLE_SHA1}"
+                STAGE_DOCKER_TAG="stage-${CIRCLE_SHA1}"
               fi
             fi
 
             if [ -n "${DOCKER_TAG}" ]; then
-              echo ${DOCKERHUB_REPO}:${DOCKER_TAG}
-              docker tag app:build ${DOCKERHUB_REPO}:${DOCKER_TAG}
+              echo ${DOCKERHUB_REPO}:${STAGE_DOCKER_TAG} ${DOCKERHUB_REPO}:${PROD_DOCKER_TAG}
+              docker tag app:build ${DOCKERHUB_REPO}:${STAGE_DOCKER_TAG}
+              docker tag app:build ${DOCKERHUB_REPO}:${PROD_DOCKER_TAG}
+              docker tag app:build ${DOCKERHUB_REPO}:latest
               docker images
-              docker push "${DOCKERHUB_REPO}:${DOCKER_TAG}"
+              docker push "${DOCKERHUB_REPO}:${STAGE_DOCKER_TAG}"
+              docker push "${DOCKERHUB_REPO}:${PROD_DOCKER_TAG}"
+              docker push "${DOCKERHUB_REPO}:latest"
             else
               echo "Not pushing to dockerhub for tag=${CIRCLE_TAG} branch=${CIRCLE_BRANCH}"
             fi
-  docker-image-publish-merino-locust:
+  docker-image-publish-locust:
     docker:
       - image: cimg/base:2022.08
     steps:
       - checkout
+      - run:
+          name: Check for load test directive
+          command: | 
+            if ! git log -1 "$CIRCLE_SHA1" | grep -q '\[load test: abort\|warn\]'; then
+              echo "Skipping remaining steps in this job: load test not required."
+              circleci-agent step halt
+            fi
       - setup_remote_docker:
           docker_layer_caching: true
       - run:
@@ -256,37 +259,6 @@ jobs:
             else
               echo "Not pushing to dockerhub for tag=${CIRCLE_TAG} branch=${CIRCLE_BRANCH}"
             fi
-  docker-image-publish-prod:
-    docker:
-      - image: cimg/base:2022.08
-    steps:
-      - checkout
-      - skip-if-do-not-deploy
-      - attach_workspace:
-          at: /tmp/workspace
-      - setup_remote_docker
-      - run:
-          name: Load Docker image from workspace
-          command: docker load -i /tmp/workspace/merinopy.tar.gz
-      - dockerhub-login
-      - run:
-          name: Push to Docker Hub (prod)
-          # Using a different tag `prod-{CIRCLE_SHA1}` for production deployments
-          command: |
-            if [ "${CIRCLE_BRANCH}" == "main" ]; then
-              DOCKER_TAG="prod-${CIRCLE_SHA1}"
-            fi
-
-            if [ -n "${DOCKER_TAG}" ]; then
-              echo ${DOCKERHUB_REPO}:${DOCKER_TAG}
-              docker tag app:build ${DOCKERHUB_REPO}:${DOCKER_TAG}
-              docker tag app:build ${DOCKERHUB_REPO}:latest
-              docker images
-              docker push "${DOCKERHUB_REPO}:${DOCKER_TAG}"
-              docker push "${DOCKERHUB_REPO}:latest"
-            else
-              echo "Not pushing to dockerhub for tag=${CIRCLE_TAG} branch=${CIRCLE_BRANCH}"
-            fi
   docs-build:
     docker:
       - image: cimg/rust:1.64

docs/dev/circleci_main_workflow.jpg

@@ -1,11 +1,15 @@
 # The Release Process
 
-This project currently follows a [Continuous Delivery][continuous_delivery] process, but it's gradually moving toward [Continuous Deployment][continuous_deployment].
+This project currently follows a [Continuous Deployment][continuous_deployment] process.
 
-[continuous_delivery]: https://en.wikipedia.org/wiki/Continuous_delivery
 [continuous_deployment]: https://en.wikipedia.org/wiki/Continuous_deployment
 
-Whenever a commit is pushed to this repository's `main` branch, the deployment pipeline kicks in, deploying the changeset to the [`stage` environment](../firefox.md#stage).
+Whenever a commit is pushed to this repository's `main` branch, a CircleCI workflow is triggered which performs code checks and runs automated tests. The workflow additionally builds a new Docker image of the service and pushes that Docker image to the Docker Hub registry (this requires all previous jobs to pass).
+
+Pushing a new Docker image to the Docker Hub registry triggers a webhook that starts the Jenkins deployment pipeline (the Docker image tag determines the target environment). The deployment pipeline first deploys to the [`stage` environment](../firefox.md#stage) and subsequently to the [`production` environment](../firefox.md#production).
+
+![Activity diagram of CircleCI main-workflow][activity_circleci_main_workflow]
+
 After the deployment is complete, accessing the [`__version__` endpoint][stage_version] will show the commit hash of the deployed version, which will eventually match to the one of the latest commit on the `main` branch (a node with an older version might still serve the request before it is shut down).
 
 [stage_version]: https://stage.merino.nonprod.cloudops.mozgcp.net/__version__
@@ -37,18 +41,12 @@ Load testing can be run locally or as a part of the deployment process. Local ex
 Abort will prevent deployment should the load testing fail while warn will simply warn via Slack and continue deployment. For detailed specifics on load testing and this convention, please see the relevant documentation: [load-testing-docs]: /tests/load/README.md].
 
 ## Releasing to production
-Developers with write access to the Merino repository can initiate a deployment to production after a Pull-Request on the Merino GitHub repository is merged to the `main` branch.
+Developers with write access to the Merino repository will initiate a deployment to production when a Pull-Request on the Merino GitHub repository is merged to the `main` branch.
+Developers **must** monitor the [Merino Application & Infrastructure][merino_app_info] dashboard for any anomaly, for example significant changes in HTTP response codes, increase in latency, cpu/memory usage (most things under the infrastructure heading).
+
 While any developer with write access can trigger the deployment to production, the _expectation_ is that individual(s) who authored and merged the Pull-Request should do so, as they are the ones most familiar with their changes and who can tell, by looking at the data, if anything looks anomalous.
 In general authors should feel _responsible_ for the changes they make and shepherd these changes through to deployment.
 
-Releasing to production can be done by:
-
-1. opening the [CircleCI dashboard][circleci_dashboard];
-2. looking up the pipeline named `merino <PR NUMBER>` running in the `main-workflow`; this pipeline should either be in a running status (if the required test jobs are still running) or in the "on hold" status, with the `unhold-to-deploy-to-prod` being held;
-3. once in the "on hold" status, with all the other jobs successfully completed, clicking on the "thumbs up" action on the `unhold-to-deploy-to-prod` job row will approve it and trigger the deployment, unblocking the `deploy-to-prod` job;
-4. developers **must** monitor the [Merino Application & Infrastructure][merino_app_info] dashboard for any anomaly, for example significant changes in HTTP response codes, increase in latency, cpu/memory usage (most things under the infrastructure heading).
-
-[circleci_dashboard]: https://app.circleci.com/pipelines/github/mozilla-services/merino-py?branch=main&filter=all
 [merino_app_info]: https://earthangel-b40313e5.influxcloud.net/d/rQAfYKIVk/wip-merino-py-application-and-infrastructure?orgId=1&from=now-24h&to=now&var-environment=prodpy&refresh=1m
 
 ## What to do if production breaks?
@@ -64,4 +62,5 @@ don't panic and follow the instructions below:
 
 [incident_docs]: https://mozilla-hub.atlassian.net/wiki/spaces/MIR/overview
 [contributing]: ../../CONTRIBUTING.md
-[load-testing-docs]: /tests/load/README.md
+[load-testing-docs]: /tests/load/README.md
+[activity_circleci_main_workflow]: ./circleci_main_workflow.jpg