GODRIVER-2792 Migrate CI to RHEL 8.7 (#1362)

Author: prestonvasquez

.evergreen/config.yml

@@ -141,8 +141,8 @@ functions:
              export UPLOAD_BUCKET="$UPLOAD_BUCKET"
              export PROJECT="$PROJECT"
              export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
-             export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
-             export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
+             export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
+             export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
              export PATH="$PATH"
           EOT
           # See what we variables we've set.
@@ -245,6 +245,7 @@ functions:
       params:
         shell: "bash"
         script: |
+          set -x
           ${PREPARE_SHELL}
 
           MONGODB_VERSION=${VERSION} \
@@ -2082,7 +2083,7 @@ tasks:
           export GCPKMS_PROJECT=${GCPKMS_PROJECT}
           export GCPKMS_ZONE=${GCPKMS_ZONE}
           export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
-          tar czf testgcpkms.tgz ./testkms ./install/libmongocrypt/lib/libmongocrypt.*
+          tar czf testgcpkms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.*
           GCPKMS_SRC=testgcpkms.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh
           echo "Copying files ... end"
 
@@ -2101,7 +2102,7 @@ tasks:
           export GCPKMS_PROJECT=${GCPKMS_PROJECT}
           export GCPKMS_ZONE=${GCPKMS_ZONE}
           export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
-          GCPKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib MONGODB_URI='mongodb://localhost:27017' PROVIDER='gcp' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
+          GCPKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='gcp' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh
 
   - name: "testgcpkms-fail-task"
     # testgcpkms-fail-task runs in a non-GCE environment.
@@ -2119,7 +2120,7 @@ tasks:
             PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
               make build-kms-test
             echo "Building build-kms-test ... end"
-            LD_LIBRARY_PATH=./install/libmongocrypt/lib \
+            LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
             MONGODB_URI='mongodb://localhost:27017/' \
             EXPECT_ERROR='unable to retrieve GCP credentials' \
             PROVIDER='gcp' \
@@ -2143,7 +2144,7 @@ tasks:
           export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
           export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
 
-          LD_LIBRARY_PATH=./install/libmongocrypt/lib \
+          LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
           MONGODB_URI='${atlas_free_tier_uri}' \
           PROVIDER='aws' \
             ./testkms
@@ -2165,9 +2166,9 @@ tasks:
             make build-kms-test
           echo "Building build-kms-test ... end"
 
-          LD_LIBRARY_PATH=./install/libmongocrypt/lib \
+          LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
           MONGODB_URI='${atlas_free_tier_uri}' \
-          EXPECT_ERROR='unable to retrieve aws credentials' \
+          EXPECT_ERROR='status=400' \
           PROVIDER='aws' \
             ./testkms
 
@@ -2191,7 +2192,7 @@ tasks:
           export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
           echo '${testazurekms_privatekey}' > /tmp/testazurekms.prikey
           export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms.prikey
-          tar czf testazurekms.tgz ./testkms ./install/libmongocrypt/lib/libmongocrypt.*
+          tar czf testazurekms.tgz ./testkms ./install/libmongocrypt/lib64/libmongocrypt.*
           AZUREKMS_SRC=testazurekms.tgz AZUREKMS_DST=/tmp $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh
           echo "Copying files ... end"
           echo "Untarring file ... begin"
@@ -2209,7 +2210,7 @@ tasks:
           export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
           echo '${testazurekms_privatekey}' > /tmp/testazurekms.prikey
           export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms.prikey
-          AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
+          AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib64 MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 
   - name: "testazurekms-fail-task"
     # testazurekms-fail-task runs without environment variables.
@@ -2228,7 +2229,7 @@ tasks:
             make build-kms-test
           echo "Building build-kms-test ... end"
 
-          LD_LIBRARY_PATH=./install/libmongocrypt/lib \
+          LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \
           MONGODB_URI='mongodb://localhost:27017' \
           EXPECT_ERROR='unable to retrieve azure credentials' \
           PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' \
@@ -2311,9 +2312,9 @@ axes:
           GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
           GO_DIST: "C:\\golang\\go1.20"
           VENV_BIN_DIR: "Scripts"
-      - id: "ubuntu1604-64-go-1-20"
-        display_name: "Ubuntu 16.04"
-        run_on: ubuntu1604-build
+      - id: "rhel87-64-go-1-20"
+        display_name: "RHEL 8.7"
+        run_on: rhel8.7-large 
         variables:
           GO_DIST: "/opt/golang/go1.20"
       - id: "macos11-go-1-20"
@@ -2335,9 +2336,9 @@ axes:
           GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
           GO_DIST: "C:\\golang\\go1.20"
           VENV_BIN_DIR: "Scripts"
-      - id: "ubuntu1804-64-go-1-20"
-        display_name: "Ubuntu 18.04"
-        run_on: ubuntu1804-build
+      - id: "rhel87-64-go-1-20"
+        display_name: "RHEL 8.7"
+        run_on: rhel8.7-large
         variables:
           GO_DIST: "/opt/golang/go1.20"
       - id: "macos11-go-1-20"
@@ -2347,13 +2348,12 @@ axes:
         variables:
           GO_DIST: "/opt/golang/go1.20"
 
-  # OCSP linux tasks need to run against this OS since stapling is disabled on Ubuntu 18.04 (SERVER-51364)
-  - id: ocsp-rhel-70
+  - id: ocsp-rhel-87
     display_name: OS
     values:
-      - id: "rhel70-go-1-20"
-        display_name: "RHEL 7.0"
-        run_on: rhel70-build
+      - id: "rhel87-go-1-20"
+        display_name: "RHEL 8.7"
+        run_on: rhel8.7-large
         variables:
           GO_DIST: "/opt/golang/go1.20"
 
@@ -2368,6 +2368,8 @@ axes:
           GCC_PATH: "/cygdrive/c/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin"
           GO_DIST: "C:\\golang\\go1.20"
           SKIP_ECS_AUTH_TEST: true
+      # TODO(BUILD-17329): Update this to Ubuntu 22 after we add a new ECS task
+      # definition.
       - id: "ubuntu1804-64-go-1-20"
         display_name: "Ubuntu 18.04"
         run_on: ubuntu1804-test
@@ -2386,18 +2388,18 @@ axes:
   - id: os-faas-80
     display_name: OS
     values:
-      - id: "rhel80-large-go-1-20"
-        display_name: "RHEL 8.0"
-        run_on: rhel80-large
+      - id: "rhel87-large-go-1-20"
+        display_name: "RHEL 8.7"
+        run_on: rhel8.7-large
         variables:
           GO_DIST: "/opt/golang/go1.20"
 
   - id: os-serverless
     display_name: OS
     values:
-      - id: "ubuntu2204-go-1-20"
-        display_name: "Ubuntu 22.04"
-        run_on: ubuntu2204-small
+      - id: "rhel87-go-1-20"
+        display_name: "RHEL 8.7"
+        run_on: rhel8.7-small
         variables:
           GO_DIST: "/opt/golang/go1.20"
 
@@ -2620,7 +2622,7 @@ buildvariants:
   - name: static-analysis
     display_name: "Static Analysis"
     run_on:
-      - ubuntu1804-build
+      - rhel8.7-large
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:
@@ -2629,7 +2631,7 @@ buildvariants:
   - name: perf
     display_name: "Performance"
     run_on:
-      - ubuntu1804-build
+      - rhel8.7-large
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:
@@ -2638,16 +2640,16 @@ buildvariants:
   - name: build-check
     display_name: "Compile Only Checks"
     run_on:
-      - ubuntu1804-test
+      - rhel8.7-large
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:
       - name: ".compile-check"
-
+  
   - name: atlas-test
     display_name: "Atlas test"
     run_on:
-      - ubuntu1804-build
+      - rhel8.7-large 
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:
@@ -2656,7 +2658,7 @@ buildvariants:
   - name: atlas-data-lake-test
     display_name: "Atlas Data Lake Test"
     run_on:
-      - ubuntu1804-build
+      - rhel8.7-large 
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:
@@ -2693,8 +2695,8 @@ buildvariants:
       - name: "aws-auth-test"
 
   - matrix_name: "ocsp-test"
-    matrix_spec: { version: ["4.4", "5.0", "6.0", "7.0", "latest"], ocsp-rhel-70: ["rhel70-go-1-20"] }
-    display_name: "OCSP ${version} ${ocsp-rhel-70}"
+    matrix_spec: { version: ["4.4", "5.0", "6.0", "7.0", "latest"], ocsp-rhel-87: ["rhel87-go-1-20"] }
+    display_name: "OCSP ${version} ${ocsp-rhel-87}"
     batchtime: 20160 # Use a batchtime of 14 days as suggested by the OCSP test README
     tasks:
       - name: ".ocsp"
@@ -2716,7 +2718,7 @@ buildvariants:
       - name: ".ocsp-rsa !.ocsp-staple"
 
   - matrix_name: "race-test"
-    matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
+    matrix_spec: { version: ["latest"], os-ssl-40: ["rhel87-64-go-1-20"] }
     display_name: "Race Detector Test"
     tasks:
       - name: ".race"
@@ -2728,14 +2730,13 @@ buildvariants:
       - name: ".versioned-api"
 
   - matrix_name: "kms-tls-test"
-    matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
+    matrix_spec: { version: ["latest"], os-ssl-40: ["rhel87-64-go-1-20"] }
     display_name: "KMS TLS ${os-ssl-40}"
     tasks:
       - name: ".kms-tls"
 
   - matrix_name: "load-balancer-test"
-    # The LB software is only available on Ubuntu 18.04, so we don't test on all OSes.
-    matrix_spec: { version: ["5.0", "6.0", "7.0", "latest", "rapid"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
+    matrix_spec: { version: ["5.0", "6.0", "7.0", "latest", "rapid"], os-ssl-40: ["rhel87-64-go-1-20"] }
     display_name: "Load Balancer Support ${version} ${os-ssl-40}"
     tasks:
       - name: ".load-balancer"
@@ -2747,28 +2748,28 @@ buildvariants:
       - "serverless_task_group"
 
   - matrix_name: "kms-kmip-test"
-    matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
+    matrix_spec: { version: ["latest"], os-ssl-40: ["rhel87-64-go-1-20"] }
     display_name: "KMS KMIP ${os-ssl-40}"
     tasks:
       - name: ".kms-kmip"
 
   - matrix_name: "fuzz-test"
-    matrix_spec: { version: ["5.0"], os-ssl-40: ["ubuntu1804-64-go-1-20"] }
+    matrix_spec: { version: ["5.0"], os-ssl-40: ["rhel87-64-go-1-20"] }
     display_name: "Fuzz ${version} ${os-ssl-40}"
     tasks:
       - name: "test-fuzz"
         batchtime: 1440 # Run at most once per 24 hours.
 
   - matrix_name: "faas-test"
-    matrix_spec: { version: ["latest"], os-faas-80: ["rhel80-large-go-1-20"] }
+    matrix_spec: { version: ["latest"], os-faas-80: ["rhel87-large-go-1-20"] }
     display_name: "FaaS ${version} ${os-faas-80}"
     tasks:
       - test-aws-lambda-task-group
 
   - name: testgcpkms-variant
     display_name: "GCP KMS"
     run_on:
-      - debian11-small
+      - rhel8.7-small
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:
@@ -2779,7 +2780,7 @@ buildvariants:
   - name: testawskms-variant
     display_name: "AWS KMS"
     run_on:
-      - debian11-small
+      - rhel8.7-small
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:
@@ -2789,7 +2790,7 @@ buildvariants:
   - name: testazurekms-variant
     display_name: "AZURE KMS"
     run_on:
-      - debian11-small
+      - rhel8.7-small
     expansions:
       GO_DIST: "/opt/golang/go1.20"
     tasks:

.evergreen/run-tests.sh

@@ -15,8 +15,8 @@ fi
 export GOROOT="${GOROOT}"
 export PATH="${GOROOT}/bin:${GCC_PATH}:$GOPATH/bin:$PATH"
 export PROJECT="${project}"
-export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
-export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
+export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
+export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
 export GOFLAGS=-mod=vendor
 
 SSL=${SSL:-nossl}