GODRIVER-1931 Run tests against LBs in Evergreen (#648)

Author: Divjot Arora

.evergreen/config.yml

@@ -518,6 +518,46 @@ functions:
           PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH
 
+  run-load-balancer-tests:
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src/go.mongodb.org/mongo-driver
+        script: |
+          ${PREPARE_SHELL}
+
+          if [ ${SSL} = "ssl" ]; then
+              export MONGO_GO_DRIVER_CA_FILE="$PROJECT_DIRECTORY/data/certificates/ca.pem"
+              if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+                  export MONGO_GO_DRIVER_CA_FILE=$(cygpath -m $MONGO_GO_DRIVER_CA_FILE)
+              fi
+          fi
+
+          # Verify that the required LB URI expansions are set to ensure that the test runner can correctly connect to
+          # the LBs.
+          if [ -z "${SINGLE_MONGOS_LB_URI}" ]; then
+            echo "SINGLE_MONGOS_LB_URI must be set for testing against LBs"
+            exit 1
+          fi
+          if [ -z "${MULTI_MONGOS_LB_URI}" ]; then
+            echo "MULTI_MONGOS_LB_URI must be set for testing against LBs"
+            exit 1
+          fi
+
+          # Per the LB testing spec, the URI of an LB fronting a single mongos should be used to configure internal
+          # testing Client instances, so we set MONGODB_URI to SINGLE_MONGOS_LB_URI.
+
+          export GOFLAGS=-mod=vendor
+          set +o xtrace
+          AUTH="${AUTH}" \
+          SSL="${SSL}" \
+          MONGODB_URI="${SINGLE_MONGOS_LB_URI}" \
+          SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \
+          MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \
+          TOPOLOGY="${TOPOLOGY}" \
+          MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
+          make evg-test-load-balancers
+
   run-atlas-data-lake-test:
     - command: shell.exec
       type: test
@@ -612,6 +652,21 @@ functions:
           -v \
           --fault revoked
 
+  run-load-balancer:
+    - command: shell.exec
+      params:
+        script: |
+          DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start
+    - command: expansions.update
+      params:
+        file: lb-expansion.yml
+
+  stop-load-balancer:
+    - command: shell.exec
+      params:
+        script: |
+          DRIVERS_TOOLS=${DRIVERS_TOOLS} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop
+
   add-aws-auth-variables-to-file:
     - command: shell.exec
       type: test
@@ -866,6 +921,7 @@ post:
       files:
         - "src/go.mongodb.org/mongo-driver/*.suite"
   - func: upload-mo-artifacts
+  - func: stop-load-balancer
   - func: cleanup
 
 tasks:
@@ -1379,6 +1435,28 @@ tasks:
       - func: bootstrap-mongohoused
       - func: run-atlas-data-lake-test
 
+  - name: test-load-balancer-noauth-nossl
+    tags: ["load-balancer"]
+    commands:
+      - func: bootstrap-mongo-orchestration
+        vars:
+          TOPOLOGY: "sharded_cluster"
+          AUTH: "noauth"
+          SSL: "nossl"
+      - func: run-load-balancer
+      - func: run-load-balancer-tests
+
+  - name: test-load-balancer-auth-ssl
+    tags: ["load-balancer"]
+    commands:
+      - func: bootstrap-mongo-orchestration
+        vars:
+          TOPOLOGY: "sharded_cluster"
+          AUTH: "auth"
+          SSL: "ssl"
+      - func: run-load-balancer
+      - func: run-load-balancer-tests
+
   - name: test-replicaset-noauth-nossl
     tags: ["test", "replicaset"]
     commands:
@@ -1974,3 +2052,10 @@ buildvariants:
     display_name: "KMS TLS ${version} ${os-ssl-40}"
     tasks:
       - name: ".kms-tls"
+
+  - matrix_name: "load-balancer-test"
+    # The LB software is only available on Ubuntu 18.04, so we don't test on all OSes.
+    matrix_spec: { version: ["latest"], os-ssl-40: ["ubuntu1804-64-go-1-15"] }
+    display_name: "Load Balancer Support ${version} ${os-ssl-40}"
+    tasks:
+      - name: ".load-balancer"

Makefile

@@ -165,6 +165,15 @@ evg-test-versioned-api:
 		go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s $$TEST_PKG >> test.suite ; \
 	done
 
+.PHONY: evg-test-load-balancers
+evg-test-load-balancers:
+	go test $(BUILD_TAGS) ./mongo/integration -run TestUnifiedSpecs/retryable-reads -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BUILD_TAGS) ./mongo/integration -run TestRetryableWritesSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BUILD_TAGS) ./mongo/integration -run TestChangeStreamSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BULID_TAGS) ./mongo/integration -run TestInitialDNSSeedlistDiscoverySpec/load_balanced -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BUILD_TAGS) ./mongo/integration -run TestLoadBalancerSupport -v -timeout $(TEST_TIMEOUT)s >> test.suite
+	go test $(BUILD_TAGS) ./mongo/integration/unified -run TestUnifiedSpec -v -timeout $(TEST_TIMEOUT)s >> test.suite
+
 .PHONY: evg-test-kms
 evg-test-kms:
 	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s ./mongo/integration -run TestClientSideEncryptionProse/kms_tls_tests >> test.suite

internal/const.go

@@ -8,3 +8,8 @@ package internal // import "go.mongodb.org/mongo-driver/internal"
 
 // Version is the current version of the driver.
 var Version = "local build"
+
+// SetMockServiceID enables a mode in which the driver mocks server support for returning a "serviceId" field in "hello"
+// command responses by using the value of "topologyVersion.processId".  This is used for testing load balancer support
+// until an upstream service can support running behind a load balancer.
+var SetMockServiceID = false

mongo/description/server.go

@@ -266,6 +266,10 @@ func NewServer(addr address.Address, response bson.Raw) Server {
 				desc.LastError = err
 				return desc
 			}
+
+			if internal.SetMockServiceID {
+				desc.ServiceID = &desc.TopologyVersion.ProcessID
+			}
 		}
 	}
 

mongo/integration/load_balancer_prose_test.go

@@ -0,0 +1,101 @@
+// Copyright (C) MongoDB, Inc. 2017-present.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License. You may obtain
+// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/internal/testutil/assert"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/integration/mtest"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func TestLoadBalancerSupport(t *testing.T) {
+	mt := mtest.New(t, mtest.NewOptions().Topologies(mtest.LoadBalanced).CreateClient(false))
+	defer mt.Close()
+
+	mt.Run("RunCommandCursor pins to a connection", func(mt *mtest.T) {
+		// The LB spec tests cover the behavior for cursors created by CRUD operations, but RunCommandCursor is
+		// Go-specific so there is no spec test coverage for it.
+
+		initCollection(mt, mt.Coll)
+		findCmd := bson.D{
+			{"find", mt.Coll.Name()},
+			{"filter", bson.D{}},
+			{"batchSize", 2},
+		}
+		cursor, err := mt.DB.RunCommandCursor(mtest.Background, findCmd)
+		assert.Nil(mt, err, "RunCommandCursor error: %v", err)
+		defer func() {
+			_ = cursor.Close(mtest.Background)
+		}()
+
+		assert.True(mt, cursor.ID() > 0, "expected cursor ID to be non-zero")
+		assert.Equal(mt, 1, mt.NumberConnectionsCheckedOut(),
+			"expected one connection to be checked out, got %d", mt.NumberConnectionsCheckedOut())
+	})
+
+	mt.RunOpts("wait queue timeout errors include extra information", noClientOpts, func(mt *mtest.T) {
+		// There are spec tests to assert this behavior, but they rely on the waitQueueTimeoutMS Client option, which is
+		// not supported in Go, so we have to skip them. These prose tests make the same assertions, but use context
+		// deadlines to force wait queue timeout errors.
+
+		assertErrorHasInfo := func(mt *mtest.T, err error, numCursorConns, numTxnConns, numOtherConns int) {
+			mt.Helper()
+
+			assert.NotNil(mt, err, "expected wait queue timeout error, got nil")
+			expectedMsg := fmt.Sprintf("maxPoolSize: 1, "+
+				"connections in use by cursors: %d, "+
+				"connections in use by transactions: %d, "+
+				"connections in use by other operations: %d",
+				numCursorConns, numTxnConns, numOtherConns,
+			)
+			assert.True(mt, strings.Contains(err.Error(), expectedMsg),
+				"expected error %q to contain substring %q", err, expectedMsg)
+		}
+		maxPoolSizeMtOpts := mtest.NewOptions().
+			ClientOptions(options.Client().SetMaxPoolSize(1))
+
+		mt.RunOpts("cursors", maxPoolSizeMtOpts, func(mt *mtest.T) {
+			initCollection(mt, mt.Coll)
+			findOpts := options.Find().SetBatchSize(2)
+			cursor, err := mt.Coll.Find(mtest.Background, bson.M{}, findOpts)
+			assert.Nil(mt, err, "Find error: %v", err)
+			defer func() {
+				_ = cursor.Close(mtest.Background)
+			}()
+
+			ctx, cancel := context.WithTimeout(mtest.Background, 5*time.Millisecond)
+			defer cancel()
+			_, err = mt.Coll.InsertOne(ctx, bson.M{"x": 1})
+			assertErrorHasInfo(mt, err, 1, 0, 0)
+		})
+		mt.RunOpts("transactions", maxPoolSizeMtOpts, func(mt *mtest.T) {
+			sess, err := mt.Client.StartSession()
+			assert.Nil(mt, err, "StartSession error: %v", err)
+			defer sess.EndSession(mtest.Background)
+			sessCtx := mongo.NewSessionContext(context.Background(), sess)
+
+			// Start a transaction and perform one transactional operation to pin a connection.
+			err = sess.StartTransaction()
+			assert.Nil(mt, err, "StartTransaction error: %v", err)
+			_, err = mt.Coll.InsertOne(sessCtx, bson.M{"x": 1})
+			assert.Nil(mt, err, "InsertOne error: %v", err)
+
+			ctx, cancel := context.WithTimeout(mtest.Background, 5*time.Millisecond)
+			defer cancel()
+			_, err = mt.Coll.InsertOne(ctx, bson.M{"x": 1})
+			assertErrorHasInfo(mt, err, 0, 1, 0)
+		})
+	})
+}

mongo/integration/main_test.go

@@ -9,12 +9,22 @@ package integration
 import (
 	"log"
 	"os"
+	"strings"
 	"testing"
 
+	"go.mongodb.org/mongo-driver/internal"
 	"go.mongodb.org/mongo-driver/mongo/integration/mtest"
 )
 
 func TestMain(m *testing.M) {
+	// If the cluster is behind a load balancer, enable the SetMockServiceID flag to mock server-side LB support.
+	if strings.Contains(os.Getenv("MONGODB_URI"), "loadBalanced=true") {
+		internal.SetMockServiceID = true
+		defer func() {
+			internal.SetMockServiceID = false
+		}()
+	}
+
 	if err := mtest.Setup(); err != nil {
 		log.Fatal(err)
 	}

mongo/integration/mtest/global_state.go

@@ -35,6 +35,18 @@ func ClusterURI() string {
 	return testContext.connString.Original
 }
 
+// SingleMongosLoadBalancerURI returns the URI for a load balancer fronting a single mongos. This will only be set
+// if the cluster is load balanced.
+func SingleMongosLoadBalancerURI() string {
+	return testContext.singleMongosLoadBalancerURI
+}
+
+// MultiMongosLoadBalancerURI returns the URI for a load balancer fronting multiple mongoses. This will only be set
+// if the cluster is load balanced.
+func MultiMongosLoadBalancerURI() string {
+	return testContext.multiMongosLoadBalancerURI
+}
+
 // ClusterConnString returns the parsed ConnString for the cluster.
 func ClusterConnString() connstring.ConnString {
 	return testContext.connString

mongo/integration/mtest/mongotest.go

@@ -359,6 +359,11 @@ func (t *T) GetProxiedMessages() []*ProxyMessage {
 	return t.proxyDialer.Messages()
 }
 
+// NumberConnectionsCheckedOut returns the number of connections checked out from the test Client.
+func (t *T) NumberConnectionsCheckedOut() int {
+	return t.connsCheckedOut
+}
+
 // ClearEvents clears the existing command monitoring events.
 func (t *T) ClearEvents() {
 	t.started = t.started[:0]

mongo/integration/mtest/setup.go

@@ -43,15 +43,17 @@ var testContext struct {
 	// shardedReplicaSet will be true if we're connected to a sharded cluster and each shard is backed by a replica set.
 	// We track this as a separate boolean rather than setting topoKind to ShardedReplicaSet because a general
 	// "Sharded" constraint in a test should match both Sharded and ShardedReplicaSet.
-	shardedReplicaSet bool
-	client            *mongo.Client // client used for setup and teardown
-	serverVersion     string
-	authEnabled       bool
-	sslEnabled        bool
-	enterpriseServer  bool
-	dataLake          bool
-	requireAPIVersion bool
-	serverParameters  bson.Raw
+	shardedReplicaSet           bool
+	client                      *mongo.Client // client used for setup and teardown
+	serverVersion               string
+	authEnabled                 bool
+	sslEnabled                  bool
+	enterpriseServer            bool
+	dataLake                    bool
+	requireAPIVersion           bool
+	serverParameters            bson.Raw
+	singleMongosLoadBalancerURI string
+	multiMongosLoadBalancerURI  string
 }
 
 func setupClient(cs connstring.ConnString, opts *options.ClientOptions) (*mongo.Client, error) {
@@ -75,7 +77,7 @@ func Setup(setupOpts ...*SetupOptions) error {
 	case opts.URI != nil:
 		testContext.connString, err = connstring.ParseAndValidate(*opts.URI)
 	default:
-		testContext.connString, err = getConnString()
+		testContext.connString, err = getClusterConnString()
 	}
 	if err != nil {
 		return fmt.Errorf("error getting connection string: %v", err)
@@ -176,6 +178,22 @@ func Setup(setupOpts ...*SetupOptions) error {
 		}
 	}
 
+	// For load balanced clusters, retrieve the required LB URIs and add additional information (e.g. TLS options) to
+	// them if necessary.
+	if testContext.topoKind == LoadBalanced {
+		singleMongosURI := os.Getenv("SINGLE_MONGOS_LB_URI")
+		if singleMongosURI == "" {
+			return errors.New("SINGLE_MONGOS_LB_URI must be set when running against load balanced clusters")
+		}
+		testContext.singleMongosLoadBalancerURI = addNecessaryParamsToURI(singleMongosURI)
+
+		multiMongosURI := os.Getenv("MULTI_MONGOS_LB_URI")
+		if multiMongosURI == "" {
+			return errors.New("MULTI_MONGOS_LB_URI must be set when running against load balanced clusters")
+		}
+		testContext.multiMongosLoadBalancerURI = addNecessaryParamsToURI(multiMongosURI)
+	}
+
 	testContext.authEnabled = os.Getenv("AUTH") == "auth"
 	testContext.sslEnabled = os.Getenv("SSL") == "ssl"
 	biRes, err := testContext.client.Database("admin").RunCommand(Background, bson.D{{"buildInfo", 1}}).DecodeBytes()
@@ -282,17 +300,21 @@ func addCompressors(uri string) string {
 	return addOptions(uri, "compressors=", comp)
 }
 
-// ConnString gets the globally configured connection string.
-func getConnString() (connstring.ConnString, error) {
+// getClusterConnString gets the globally configured connection string.
+func getClusterConnString() (connstring.ConnString, error) {
 	uri := os.Getenv("MONGODB_URI")
 	if uri == "" {
 		uri = "mongodb://localhost:27017"
 	}
-	uri = addTLSConfig(uri)
-	uri = addCompressors(uri)
+	uri = addNecessaryParamsToURI(uri)
 	return connstring.ParseAndValidate(uri)
 }
 
+func addNecessaryParamsToURI(uri string) string {
+	uri = addTLSConfig(uri)
+	return addCompressors(uri)
+}
+
 // CompareServerVersions compares two version number strings (i.e. positive integers separated by
 // periods). Comparisons are done to the lesser precision of the two versions. For example, 3.2 is
 // considered equal to 3.2.11, whereas 3.2.0 is considered less than 3.2.11.