PHPLIB-671: Load balancer testing

Add load balancer task to Evergreen. This cleans up some unused variables in run-tests.sh and copies over SSL URI handling from PHPC (needed since LB appears to be the first task using SSL).

Sync all spec tests. Synced with mongodb/specifications@b508f6d

Added skips for some CSFLE and command monitoring tests, which replaces a previous manual edit to azureKMS.json.

Various test suite fixes for load balancers and/or sharded clusters.

Author: jmikola

.evergreen/config.yml

@@ -246,7 +246,7 @@ functions:
           export GCP_EMAIL="${client_side_encryption_gcp_email}"
           export GCP_PRIVATEKEY="${client_side_encryption_gcp_privatekey}"
           export PATH="${PHP_PATH}/bin:$PATH"
-          API_VERSION=${API_VERSION} PHP_VERSION=${PHP_VERSION} AUTH=${AUTH} SSL=${SSL} MONGODB_URI="${MONGODB_URI}" sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
+          MOCK_SERVICE_ID=${MOCK_SERVICE_ID} API_VERSION=${API_VERSION} PHP_VERSION=${PHP_VERSION} AUTH=${AUTH} SSL=${SSL} MONGODB_URI="${MONGODB_URI}" sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
   "run atlas data lake test":
      - command: shell.exec
@@ -323,6 +323,24 @@ functions:
       params:
         file: src/php-expansion.yml
 
+  "start load balancer":
+    - command: shell.exec
+      params:
+        script: |
+          MONGODB_URI="${MONGODB_URI}" ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start
+    - command: expansions.update
+      params:
+        file: lb-expansion.yml
+
+  "stop load balancer":
+    - command: shell.exec
+      params:
+        script: |
+          # Only run if a load balancer was started
+          if [ -n "${SINGLE_MONGOS_LB_URI}" ]; then
+            ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop
+          fi
+
 pre:
   - func: "fetch source"
   - func: "prepare resources"
@@ -336,6 +354,7 @@ post:
   # - func: "upload working dir"
   - func: "upload mo artifacts"
   - func: "upload test results"
+  - func: "stop load balancer"
   - func: "stop mongo-orchestration"
   - func: "cleanup"
 
@@ -432,6 +451,23 @@ tasks:
             PROJECT: "mongo-php-library"
         - func: "run serverless tests"
         - func: "delete serverless instance"
+
+    - name: "test-loadBalanced"
+      tags: ["loadbalanced"]
+      commands:
+        - func: "bootstrap mongo-orchestration"
+          vars:
+            TOPOLOGY: "sharded_cluster"
+            SSL: "yes"
+        - func: "start load balancer"
+        - func: "run tests"
+          vars:
+            # Testing with HAProxy requires service ID mocking
+            MOCK_SERVICE_ID: 1
+            # Note: loadBalanced=true should already be appended to SINGLE_MONGOS_LB_URI
+            MONGODB_URI: "${SINGLE_MONGOS_LB_URI}"
+            SSL: "yes"
+        # Note: "stop load balancer" will be called from "post"
 # }}}
 
 
@@ -693,3 +729,9 @@ buildvariants:
   run_on: rhel70
   tasks:
     - .serverless
+
+- matrix_name: "test-loadBalanced"
+  matrix_spec: { "os-php7": "debian92-test", "php-versions": "7.3", "versions": ["5.0", "latest"], "driver-versions": "latest-dev" }
+  display_name: "Load balanced - ${versions}"
+  tasks:
+    - name: "test-loadBalanced"

.evergreen/run-tests.sh

@@ -2,18 +2,17 @@
 set -o errexit  # Exit the script with error if any of the commands fail
 
 # Supported/used environment variables:
-#       AUTH                    Set to enable authentication. Defaults to "noauth"
-#       SSL                     Set to enable SSL. Defaults to "nossl"
-#       MONGODB_URI             Set the suggested connection MONGODB_URI (including credentials and topology info)
-#       MARCH                   Machine Architecture. Defaults to lowercase uname -m
+#   SSL               Set to "yes" to enable SSL. Defaults to "nossl"
+#   MONGODB_URI       Set the suggested connection MONGODB_URI (including credentials and topology info)
+#   API_VERSION       Optional API_VERSION environment variable for run-tests.php
+#   IS_MATRIX_TESTING Set to "true" to enable matrix testing. Defaults to empty string. If "true", DRIVER_MONGODB_VERSION and MONGODB_VERSION will also be checked.
+#   MOCK_SERVICE_ID   Set to "1" to enable service ID mocking for load balancers. Defaults to empty string.
 
-
-AUTH=${AUTH:-noauth}
 SSL=${SSL:-nossl}
 MONGODB_URI=${MONGODB_URI:-}
-TESTS=${TESTS:-}
 API_VERSION=${API_VERSION:-}
 IS_MATRIX_TESTING=${IS_MATRIX_TESTING:-}
+MOCK_SERVICE_ID=${MOCK_SERVICE_ID:-}
 
 # For matrix testing, we have to determine the correct driver version
 if [ "${IS_MATRIX_TESTING}" = "true" ]; then
@@ -44,10 +43,23 @@ if [ "${IS_MATRIX_TESTING}" = "true" ]; then
    . $DIR/install-dependencies.sh
 fi
 
-OS=$(uname -s | tr '[:upper:]' '[:lower:]')
-[ -z "$MARCH" ] && MARCH=$(uname -m | tr '[:upper:]' '[:lower:]')
+# For load balancer testing, we need to enable service ID mocking
+if [ "${MOCK_SERVICE_ID}" = "1" ]; then
+   PHPUNIT_OPTS="${PHPUNIT_OPTS} -d mongodb.mock_service_id=1"
+fi
+
+# Determine if MONGODB_URI already has a query string
+SUFFIX=$(echo "$MONGODB_URI" | grep -Eo "\?(.*)" | cat)
+
+if [ "$SSL" = "yes" ]; then
+   if [ -z "$SUFFIX" ]; then
+      MONGODB_URI="${MONGODB_URI}/?ssl=true&sslallowinvalidcertificates=true"
+   else
+      MONGODB_URI="${MONGODB_URI}&ssl=true&sslallowinvalidcertificates=true"
+   fi
+fi
 
-echo "Running tests with $AUTH and $SSL, connecting to: $MONGODB_URI"
+echo "Running tests with URI: $MONGODB_URI"
 
 # Disable failing PHPUnit due to deprecations
 export SYMFONY_DEPRECATIONS_HELPER=999999

tests/Collection/CollectionFunctionalTest.php

@@ -356,8 +356,8 @@ public function testRenameToDifferentDatabase(): void
         $toDatabase = new Database($this->manager, $toDatabaseName);
 
         /* When renaming an unsharded collection, mongos requires the source
-        * and target database to both exist on the primary shard. In practice, this
-        * means we need to create the target database explicitly.
+        * and target database to both exist on the primary shard. In practice,
+        * this means we need to create the target database explicitly.
         * See: https://docs.mongodb.com/manual/reference/command/renameCollection/#unsharded-collections
         */
         if ($this->isShardedCluster()) {

tests/FunctionalTestCase.php

@@ -390,11 +390,21 @@ protected function getServerStorageEngine(?ReadPreference $readPreference = null
         throw new UnexpectedValueException('Could not determine server storage engine');
     }
 
+    protected function isLoadBalanced()
+    {
+        return $this->getPrimaryServer()->getType() == Server::TYPE_LOAD_BALANCER;
+    }
+
     protected function isReplicaSet()
     {
         return $this->getPrimaryServer()->getType() == Server::TYPE_RS_PRIMARY;
     }
 
+    protected function isMongos()
+    {
+        return $this->getPrimaryServer()->getType() == Server::TYPE_MONGOS;
+    }
+
     /**
      * Return whether serverless (i.e. proxy as mongos) is being utilized.
      */
@@ -407,7 +417,18 @@ protected static function isServerless(): bool
 
     protected function isShardedCluster()
     {
-        return $this->getPrimaryServer()->getType() == Server::TYPE_MONGOS;
+        $type = $this->getPrimaryServer()->getType();
+
+        if ($type == Server::TYPE_MONGOS) {
+            return true;
+        }
+
+        // Assume that load balancers are properly configured and front mongos
+        if ($type == Server::TYPE_LOAD_BALANCER) {
+            return true;
+        }
+
+        return false;
     }
 
     protected function isShardedClusterUsingReplicasets()
@@ -436,6 +457,7 @@ protected function skipIfChangeStreamIsNotSupported(): void
     {
         switch ($this->getPrimaryServer()->getType()) {
             case Server::TYPE_MONGOS:
+            case Server::TYPE_LOAD_BALANCER:
                 if (version_compare($this->getServerVersion(), '3.6.0', '<')) {
                     $this->markTestSkipped('$changeStream is only supported on MongoDB 3.6 or higher');
                 }
@@ -462,6 +484,7 @@ protected function skipIfCausalConsistencyIsNotSupported(): void
     {
         switch ($this->getPrimaryServer()->getType()) {
             case Server::TYPE_MONGOS:
+            case Server::TYPE_LOAD_BALANCER:
                 if (version_compare($this->getServerVersion(), '3.6.0', '<')) {
                     $this->markTestSkipped('Causal Consistency is only supported on MongoDB 3.6 or higher');
                 }

tests/Operation/RenameCollectionFunctionalTest.php

@@ -106,7 +106,12 @@ public function testRenameCollectionExistingTarget(): void
         $this->assertEquals(1, $writeResult->getInsertedCount());
 
         $this->expectException(CommandException::class);
-        $this->expectExceptionCode(self::$errorCodeNamespaceExists);
+
+        // mongos returns an inconsistent error code (see: SERVER-60632)
+        if (! $this->isShardedCluster()) {
+            $this->expectExceptionCode(self::$errorCodeNamespaceExists);
+        }
+
         $operation = new RenameCollection(
             $this->getDatabaseName(),
             $this->getCollectionName(),

tests/Operation/WatchFunctionalTest.php

@@ -1178,6 +1178,10 @@ function (array $event) use (&$sessionAfterResume, &$commands): void {
 
     public function testSessionFreed(): void
     {
+        if ($this->isShardedCluster() && version_compare($this->getServerVersion(), '5.1.0', '>=')) {
+            $this->markTestSkipped('mongos still reports non-zero cursor ID for invalidated change stream (SERVER-60764)');
+        }
+
         $operation = new Watch($this->manager, $this->getDatabaseName(), $this->getCollectionName(), [], $this->defaultOptions);
         $changeStream = $operation->execute($this->getPrimaryServer());
 

tests/SpecTests/ClientSideEncryptionSpecTest.php

@@ -42,6 +42,13 @@ class ClientSideEncryptionSpecTest extends FunctionalTestCase
 {
     public const LOCAL_MASTERKEY = 'Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk';
 
+    /** @var array */
+    private static $incompleteTests = [
+        'awsTemporary: Insert a document with auto encryption using the AWS provider with temporary credentials' => 'Not yet implemented (PHPC-1751)',
+        'awsTemporary: Insert with invalid temporary credentials' => 'Not yet implemented (PHPC-1751)',
+        'azureKMS: Insert a document with auto encryption using Azure KMS provider' => 'RHEL platform is missing Azure root certificate (PHPLIB-619)',
+    ];
+
     public function setUp(): void
     {
         parent::setUp();
@@ -74,6 +81,10 @@ public static function assertCommandMatches(stdClass $expected, stdClass $actual
      */
     public function testClientSideEncryption(stdClass $test, ?array $runOn = null, array $data, ?array $keyVaultData = null, $jsonSchema = null, ?string $databaseName = null, ?string $collectionName = null): void
     {
+        if (isset(self::$incompleteTests[$this->dataDescription()])) {
+            $this->markTestIncomplete(self::$incompleteTests[$this->dataDescription()]);
+        }
+
         if (isset($runOn)) {
             $this->checkServerRequirements($runOn);
         }

tests/SpecTests/FunctionalTestCase.php

@@ -29,6 +29,7 @@ class FunctionalTestCase extends BaseFunctionalTestCase
     public const TOPOLOGY_SINGLE = 'single';
     public const TOPOLOGY_REPLICASET = 'replicaset';
     public const TOPOLOGY_SHARDED = 'sharded';
+    public const TOPOLOGY_LOAD_BALANCED = 'load-balanced';
 
     public const SERVERLESS_ALLOW = 'allow';
     public const SERVERLESS_FORBID = 'forbid';
@@ -265,6 +266,7 @@ private function getTopology(): string
             Server::TYPE_STANDALONE => self::TOPOLOGY_SINGLE,
             Server::TYPE_RS_PRIMARY => self::TOPOLOGY_REPLICASET,
             Server::TYPE_MONGOS => self::TOPOLOGY_SHARDED,
+            Server::TYPE_LOAD_BALANCER => self::TOPOLOGY_LOAD_BALANCED,
         ];
 
         $primaryType = $this->getPrimaryServer()->getType();
@@ -273,7 +275,7 @@ private function getTopology(): string
             return $topologyTypeMap[$primaryType];
         }
 
-        throw new UnexpectedValueException(sprintf('Cannot find topology for primary of type "%s".', $primaryType));
+        throw new UnexpectedValueException(sprintf('Cannot find topology for primary of type "%d".', $primaryType));
     }
 
     private function isServerlessRequirementSatisfied(?string $serverlessMode): bool

tests/SpecTests/RetryableWritesSpecTest.php

@@ -30,7 +30,7 @@ public function testRetryableWrites(stdClass $test, ?array $runOn = null, array
             $this->markTestSkipped('Transaction numbers are only allowed on a replica set member or mongos (PHPC-1415)');
         }
 
-        $useMultipleMongoses = isset($test->useMultipleMongoses) && $test->useMultipleMongoses && $this->isShardedCluster();
+        $useMultipleMongoses = isset($test->useMultipleMongoses) && $test->useMultipleMongoses && $this->isMongos();
 
         if (isset($runOn)) {
             $this->checkServerRequirements($runOn);

tests/SpecTests/TransactionsSpecTest.php

@@ -152,7 +152,7 @@ private function runTransactionTest(stdClass $test, ?array $runOn = null, array
             $this->markTestIncomplete(self::$incompleteTests[$this->dataDescription()]);
         }
 
-        $useMultipleMongoses = isset($test->useMultipleMongoses) && $test->useMultipleMongoses && $this->isShardedCluster();
+        $useMultipleMongoses = isset($test->useMultipleMongoses) && $test->useMultipleMongoses && $this->isMongos();
 
         if (isset($runOn)) {
             $this->checkServerRequirements($runOn);
@@ -229,8 +229,10 @@ private function provideTests(string $dir): array
      */
     public function testStartingNewTransactionOnPinnedSessionUnpinsSession(): void
     {
-        if (! $this->isShardedClusterUsingReplicasets()) {
-            $this->markTestSkipped('Mongos pinning tests can only run on sharded clusters using replica sets');
+        $this->skipIfTransactionsAreNotSupported();
+
+        if (! $this->isMongos()) {
+            $this->markTestSkipped('Pinning tests require mongos');
         }
 
         $client = self::createTestClient($this->getUri(true));
@@ -267,8 +269,10 @@ public function testStartingNewTransactionOnPinnedSessionUnpinsSession(): void
      */
     public function testRunningNonTransactionOperationOnPinnedSessionUnpinsSession(): void
     {
-        if (! $this->isShardedClusterUsingReplicasets()) {
-            $this->markTestSkipped('Mongos pinning tests can only run on sharded clusters using replica sets');
+        $this->skipIfTransactionsAreNotSupported();
+
+        if (! $this->isMongos()) {
+            $this->markTestSkipped('Pinning tests require mongos');
         }
 
         $client = self::createTestClient($this->getUri(true));