Merge branch 'master' into PYTHON-4069-1

Author: Jibola

.evergreen/config.yml

@@ -284,14 +284,67 @@ functions:
            PYTHON_BINARY=${PYTHON_BINARY} bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh doctest:test
 
   "run tests":
-    - command: subprocess.exec
+    - command: shell.exec
       type: test
       params:
         include_expansions_in_env:  ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         working_dir: "src"
-        binary: bash
-        args:
-          - .evergreen/scripts/run-tests.sh
+        shell: bash
+        script: |
+          # Disable xtrace
+          set +x
+          . .evergreen/scripts/env.sh
+          if [ -n "${MONGODB_STARTED}" ]; then
+            export PYMONGO_MUST_CONNECT=true
+          fi
+          if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
+            export PYMONGO_DISABLE_TEST_COMMANDS=1
+          fi
+          if [ -n "${test_encryption}" ]; then
+            # Disable xtrace (just in case it was accidentally set).
+            set +x
+            bash ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh
+            export TEST_ENCRYPTION=1
+            if [ -n "${test_encryption_pyopenssl}" ]; then
+              export TEST_ENCRYPTION_PYOPENSSL=1
+            fi
+          fi
+          if [ -n "${test_crypt_shared}" ]; then
+            export TEST_CRYPT_SHARED=1
+            export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
+          fi
+          if [ -n "${test_pyopenssl}" ]; then
+            export TEST_PYOPENSSL=1
+          fi
+          if [ -n "${SETDEFAULTENCODING}" ]; then
+            export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
+          fi
+          if [ -n "${test_loadbalancer}" ]; then
+            export TEST_LOADBALANCER=1
+            export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
+            export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
+          fi
+          if [ -n "${test_serverless}" ]; then
+            export TEST_SERVERLESS=1
+          fi
+          if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then
+            export TEST_INDEX_MANAGEMENT=1
+          fi
+          if [ -n "${SKIP_CSOT_TESTS}" ]; then
+            export SKIP_CSOT_TESTS=1
+          fi
+
+          GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
+            PYTHON_BINARY=${PYTHON_BINARY} \
+            NO_EXT=${NO_EXT} \
+            COVERAGE=${COVERAGE} \
+            COMPRESSORS=${COMPRESSORS} \
+            AUTH=${AUTH} \
+            SSL=${SSL} \
+            TEST_DATA_LAKE=${TEST_DATA_LAKE} \
+            MONGODB_API_VERSION=${MONGODB_API_VERSION} \
+            SKIP_HATCH=${SKIP_HATCH} \
+            bash ${PROJECT_DIRECTORY}/.evergreen/hatch.sh test:test-eg
 
   "run enterprise auth tests":
     - command: shell.exec
@@ -430,7 +483,9 @@ functions:
         working_dir: "src"
         script: |
           . .evergreen/scripts/env.sh
-          . .evergreen/hatch.sh encryption:teardown
+          if [ -f $DRIVERS_TOOLS/.evergreen/csfle/secrets-export.sh ]; then
+            . .evergreen/hatch.sh encryption:teardown
+          fi
           rm -rf ${DRIVERS_TOOLS} || true
           rm -f ./secrets-export.sh || true
 
@@ -1931,10 +1986,14 @@ axes:
         display_name: "RHEL 8.3 (zSeries)"
         run_on: rhel83-zseries-small
         batchtime: 10080  # 7 days
+        variables:
+          SKIP_HATCH: true
       - id: rhel81-power8
         display_name: "RHEL 8.1 (POWER8)"
         run_on: rhel81-power8-small
         batchtime: 10080  # 7 days
+        variables:
+          SKIP_HATCH: true
       - id: rhel82-arm64
         display_name: "RHEL 8.2 (ARM64)"
         run_on: rhel82-arm64-small

.evergreen/hatch.sh

@@ -8,7 +8,17 @@ if [ -z "$PYTHON_BINARY" ]; then
     PYTHON_BINARY=$(find_python3)
 fi
 
-if $PYTHON_BINARY -m hatch --version; then
+# Check if we should skip hatch and run the tests directly.
+if [ -n "$SKIP_HATCH" ]; then
+    ENV_NAME=testenv-$RANDOM
+    createvirtualenv "$PYTHON_BINARY" $ENV_NAME
+    # shellcheck disable=SC2064
+    trap "deactivate; rm -rf $ENV_NAME" EXIT HUP
+    python -m pip install -e ".[test]"
+    run_hatch() {
+      bash ./.evergreen/run-tests.sh
+    }
+elif $PYTHON_BINARY -m hatch --version; then
     run_hatch() {
       $PYTHON_BINARY -m hatch run "$@"
     }

bson/_cbsonmodule.c

@@ -383,10 +383,11 @@ static int millis_from_datetime_ms(PyObject* dt, long long* out){
 static PyObject* decode_datetime(PyObject* self, long long millis, const codec_options_t* options){
     PyObject* naive = NULL;
     PyObject* replace = NULL;
-    PyObject* args = NULL;
-    PyObject* kwargs = NULL;
     PyObject* value = NULL;
     struct module_state *state = GETSTATE(self);
+    if (!state) {
+        goto invalid;
+    }
     if (options->datetime_conversion == DATETIME_MS){
         return datetime_ms_from_millis(self, millis);
     }
@@ -414,8 +415,8 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
                     Py_DECREF(utcoffset);
                     return 0;
                 }
-                min_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * 86400 +
-                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * 1000 +
+                min_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 +
+                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 +
                                      (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000);
             }
             Py_DECREF(utcoffset);
@@ -433,8 +434,8 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
                     Py_DECREF(utcoffset);
                     return 0;
                 }
-                max_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * 86400 +
-                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * 1000 +
+                max_millis_offset = (PyDateTime_DELTA_GET_DAYS(utcoffset) * (int64_t)86400 +
+                                     PyDateTime_DELTA_GET_SECONDS(utcoffset)) * (int64_t)1000 +
                                      (PyDateTime_DELTA_GET_MICROSECONDS(utcoffset) / 1000);
             }
             Py_DECREF(utcoffset);
@@ -487,8 +488,6 @@ static PyObject* decode_datetime(PyObject* self, long long millis, const codec_o
 invalid:
     Py_XDECREF(naive);
     Py_XDECREF(replace);
-    Py_XDECREF(args);
-    Py_XDECREF(kwargs);
     return value;
 }
 

test/asynchronous/test_client.py

@@ -617,7 +617,7 @@ def test_detected_environment_logging(self, mock_get_hosts):
                 mock_get_hosts.return_value = [(host, 1)]
                 AsyncMongoClient(host)
             AsyncMongoClient(multi_host)
-            logs = [record.message for record in cm.records if record.name == "pymongo.client"]
+            logs = [record.getMessage() for record in cm.records if record.name == "pymongo.client"]
             self.assertEqual(len(logs), 7)
 
     @patch("pymongo.srv_resolver._SrvResolver.get_hosts")

test/asynchronous/test_logger.py

@@ -37,15 +37,15 @@ async def test_default_truncation_limit(self):
             with self.assertLogs("pymongo.command", level="DEBUG") as cm:
                 await db.test.insert_many(docs)
 
-                cmd_started_log = json_util.loads(cm.records[0].message)
+                cmd_started_log = json_util.loads(cm.records[0].getMessage())
                 self.assertEqual(len(cmd_started_log["command"]), _DEFAULT_DOCUMENT_LENGTH + 3)
 
-                cmd_succeeded_log = json_util.loads(cm.records[1].message)
+                cmd_succeeded_log = json_util.loads(cm.records[1].getMessage())
                 self.assertLessEqual(len(cmd_succeeded_log["reply"]), _DEFAULT_DOCUMENT_LENGTH + 3)
 
             with self.assertLogs("pymongo.command", level="DEBUG") as cm:
                 await db.test.find({}).to_list()
-                cmd_succeeded_log = json_util.loads(cm.records[1].message)
+                cmd_succeeded_log = json_util.loads(cm.records[1].getMessage())
                 self.assertEqual(len(cmd_succeeded_log["reply"]), _DEFAULT_DOCUMENT_LENGTH + 3)
 
     async def test_configured_truncation_limit(self):
@@ -55,14 +55,14 @@ async def test_configured_truncation_limit(self):
             with self.assertLogs("pymongo.command", level="DEBUG") as cm:
                 await db.command(cmd)
 
-                cmd_started_log = json_util.loads(cm.records[0].message)
+                cmd_started_log = json_util.loads(cm.records[0].getMessage())
                 self.assertEqual(len(cmd_started_log["command"]), 5 + 3)
 
-                cmd_succeeded_log = json_util.loads(cm.records[1].message)
+                cmd_succeeded_log = json_util.loads(cm.records[1].getMessage())
                 self.assertLessEqual(len(cmd_succeeded_log["reply"]), 5 + 3)
                 with self.assertRaises(OperationFailure):
                     await db.command({"notARealCommand": True})
-                cmd_failed_log = json_util.loads(cm.records[-1].message)
+                cmd_failed_log = json_util.loads(cm.records[-1].getMessage())
                 self.assertEqual(len(cmd_failed_log["failure"]), 5 + 3)
 
     async def test_truncation_multi_byte_codepoints(self):
@@ -78,7 +78,7 @@ async def test_truncation_multi_byte_codepoints(self):
             with patch.dict("os.environ", {"MONGOB_LOG_MAX_DOCUMENT_LENGTH": length}):
                 with self.assertLogs("pymongo.command", level="DEBUG") as cm:
                     await self.db.test.insert_one({"x": multi_byte_char_str})
-                    cmd_started_log = json_util.loads(cm.records[0].message)["command"]
+                    cmd_started_log = json_util.loads(cm.records[0].getMessage())["command"]
 
                     cmd_started_log = cmd_started_log[:-3]
                     last_3_bytes = cmd_started_log.encode()[-3:].decode()