diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/README.md b/test/spec/connection-monitoring-and-pooling/cmap-format/README.md new file mode 100644 index 00000000000..ced96961f5a --- /dev/null +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/README.md @@ -0,0 +1,167 @@ +# Connection Monitoring and Pooling (CMAP) Unit and Integration Tests + +______________________________________________________________________ + +## Introduction + +The YAML and JSON files in this directory are platform-independent tests that drivers can use to prove their conformance +to the Connection Monitoring and Pooling (CMAP) Spec. + +## Common Test Format + +Each YAML file has the following keys: + +- `version`: A version number indicating the expected format of the spec tests (current version = 1) +- `style`: A string indicating what style of tests this file contains. Contains one of the following: + - `"unit"`: a test that may be run without connecting to a MongoDB deployment. + - `"integration"`: a test that MUST be run against a real MongoDB deployment. +- `description`: A text description of what the test is meant to assert + +## Unit Test Format: + +All Unit Tests have some of the following fields: + +- `poolOptions`: If present, connection pool options to use when creating a pool; both + [standard ConnectionPoolOptions](../../connection-monitoring-and-pooling.md#connection-pool-options) and the + following test-specific options are allowed: + - `backgroundThreadIntervalMS`: A time interval between the end of a + [Background Thread Run](../../connection-monitoring-and-pooling.md#background-thread) and the beginning of the + next Run. If a Connection Pool does not implement a Background Thread, the Test Runner MUST ignore the option. If + the option is not specified, an implementation is free to use any value it finds reasonable. + + Possible values (0 is not allowed): + + - A negative value: never begin a Run. + - A positive value: the interval between Runs in milliseconds. +- `operations`: A list of operations to perform. All operations support the following fields: + - `name`: A string describing which operation to issue. + - `thread`: The name of the thread in which to run this operation. If not specified, runs in the default thread +- `error`: Indicates that the main thread is expected to error during this test. An error may include of the following + fields: + - `type`: the type of error emitted + - `message`: the message associated with that error + - `address`: Address of pool emitting error +- `events`: An array of all connection monitoring events expected to occur while running `operations`. An event may + contain any of the following fields + - `type`: The type of event emitted + - `address`: The address of the pool emitting the event + - `connectionId`: The id of a connection associated with the event + - `duration`: The event duration + - `options`: Options used to create the pool + - `reason`: A reason giving more information on why the event was emitted +- `ignore`: An array of event names to ignore + +Valid Unit Test Operations are the following: + +- `start(target)`: Starts a new thread named `target` + - `target`: The name of the new thread to start +- `wait(ms)`: Sleep the current thread for `ms` milliseconds + - `ms`: The number of milliseconds to sleep the current thread for +- `waitForThread(target)`: wait for thread `target` to finish executing. Propagate any errors to the main thread. + - `target`: The name of the thread to wait for. +- `waitForEvent(event, count, timeout)`: block the current thread until `event` has occurred `count` times + - `event`: The name of the event + - `count`: The number of times the event must occur (counting from the start of the test) + - `timeout`: If specified, time out with an error after waiting for this many milliseconds without seeing the required + events +- `label = pool.checkOut()`: call `checkOut` on pool, returning the checked out connection + - `label`: If specified, associate this label with the returned connection, so that it may be referenced in later + operations +- `pool.checkIn(connection)`: call `checkIn` on pool + - `connection`: A string label identifying which connection to check in. Should be a label that was previously set + with `checkOut` +- `pool.clear()`: call `clear` on Pool + - `interruptInUseConnections`: Determines whether "in use" connections should be also interrupted +- `pool.close()`: call `close` on Pool +- `pool.ready()`: call `ready` on Pool + +## Integration Test Format + +The integration test format is identical to the unit test format with the addition of the following fields to each test: + +- `runOn` (optional): An array of server version and/or topology requirements for which the tests can be run. If the + test environment satisfies one or more of these requirements, the tests may be executed; otherwise, this test should + be skipped. If this field is omitted, the tests can be assumed to have no particular requirements and should be + executed. Each element will have some or all of the following fields: + - `minServerVersion` (optional): The minimum server version (inclusive) required to successfully run the tests. If + this field is omitted, it should be assumed that there is no lower bound on the required server version. + - `maxServerVersion` (optional): The maximum server version (inclusive) against which the tests can be run + successfully. If this field is omitted, it should be assumed that there is no upper bound on the required server + version. +- `failPoint`: optional, a document containing a `configureFailPoint` command to run against the endpoint being used for + the test. +- `poolOptions.appName` (optional): appName attribute to be set in connections, which will be affected by the fail + point. + +## Spec Test Match Function + +The definition of MATCH or MATCHES in the Spec Test Runner is as follows: + +- MATCH takes two values, `expected` and `actual` +- Notation is "Assert `actual` MATCHES `expected`" +- Assertion passes if `expected` is a subset of `actual`, with the values `42` and `"42"` acting as placeholders for + "any value" + +Pseudocode implementation of `actual` MATCHES `expected`: + +```text +If expected is "42" or 42: + Assert that actual exists (is not null or undefined) +Else: + Assert that actual is of the same JSON type as expected + If expected is a JSON array: + For every idx/value in expected: + Assert that actual[idx] MATCHES value + Else if expected is a JSON object: + For every key/value in expected + Assert that actual[key] MATCHES value + Else: + Assert that expected equals actual +``` + +## Unit Test Runner: + +For the unit tests, the behavior of a Connection is irrelevant beyond the need to asserting `connection.id`. Drivers MAY +use a mock connection class for testing the pool behavior in unit tests + +For each YAML file with `style: unit`: + +- Create a Pool `pool`, subscribe and capture any Connection Monitoring events emitted in order. + - If `poolOptions` is specified, use those options to initialize both pools + - The returned pool must have an `address` set as a string value. +- Process each `operation` in `operations` (on the main thread) + - If a `thread` is specified, the main thread MUST schedule the operation to execute in the corresponding thread. + Otherwise, execute the operation directly in the main thread. +- If `error` is presented + - Assert that an actual error `actualError` was thrown by the main thread + - Assert that `actualError` MATCHES `error` +- Else: + - Assert that no errors were thrown by the main thread +- calculate `actualEvents` as every Connection Event emitted whose `type` is not in `ignore` +- if `events` is not empty, then for every `idx`/`expectedEvent` in `events` + - Assert that `actualEvents[idx]` exists + - Assert that `actualEvents[idx]` MATCHES `expectedEvent` + +It is important to note that the `ignore` list is used for calculating `actualEvents`, but is NOT used for the +`waitForEvent` command + +## Integration Test Runner + +The steps to run the integration tests are the same as those used to run the unit tests with the following +modifications: + +- The integration tests MUST be run against an actual endpoint. If the deployment being tested contains multiple + endpoints, then the runner MUST only use one of them to run the tests against. + +- For each test, if `failPoint` is specified, its value is a `configureFailPoint` command. Run the command on the admin + database of the endpoint being tested to enable the fail point. + +- At the end of each test, any enabled fail point MUST be disabled to avoid spurious failures in subsequent tests. The + fail point may be disabled like so: + + ```javascript + db.adminCommand({ + configureFailPoint: "", + mode: "off" + }); + ``` diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.json index 41c522ae672..3f37f188c0f 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.json @@ -22,7 +22,8 @@ { "type": "ConnectionCheckedOut", "connectionId": 1, - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionCheckedIn", @@ -32,7 +33,8 @@ { "type": "ConnectionCheckedOut", "connectionId": 1, - "address": 42 + "address": 42, + "duration": 42 } ], "ignore": [ diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.yml index 5179432788e..9dbd5aebe98 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkin-make-available.yml @@ -12,12 +12,14 @@ events: - type: ConnectionCheckedOut connectionId: 1 address: 42 + duration: 42 - type: ConnectionCheckedIn connectionId: 1 address: 42 - type: ConnectionCheckedOut connectionId: 1 address: 42 + duration: 42 ignore: - ConnectionPoolCreated - ConnectionPoolReady diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.json index d89b342605c..c7e8914d45d 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.json @@ -23,12 +23,14 @@ { "type": "ConnectionReady", "connectionId": 1, - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionCheckedOut", "connectionId": 1, - "address": 42 + "address": 42, + "duration": 42 } ], "ignore": [ diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.yml index bbbd03ff5cf..1d94778dd40 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-connection.yml @@ -13,9 +13,11 @@ events: - type: ConnectionReady connectionId: 1 address: 42 + duration: 42 - type: ConnectionCheckedOut connectionId: 1 address: 42 + duration: 42 ignore: - ConnectionPoolReady - ConnectionPoolCreated diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.json index ee2926e1c06..614403ef509 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.json @@ -38,7 +38,8 @@ { "type": "ConnectionCheckedOut", "address": 42, - "connectionId": 42 + "connectionId": 42, + "duration": 42 }, { "type": "ConnectionCheckedIn", @@ -56,6 +57,7 @@ { "type": "ConnectionCheckOutFailed", "address": 42, + "duration": 42, "reason": "poolClosed" } ], diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.yml index 4d1b0f3b292..2d0ce8d1111 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-error-closed.yml @@ -21,6 +21,7 @@ events: - type: ConnectionCheckedOut address: 42 connectionId: 42 + duration: 42 - type: ConnectionCheckedIn address: 42 connectionId: 42 @@ -30,6 +31,7 @@ events: address: 42 - type: ConnectionCheckOutFailed address: 42 + duration: 42 reason: poolClosed ignore: - ConnectionPoolReady diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.json index 84ddf8fdbad..4d9fda1a68f 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.json @@ -89,7 +89,8 @@ { "type": "ConnectionCheckOutFailed", "reason": "timeout", - "address": 42 + "address": 42, + "duration": 42 } ], "ignore": [ diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.yml index 383f666ad06..3c6fb5da21b 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-checkout-maxConnecting-timeout.yml @@ -60,6 +60,7 @@ events: - type: ConnectionCheckOutFailed reason: timeout address: 42 + duration: 42 ignore: - ConnectionCreated - ConnectionCheckedIn diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.json index d4aef928c7c..e6077f12a54 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.json @@ -59,7 +59,8 @@ }, { "type": "ConnectionCheckedOut", - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionCheckOutStarted", @@ -76,17 +77,20 @@ { "type": "ConnectionCheckOutFailed", "reason": "connectionError", - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionCheckOutFailed", "reason": "connectionError", - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionCheckOutFailed", "reason": "connectionError", - "address": 42 + "address": 42, + "duration": 42 } ], "ignore": [ diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.yml index 521f8ed2455..388056f4fd3 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-clears-waitqueue.yml @@ -38,6 +38,7 @@ events: address: 42 - type: ConnectionCheckedOut address: 42 + duration: 42 - type: ConnectionCheckOutStarted address: 42 - type: ConnectionCheckOutStarted @@ -47,12 +48,15 @@ events: - type: ConnectionCheckOutFailed reason: connectionError address: 42 + duration: 42 - type: ConnectionCheckOutFailed reason: connectionError address: 42 + duration: 42 - type: ConnectionCheckOutFailed reason: connectionError address: 42 + duration: 42 ignore: - ConnectionPoolReady - ConnectionPoolCleared diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.json index ceae07a1c76..c1fd7463294 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.json @@ -17,7 +17,7 @@ ], "closeConnection": false, "blockConnection": true, - "blockTimeMS": 1000 + "blockTimeMS": 10000 } }, "poolOptions": { diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.yml index 4726bca1fa4..ea0bbc7d4ef 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-interrupting-pending-connections.yml @@ -11,7 +11,7 @@ failPoint: failCommands: ["isMaster","hello"] closeConnection: false blockConnection: true - blockTimeMS: 1000 + blockTimeMS: 10000 poolOptions: minPoolSize: 0 operations: diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.json index 800c3545ad5..88c2988ac5c 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.json @@ -40,7 +40,8 @@ { "type": "ConnectionCheckedOut", "address": 42, - "connectionId": 42 + "connectionId": 42, + "duration": 42 }, { "type": "ConnectionPoolCleared", @@ -49,6 +50,7 @@ { "type": "ConnectionCheckOutFailed", "address": 42, + "duration": 42, "reason": "connectionError" }, { @@ -57,7 +59,8 @@ }, { "type": "ConnectionCheckedOut", - "address": 42 + "address": 42, + "duration": 42 } ], "ignore": [ diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.yml index c783d4d0965..93c85bfbef5 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-clear-ready.yml @@ -20,15 +20,18 @@ events: - type: ConnectionCheckedOut address: 42 connectionId: 42 + duration: 42 - type: ConnectionPoolCleared address: 42 - type: ConnectionCheckOutFailed address: 42 + duration: 42 reason: connectionError - type: ConnectionPoolReady address: 42 - type: ConnectionCheckedOut address: 42 + duration: 42 ignore: - ConnectionPoolCreated - ConnectionReady diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.json b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.json index 29ce7326cfe..a90aed04d04 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.json @@ -31,7 +31,8 @@ { "type": "ConnectionCheckOutFailed", "reason": "connectionError", - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionPoolReady", @@ -47,7 +48,8 @@ }, { "type": "ConnectionCheckedOut", - "address": 42 + "address": 42, + "duration": 42 } ], "ignore": [ diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.yml index 730d4d27b76..23320993933 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/pool-ready.yml @@ -17,6 +17,7 @@ events: - type: ConnectionCheckOutFailed reason: connectionError address: 42 + duration: 42 - type: ConnectionPoolReady address: 42 - type: ConnectionCheckOutStarted @@ -25,6 +26,7 @@ events: address: 42 - type: ConnectionCheckedOut address: 42 + duration: 42 ignore: - ConnectionPoolCreated - ConnectionReady diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.json b/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.json index fbcbdfb04d0..8bd7c494991 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.json +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.json @@ -48,7 +48,8 @@ { "type": "ConnectionCheckedOut", "connectionId": 42, - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionCheckOutStarted", @@ -57,7 +58,8 @@ { "type": "ConnectionCheckOutFailed", "reason": "timeout", - "address": 42 + "address": 42, + "duration": 42 }, { "type": "ConnectionCheckedIn", diff --git a/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.yml b/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.yml index 5433c148981..fdb3b586293 100644 --- a/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.yml +++ b/test/spec/connection-monitoring-and-pooling/cmap-format/wait-queue-timeout.yml @@ -32,11 +32,13 @@ events: - type: ConnectionCheckedOut connectionId: 42 address: 42 + duration: 42 - type: ConnectionCheckOutStarted address: 42 - type: ConnectionCheckOutFailed reason: timeout address: 42 + duration: 42 - type: ConnectionCheckedIn connectionId: 42 address: 42 diff --git a/test/tools/cmap_spec_runner.ts b/test/tools/cmap_spec_runner.ts index f6d7e68bedc..56fd5ba92c6 100644 --- a/test/tools/cmap_spec_runner.ts +++ b/test/tools/cmap_spec_runner.ts @@ -168,8 +168,14 @@ const compareInputToSpec = (input, expected, message) => { return; } + const expectedEntries: [string, unknown][] = Object.entries(expected).map(([k, v]) => { + // Node uses `durationMS` instead of `duration` on CMAP events. + if (k === 'duration') return ['durationMS', v]; + return [k, v]; + }); + if (expected && typeof expected === 'object') { - for (const [expectedPropName, expectedValue] of Object.entries(expected)) { + for (const [expectedPropName, expectedValue] of expectedEntries) { expect(input, message).to.have.property(expectedPropName); compareInputToSpec( input[expectedPropName],