Remove most field name validation (#725)

The driver no longer restricts field names containing ".".
The driver no longer restricts field names starting with "$", except for
top-level fields in documents being saved via replaceOne, findOneAndReplace,
and ReplaceOneModel in bulkWrite.  This is to ensure that applications don't
accidentally use replace when they meant to use update.

JAVA-3996

Author: jyemin

bson/src/test/resources/bson/document.json

@@ -17,6 +17,26 @@
             "description": "Single-character key subdoc",
             "canonical_bson": "160000000378000E0000000261000200000062000000",
             "canonical_extjson": "{\"x\" : {\"a\" : \"b\"}}"
+        },
+        {
+            "description": "Dollar-prefixed key in sub-document",
+            "canonical_bson": "170000000378000F000000022461000200000062000000",
+            "canonical_extjson": "{\"x\" : {\"$a\" : \"b\"}}"
+        },
+        {
+            "description": "Dollar as key in sub-document",
+            "canonical_bson": "160000000378000E0000000224000200000061000000",
+            "canonical_extjson": "{\"x\" : {\"$\" : \"a\"}}"
+        },
+        {
+            "description": "Dotted key in sub-document",
+            "canonical_bson": "180000000378001000000002612E62000200000063000000",
+            "canonical_extjson": "{\"x\" : {\"a.b\" : \"c\"}}"
+        },
+        {
+            "description": "Dot as key in sub-document",
+            "canonical_bson": "160000000378000E000000022E000200000061000000",
+            "canonical_extjson": "{\"x\" : {\".\" : \"a\"}}"
         }
     ],
     "decodeErrors": [

bson/src/test/resources/bson/top.json

@@ -3,9 +3,24 @@
     "bson_type": "0x00",
     "valid": [
         {
-            "description": "Document with keys that start with $",
+            "description": "Dollar-prefixed key in top-level document",
             "canonical_bson": "0F00000010246B6579002A00000000",
             "canonical_extjson": "{\"$key\": {\"$numberInt\": \"42\"}}"
+        },
+        {
+            "description": "Dollar as key in top-level document",
+            "canonical_bson": "0E00000002240002000000610000",
+            "canonical_extjson": "{\"$\": \"a\"}"
+        },
+        {
+            "description": "Dotted key in top-level document",
+            "canonical_bson": "1000000002612E620002000000630000",
+            "canonical_extjson": "{\"a.b\": \"c\"}"
+        },
+        {
+            "description": "Dot as key in top-level document",
+            "canonical_bson": "0E000000022E0002000000610000",
+            "canonical_extjson": "{\".\": \"a\"}"
         }
     ],
     "decodeErrors": [
@@ -69,11 +84,11 @@
     "parseErrors": [
         {
             "description" : "Bad $regularExpression (extra field)",
-            "string" : "{\"a\" : \"$regularExpression\": {\"pattern\": \"abc\", \"options\": \"\", \"unrelated\": true}}}"
+            "string" : "{\"a\" : {\"$regularExpression\": {\"pattern\": \"abc\", \"options\": \"\", \"unrelated\": true}}}"
         },
         {
             "description" : "Bad $regularExpression (missing options field)",
-            "string" : "{\"a\" : \"$regularExpression\": {\"pattern\": \"abc\"}}}"
+            "string" : "{\"a\" : {\"$regularExpression\": {\"pattern\": \"abc\"}}}"
         },
         {
             "description": "Bad $regularExpression (pattern is number, not string)",
@@ -85,7 +100,7 @@
         },
         {
             "description" : "Bad $regularExpression (missing pattern field)",
-            "string" : "{\"a\" : \"$regularExpression\": {\"options\":\"ix\"}}}"
+            "string" : "{\"a\" : {\"$regularExpression\": {\"options\":\"ix\"}}}"
         },
         {
             "description": "Bad $oid (number, not string)",

docs/reference/content/whats-new.md

@@ -15,6 +15,15 @@ New features of the 4.3 Java driver release include:
 
 * Added support for the MongoDB Versioned API.  See the 
   [`ServerApi`]({{< apiref "mongodb-driver-core" "com/mongodb/ServerApi.html" >}}) API documentation for details.
+* Removed most restrictions on allowed characters in field names of documents being inserted or replaced.  This is a behavioral change 
+  for any application that is relying on client-side enforcement of these restrictions. In particular: 
+  * Restrictions on field names containing the "." character have been removed. This affects all insert and replace operations.
+  * Restrictions on field names starting with the "$" character have been removed for all insert operations.
+  * Restrictions in nested documents on field names starting with the "$" character have been removed for all replace operations.  
+  * Restrictions in the top-level document on field names starting with the "$" character remain for all replace operations. This is 
+    primarily to prevent accidental use of a replace operation when the intention was to use an update operation.
+  * Note that unacknowledged writes using dollar-prefixed or dotted keys may be silently rejected by pre-5.0 servers, where some 
+    restrictions on field names are still enforced in the server.
 
 # What's new in 4.2
 

driver-core/src/main/com/mongodb/internal/connection/InsertMessage.java

@@ -17,7 +17,7 @@
 package com.mongodb.internal.connection;
 
 import com.mongodb.internal.bulk.InsertRequest;
-import com.mongodb.internal.validator.CollectibleDocumentFieldNameValidator;
+import com.mongodb.internal.validator.NoOpFieldNameValidator;
 import org.bson.io.BsonOutput;
 
 /**
@@ -38,7 +38,7 @@ class InsertMessage extends LegacyMessage {
     protected EncodingMetadata encodeMessageBodyWithMetadata(final BsonOutput outputStream) {
         writeInsertPrologue(outputStream);
         int firstDocumentPosition = outputStream.getPosition();
-        addCollectibleDocument(insertRequest.getDocument(), outputStream, new CollectibleDocumentFieldNameValidator());
+        addCollectibleDocument(insertRequest.getDocument(), outputStream, new NoOpFieldNameValidator());
         return new EncodingMetadata(firstDocumentPosition);
     }
 

driver-core/src/main/com/mongodb/internal/connection/UpdateMessage.java

@@ -17,7 +17,7 @@
 package com.mongodb.internal.connection;
 
 import com.mongodb.internal.bulk.UpdateRequest;
-import com.mongodb.internal.validator.CollectibleDocumentFieldNameValidator;
+import com.mongodb.internal.validator.ReplacingDocumentFieldNameValidator;
 import com.mongodb.internal.validator.NoOpFieldNameValidator;
 import com.mongodb.internal.validator.UpdateFieldNameValidator;
 import org.bson.BsonValue;
@@ -56,7 +56,7 @@ protected EncodingMetadata encodeMessageBodyWithMetadata(final BsonOutput bsonOu
 
         addDocument(updateRequest.getFilter(), bsonOutput, new NoOpFieldNameValidator());
         if (updateRequest.getType() == REPLACE && updateRequest.getUpdateValue().isDocument()) {
-            addDocument(updateRequest.getUpdateValue().asDocument(), bsonOutput, new CollectibleDocumentFieldNameValidator());
+            addDocument(updateRequest.getUpdateValue().asDocument(), bsonOutput, new ReplacingDocumentFieldNameValidator());
         } else {
             int bufferPosition = bsonOutput.getPosition();
             BsonValue update = updateRequest.getUpdateValue();

driver-core/src/main/com/mongodb/internal/operation/BulkWriteBatch.java

@@ -36,9 +36,9 @@
 import com.mongodb.internal.connection.IndexMap;
 import com.mongodb.internal.connection.SplittablePayload;
 import com.mongodb.internal.session.SessionContext;
-import com.mongodb.internal.validator.CollectibleDocumentFieldNameValidator;
 import com.mongodb.internal.validator.MappedFieldNameValidator;
 import com.mongodb.internal.validator.NoOpFieldNameValidator;
+import com.mongodb.internal.validator.ReplacingDocumentFieldNameValidator;
 import com.mongodb.internal.validator.UpdateFieldNameValidator;
 import org.bson.BsonArray;
 import org.bson.BsonBoolean;
@@ -260,12 +260,10 @@ public BulkWriteBatch getNextBatch() {
     }
 
     public FieldNameValidator getFieldNameValidator() {
-        if (batchType == INSERT) {
-            return new CollectibleDocumentFieldNameValidator();
-        } else if (batchType == UPDATE || batchType == REPLACE) {
+        if (batchType == UPDATE || batchType == REPLACE) {
             Map<String, FieldNameValidator> rootMap = new HashMap<String, FieldNameValidator>();
             if (batchType == WriteRequest.Type.REPLACE) {
-                rootMap.put("u", new CollectibleDocumentFieldNameValidator());
+                rootMap.put("u", new ReplacingDocumentFieldNameValidator());
             } else {
                 rootMap.put("u", new UpdateFieldNameValidator());
             }

driver-core/src/main/com/mongodb/internal/operation/FindAndReplaceOperation.java

@@ -21,10 +21,10 @@
 import com.mongodb.client.model.Collation;
 import com.mongodb.connection.ConnectionDescription;
 import com.mongodb.connection.ServerDescription;
-import com.mongodb.internal.validator.CollectibleDocumentFieldNameValidator;
 import com.mongodb.internal.validator.MappedFieldNameValidator;
 import com.mongodb.internal.validator.NoOpFieldNameValidator;
 import com.mongodb.internal.session.SessionContext;
+import com.mongodb.internal.validator.ReplacingDocumentFieldNameValidator;
 import com.mongodb.lang.Nullable;
 import org.bson.BsonBoolean;
 import org.bson.BsonDocument;
@@ -392,7 +392,7 @@ private BsonDocument createCommand(final SessionContext sessionContext, final Se
     @Override
     protected FieldNameValidator getFieldNameValidator() {
         Map<String, FieldNameValidator> map = new HashMap<String, FieldNameValidator>();
-        map.put("update", new CollectibleDocumentFieldNameValidator());
+        map.put("update", new ReplacingDocumentFieldNameValidator());
         return new MappedFieldNameValidator(new NoOpFieldNameValidator(), map);
     }
 }

driver-core/src/main/com/mongodb/internal/validator/CollectibleDocumentFieldNameValidator.java renamed to driver-core/src/main/com/mongodb/internal/validator/ReplacingDocumentFieldNameValidator.java

@@ -22,12 +22,13 @@
 import java.util.List;
 
 /**
- * A field name validator for document that are meant for storage in MongoDB collections.  It ensures that no fields contain a '.',
- * or start with '$' (with the exception of "$db", "$ref", and "$id", so that DBRefs are not rejected).
+ * A field name validator for documents that are meant for storage in MongoDB collections via replace operations. It ensures that no
+ * top-level fields start with '$' (with the exception of "$db", "$ref", and "$id", so that DBRefs are not rejected).
  *
  * <p>This class should not be considered a part of the public API.</p>
  */
-public class CollectibleDocumentFieldNameValidator implements FieldNameValidator {
+public class ReplacingDocumentFieldNameValidator implements FieldNameValidator {
+    private static final NoOpFieldNameValidator NO_OP_FIELD_NAME_VALIDATOR = new NoOpFieldNameValidator();
     // Have to support DBRef fields
     private static final List<String> EXCEPTIONS = Arrays.asList("$db", "$ref", "$id");
 
@@ -37,18 +38,12 @@ public boolean validate(final String fieldName) {
             throw new IllegalArgumentException("Field name can not be null");
         }
 
-        if (fieldName.contains(".")) {
-            return false;
-        }
-
-        if (fieldName.startsWith("$") && !EXCEPTIONS.contains(fieldName)) {
-            return false;
-        }
-        return true;
+        return !fieldName.startsWith("$") || EXCEPTIONS.contains(fieldName);
     }
 
     @Override
     public FieldNameValidator getValidatorForField(final String fieldName) {
-        return this;
+        // Only top-level fields are validated
+        return NO_OP_FIELD_NAME_VALIDATOR;
     }
 }

driver-core/src/test/functional/com/mongodb/internal/connection/WriteProtocolCommandEventSpecification.groovy

@@ -201,8 +201,8 @@ class WriteProtocolCommandEventSpecification extends OperationFunctionalSpecific
 
     def 'should not deliver any events if encoding fails'() {
         given:
-        def insertRequest = new InsertRequest(new BsonDocument('$set', new BsonInt32(1)))
-        def protocol = new InsertProtocol(getNamespace(), true, insertRequest)
+        def updateRequest = new UpdateRequest(new BsonDocument(), new BsonDocument('$set', new BsonInt32(1)), REPLACE)
+        def protocol = new UpdateProtocol(getNamespace(), true, updateRequest)
         def commandListener = new TestCommandListener()
         protocol.commandListener = commandListener
 

driver-core/src/test/resources/transactions/errors-client.json

@@ -25,14 +25,15 @@
           "object": "session0"
         },
         {
-          "name": "insertOne",
+          "name": "updateOne",
           "object": "collection",
           "arguments": {
             "session": "session0",
-            "document": {
-              "_id": {
-                ".": "."
-              }
+            "filter": {
+              "_id": 1
+            },
+            "update": {
+              "x": 1
             }
           },
           "error": true
@@ -60,22 +61,23 @@
           "arguments": {
             "session": "session0",
             "document": {
-              "_id": 4
+              "_id": 1
             }
           },
           "result": {
-            "insertedId": 4
+            "insertedId": 1
           }
         },
         {
-          "name": "insertOne",
+          "name": "updateOne",
           "object": "collection",
           "arguments": {
             "session": "session0",
-            "document": {
-              "_id": {
-                ".": "."
-              }
+            "filter": {
+              "_id": 1
+            },
+            "update": {
+              "x": 1
             }
           },
           "error": true