REP-6129 Accommodate type bracketing for pre-v5 find queries (#118)

Pre-v5 server versions don’t recognize indexes in $expr-using partition queries. As a result, such queries entail a collection scan. To prevent the performance problems that that would create, migration-verifier avoids $expr when querying partitions against these server versions. Thus, migration-verifier’s partition queries have been type-bracketed; i.e., they only return results for types in the query itself.

For example, consider a query for all documents with `_id` between MinKey and ObjectID(...). An $expr query will return any document with an `_id` that sorts between those two values. A non-$expr query, though, will only return documents with an `_id` of one of those 2 types, despite the fact that many types—including strings and all numeric types—sort between MinKey and ObjectID.

As a result of its non-$expr queries, migration-verifier has been quietly skipping documents when querying pre-v5 servers. In migrations to v5+ it’s a minor concern because whatever documents migration-verifier skips on the source will be recorded as “missing” in generation 0 but will “magically” appear in generation 1. (The only problem is when large numbers of “missing” documents happen, which caused REP-6088.)

In migrations to pre-v5 versions, though, migration-verifier will simply skip documents.

This changeset fixes this problem in pre-v5 partition queries by:
- determining the partition bounds’ BSON types
- determining the BSON types that type bracketing will exclude
- adding an $or clause that includes documents with `_id`s of those types

The above example, then, becomes:
```
(_id >= MinKey OR type(_id) > MinKey)
AND
(_id <= ObjectID(...) OR type(_id) < ObjectID)
```

NB: mongosync solves this problem differently. It uses $expr but adds hint, min, and max to the query. This properly constrains the query, but it excludes the partition’s upper bound. To compensate for that, monosync adds a separate, single-value partition to check the partition’s upper bound. This changeset uses the above-described approach instead because it seems simpler and less disruptive.

Author: FGasper

internal/partitions/bson.go

@@ -0,0 +1,133 @@
+package partitions
+
+import (
+	"slices"
+
+	"github.com/10gen/migration-verifier/mslices"
+	"github.com/pkg/errors"
+	"github.com/samber/lo"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+)
+
+// -------------------------------------------------------------------------
+// The sort order defined here derives from:
+//
+// https://www.mongodb.com/docs/manual/reference/bson-type-comparison-order/
+//
+// Everything there was verified empirically as part of writing this code.
+// Note that, at least as of this writing, that page erroneously omits
+// JS Code with Scope in the sort order. The observed behavior in MongoDB 4.4
+// is that that type sorts between JavaScript Code and MaxKey.
+// ------------------------------------------------------------------------
+
+var numericTypes = mslices.Of(
+	bson.TypeInt32,
+	bson.TypeInt64,
+	bson.TypeDouble,
+	bson.TypeDecimal128,
+)
+
+var stringTypes = mslices.Of(
+	bson.TypeString,
+	bson.TypeSymbol,
+)
+
+// NB: The server forbids arrays, undefined, and regexes as _id values.
+// That simplifies the following greatly because all of those behave
+// weirdly at various times:
+// - 0-element arrays sort as null.
+// - 1-element arrays sort as their contained element.
+// - undefined & null sometimes match strangely. (This changed in 8.0.)
+// - simple matches against regex actually run the regex.
+var bsonTypeSortOrder = lo.Flatten(mslices.Of(
+	mslices.Of(
+		bson.TypeMinKey,
+		bson.TypeNull,
+	),
+	numericTypes,
+	stringTypes,
+	mslices.Of(
+		bson.TypeEmbeddedDocument,
+		bson.TypeBinary,
+		bson.TypeObjectID,
+		bson.TypeBoolean,
+		bson.TypeDateTime,
+		bson.TypeTimestamp,
+		bson.TypeDBPointer,
+		bson.TypeJavaScript,
+		bson.TypeCodeWithScope,
+		bson.TypeMaxKey,
+	),
+))
+
+var bsonTypeString = map[bsontype.Type]string{
+	bson.TypeMinKey:           "minKey",
+	bson.TypeNull:             "null",
+	bson.TypeBoolean:          "bool",
+	bson.TypeInt32:            "int",
+	bson.TypeInt64:            "long",
+	bson.TypeDouble:           "double",
+	bson.TypeDecimal128:       "decimal",
+	bson.TypeString:           "string",
+	bson.TypeSymbol:           "symbol",
+	bson.TypeObjectID:         "objectId",
+	bson.TypeDateTime:         "date",
+	bson.TypeTimestamp:        "timestamp",
+	bson.TypeJavaScript:       "javascript",
+	bson.TypeCodeWithScope:    "javascriptWithScope",
+	bson.TypeEmbeddedDocument: "object",
+	bson.TypeBinary:           "binData",
+	bson.TypeDBPointer:        "dbPointer",
+	bson.TypeMaxKey:           "maxKey",
+}
+
+// This returns BSON types that the server’s type bracketing excludes from
+// query results when matching against the given value.
+//
+// The returned slices are types before & after, respectively. They are
+// strings rather than bsontype.Type to facilitate easy insertion into queries.
+//
+// This is kind of like strings.Cut() but against the sort-ordered list of BSON
+// types, except that if the given value is a number or string-like, then other
+// “like” types will not be in the returned slices.
+func getTypeBracketExcludedBSONTypes(val any) ([]string, []string, error) {
+	bsonType, _, err := bson.MarshalValue(val)
+	if err != nil {
+		return nil, nil, errors.Wrapf(err, "marshaling min value (%v)", val)
+	}
+
+	curSortOrder := slices.Index(bsonTypeSortOrder, bsonType)
+	if curSortOrder < 0 {
+		return nil, nil, errors.Errorf("go value (%T: %v) marshaled to BSON %s, which is invalid", val, val, bsonType)
+	}
+
+	earlier := bsonTypeSortOrder[:curSortOrder]
+	later := bsonTypeSortOrder[1+curSortOrder:]
+
+	// If the given value is, e.g., an int, then we need to omit
+	// other numeric types from the returned slices. If we don’t, then we’re
+	// telling the caller to query on, e.g., [_id >= 123 OR type is double],
+	// which would match something like float64(12), which, of course, we
+	// don’t want.
+	//
+	// For the same reason, we need the same exclusion for string vs. symbol.
+	if slices.Contains(numericTypes, bsonType) {
+		earlier = lo.Without(earlier, numericTypes...)
+		later = lo.Without(later, numericTypes...)
+	} else if slices.Contains(stringTypes, bsonType) {
+		earlier = lo.Without(earlier, stringTypes...)
+		later = lo.Without(later, stringTypes...)
+	}
+
+	return typesToStrings(earlier), typesToStrings(later), nil
+}
+
+func typesToStrings(in []bsontype.Type) []string {
+	return lo.Map(
+		in,
+		func(t bsontype.Type, _ int) string {
+			return bsonTypeString[t]
+		},
+	)
+}