Skip to content

Commit 1f763d0

Browse files
qingyang-huqingyang-hu
committed
GODRIVER-3421 Remove the BSON document size validation.
1 parent acca80b commit 1f763d0

File tree

6 files changed

+61
-57
lines changed

6 files changed

+61
-57
lines changed

‎internal/integration/collection_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ func TestCollection(t *testing.T) {
167167
mt.Run("large document batches", func(mt *mtest.T) {
168168
mt.Parallel()
169169

170-
docs := []interface{}{create16MBDocument(mt), create16MBDocument(mt)}
170+
docs := []interface{}{create16MBDocument(mt), create16MBDocument(mt), create16MBDocument(mt)}
171171
_, err := mt.Coll.InsertMany(context.Background(), docs)
172172
assert.Nil(mt, err, "InsertMany error: %v", err)
173173
evt := mt.GetStartedEvent()

‎mongo/client_bulk_write.go‎

Lines changed: 26 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,8 @@ type modelBatches struct {
191191
writeErrors map[int]WriteError
192192
}
193193

194+
var _ driver.OperationBatches = &modelBatches{}
195+
194196
func (mb *modelBatches) IsOrdered() *bool {
195197
return &mb.ordered
196198
}
@@ -209,7 +211,7 @@ func (mb *modelBatches) Size() int {
209211
return len(mb.models) - mb.offset
210212
}
211213

212-
func (mb *modelBatches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, totalSize int) (int, []byte, error) {
214+
func (mb *modelBatches) AppendBatchSequence(dst []byte, maxCount, totalSize int) (int, []byte, error) {
213215
fn := functionSet{
214216
appendStart: func(dst []byte, identifier string) (int32, []byte) {
215217
var idx int32
@@ -228,10 +230,10 @@ func (mb *modelBatches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, to
228230
return dst
229231
},
230232
}
231-
return mb.appendBatches(fn, dst, maxCount, maxDocSize, totalSize)
233+
return mb.appendBatches(fn, dst, maxCount, totalSize)
232234
}
233235

234-
func (mb *modelBatches) AppendBatchArray(dst []byte, maxCount, maxDocSize, totalSize int) (int, []byte, error) {
236+
func (mb *modelBatches) AppendBatchArray(dst []byte, maxCount, totalSize int) (int, []byte, error) {
235237
fn := functionSet{
236238
appendStart: bsoncore.AppendArrayElementStart,
237239
appendDocument: bsoncore.AppendDocumentElement,
@@ -240,7 +242,7 @@ func (mb *modelBatches) AppendBatchArray(dst []byte, maxCount, maxDocSize, total
240242
return dst
241243
},
242244
}
243-
return mb.appendBatches(fn, dst, maxCount, maxDocSize, totalSize)
245+
return mb.appendBatches(fn, dst, maxCount, totalSize)
244246
}
245247

246248
type functionSet struct {
@@ -249,7 +251,7 @@ type functionSet struct {
249251
updateLength func([]byte, int32, int32) []byte
250252
}
251253

252-
func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxDocSize, totalSize int) (int, []byte, error) {
254+
func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, totalSize int) (int, []byte, error) {
253255
if mb.Size() == 0 {
254256
return 0, dst, io.EOF
255257
}
@@ -269,7 +271,7 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
269271
}
270272

271273
canRetry := true
272-
checkSize := true
274+
// checkSize := true
273275

274276
l := len(dst)
275277

@@ -291,13 +293,13 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
291293
var err error
292294
switch model := mb.models[i].model.(type) {
293295
case *ClientInsertOneModel:
294-
checkSize = false
296+
// checkSize = false
295297
mb.cursorHandlers = append(mb.cursorHandlers, mb.appendInsertResult)
296298
var id interface{}
297299
id, doc, err = (&clientInsertDoc{
298300
namespace: nsIdx,
299301
document: model.Document,
300-
sizeLimit: maxDocSize,
302+
// sizeLimit: maxDocSize,
301303
}).marshal(mb.client.bsonOpts, mb.client.registry)
302304
if err != nil {
303305
break
@@ -331,7 +333,7 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
331333
checkDollarKey: true,
332334
}).marshal(mb.client.bsonOpts, mb.client.registry)
333335
case *ClientReplaceOneModel:
334-
checkSize = false
336+
// checkSize = false
335337
mb.cursorHandlers = append(mb.cursorHandlers, mb.appendUpdateResult)
336338
doc, err = (&clientUpdateDoc{
337339
namespace: nsIdx,
@@ -343,7 +345,7 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
343345
upsert: model.Upsert,
344346
multi: false,
345347
checkDollarKey: false,
346-
sizeLimit: maxDocSize,
348+
// sizeLimit: maxDocSize,
347349
}).marshal(mb.client.bsonOpts, mb.client.registry)
348350
case *ClientDeleteOneModel:
349351
mb.cursorHandlers = append(mb.cursorHandlers, mb.appendDeleteResult)
@@ -371,9 +373,9 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
371373
return 0, nil, err
372374
}
373375
length := len(doc)
374-
if maxDocSize > 0 && length > maxDocSize+16*1024 {
375-
return 0, nil, driver.ErrDocumentTooLarge
376-
}
376+
// if maxDocSize > 0 && length > maxDocSize+16*1024 {
377+
// return 0, nil, driver.ErrDocumentTooLarge
378+
// }
377379
if !exists {
378380
length += len(ns)
379381
}
@@ -398,9 +400,9 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
398400
dst = fn.updateLength(dst, opsIdx, int32(len(dst[opsIdx:])))
399401
nsDst = fn.updateLength(nsDst, nsIdx, int32(len(nsDst[nsIdx:])))
400402
dst = append(dst, nsDst...)
401-
if checkSize && maxDocSize > 0 && len(dst)-l > maxDocSize+16*1024 {
402-
return 0, nil, driver.ErrDocumentTooLarge
403-
}
403+
// if checkSize && maxDocSize > 0 && len(dst)-l > maxDocSize+16*1024 {
404+
// return 0, nil, driver.ErrDocumentTooLarge
405+
// }
404406

405407
mb.retryMode = driver.RetryNone
406408
if mb.client.retryWrites && canRetry {
@@ -585,7 +587,7 @@ type clientInsertDoc struct {
585587
namespace int
586588
document interface{}
587589

588-
sizeLimit int
590+
// sizeLimit int
589591
}
590592

591593
func (d *clientInsertDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.Registry) (interface{}, bsoncore.Document, error) {
@@ -596,9 +598,9 @@ func (d *clientInsertDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.
596598
if err != nil {
597599
return nil, nil, err
598600
}
599-
if d.sizeLimit > 0 && len(f) > d.sizeLimit {
600-
return nil, nil, driver.ErrDocumentTooLarge
601-
}
601+
// if d.sizeLimit > 0 && len(f) > d.sizeLimit {
602+
// return nil, nil, driver.ErrDocumentTooLarge
603+
// }
602604
var id interface{}
603605
f, id, err = ensureID(f, bson.NilObjectID, bsonOpts, registry)
604606
if err != nil {
@@ -620,7 +622,7 @@ type clientUpdateDoc struct {
620622
multi bool
621623
checkDollarKey bool
622624

623-
sizeLimit int
625+
// sizeLimit int
624626
}
625627

626628
func (d *clientUpdateDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.Registry) (bsoncore.Document, error) {
@@ -641,9 +643,9 @@ func (d *clientUpdateDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.
641643
if err != nil {
642644
return nil, err
643645
}
644-
if d.sizeLimit > 0 && len(u.Data) > d.sizeLimit {
645-
return nil, driver.ErrDocumentTooLarge
646-
}
646+
// if d.sizeLimit > 0 && len(u.Data) > d.sizeLimit {
647+
// return nil, driver.ErrDocumentTooLarge
648+
// }
647649
doc = bsoncore.AppendValueElement(doc, "updateMods", u)
648650
doc = bsoncore.AppendBooleanElement(doc, "multi", d.multi)
649651

‎mongo/client_bulk_write_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ func TestBatches(t *testing.T) {
5555
var n int
5656
const limitBigEnough = 16_000
5757
// test the "maxCount" that truncates the output
58-
n, _, err = batches.AppendBatchSequence(nil, 4, limitBigEnough, limitBigEnough)
58+
n, _, err = batches.AppendBatchSequence(nil, 4, limitBigEnough)
5959
require.NoError(t, err, "AppendBatchSequence error: %v", err)
6060
assert.Equal(t, 3, n, "expected %d appendings, got: %d", 3, n)
6161

‎x/mongo/driver/batches.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@ type Batches struct {
2424
offset int
2525
}
2626

27+
var _ OperationBatches = &Batches{}
28+
2729
// AppendBatchSequence appends dst with document sequence of batches as long as the limits of max count, max
2830
// document size, or total size allows. It returns the number of batches appended, the new appended slice, and
2931
// any error raised. It returns the origenal input slice if nothing can be appends within the limits.
30-
func (b *Batches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, _ int) (int, []byte, error) {
32+
func (b *Batches) AppendBatchSequence(dst []byte, maxCount, totalSize int) (int, []byte, error) {
3133
if b.Size() == 0 {
3234
return 0, dst, io.EOF
3335
}
@@ -44,11 +46,11 @@ func (b *Batches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, _ int) (
4446
break
4547
}
4648
doc := b.Documents[i]
47-
if len(doc) > maxDocSize {
48-
break
49-
}
49+
// if len(doc) > maxDocSize {
50+
// break
51+
// }
5052
size += len(doc)
51-
if size > maxDocSize {
53+
if size > totalSize {
5254
break
5355
}
5456
dst = append(dst, doc...)
@@ -64,7 +66,7 @@ func (b *Batches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, _ int) (
6466
// AppendBatchArray appends dst with array of batches as long as the limits of max count, max document size, or
6567
// total size allows. It returns the number of batches appended, the new appended slice, and any error raised. It
6668
// returns the origenal input slice if nothing can be appends within the limits.
67-
func (b *Batches) AppendBatchArray(dst []byte, maxCount, maxDocSize, _ int) (int, []byte, error) {
69+
func (b *Batches) AppendBatchArray(dst []byte, maxCount, totalSize int) (int, []byte, error) {
6870
if b.Size() == 0 {
6971
return 0, dst, io.EOF
7072
}
@@ -77,11 +79,11 @@ func (b *Batches) AppendBatchArray(dst []byte, maxCount, maxDocSize, _ int) (int
7779
break
7880
}
7981
doc := b.Documents[i]
80-
if len(doc) > maxDocSize {
81-
break
82-
}
82+
// if len(doc) > maxDocSize {
83+
// break
84+
// }
8385
size += len(doc)
84-
if size > maxDocSize {
86+
if size > totalSize {
8587
break
8688
}
8789
dst = bsoncore.AppendDocumentElement(dst, strconv.Itoa(n), doc)

‎x/mongo/driver/batches_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111

1212
"go.mongodb.org/mongo-driver/v2/internal/assert"
1313
"go.mongodb.org/mongo-driver/v2/x/bsonx/bsoncore"
14-
"go.mongodb.org/mongo-driver/v2/x/mongo/driver/wiremessage"
1514
)
1615

1716
func newTestBatches(t *testing.T) *Batches {
@@ -32,6 +31,7 @@ func TestAdvancing(t *testing.T) {
3231
assert.Equal(t, 0, size, "expected Size(): %d, got: %d", 1, size)
3332
}
3433

34+
/*
3535
func TestAppendBatchSequence(t *testing.T) {
3636
t.Run("Append 0", func(t *testing.T) {
3737
batches := newTestBatches(t)
@@ -99,3 +99,4 @@ func TestAppendBatchArray(t *testing.T) {
9999
assert.Equal(t, dst, got)
100100
})
101101
}
102+
*/

‎x/mongo/driver/operation.go‎

Lines changed: 19 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,16 @@ func redactFinishedInformationResponse(info finishedInformation) bson.Raw {
184184
return bson.Raw{}
185185
}
186186

187+
// OperationBatches contains the documents that are split when executing a write command that potentially
188+
// has more documents than can fit in a single command.
189+
type OperationBatches interface {
190+
AppendBatchSequence(dst []byte, maxCount int, totalSize int) (int, []byte, error)
191+
AppendBatchArray(dst []byte, maxCount int, totalSize int) (int, []byte, error)
192+
IsOrdered() *bool
193+
AdvanceBatches(n int)
194+
Size() int
195+
}
196+
187197
// Operation is used to execute an operation. It contains all of the common code required to
188198
// select a server, transform an operation into a command, write the command to a connection from
189199
// the selected server, read a response from that connection, process the response, and potentially
@@ -269,15 +279,8 @@ type Operation struct {
269279

270280
// Batches contains the documents that are split when executing a write command that potentially
271281
// has more documents than can fit in a single command. This should only be specified for
272-
// commands that are batch compatible. For more information, please refer to the definition of
273-
// Batches.
274-
Batches interface {
275-
AppendBatchSequence(dst []byte, maxCount int, maxDocSize int, totalSize int) (int, []byte, error)
276-
AppendBatchArray(dst []byte, maxCount int, maxDocSize int, totalSize int) (int, []byte, error)
277-
IsOrdered() *bool
278-
AdvanceBatches(n int)
279-
Size() int
280-
}
282+
// commands that are batch compatible.
283+
Batches OperationBatches
281284

282285
// Legacy sets the legacy type for this operation. There are only 3 types that require legacy
283286
// support: find, getMore, and killCursors. For more information about LegacyOperationKind,
@@ -1371,7 +1374,7 @@ func (op Operation) createWireMessage(
13711374
if err == nil && op.Batches != nil {
13721375
batchOffset = len(dst)
13731376
info.processedBatches, dst, err = op.Batches.AppendBatchSequence(dst,
1374-
int(desc.MaxBatchCount), int(desc.MaxDocumentSize), int(desc.MaxDocumentSize),
1377+
int(desc.MaxBatchCount), int(desc.MaxMessageSize),
13751378
)
13761379
if err != nil {
13771380
break
@@ -1383,12 +1386,8 @@ func (op Operation) createWireMessage(
13831386
default:
13841387
var batches []byte
13851388
if op.Batches != nil {
1386-
maxDocSize := -1
1387-
if unacknowledged {
1388-
maxDocSize = int(desc.MaxDocumentSize)
1389-
}
13901389
info.processedBatches, batches, err = op.Batches.AppendBatchSequence(batches,
1391-
int(desc.MaxBatchCount), maxDocSize, int(desc.MaxMessageSize),
1390+
int(desc.MaxBatchCount), int(desc.MaxMessageSize),
13921391
)
13931392
if err != nil {
13941393
break
@@ -1443,14 +1442,14 @@ func (op Operation) addEncryptCommandFields(ctx context.Context, dst []byte, des
14431442
var n int
14441443
if op.Batches != nil {
14451444
if maxBatchCount := int(desc.MaxBatchCount); maxBatchCount > 1 {
1446-
n, cmdDst, err = op.Batches.AppendBatchArray(cmdDst, maxBatchCount, cryptMaxBsonObjectSize, cryptMaxBsonObjectSize)
1445+
n, cmdDst, err = op.Batches.AppendBatchArray(cmdDst, maxBatchCount, cryptMaxBsonObjectSize)
14471446
if err != nil {
14481447
return 0, nil, err
14491448
}
14501449
}
14511450
if n == 0 {
1452-
maxDocumentSize := int(desc.MaxDocumentSize)
1453-
n, cmdDst, err = op.Batches.AppendBatchArray(cmdDst, 1, maxDocumentSize, maxDocumentSize)
1451+
// maxDocumentSize := int(desc.MaxDocumentSize)
1452+
n, cmdDst, err = op.Batches.AppendBatchArray(cmdDst, 1, int(desc.MaxMessageSize))
14541453
if err != nil {
14551454
return 0, nil, err
14561455
}
@@ -1483,8 +1482,8 @@ func (op Operation) addLegacyCommandFields(dst []byte, desc description.Selected
14831482
return 0, dst, nil
14841483
}
14851484
var n int
1486-
maxDocumentSize := int(desc.MaxDocumentSize)
1487-
n, dst, err = op.Batches.AppendBatchArray(dst, int(desc.MaxBatchCount), maxDocumentSize, maxDocumentSize)
1485+
// maxDocumentSize := int(desc.MaxDocumentSize)
1486+
n, dst, err = op.Batches.AppendBatchArray(dst, int(desc.MaxBatchCount), int(desc.MaxMessageSize))
14881487
if err != nil {
14891488
return 0, nil, err
14901489
}

0 commit comments

Comments
 (0)