Skip to content

Commit fc91428

Browse files
qingyang-huqingyang-hu
committed
GODRIVER-3421 Remove the BSON document size validation.
1 parent acca80b commit fc91428

File tree

8 files changed

+152
-158
lines changed

8 files changed

+152
-158
lines changed

‎internal/integration/collection_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ func TestCollection(t *testing.T) {
167167
mt.Run("large document batches", func(mt *mtest.T) {
168168
mt.Parallel()
169169

170-
docs := []interface{}{create16MBDocument(mt), create16MBDocument(mt)}
170+
docs := []interface{}{create16MBDocument(mt), create16MBDocument(mt), create16MBDocument(mt)}
171171
_, err := mt.Coll.InsertMany(context.Background(), docs)
172172
assert.Nil(mt, err, "InsertMany error: %v", err)
173173
evt := mt.GetStartedEvent()

‎internal/integration/crud_prose_test.go‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -745,32 +745,6 @@ func TestClientBulkWrite(t *testing.T) {
745745
assert.Equal(mt, 1, killCursorsCalled, "expected %d killCursors call, got: %d", 1, killCursorsCalled)
746746
})
747747

748-
mt.Run("bulkWrite returns error for unacknowledged too-large insert", func(mt *mtest.T) {
749-
mt.ResetClient(options.Client())
750-
var hello struct {
751-
MaxBsonObjectSize int
752-
}
753-
err := mt.DB.RunCommand(context.Background(), bson.D{{"hello", 1}}).Decode(&hello)
754-
require.NoError(mt, err, "Hello error: %v", err)
755-
mt.Run("insert", func(mt *mtest.T) {
756-
models := (&mongo.ClientWriteModels{}).
757-
AppendInsertOne("db", "coll", &mongo.ClientInsertOneModel{
758-
Document: bson.D{{"a", strings.Repeat("b", hello.MaxBsonObjectSize)}},
759-
})
760-
_, err := mt.Client.BulkWrite(context.Background(), models, options.ClientBulkWrite().SetOrdered(false).SetWriteConcern(writeconcern.Unacknowledged()))
761-
require.EqualError(mt, err, driver.ErrDocumentTooLarge.Error())
762-
})
763-
mt.Run("replace", func(mt *mtest.T) {
764-
models := (&mongo.ClientWriteModels{}).
765-
AppendReplaceOne("db", "coll", &mongo.ClientReplaceOneModel{
766-
Filter: bson.D{},
767-
Replacement: bson.D{{"a", strings.Repeat("b", hello.MaxBsonObjectSize)}},
768-
})
769-
_, err := mt.Client.BulkWrite(context.Background(), models, options.ClientBulkWrite().SetOrdered(false).SetWriteConcern(writeconcern.Unacknowledged()))
770-
require.EqualError(mt, err, driver.ErrDocumentTooLarge.Error())
771-
})
772-
})
773-
774748
mt.Run("bulkWrite batch splits when the addition of a new namespace exceeds the maximum message size", func(mt *mtest.T) {
775749
type cmd struct {
776750
Ops []bson.D

‎mongo/client_bulk_write.go‎

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,8 @@ type modelBatches struct {
191191
writeErrors map[int]WriteError
192192
}
193193

194+
var _ driver.OperationBatches = &modelBatches{}
195+
194196
func (mb *modelBatches) IsOrdered() *bool {
195197
return &mb.ordered
196198
}
@@ -209,7 +211,7 @@ func (mb *modelBatches) Size() int {
209211
return len(mb.models) - mb.offset
210212
}
211213

212-
func (mb *modelBatches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, totalSize int) (int, []byte, error) {
214+
func (mb *modelBatches) AppendBatchSequence(dst []byte, maxCount, totalSize int) (int, []byte, error) {
213215
fn := functionSet{
214216
appendStart: func(dst []byte, identifier string) (int32, []byte) {
215217
var idx int32
@@ -228,10 +230,10 @@ func (mb *modelBatches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, to
228230
return dst
229231
},
230232
}
231-
return mb.appendBatches(fn, dst, maxCount, maxDocSize, totalSize)
233+
return mb.appendBatches(fn, dst, maxCount, totalSize)
232234
}
233235

234-
func (mb *modelBatches) AppendBatchArray(dst []byte, maxCount, maxDocSize, totalSize int) (int, []byte, error) {
236+
func (mb *modelBatches) AppendBatchArray(dst []byte, maxCount, totalSize int) (int, []byte, error) {
235237
fn := functionSet{
236238
appendStart: bsoncore.AppendArrayElementStart,
237239
appendDocument: bsoncore.AppendDocumentElement,
@@ -240,7 +242,7 @@ func (mb *modelBatches) AppendBatchArray(dst []byte, maxCount, maxDocSize, total
240242
return dst
241243
},
242244
}
243-
return mb.appendBatches(fn, dst, maxCount, maxDocSize, totalSize)
245+
return mb.appendBatches(fn, dst, maxCount, totalSize)
244246
}
245247

246248
type functionSet struct {
@@ -249,7 +251,7 @@ type functionSet struct {
249251
updateLength func([]byte, int32, int32) []byte
250252
}
251253

252-
func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxDocSize, totalSize int) (int, []byte, error) {
254+
func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, totalSize int) (int, []byte, error) {
253255
if mb.Size() == 0 {
254256
return 0, dst, io.EOF
255257
}
@@ -269,8 +271,6 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
269271
}
270272

271273
canRetry := true
272-
checkSize := true
273-
274274
l := len(dst)
275275

276276
opsIdx, dst := fn.appendStart(dst, "ops")
@@ -291,13 +291,11 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
291291
var err error
292292
switch model := mb.models[i].model.(type) {
293293
case *ClientInsertOneModel:
294-
checkSize = false
295294
mb.cursorHandlers = append(mb.cursorHandlers, mb.appendInsertResult)
296295
var id interface{}
297296
id, doc, err = (&clientInsertDoc{
298297
namespace: nsIdx,
299298
document: model.Document,
300-
sizeLimit: maxDocSize,
301299
}).marshal(mb.client.bsonOpts, mb.client.registry)
302300
if err != nil {
303301
break
@@ -331,7 +329,6 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
331329
checkDollarKey: true,
332330
}).marshal(mb.client.bsonOpts, mb.client.registry)
333331
case *ClientReplaceOneModel:
334-
checkSize = false
335332
mb.cursorHandlers = append(mb.cursorHandlers, mb.appendUpdateResult)
336333
doc, err = (&clientUpdateDoc{
337334
namespace: nsIdx,
@@ -343,7 +340,6 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
343340
upsert: model.Upsert,
344341
multi: false,
345342
checkDollarKey: false,
346-
sizeLimit: maxDocSize,
347343
}).marshal(mb.client.bsonOpts, mb.client.registry)
348344
case *ClientDeleteOneModel:
349345
mb.cursorHandlers = append(mb.cursorHandlers, mb.appendDeleteResult)
@@ -371,9 +367,6 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
371367
return 0, nil, err
372368
}
373369
length := len(doc)
374-
if maxDocSize > 0 && length > maxDocSize+16*1024 {
375-
return 0, nil, driver.ErrDocumentTooLarge
376-
}
377370
if !exists {
378371
length += len(ns)
379372
}
@@ -398,9 +391,6 @@ func (mb *modelBatches) appendBatches(fn functionSet, dst []byte, maxCount, maxD
398391
dst = fn.updateLength(dst, opsIdx, int32(len(dst[opsIdx:])))
399392
nsDst = fn.updateLength(nsDst, nsIdx, int32(len(nsDst[nsIdx:])))
400393
dst = append(dst, nsDst...)
401-
if checkSize && maxDocSize > 0 && len(dst)-l > maxDocSize+16*1024 {
402-
return 0, nil, driver.ErrDocumentTooLarge
403-
}
404394

405395
mb.retryMode = driver.RetryNone
406396
if mb.client.retryWrites && canRetry {
@@ -584,8 +574,6 @@ func (mb *modelBatches) appendUpdateResult(cur *cursorInfo, raw bson.Raw) bool {
584574
type clientInsertDoc struct {
585575
namespace int
586576
document interface{}
587-
588-
sizeLimit int
589577
}
590578

591579
func (d *clientInsertDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.Registry) (interface{}, bsoncore.Document, error) {
@@ -596,9 +584,6 @@ func (d *clientInsertDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.
596584
if err != nil {
597585
return nil, nil, err
598586
}
599-
if d.sizeLimit > 0 && len(f) > d.sizeLimit {
600-
return nil, nil, driver.ErrDocumentTooLarge
601-
}
602587
var id interface{}
603588
f, id, err = ensureID(f, bson.NilObjectID, bsonOpts, registry)
604589
if err != nil {
@@ -619,8 +604,6 @@ type clientUpdateDoc struct {
619604
upsert *bool
620605
multi bool
621606
checkDollarKey bool
622-
623-
sizeLimit int
624607
}
625608

626609
func (d *clientUpdateDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.Registry) (bsoncore.Document, error) {
@@ -641,9 +624,6 @@ func (d *clientUpdateDoc) marshal(bsonOpts *options.BSONOptions, registry *bson.
641624
if err != nil {
642625
return nil, err
643626
}
644-
if d.sizeLimit > 0 && len(u.Data) > d.sizeLimit {
645-
return nil, driver.ErrDocumentTooLarge
646-
}
647627
doc = bsoncore.AppendValueElement(doc, "updateMods", u)
648628
doc = bsoncore.AppendBooleanElement(doc, "multi", d.multi)
649629

‎mongo/client_bulk_write_test.go‎

Lines changed: 64 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,22 +17,23 @@ import (
1717
func TestBatches(t *testing.T) {
1818
t.Parallel()
1919

20-
t.Run("test Addvancing", func(t *testing.T) {
21-
t.Parallel()
20+
t.Parallel()
2221

23-
batches := &modelBatches{
24-
models: make([]clientWriteModel, 2),
25-
}
26-
batches.AdvanceBatches(3)
27-
size := batches.Size()
28-
assert.Equal(t, 0, size, "expected: %d, got: %d", 1, size)
29-
})
30-
t.Run("test appendBatches", func(t *testing.T) {
31-
t.Parallel()
22+
batches := &modelBatches{
23+
models: make([]clientWriteModel, 2),
24+
}
25+
batches.AdvanceBatches(3)
26+
size := batches.Size()
27+
assert.Equal(t, 0, size, "expected: %d, got: %d", 1, size)
28+
}
29+
30+
func TestAppendBatchSequence(t *testing.T) {
31+
t.Parallel()
3232

33+
newBatches := func(t *testing.T) *modelBatches {
3334
client, err := newClient()
3435
require.NoError(t, err, "NewClient error: %v", err)
35-
batches := &modelBatches{
36+
return &modelBatches{
3637
client: client,
3738
models: []clientWriteModel{
3839
{"ns0", nil},
@@ -52,12 +53,15 @@ func TestBatches(t *testing.T) {
5253
Acknowledged: true,
5354
},
5455
}
55-
var n int
56+
}
57+
t.Run("test appendBatches", func(t *testing.T) {
58+
t.Parallel()
59+
60+
batches := newBatches(t)
5661
const limitBigEnough = 16_000
57-
// test the "maxCount" that truncates the output
58-
n, _, err = batches.AppendBatchSequence(nil, 4, limitBigEnough, limitBigEnough)
62+
n, _, err := batches.AppendBatchSequence(nil, 4, limitBigEnough)
5963
require.NoError(t, err, "AppendBatchSequence error: %v", err)
60-
assert.Equal(t, 3, n, "expected %d appendings, got: %d", 3, n)
64+
require.Equal(t, 3, n, "expected %d appendings, got: %d", 3, n)
6165

6266
_ = batches.cursorHandlers[0](&cursorInfo{Ok: true, Idx: 0}, nil)
6367
_ = batches.cursorHandlers[1](&cursorInfo{Ok: true, Idx: 1}, nil)
@@ -71,6 +75,50 @@ func TestBatches(t *testing.T) {
7175
assert.True(t, ok, "expected an insert results")
7276

7377
_, ok = batches.result.DeleteResults[3]
78+
assert.True(t, ok, "expected an delete results")
79+
})
80+
t.Run("test appendBatches with maxCount", func(t *testing.T) {
81+
t.Parallel()
82+
83+
batches := newBatches(t)
84+
const limitBigEnough = 16_000
85+
n, _, err := batches.AppendBatchSequence(nil, 2, limitBigEnough)
86+
require.NoError(t, err, "AppendBatchSequence error: %v", err)
87+
require.Equal(t, 2, n, "expected %d appendings, got: %d", 2, n)
88+
89+
_ = batches.cursorHandlers[0](&cursorInfo{Ok: true, Idx: 0}, nil)
90+
_ = batches.cursorHandlers[1](&cursorInfo{Ok: true, Idx: 1}, nil)
91+
92+
ins, ok := batches.result.InsertResults[1]
93+
assert.True(t, ok, "expected an insert results")
94+
assert.NotNil(t, ins.InsertedID, "expected an ID")
95+
96+
_, ok = batches.result.UpdateResults[2]
97+
assert.True(t, ok, "expected an insert results")
98+
99+
_, ok = batches.result.DeleteResults[3]
100+
assert.False(t, ok, "expected an delete results")
101+
})
102+
t.Run("test appendBatches with totalSize", func(t *testing.T) {
103+
t.Parallel()
104+
105+
batches := newBatches(t)
106+
const limit = 1200 // > ( 166 first two batches + 1000 overhead )
107+
n, _, err := batches.AppendBatchSequence(nil, 4, limit)
108+
require.NoError(t, err, "AppendBatchSequence error: %v", err)
109+
require.Equal(t, 2, n, "expected %d appendings, got: %d", 2, n)
110+
111+
_ = batches.cursorHandlers[0](&cursorInfo{Ok: true, Idx: 0}, nil)
112+
_ = batches.cursorHandlers[1](&cursorInfo{Ok: true, Idx: 1}, nil)
113+
114+
ins, ok := batches.result.InsertResults[1]
74115
assert.True(t, ok, "expected an insert results")
116+
assert.NotNil(t, ins.InsertedID, "expected an ID")
117+
118+
_, ok = batches.result.UpdateResults[2]
119+
assert.True(t, ok, "expected an insert results")
120+
121+
_, ok = batches.result.DeleteResults[3]
122+
assert.False(t, ok, "expected an delete results")
75123
})
76124
}

‎mongo/client_test.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"errors"
1212
"math"
1313
"os"
14+
"strings"
1415
"testing"
1516
"time"
1617

@@ -516,4 +517,30 @@ func TestClient(t *testing.T) {
516517
errmsg := `invalid value "-1s" for "Timeout": value must be positive`
517518
assert.Equal(t, errmsg, err.Error(), "expected error %v, got %v", errmsg, err.Error())
518519
})
520+
t.Run("bulkWrite with large messages", func(t *testing.T) {
521+
var bulkWrites int
522+
cmdMonitor := &event.CommandMonitor{
523+
Started: func(_ context.Context, evt *event.CommandStartedEvent) {
524+
if evt.CommandName == "bulkWrite" {
525+
bulkWrites++
526+
}
527+
},
528+
}
529+
cs := integtest.ConnString(t)
530+
clientOpts := options.Client().ApplyURI(cs.Original).SetMonitor(cmdMonitor)
531+
client, err := Connect(clientOpts)
532+
assert.Nil(t, err, "Connect error: %v", err)
533+
defer func() {
534+
_ = client.Disconnect(bgCtx)
535+
}()
536+
document := bson.D{{"largeField", strings.Repeat("a", 16777216-100)}} // Adjust size to account for BSON overhead
537+
models := &ClientWriteModels{}
538+
models = models.AppendInsertOne("db", "x", NewClientInsertOneModel().SetDocument(document))
539+
models = models.AppendInsertOne("db", "x", NewClientInsertOneModel().SetDocument(document))
540+
models = models.AppendInsertOne("db", "x", NewClientInsertOneModel().SetDocument(document))
541+
542+
_, err = client.BulkWrite(context.Background(), models)
543+
require.NoError(t, err)
544+
assert.Equal(t, 2, bulkWrites, "expected %d bulkWrites, got %d", 2, bulkWrites)
545+
})
519546
}

‎x/mongo/driver/batches.go‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,12 @@ type Batches struct {
2424
offset int
2525
}
2626

27+
var _ OperationBatches = &Batches{}
28+
2729
// AppendBatchSequence appends dst with document sequence of batches as long as the limits of max count, max
2830
// document size, or total size allows. It returns the number of batches appended, the new appended slice, and
2931
// any error raised. It returns the origenal input slice if nothing can be appends within the limits.
30-
func (b *Batches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, _ int) (int, []byte, error) {
32+
func (b *Batches) AppendBatchSequence(dst []byte, maxCount, totalSize int) (int, []byte, error) {
3133
if b.Size() == 0 {
3234
return 0, dst, io.EOF
3335
}
@@ -44,11 +46,8 @@ func (b *Batches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, _ int) (
4446
break
4547
}
4648
doc := b.Documents[i]
47-
if len(doc) > maxDocSize {
48-
break
49-
}
5049
size += len(doc)
51-
if size > maxDocSize {
50+
if size > totalSize {
5251
break
5352
}
5453
dst = append(dst, doc...)
@@ -64,7 +63,7 @@ func (b *Batches) AppendBatchSequence(dst []byte, maxCount, maxDocSize, _ int) (
6463
// AppendBatchArray appends dst with array of batches as long as the limits of max count, max document size, or
6564
// total size allows. It returns the number of batches appended, the new appended slice, and any error raised. It
6665
// returns the origenal input slice if nothing can be appends within the limits.
67-
func (b *Batches) AppendBatchArray(dst []byte, maxCount, maxDocSize, _ int) (int, []byte, error) {
66+
func (b *Batches) AppendBatchArray(dst []byte, maxCount, totalSize int) (int, []byte, error) {
6867
if b.Size() == 0 {
6968
return 0, dst, io.EOF
7069
}
@@ -77,11 +76,8 @@ func (b *Batches) AppendBatchArray(dst []byte, maxCount, maxDocSize, _ int) (int
7776
break
7877
}
7978
doc := b.Documents[i]
80-
if len(doc) > maxDocSize {
81-
break
82-
}
8379
size += len(doc)
84-
if size > maxDocSize {
80+
if size > totalSize {
8581
break
8682
}
8783
dst = bsoncore.AppendDocumentElement(dst, strconv.Itoa(n), doc)

0 commit comments

Comments
 (0)