Fixed bugs in gridfs implementation

GODRIVER-724

Change-Id: I5afcd5b7dc810f1696a7ba1f258f3a80c4d0284c

Author: dmhilly

data/gridfs/download.json

@@ -355,97 +355,6 @@
         "error": "ChunkIsMissing"
       }
     },
-    {
-      "description": "Download when an intermediate chunk is the wrong size",
-      "arrange": {
-        "data": [
-          {
-            "update": "fs.chunks",
-            "updates": [
-              {
-                "q": {
-                  "files_id": {
-                    "$oid": "000000000000000000000005"
-                  },
-                  "n": 1
-                },
-                "u": {
-                  "$set": {
-                    "data": {
-                      "$hex": "556677"
-                    }
-                  }
-                }
-              },
-              {
-                "q": {
-                  "files_id": {
-                    "$oid": "000000000000000000000005"
-                  },
-                  "n": 2
-                },
-                "u": {
-                  "$set": {
-                    "data": {
-                      "$hex": "8899aa"
-                    }
-                  }
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "act": {
-        "operation": "download",
-        "arguments": {
-          "id": {
-            "$oid": "000000000000000000000005"
-          }
-        }
-      },
-      "assert": {
-        "error": "ChunkIsWrongSize"
-      }
-    },
-    {
-      "description": "Download when final chunk is the wrong size",
-      "arrange": {
-        "data": [
-          {
-            "update": "fs.chunks",
-            "updates": [
-              {
-                "q": {
-                  "files_id": {
-                    "$oid": "000000000000000000000005"
-                  },
-                  "n": 2
-                },
-                "u": {
-                  "$set": {
-                    "data": {
-                      "$hex": "99"
-                    }
-                  }
-                }
-              }
-            ]
-          }
-        ]
-      },
-      "act": {
-        "operation": "download",
-        "arguments": {
-          "id": {
-            "$oid": "000000000000000000000005"
-          }
-        }
-      },
-      "assert": {
-        "error": "ChunkIsWrongSize"
-      }
-    },
     {
       "description": "Download legacy file with no name",
       "act": {

data/gridfs/download.yml

@@ -152,35 +152,6 @@ tests:
                 id: { "$oid" : "000000000000000000000005" }
         assert:
             error: "ChunkIsMissing"
-    -
-        description: "Download when an intermediate chunk is the wrong size"
-        arrange:
-            data:
-                -
-                    { update : "fs.chunks", updates : [
-                        { q : { files_id : { "$oid" : "000000000000000000000005" }, n : 1 }, u : { $set : { data : { $hex : "556677" } } } },
-                        { q : { files_id : { "$oid" : "000000000000000000000005" }, n : 2 }, u : { $set : { data : { $hex : "8899aa" } } } }
-                    ] }
-        act:
-            operation: download
-            arguments:
-                id: { "$oid" : "000000000000000000000005" }
-        assert:
-            error: "ChunkIsWrongSize"
-    -
-        description: "Download when final chunk is the wrong size"
-        arrange:
-            data:
-                -
-                    { update : "fs.chunks", updates : [
-                        { q : { files_id : { "$oid" : "000000000000000000000005" }, n : 2 }, u : { $set : { data : { $hex : "99" } } } }
-                    ] }
-        act:
-            operation: download
-            arguments:
-                id: { "$oid" : "000000000000000000000005" }
-        assert:
-            error: "ChunkIsWrongSize"
     -
         description: "Download legacy file with no name"
         act:

mongo/gridfs/bucket.go

@@ -458,31 +458,34 @@ func (b *Bucket) createIndexes(ctx context.Context) error {
 
 	docRes := cloned.FindOne(ctx, bsonx.Doc{}, options.FindOne().SetProjection(bsonx.Doc{{"_id", bsonx.Int32(1)}}))
 
-	err = docRes.Err()
-	if err == mongo.ErrNoDocuments {
-		filesIv := b.filesColl.Indexes()
-		chunksIv := b.chunksColl.Indexes()
-
-		filesModel := mongo.IndexModel{
-			Keys: bson.D{
-				{"filename", int32(1)},
-				{"uploadDate", int32(1)},
-			},
-		}
+	_, err = docRes.DecodeBytes()
+	if err != mongo.ErrNoDocuments {
+		// nil, or error that occured during the FindOne operation
+		return err
+	}
 
-		chunksModel := mongo.IndexModel{
-			Keys: bson.D{
-				{"files_id", int32(1)},
-				{"n", int32(1)},
-			},
-		}
+	filesIv := b.filesColl.Indexes()
+	chunksIv := b.chunksColl.Indexes()
 
-		if err = createIndexIfNotExists(ctx, filesIv, filesModel); err != nil {
-			return err
-		}
-		if err = createIndexIfNotExists(ctx, chunksIv, chunksModel); err != nil {
-			return err
-		}
+	filesModel := mongo.IndexModel{
+		Keys: bson.D{
+			{"filename", int32(1)},
+			{"uploadDate", int32(1)},
+		},
+	}
+
+	chunksModel := mongo.IndexModel{
+		Keys: bson.D{
+			{"files_id", int32(1)},
+			{"n", int32(1)},
+		},
+	}
+
+	if err = createIndexIfNotExists(ctx, filesIv, filesModel); err != nil {
+		return err
+	}
+	if err = createIndexIfNotExists(ctx, chunksIv, chunksModel); err != nil {
+		return err
 	}
 
 	return nil

mongo/gridfs/download_stream.go

@@ -36,6 +36,7 @@ type DownloadStream struct {
 	closed        bool
 	buffer        []byte // store up to 1 chunk if the user provided buffer isn't big enough
 	bufferStart   int
+	bufferEnd     int
 	expectedChunk int32 // index of next expected chunk
 	readDeadline  time.Time
 	fileLen       int64
@@ -93,21 +94,21 @@ func (ds *DownloadStream) Read(p []byte) (int, error) {
 	var err error
 
 	for bytesCopied < len(p) {
-		if ds.bufferStart == 0 {
-			// buffer empty
+		if ds.bufferStart >= ds.bufferEnd {
+			// Buffer is empty and can load in data from new chunk.
 			err = ds.fillBuffer(ctx)
 			if err != nil {
 				if err == errNoMoreChunks {
 					return bytesCopied, nil
 				}
-
 				return bytesCopied, err
 			}
 		}
 
-		copied := copy(p, ds.buffer[ds.bufferStart:])
+		copied := copy(p[bytesCopied:], ds.buffer[ds.bufferStart:ds.bufferEnd])
+
 		bytesCopied += copied
-		ds.bufferStart = (ds.bufferStart + copied) % int(ds.chunkSize)
+		ds.bufferStart += copied
 	}
 
 	return len(p), nil
@@ -187,22 +188,10 @@ func (ds *DownloadStream) fillBuffer(ctx context.Context) error {
 	}
 
 	_, dataBytes := data.Binary()
+	copied := copy(ds.buffer, dataBytes)
 
-	bytesLen := int32(len(dataBytes))
-	if ds.expectedChunk == ds.numChunks {
-		// final chunk can be fewer than ds.chunkSize bytes
-		bytesDownloaded := ds.chunkSize * (ds.expectedChunk - 1)
-		bytesRemaining := ds.fileLen - int64(bytesDownloaded)
-
-		if int64(bytesLen) != bytesRemaining {
-			return ErrWrongSize
-		}
-	} else if bytesLen != ds.chunkSize {
-		// all intermediate chunks must have size ds.chunkSize
-		return ErrWrongSize
-	}
-
-	copy(ds.buffer, dataBytes)
 	ds.bufferStart = 0
+	ds.bufferEnd = copied
+
 	return nil
 }

mongo/gridfs/upload_stream.go

@@ -106,24 +106,22 @@ func (us *UploadStream) Write(p []byte) (int, error) {
 
 	origLen := len(p)
 	for {
-		if us.bufferIndex == UploadBufferSize {
-			err := us.uploadChunks(ctx)
-			if err != nil {
-				return 0, err
-			}
-			us.bufferIndex = 0
-		}
-
 		if len(p) == 0 {
 			break
 		}
 
 		n := copy(us.buffer[us.bufferIndex:], p) // copy as much as possible
 		p = p[n:]
 		us.bufferIndex += n
-		break
-	}
 
+		if us.bufferIndex == UploadBufferSize {
+			err := us.uploadChunks(ctx)
+			if err != nil {
+				return 0, err
+			}
+			us.bufferIndex = 0
+		}
+	}
 	return origLen, nil
 }
 
@@ -151,16 +149,15 @@ func (us *UploadStream) uploadChunks(ctx context.Context) error {
 	numChunks := math.Ceil(float64(us.bufferIndex) / float64(us.chunkSize))
 
 	docs := make([]interface{}, int(numChunks))
-
+	begChunkIndex := us.chunkIndex
 	for i := 0; i < us.bufferIndex; i += int(us.chunkSize) {
 		var chunkData []byte
 		if us.bufferIndex-i < int(us.chunkSize) {
 			chunkData = us.buffer[i:us.bufferIndex]
 		} else {
 			chunkData = us.buffer[i : i+int(us.chunkSize)]
 		}
-
-		docs[us.chunkIndex] = bsonx.Doc{
+		docs[us.chunkIndex-begChunkIndex] = bsonx.Doc{
 			{"_id", bsonx.ObjectID(primitive.NewObjectID())},
 			{"files_id", bsonx.ObjectID(us.FileID)},
 			{"n", bsonx.Int32(int32(us.chunkIndex))},

mongo/options/gridfsoptions.go

@@ -109,9 +109,7 @@ type UploadOptions struct {
 
 // GridFSUpload creates a new *UploadOptions
 func GridFSUpload() *UploadOptions {
-	return &UploadOptions{
-		ChunkSizeBytes: &DefaultChunkSize,
-	}
+	return &UploadOptions{}
 }
 
 // SetChunkSizeBytes sets the chunk size in bytes for the upload. Defaults to 255KB if not set.