Add configurable chunk size for GCS uploads through SetUploadChunkSizeByte

Author: jacobfederer

gcsfs/file.go

@@ -90,6 +90,12 @@ func NewGcsFileFromOldFH(
 	return res
 }
 
+// SetUploadChunkSizeByte The default chunk size for uploading files to GCS is 16MB.
+// This buffer size may use memory excessively. You can adjust it to a value that is a multiple of 256KiB.
+func (o *GcsFile) SetUploadChunkSizeByte(size int) {
+	o.resource.uploadChunkSizeByte = size
+}
+
 func (o *GcsFile) Close() error {
 	if o.closed {
 		// the afero spec expects the call to Close on a closed file to return an error

gcsfs/file_resource.go

@@ -46,6 +46,8 @@ type gcsFileResource struct {
 	name     string
 	fileMode os.FileMode
 
+	uploadChunkSizeByte int
+
 	currentGcsSize int64
 	offset         int64
 	reader         io.ReadCloser
@@ -185,6 +187,16 @@ func (o *gcsFileResource) WriteAt(b []byte, off int64) (n int, err error) {
 	//
 	// It will however require a download and upload of the original file but it
 	// can't be avoided if we should support seek-write-operations on GCS.
+
+	chunkSize := o.fs.UploadChunkSizeByte
+	if o.uploadChunkSizeByte != 0 {
+		chunkSize = o.uploadChunkSizeByte
+	}
+
+	if chunkSize != 0 {
+		w.SetChunkSize(chunkSize)
+	}
+
 	objAttrs, err := o.obj.Attrs(o.ctx)
 	if err != nil {
 		if off > 0 {

gcsfs/fs.go

@@ -43,6 +43,7 @@ type Fs struct {
 	rawGcsObjects map[string]*GcsFile
 
 	autoRemoveEmptyFolders bool // trigger for creating "virtual folders" (not required by GCSs)
+	UploadChunkSizeByte    int
 }
 
 func NewGcsFs(ctx context.Context, client stiface.Client) *Fs {
@@ -60,6 +61,12 @@ func NewGcsFsWithSeparator(ctx context.Context, client stiface.Client, folderSep
 	}
 }
 
+// SetUploadChunkSizeByte The default chunk size for uploading files to GCS is 16MB.
+// This buffer size may use memory excessively. You can adjust it to a value that is a multiple of 256KiB.
+func (fs *Fs) SetUploadChunkSizeByte(uploadChunkSize int) {
+	fs.UploadChunkSizeByte = uploadChunkSize
+}
+
 // normSeparators will normalize all "\\" and "/" to the provided separator
 func (fs *Fs) normSeparators(s string) string {
 	return strings.Replace(strings.Replace(s, "\\", fs.separator, -1), "/", fs.separator, -1)
@@ -147,6 +154,7 @@ func (fs *Fs) Create(name string) (*GcsFile, error) {
 		return nil, err
 	}
 	w := obj.NewWriter(fs.ctx)
+
 	err = w.Close()
 	if err != nil {
 		return nil, err

gcsfs/gcs.go

@@ -85,6 +85,10 @@ func (fs *GcsFs) Name() string {
 	return fs.source.Name()
 }
 
+func (fs *GcsFs) SetUploadChunkSizeByte(size int) {
+	fs.source.SetUploadChunkSizeByte(size)
+}
+
 func (fs *GcsFs) Create(name string) (afero.File, error) {
 	return fs.source.Create(name)
 }