Skip to content

Commit 00bc1c8

Browse files
authored
staging put: stream file uploads instead of loading all to memory (#197)
`os.ReadFile` reads all of the content of the file into a byte array in memory, which can cause memory consumption pressure for users. Instead, an `os.File` instance is itself a byte reader, and we can provide the file directly to `http.NewRequest` so it can read the file in chunks and upload it as a stream, thus not holding the whole file in memory.
1 parent bb10f7a commit 00bc1c8

File tree

1 file changed

+3
-4
lines changed

1 file changed

+3
-4
lines changed

connection.go

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package dbsql
22

33
import (
4-
"bytes"
54
"context"
65
"database/sql/driver"
76
"encoding/json"
@@ -433,13 +432,13 @@ func (c *conn) handleStagingPut(ctx context.Context, presignedUrl string, header
433432
}
434433
client := &http.Client{}
435434

436-
dat, err := os.ReadFile(localFile)
437-
435+
dat, err := os.Open(localFile)
438436
if err != nil {
439437
return dbsqlerrint.NewDriverError(ctx, "error reading local file", err)
440438
}
439+
defer dat.Close()
441440

442-
req, _ := http.NewRequest("PUT", presignedUrl, bytes.NewReader(dat))
441+
req, _ := http.NewRequest("PUT", presignedUrl, dat)
443442

444443
for k, v := range headers {
445444
req.Header.Set(k, v)

0 commit comments

Comments
 (0)