Merge branch 'main' of https://github.com/tus/tus-node-server

* 'main' of https://github.com/tus/tus-node-server:
  @tus/s3-store: add `maxMultipartParts` option (#712)
  [ci] release (#701)
  @tus/s3-store: add `minPartSize` option (#703)

Author: Murderlon

.changeset/gold-adults-warn.md

@@ -0,0 +1,5 @@
+---
+"@tus/s3-store": minor
+---
+
+Add `maxMultipartParts` option. This can be used when using S3-compatible storage provider with different part number limitations.

.changeset/proud-terms-swim.md

@@ -1,5 +1,11 @@
 # @tus/gcs-store
 
+## 1.4.2
+
+### Patch Changes
+
+- 8217f5e: Correctly pass the content type from upload.metadata to GCS.
+
 ## 1.4.1
 
 ### Patch Changes

.changeset/tricky-emus-fail.md

@@ -1,14 +1,19 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "@tus/gcs-store",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "Google Cloud Storage for @tus/server",
   "main": "dist/index.js",
   "homepage": "https://github.com/tus/tus-node-server#readme",
   "bugs": "https://github.com/tus/tus-node-server/issues",
   "repository": "tus/tus-node-server",
   "license": "MIT",
-  "files": ["README.md", "LICENSE", "dist", "src"],
+  "files": [
+    "README.md",
+    "LICENSE",
+    "dist",
+    "src"
+  ],
   "scripts": {
     "build": "tsc --build",
     "test": "mocha --timeout 30000 --exit --extension ts --require ts-node/register"

packages/gcs-store/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @tus/s3-store
 
+## 1.8.0
+
+### Minor Changes
+
+- 6351485: Add `minPartSize` option. This can be used alongside `partSize` to guarantee that all non-trailing parts are _exactly_ the same size, which is required for Cloudflare R2.
+
+### Patch Changes
+
+- c970858: Fix zero byte files only storing a .info file. Now correctly stores an empty file.
+
 ## 1.7.0
 
 ### Minor Changes

packages/gcs-store/package.json

@@ -13,6 +13,7 @@
 - [Extensions](#extensions)
 - [Examples](#examples)
   - [Example: using `credentials` to fetch credentials inside a AWS container](#example-using-credentials-to-fetch-credentials-inside-a-aws-container)
+  - [Example: use with Cloudflare R2](#example-use-with-cloudflare-r2)
 - [Types](#types)
 - [Compatibility](#compatibility)
 - [Contribute](#contribute)
@@ -61,10 +62,20 @@ The bucket name.
 
 #### `options.partSize`
 
-The preferred part size for parts send to S3. Can not be lower than 5MiB or more than
+The **preferred** part size for parts send to S3. Can not be lower than 5MiB or more than
 5GiB. The server calculates the optimal part size, which takes this size into account, but
 may increase it to not exceed the S3 10K parts limit.
 
+#### `options.minPartSize`
+
+The minimal part size for parts.
+Can be used to ensure that all non-trailing parts are exactly the same size
+by setting `partSize` and `minPartSize` to the same value.
+Can not be lower than 5MiB or more than 5GiB.
+
+The server calculates the optimal part size, which takes this size into account, but
+may increase it to not exceed the S3 10K parts limit.
+
 #### `options.s3ClientConfig`
 
 Options to pass to the AWS S3 SDK. Checkout the
@@ -182,7 +193,7 @@ docs for the supported values of
 ```js
 const aws = require('aws-sdk')
 const {Server} = require('@tus/server')
-const {FileStore} = require('@tus/s3-store')
+const {S3Store} = require('@tus/s3-store')
 
 const s3Store = new S3Store({
   partSize: 8 * 1024 * 1024,
@@ -199,6 +210,22 @@ const server = new Server({path: '/files', datastore: s3Store})
 // ...
 ```
 
+### Example: use with Cloudflare R2
+
+`@tus/s3-store` can be used with all S3-compatible storage solutions, including Cloudflare R2.
+However R2 requires that all non-trailing parts are _exactly_ the same size.
+This can be achieved by setting `partSize` and `minPartSize` to the same value.
+
+```ts
+// ...
+
+const s3Store = new S3Store({
+  partSize: 8 * 1024 * 1024,
+  minPartSize: 8 * 1024 * 1024,
+  // ...
+})
+```
+
 ## Types
 
 This package is fully typed with TypeScript.

packages/s3-store/CHANGELOG.md

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json.schemastore.org/package.json",
   "name": "@tus/s3-store",
-  "version": "1.7.0",
+  "version": "1.8.0",
   "description": "AWS S3 store for @tus/server",
   "main": "dist/index.js",
   "homepage": "https://github.com/tus/tus-node-server#readme",

packages/s3-store/README.md

@@ -25,10 +25,22 @@ import path from 'node:path'
 const log = debug('tus-node-server:stores:s3store')
 
 export type Options = {
-  // The preferred part size for parts send to S3. Can not be lower than 5MiB or more than 5GiB.
-  // The server calculates the optimal part size, which takes this size into account,
-  // but may increase it to not exceed the S3 10K parts limit.
+  /**
+   * The preferred part size for parts send to S3. Can not be lower than 5MiB or more than 5GiB.
+   * The server calculates the optimal part size, which takes this size into account,
+   * but may increase it to not exceed the S3 10K parts limit.
+   */
   partSize?: number
+  /**
+   * The minimal part size for parts.
+   * Can be used to ensure that all non-trailing parts are exactly the same size.
+   * Can not be lower than 5MiB or more than 5GiB.
+   */
+  minPartSize?: number
+  /**
+   * The maximum number of parts allowed in a multipart upload. Defaults to 10,000.
+   */
+  maxMultipartParts?: number
   useTags?: boolean
   maxConcurrentPartUploads?: number
   cache?: KvStore<MetadataValue>
@@ -89,13 +101,13 @@ export class S3Store extends DataStore {
   protected expirationPeriodInMilliseconds = 0
   protected useTags = true
   protected partUploadSemaphore: Semaphore
-  public maxMultipartParts = 10_000 as const
-  public minPartSize = 5_242_880 as const // 5MiB
+  public maxMultipartParts = 10_000
+  public minPartSize = 5_242_880 // 5MiB
   public maxUploadSize = 5_497_558_138_880 as const // 5TiB
 
   constructor(options: Options) {
     super()
-    const {partSize, s3ClientConfig} = options
+    const {maxMultipartParts, partSize, minPartSize, s3ClientConfig} = options
     const {bucket, ...restS3ClientConfig} = s3ClientConfig
     this.extensions = [
       'creation',
@@ -106,6 +118,12 @@ export class S3Store extends DataStore {
     ]
     this.bucket = bucket
     this.preferredPartSize = partSize || 8 * 1024 * 1024
+    if (minPartSize) {
+      this.minPartSize = minPartSize
+    }
+    if (maxMultipartParts) {
+      this.maxMultipartParts = maxMultipartParts
+    }
     this.expirationPeriodInMilliseconds = options.expirationPeriodInMilliseconds ?? 0
     this.useTags = options.useTags ?? true
     this.cache = options.cache ?? new MemoryKvStore<MetadataValue>()
@@ -509,7 +527,8 @@ export class S3Store extends DataStore {
       optimalPartSize = Math.ceil(size / this.maxMultipartParts)
     }
 
-    return optimalPartSize
+    // Always ensure the part size is at least minPartSize
+    return Math.max(optimalPartSize, this.minPartSize)
   }
 
   /**

packages/s3-store/package.json

@@ -278,6 +278,22 @@ describe('S3DataStore', () => {
     }
   })
 
+  it('should use default maxMultipartParts when not specified', function () {
+    const store = new S3Store({
+      s3ClientConfig,
+    })
+    assert.equal(store.maxMultipartParts, 10000)
+  })
+
+  it('should use custom maxMultipartParts when specified', function () {
+    const customMaxParts = 5000
+    const store = new S3Store({
+      s3ClientConfig,
+      maxMultipartParts: customMaxParts,
+    })
+    assert.equal(store.maxMultipartParts, customMaxParts)
+  })
+
   shared.shouldHaveStoreMethods()
   shared.shouldCreateUploads()
   shared.shouldRemoveUploads() // Termination extension