fix: respect useAuthWithCustomEndpoint flag for resumable uploads (#2637)

Fixes resumable uploads to respect the useAuthWithCustomEndpoint configuration flag when using custom API endpoints.

Previously, resumable uploads would automatically bypass authentication for any non-googleapis.com domain, even when useAuthWithCustomEndpoint: true was explicitly set. This prevented using authenticated proxies or custom storage endpoints with resumable uploads.

Changes:
- Pass useAuthWithCustomEndpoint flag through to resumable upload configuration
- Only bypass authentication if the flag is not explicitly set to true
- Add useAuthWithCustomEndpoint property to Service class for proper access
- Maintains backward compatibility with existing behavior

Fixes #[issue-number]

Author: pm990320

src/file.ts

@@ -1835,6 +1835,7 @@ class File extends ServiceObject<File, FileMetadata> {
         retryOptions: retryOptions,
         params: options?.preconditionOpts || this.instancePreconditionOpts,
         universeDomain: this.bucket.storage.universeDomain,
+        useAuthWithCustomEndpoint: this.storage.useAuthWithCustomEndpoint,
         [GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
       },
       callback!

src/nodejs-common/service.ts

@@ -72,6 +72,11 @@ export interface ServiceConfig {
    * Set to true if the endpoint is a custom URL
    */
   customEndpoint?: boolean;
+
+  /**
+   * Controls whether or not to use authentication when using a custom endpoint.
+   */
+  useAuthWithCustomEndpoint?: boolean;
 }
 
 export interface ServiceOptions extends Omit<GoogleAuthOptions, 'authClient'> {
@@ -98,6 +103,7 @@ export class Service {
   timeout?: number;
   universeDomain: string;
   customEndpoint: boolean;
+  useAuthWithCustomEndpoint?: boolean;
 
   /**
    * Service is a base class, meant to be inherited from by a "service," like
@@ -128,6 +134,7 @@ export class Service {
     this.providedUserAgent = options.userAgent;
     this.universeDomain = options.universeDomain || DEFAULT_UNIVERSE;
     this.customEndpoint = config.customEndpoint || false;
+    this.useAuthWithCustomEndpoint = config.useAuthWithCustomEndpoint;
 
     this.makeAuthenticatedRequest = util.makeAuthenticatedRequestFactory({
       ...config,

src/resumable-upload.ts

@@ -243,6 +243,11 @@ export interface UploadConfig extends Pick<WritableOptions, 'highWaterMark'> {
    */
   retryOptions: RetryOptions;
 
+  /**
+   * Controls whether or not to use authentication when using a custom endpoint.
+   */
+  useAuthWithCustomEndpoint?: boolean;
+
   [GCCL_GCS_CMD_KEY]?: string;
 }
 
@@ -391,9 +396,12 @@ export class Upload extends Writable {
         !isSubDomainOfUniverse &&
         !isSubDomainOfDefaultUniverse
       ) {
-        // a custom, non-universe domain,
-        // use gaxios
-        this.authClient = gaxios;
+        // Check if we should use auth with custom endpoint
+        if (cfg.useAuthWithCustomEndpoint !== true) {
+          // Only bypass auth if explicitly not requested
+          this.authClient = gaxios;
+        }
+        // Otherwise keep the authenticated client
       }
     }
 

test/resumable-upload.ts

@@ -1650,6 +1650,95 @@ describe('resumable-upload', () => {
       assert.deepStrictEqual(res.headers, {});
     });
 
+    it('should use authentication with custom endpoint when useAuthWithCustomEndpoint is true', async () => {
+      up = upload({
+        bucket: BUCKET,
+        file: FILE,
+        customRequestOptions: CUSTOM_REQUEST_OPTIONS,
+        generation: GENERATION,
+        metadata: METADATA,
+        origin: ORIGIN,
+        params: PARAMS,
+        predefinedAcl: PREDEFINED_ACL,
+        userProject: USER_PROJECT,
+        authConfig: {keyFile},
+        apiEndpoint: 'https://custom-proxy.example.com',
+        useAuthWithCustomEndpoint: true,
+        retryOptions: RETRY_OPTIONS,
+      });
+
+      // Mock the authorization request
+      mockAuthorizeRequest();
+
+      // Mock the actual request with auth header expectation
+      const scopes = [
+        nock(REQ_OPTS.url!)
+          .matchHeader('authorization', /Bearer .+/)
+          .get(queryPath)
+          .reply(200, undefined, {}),
+      ];
+
+      const res = await up.makeRequest(REQ_OPTS);
+      scopes.forEach(x => x.done());
+      assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1));
+      // Headers should include authorization
+      assert.ok(res.config.headers?.['Authorization']);
+    });
+
+    it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is false', async () => {
+      up = upload({
+        bucket: BUCKET,
+        file: FILE,
+        customRequestOptions: CUSTOM_REQUEST_OPTIONS,
+        generation: GENERATION,
+        metadata: METADATA,
+        origin: ORIGIN,
+        params: PARAMS,
+        predefinedAcl: PREDEFINED_ACL,
+        userProject: USER_PROJECT,
+        authConfig: {keyFile},
+        apiEndpoint: 'https://storage-emulator.local',
+        useAuthWithCustomEndpoint: false,
+        retryOptions: RETRY_OPTIONS,
+      });
+
+      const scopes = [
+        nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}),
+      ];
+      const res = await up.makeRequest(REQ_OPTS);
+      scopes.forEach(x => x.done());
+      assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1));
+      // When auth is bypassed, no auth headers should be present
+      assert.deepStrictEqual(res.headers, {});
+    });
+
+    it('should bypass authentication with custom endpoint when useAuthWithCustomEndpoint is undefined (backward compatibility)', async () => {
+      up = upload({
+        bucket: BUCKET,
+        file: FILE,
+        customRequestOptions: CUSTOM_REQUEST_OPTIONS,
+        generation: GENERATION,
+        metadata: METADATA,
+        origin: ORIGIN,
+        params: PARAMS,
+        predefinedAcl: PREDEFINED_ACL,
+        userProject: USER_PROJECT,
+        authConfig: {keyFile},
+        apiEndpoint: 'https://storage-emulator.local',
+        // useAuthWithCustomEndpoint is intentionally not set
+        retryOptions: RETRY_OPTIONS,
+      });
+
+      const scopes = [
+        nock(REQ_OPTS.url!).get(queryPath).reply(200, undefined, {}),
+      ];
+      const res = await up.makeRequest(REQ_OPTS);
+      scopes.forEach(x => x.done());
+      assert.strictEqual(res.config.url, REQ_OPTS.url + queryPath.slice(1));
+      // When auth is bypassed (backward compatibility), no auth headers should be present
+      assert.deepStrictEqual(res.headers, {});
+    });
+
     it('should combine customRequestOptions', done => {
       const up = upload({
         bucket: BUCKET,