test: reproduce commonly cited timeout error

mock-server/datastore-server.ts

@@ -12,6 +12,9 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import {ServiceError} from 'google-gax';
+import {Server} from '@grpc/grpc-js';
+
 const {dirname, resolve} = require('node:path');
 
 const PROTO_PATH = __dirname + '/../protos/google/datastore/v1/datastore.proto';
@@ -37,25 +40,38 @@
 });
 const descriptor = grpc.loadPackageDefinition(packageDefinition);
 
-/**
- * Implements the runQuery RPC method.
- */
-function grpcEndpoint(
-  call: {},
-  callback: (arg1: string | null, arg2: {}) => {},
-) {
-  // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST.
-  callback(null, {message: 'Hello'});
+export type CallType = any;
+export type SuccessType = any;
+export type GrpcErrorType = ServiceError | null;
+
+type MockServiceCallback = (arg1: GrpcErrorType, arg2: SuccessType) => {};
+
+interface MockServiceConfiguration {
+  [endpoint: string]: (call: CallType, callback: MockServiceCallback) => void;
 }
 
 /**
  * Starts an RPC server that receives requests for datastore
  */
-export function startServer(cb: () => void) {
+export function startServer(
+  cb: () => void,
+  serviceConfigurationOverride?: MockServiceConfiguration,
+): Server {
+  /**
+   * Implements the runQuery RPC method.
+   */
+  function grpcEndpoint(call: CallType, callback: MockServiceCallback) {
+    // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST.
+    callback(null, {message: 'Hello'});
+  }
+
   const server = new grpc.Server();
   const service = descriptor.google.datastore.v1.Datastore.service;
   // On the next line, change runQuery to the grpc method you want to investigate
-  server.addService(service, {runQuery: grpcEndpoint});
+  const serviceConfiguration = serviceConfigurationOverride ?? {
+    runQuery: grpcEndpoint,
+  };
+  server.addService(service, serviceConfiguration);
   server.bindAsync(
     '0.0.0.0:50051',
     grpc.ServerCredentials.createInsecure(),
@@ -64,4 +80,5 @@
       cb();
     },
   );
+  return server;
 }

test/mock-server-tester.ts

@@ -0,0 +1,126 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {
+  CallType,
+  GrpcErrorType,
+  SuccessType,
+} from '../mock-server/datastore-server';
+
+const grpc = require('@grpc/grpc-js');
+import {ServiceError} from 'google-gax';
+import {Server} from '@grpc/grpc-js';
+
+/**
+ * Simulates a non-retryable error response from a gRPC service.
+ *
+ * This function is designed for use in mock server setups during testing.
+ * When invoked as a gRPC endpoint handler, it responds with a pre-configured
+ * error object representing a non-retryable error. The error includes a
+ * NOT_FOUND code (5), a fixed details message, and some metadata.
+ *
+ * @param {any} call - The gRPC call object, representing the incoming request.
+ * @param {function} callback - The callback function to be invoked with the
+ *   simulated error response. It expects two arguments: an error object and an
+ *   empty object (representing no response data).
+ */
+export function sendNonRetryableError(
+  call: CallType,
+  callback: (arg1: GrpcErrorType, arg2: SuccessType) => {},
+) {
+  const metadata = new grpc.Metadata();
+  metadata.set(
+    'grpc-server-stats-bin',
+    Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]),
+  );
+  const error = new Error('error message') as ServiceError;
+  error.code = 5;
+  error.details = 'error details';
+  error.metadata = metadata;
+  callback(error, {});
+}
+
+export class ErrorGenerator {
+  private errorSeriesCount = 0;
+
+  /**
+   * Generates an error object for testing purposes.
+   *
+   * This method creates a `ServiceError` object, simulating an error
+   * that might be returned by a gRPC service. The error includes a
+   * `DEADLINE_EXCEEDED` code (4), a details message indicating the number of
+   * errors generated so far by this instance, and some metadata.
+   *
+   * @returns {ServiceError} A `ServiceError` object representing a simulated
+   *   gRPC error.
+   */
+  generateError() {
+    // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST.
+    this.errorSeriesCount++;
+    const metadata = new grpc.Metadata();
+    metadata.set(
+      'grpc-server-stats-bin',
+      Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]),
+    );
+    const error = new Error('error message') as ServiceError;
+    error.code = 4;
+    error.details = `error details: error count: ${this.errorSeriesCount}`;
+    error.metadata = metadata;
+    return error;
+  }
+
+  /**
+   * Returns a function that simulates an error response from a gRPC service.
+   *
+   * This method is designed to be used in mock server setups for testing purposes.
+   * It returns a function that, when called, will invoke a callback with a
+   * pre-configured error object, simulating a gRPC service responding with an error.
+   * The error includes a DEADLINE_EXCEEDED code (4), a details message indicating the
+   * number of errors generated so far by this instance, and some metadata.
+   *
+   * @returns {function} A function that takes a `call` object (representing the
+   *   gRPC call) and a `callback` function, and responds to the call with a
+   *   simulated error.
+   */
+  sendErrorSeries() {
+    return (
+      call: CallType,
+      callback: (arg1: GrpcErrorType, arg2: SuccessType) => {},
+    ) => {
+      const error = this.generateError();
+      callback(error, {});
+    };
+  }
+}
+
+export function shutdownServer(server: Server) {
+  return new Promise((resolve, reject) => {
+    // Assuming 'server.tryShutdown' is a function that takes a callback.
+    // The callback is expected to be called when the shutdown attempt is complete.
+    // If 'tryShutdown' itself can throw an error or indicate an immediate failure
+    // (without calling the callback), you might need to wrap this call in a try...catch block.
+
+    server.tryShutdown((error: Error | undefined) => {
+      if (error) {
+        // If the callback is called with an error, reject the promise.
+        console.error('Server shutdown failed:', error);
+        reject(error);
+      } else {
+        // If the callback is called without an error, resolve the promise.
+        console.log('Server has been shut down successfully.');
+        resolve('done');
+      }
+    });
+  });
+}

test/try-server.ts

@@ -17,17 +17,21 @@ import {Datastore} from '../src';
 
 import {startServer} from '../mock-server/datastore-server';
 
-describe('Try server', () => {
-  it.skip('should try to connect to the running server', done => {
+describe.skip('Try server', () => {
+  it('should try to connect to the running server', done => {
     startServer(async () => {
-      const datastore = new Datastore({
-        namespace: `${Date.now()}`,
-        apiEndpoint: 'localhost:50051',
-      });
-      const postKey = datastore.key(['Post', 'post1']);
-      const query = datastore.createQuery('Post').hasAncestor(postKey);
-      const allResults = await datastore.runQuery(query);
-      done();
+      try {
+        const datastore = new Datastore({
+          namespace: `${Date.now()}`,
+          apiEndpoint: 'localhost:50051',
+        });
+        const postKey = datastore.key(['Post', 'post1']);
+        const query = datastore.createQuery('Post').hasAncestor(postKey);
+        const allResults = await datastore.runQuery(query);
+        done();
+      } catch (e) {
+        done(e);
+      }
     });
   });
 });

test/with-createreadstream-mockserver.ts

@@ -0,0 +1,51 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {describe, it} from 'mocha';
+import {Datastore} from '../src';
+import * as assert from 'assert';
+
+import {startServer} from '../mock-server/datastore-server';
+import {sendNonRetryableError} from './mock-server-tester';
+
+describe.skip('Should make calls to lookup', () => {
+  it('should report a NOT_FOUND error to the user when it occurs', done => {
+    startServer(
+      async () => {
+        try {
+          try {
+            const datastore = new Datastore({
+              namespace: `${Date.now()}`,
+              apiEndpoint: 'localhost:50051',
+            });
+            const postKey = datastore.key(['Post', 'post1']);
+            await datastore.get(postKey);
+            console.log('call failed');
+            assert.fail('The call should not have succeeded');
+          } catch (e) {
+            // The test should produce the right error message here for the user.
+            assert.strictEqual(
+              (e as Error).message,
+              '5 NOT_FOUND: error details',
+            );
+            done();
+          }
+        } catch (e) {
+          done(e);
+        }
+      },
+      {lookup: sendNonRetryableError},
+    );
+  });
+});

test/with-runquery-mockserver.ts

@@ -0,0 +1,56 @@
+// Copyright 2025 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {describe, it} from 'mocha';
+import {Datastore} from '../src';
+import * as assert from 'assert';
+
+import {startServer} from '../mock-server/datastore-server';
+import {ErrorGenerator, shutdownServer} from './mock-server-tester';
+
+describe.skip('Should make calls to runQuery', () => {
+  it('should report a DEADLINE_EXCEEDED error to the user when it occurs with the original error details', done => {
+    const errorGenerator = new ErrorGenerator();
+    const server = startServer(
+      async () => {
+        try {
+          try {
+            const datastore = new Datastore({
+              namespace: `${Date.now()}`,
+              apiEndpoint: 'localhost:50051',
+            });
+            const postKey = datastore.key(['Post', 'post1']);
+            const query = datastore.createQuery('Post').hasAncestor(postKey);
+            await datastore.runQuery(query);
+            console.log('call failed');
+            assert.fail('The call should not have succeeded');
+          } catch (e) {
+            // The test should produce the right error message here for the user.
+            // TODO: Later on we are going to decide on what the error message should be
+            // The error message is based on client library behavior.
+            assert.strictEqual(
+              (e as Error).message,
+              '4 DEADLINE_EXCEEDED: error details: error count: 1',
+            );
+            await shutdownServer(server);
+            done();
+          }
+        } catch (e) {
+          done(e);
+        }
+      },
+      {runQuery: errorGenerator.sendErrorSeries()},
+    );
+  });
+});