From a6d82eff9a5ceae788efc357cdf3f4fa4634012f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 10:49:02 -0400 Subject: [PATCH 01/43] send error back in the mock server --- mock-server/datastore-server.ts | 18 +++++++++++++++--- test/try-server.ts | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index c2e1ee5f2..e07e6e5c2 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import {ServiceError} from 'google-gax'; + const {dirname, resolve} = require('node:path'); const PROTO_PATH = __dirname + '/../protos/google/datastore/v1/datastore.proto'; @@ -41,11 +43,21 @@ const descriptor = grpc.loadPackageDefinition(packageDefinition); * Implements the runQuery RPC method. */ function grpcEndpoint( - call: {}, - callback: (arg1: string | null, arg2: {}) => {}, + call: any, + callback: (arg1: any, arg2: {}) => {}, ) { // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. - callback(null, {message: 'Hello'}); + //callback(null, {message: 'Hello'}); + const metadata = new grpc.Metadata(); + metadata.set( + 'grpc-server-stats-bin', + Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), + ); + const error = new Error('error message') as ServiceError; + error.code = 5; + error.details = 'error details'; + error.metadata = metadata; + callback(error, {}); } /** diff --git a/test/try-server.ts b/test/try-server.ts index 28f67d0d7..5bfcd821a 100644 --- a/test/try-server.ts +++ b/test/try-server.ts @@ -18,7 +18,7 @@ import {Datastore} from '../src'; import {startServer} from '../mock-server/datastore-server'; describe('Try server', () => { - it.skip('should try to connect to the running server', done => { + it.only('should try to connect to the running server', done => { startServer(async () => { const datastore = new Datastore({ namespace: `${Date.now()}`, From b09bab327268a1f8bd3158a669e289aba37fbf4e Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:39:09 -0400 Subject: [PATCH 02/43] Mock server error tests with fixes - Added tests to ensure error is passed back to user - Changed source code to use passthrough for runQuery - changed the mock server to allow overriding of the default service --- mock-server/datastore-server.ts | 33 ++++++++------- src/request.ts | 2 +- test/grpc-endpoint.ts | 16 ++++++++ test/try-server.ts | 22 +++++----- test/with-createreadstream-mockserver.ts | 50 +++++++++++++++++++++++ test/with-runquery-mockserver.ts | 51 ++++++++++++++++++++++++ 6 files changed, 147 insertions(+), 27 deletions(-) create mode 100644 test/grpc-endpoint.ts create mode 100644 test/with-createreadstream-mockserver.ts create mode 100644 test/with-runquery-mockserver.ts diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index e07e6e5c2..4c8d6e63d 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -42,32 +42,31 @@ const descriptor = grpc.loadPackageDefinition(packageDefinition); /** * Implements the runQuery RPC method. */ -function grpcEndpoint( - call: any, - callback: (arg1: any, arg2: {}) => {}, -) { +function grpcEndpoint(call: any, callback: MockServiceCallback) { // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. - //callback(null, {message: 'Hello'}); - const metadata = new grpc.Metadata(); - metadata.set( - 'grpc-server-stats-bin', - Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), - ); - const error = new Error('error message') as ServiceError; - error.code = 5; - error.details = 'error details'; - error.metadata = metadata; - callback(error, {}); + callback(null, {message: 'Hello'}); +} + +type MockServiceCallback = (arg1: any, arg2: {}) => {}; + +interface MockServiceConfiguration { + [endpoint: string]: (call: any, callback: MockServiceCallback) => void; } /** * Starts an RPC server that receives requests for datastore */ -export function startServer(cb: () => void) { +export function startServer( + cb: () => void, + serviceConfigurationOverride?: MockServiceConfiguration, +) { const server = new grpc.Server(); const service = descriptor.google.datastore.v1.Datastore.service; // On the next line, change runQuery to the grpc method you want to investigate - server.addService(service, {runQuery: grpcEndpoint}); + const serviceConfiguration = serviceConfigurationOverride ?? { + runQuery: grpcEndpoint, + }; + server.addService(service, serviceConfiguration); server.bindAsync( '0.0.0.0:50051', grpc.ServerCredentials.createInsecure(), diff --git a/src/request.ts b/src/request.ts index 132f9011c..e25d85c63 100644 --- a/src/request.ts +++ b/src/request.ts @@ -1081,7 +1081,7 @@ class DatastoreRequest { }); }; - const stream = streamEvents(new Transform({objectMode: true})); + const stream = streamEvents(new PassThrough({objectMode: true})); stream.once('reading', () => { makeRequest(query); }); diff --git a/test/grpc-endpoint.ts b/test/grpc-endpoint.ts new file mode 100644 index 000000000..628853f1b --- /dev/null +++ b/test/grpc-endpoint.ts @@ -0,0 +1,16 @@ +const grpc = require('@grpc/grpc-js'); +import {ServiceError} from 'google-gax'; + +export function grpcEndpoint(call: any, callback: (arg1: any, arg2: {}) => {}) { + // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. + const metadata = new grpc.Metadata(); + metadata.set( + 'grpc-server-stats-bin', + Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), + ); + const error = new Error('error message') as ServiceError; + error.code = 5; + error.details = 'error details'; + error.metadata = metadata; + callback(error, {}); +} diff --git a/test/try-server.ts b/test/try-server.ts index 5bfcd821a..1cebce658 100644 --- a/test/try-server.ts +++ b/test/try-server.ts @@ -18,16 +18,20 @@ import {Datastore} from '../src'; import {startServer} from '../mock-server/datastore-server'; describe('Try server', () => { - it.only('should try to connect to the running server', done => { + it('should try to connect to the running server', done => { startServer(async () => { - const datastore = new Datastore({ - namespace: `${Date.now()}`, - apiEndpoint: 'localhost:50051', - }); - const postKey = datastore.key(['Post', 'post1']); - const query = datastore.createQuery('Post').hasAncestor(postKey); - const allResults = await datastore.runQuery(query); - done(); + try { + const datastore = new Datastore({ + namespace: `${Date.now()}`, + apiEndpoint: 'localhost:50051', + }); + const postKey = datastore.key(['Post', 'post1']); + const query = datastore.createQuery('Post').hasAncestor(postKey); + const allResults = await datastore.runQuery(query); + done(); + } catch (e) { + done(e); + } }); }); }); diff --git a/test/with-createreadstream-mockserver.ts b/test/with-createreadstream-mockserver.ts new file mode 100644 index 000000000..93ea42084 --- /dev/null +++ b/test/with-createreadstream-mockserver.ts @@ -0,0 +1,50 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {describe, it} from 'mocha'; +import {Datastore} from '../src'; +import * as assert from 'assert'; + +import {startServer} from '../mock-server/datastore-server'; +import {grpcEndpoint} from './grpc-endpoint'; + +describe.only('lookup', () => { + it('should report an error to the user when it occurs', done => { + startServer( + async () => { + try { + try { + const datastore = new Datastore({ + namespace: `${Date.now()}`, + apiEndpoint: 'localhost:50051', + }); + const postKey = datastore.key(['Post', 'post1']); + await datastore.get(postKey); + assert.fail('The call should not have succeeded'); + } catch (e) { + // The test should produce the right error message here for the user. + assert.strictEqual( + (e as Error).message, + '5 NOT_FOUND: error details', + ); + done(); + } + } catch (e) { + done(e); + } + }, + {lookup: grpcEndpoint}, + ); + }); +}); diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts new file mode 100644 index 000000000..124fa7b9c --- /dev/null +++ b/test/with-runquery-mockserver.ts @@ -0,0 +1,51 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {describe, it} from 'mocha'; +import {Datastore} from '../src'; +import * as assert from 'assert'; + +import {startServer} from '../mock-server/datastore-server'; +import {grpcEndpoint} from './grpc-endpoint'; + +describe('runQuery', () => { + it('should report an error to the user when it occurs', done => { + startServer( + async () => { + try { + try { + const datastore = new Datastore({ + namespace: `${Date.now()}`, + apiEndpoint: 'localhost:50051', + }); + const postKey = datastore.key(['Post', 'post1']); + const query = datastore.createQuery('Post').hasAncestor(postKey); + await datastore.runQuery(query); + assert.fail('The call should not have succeeded'); + } catch (e) { + // The test should produce the right error message here for the user. + assert.strictEqual( + (e as Error).message, + '5 NOT_FOUND: error details', + ); + done(); + } + } catch (e) { + done(e); + } + }, + {runQuery: grpcEndpoint}, + ); + }); +}); From 68c888b8e2e6a67cf7a0a27a18c437fa20231405 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:41:34 -0400 Subject: [PATCH 03/43] Change the source code back - no problems --- src/request.ts | 2 +- test/with-createreadstream-mockserver.ts | 2 +- test/with-runquery-mockserver.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/request.ts b/src/request.ts index e25d85c63..132f9011c 100644 --- a/src/request.ts +++ b/src/request.ts @@ -1081,7 +1081,7 @@ class DatastoreRequest { }); }; - const stream = streamEvents(new PassThrough({objectMode: true})); + const stream = streamEvents(new Transform({objectMode: true})); stream.once('reading', () => { makeRequest(query); }); diff --git a/test/with-createreadstream-mockserver.ts b/test/with-createreadstream-mockserver.ts index 93ea42084..c26fdf77f 100644 --- a/test/with-createreadstream-mockserver.ts +++ b/test/with-createreadstream-mockserver.ts @@ -19,7 +19,7 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {grpcEndpoint} from './grpc-endpoint'; -describe.only('lookup', () => { +describe('lookup', () => { it('should report an error to the user when it occurs', done => { startServer( async () => { diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 124fa7b9c..2475381dc 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -19,7 +19,7 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {grpcEndpoint} from './grpc-endpoint'; -describe('runQuery', () => { +describe.only('runQuery', () => { it('should report an error to the user when it occurs', done => { startServer( async () => { From e9c0addf5a4e8bc462393dc2c0e5a80878f4a99f Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:43:26 -0400 Subject: [PATCH 04/43] Add some console logs --- test/with-createreadstream-mockserver.ts | 1 + test/with-runquery-mockserver.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/test/with-createreadstream-mockserver.ts b/test/with-createreadstream-mockserver.ts index c26fdf77f..57f2533c5 100644 --- a/test/with-createreadstream-mockserver.ts +++ b/test/with-createreadstream-mockserver.ts @@ -31,6 +31,7 @@ describe('lookup', () => { }); const postKey = datastore.key(['Post', 'post1']); await datastore.get(postKey); + console.log('call failed'); assert.fail('The call should not have succeeded'); } catch (e) { // The test should produce the right error message here for the user. diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 2475381dc..73ffddd9c 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -32,6 +32,7 @@ describe.only('runQuery', () => { const postKey = datastore.key(['Post', 'post1']); const query = datastore.createQuery('Post').hasAncestor(postKey); await datastore.runQuery(query); + console.log('call failed'); assert.fail('The call should not have succeeded'); } catch (e) { // The test should produce the right error message here for the user. From 57e5064239c3e85daaed08b8a41898105957abd7 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:51:06 -0400 Subject: [PATCH 05/43] Add mechanism for shutting down the server --- mock-server/datastore-server.ts | 1 + test/with-runquery-mockserver.ts | 36 +++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index 4c8d6e63d..c2aa366fc 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -75,4 +75,5 @@ export function startServer( cb(); }, ); + return server; } diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 73ffddd9c..8a15269ab 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -19,9 +19,42 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {grpcEndpoint} from './grpc-endpoint'; +function shutdownServer(server: any) { + return new Promise((resolve, reject) => { + // Assuming 'server.tryShutdown' is a function that takes a callback. + // The callback is expected to be called when the shutdown attempt is complete. + // If 'tryShutdown' itself can throw an error or indicate an immediate failure + // (without calling the callback), you might need to wrap this call in a try...catch block. + + server.tryShutdown((error: any) => { + if (error) { + // If the callback is called with an error, reject the promise. + console.error('Server shutdown failed:', error); + reject(error); + } else { + // If the callback is called without an error, resolve the promise. + console.log('Server has been shut down successfully.'); + resolve('done'); + } + }); + + // It's also good practice to consider scenarios where `tryShutdown` + // might not call the callback at all in certain failure cases. + // Depending on the specifics of `server.tryShutdown`, + // you might want to add a timeout to prevent the promise from hanging indefinitely. + // For example: + // const timeoutId = setTimeout(() => { + // reject(new Error('Server shutdown timed out.')); + // }, 30000); // 30 seconds timeout + // + // And then in the callback: + // clearTimeout(timeoutId); + }); +} + describe.only('runQuery', () => { it('should report an error to the user when it occurs', done => { - startServer( + const server = startServer( async () => { try { try { @@ -40,6 +73,7 @@ describe.only('runQuery', () => { (e as Error).message, '5 NOT_FOUND: error details', ); + await shutdownServer(server); done(); } } catch (e) { From cbf59bffd5e031954410174f1621d5fc586d3006 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:52:32 -0400 Subject: [PATCH 06/43] Move the shutdown server function --- test/grpc-endpoint.ts | 33 ++++++++++++++++++++++++++++++ test/with-runquery-mockserver.ts | 35 +------------------------------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/test/grpc-endpoint.ts b/test/grpc-endpoint.ts index 628853f1b..8417c63eb 100644 --- a/test/grpc-endpoint.ts +++ b/test/grpc-endpoint.ts @@ -14,3 +14,36 @@ export function grpcEndpoint(call: any, callback: (arg1: any, arg2: {}) => {}) { error.metadata = metadata; callback(error, {}); } + +export function shutdownServer(server: any) { + return new Promise((resolve, reject) => { + // Assuming 'server.tryShutdown' is a function that takes a callback. + // The callback is expected to be called when the shutdown attempt is complete. + // If 'tryShutdown' itself can throw an error or indicate an immediate failure + // (without calling the callback), you might need to wrap this call in a try...catch block. + + server.tryShutdown((error: any) => { + if (error) { + // If the callback is called with an error, reject the promise. + console.error('Server shutdown failed:', error); + reject(error); + } else { + // If the callback is called without an error, resolve the promise. + console.log('Server has been shut down successfully.'); + resolve('done'); + } + }); + + // It's also good practice to consider scenarios where `tryShutdown` + // might not call the callback at all in certain failure cases. + // Depending on the specifics of `server.tryShutdown`, + // you might want to add a timeout to prevent the promise from hanging indefinitely. + // For example: + // const timeoutId = setTimeout(() => { + // reject(new Error('Server shutdown timed out.')); + // }, 30000); // 30 seconds timeout + // + // And then in the callback: + // clearTimeout(timeoutId); + }); +} diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 8a15269ab..a84d32f88 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -17,40 +17,7 @@ import {Datastore} from '../src'; import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; -import {grpcEndpoint} from './grpc-endpoint'; - -function shutdownServer(server: any) { - return new Promise((resolve, reject) => { - // Assuming 'server.tryShutdown' is a function that takes a callback. - // The callback is expected to be called when the shutdown attempt is complete. - // If 'tryShutdown' itself can throw an error or indicate an immediate failure - // (without calling the callback), you might need to wrap this call in a try...catch block. - - server.tryShutdown((error: any) => { - if (error) { - // If the callback is called with an error, reject the promise. - console.error('Server shutdown failed:', error); - reject(error); - } else { - // If the callback is called without an error, resolve the promise. - console.log('Server has been shut down successfully.'); - resolve('done'); - } - }); - - // It's also good practice to consider scenarios where `tryShutdown` - // might not call the callback at all in certain failure cases. - // Depending on the specifics of `server.tryShutdown`, - // you might want to add a timeout to prevent the promise from hanging indefinitely. - // For example: - // const timeoutId = setTimeout(() => { - // reject(new Error('Server shutdown timed out.')); - // }, 30000); // 30 seconds timeout - // - // And then in the callback: - // clearTimeout(timeoutId); - }); -} +import {grpcEndpoint, shutdownServer} from './grpc-endpoint'; describe.only('runQuery', () => { it('should report an error to the user when it occurs', done => { From eb61f469d07c07bee026f577fdb8d66b9ff005aa Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:53:56 -0400 Subject: [PATCH 07/43] Rename the service method to sendNonRetryableError --- test/grpc-endpoint.ts | 2 +- test/with-createreadstream-mockserver.ts | 4 ++-- test/with-runquery-mockserver.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/grpc-endpoint.ts b/test/grpc-endpoint.ts index 8417c63eb..7841dd29a 100644 --- a/test/grpc-endpoint.ts +++ b/test/grpc-endpoint.ts @@ -1,7 +1,7 @@ const grpc = require('@grpc/grpc-js'); import {ServiceError} from 'google-gax'; -export function grpcEndpoint(call: any, callback: (arg1: any, arg2: {}) => {}) { +export function sendNonRetryableError(call: any, callback: (arg1: any, arg2: {}) => {}) { // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. const metadata = new grpc.Metadata(); metadata.set( diff --git a/test/with-createreadstream-mockserver.ts b/test/with-createreadstream-mockserver.ts index 57f2533c5..47d37e678 100644 --- a/test/with-createreadstream-mockserver.ts +++ b/test/with-createreadstream-mockserver.ts @@ -17,7 +17,7 @@ import {Datastore} from '../src'; import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; -import {grpcEndpoint} from './grpc-endpoint'; +import {sendNonRetryableError} from './grpc-endpoint'; describe('lookup', () => { it('should report an error to the user when it occurs', done => { @@ -45,7 +45,7 @@ describe('lookup', () => { done(e); } }, - {lookup: grpcEndpoint}, + {lookup: sendNonRetryableError}, ); }); }); diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index a84d32f88..59ed82531 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -17,7 +17,7 @@ import {Datastore} from '../src'; import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; -import {grpcEndpoint, shutdownServer} from './grpc-endpoint'; +import {sendNonRetryableError, shutdownServer} from './grpc-endpoint'; describe.only('runQuery', () => { it('should report an error to the user when it occurs', done => { @@ -47,7 +47,7 @@ describe.only('runQuery', () => { done(e); } }, - {runQuery: grpcEndpoint}, + {runQuery: sendNonRetryableError}, ); }); }); From 19fbec5f5ae750b558390b6ae2974a9432699148 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:55:01 -0400 Subject: [PATCH 08/43] Delete shutdown comment --- test/grpc-endpoint.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/grpc-endpoint.ts b/test/grpc-endpoint.ts index 7841dd29a..2f83a0fc2 100644 --- a/test/grpc-endpoint.ts +++ b/test/grpc-endpoint.ts @@ -33,17 +33,5 @@ export function shutdownServer(server: any) { resolve('done'); } }); - - // It's also good practice to consider scenarios where `tryShutdown` - // might not call the callback at all in certain failure cases. - // Depending on the specifics of `server.tryShutdown`, - // you might want to add a timeout to prevent the promise from hanging indefinitely. - // For example: - // const timeoutId = setTimeout(() => { - // reject(new Error('Server shutdown timed out.')); - // }, 30000); // 30 seconds timeout - // - // And then in the callback: - // clearTimeout(timeoutId); }); } From 4c230a1ffafa977b96e887ea8faca96a4d5d12c1 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 11:58:21 -0400 Subject: [PATCH 09/43] Only run the run query test --- test/try-server.ts | 2 +- test/with-createreadstream-mockserver.ts | 2 +- test/with-runquery-mockserver.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/try-server.ts b/test/try-server.ts index 1cebce658..458a18c07 100644 --- a/test/try-server.ts +++ b/test/try-server.ts @@ -17,7 +17,7 @@ import {Datastore} from '../src'; import {startServer} from '../mock-server/datastore-server'; -describe('Try server', () => { +describe.skip('Try server', () => { it('should try to connect to the running server', done => { startServer(async () => { try { diff --git a/test/with-createreadstream-mockserver.ts b/test/with-createreadstream-mockserver.ts index 47d37e678..104b606cd 100644 --- a/test/with-createreadstream-mockserver.ts +++ b/test/with-createreadstream-mockserver.ts @@ -19,7 +19,7 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {sendNonRetryableError} from './grpc-endpoint'; -describe('lookup', () => { +describe.skip('lookup', () => { it('should report an error to the user when it occurs', done => { startServer( async () => { diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 59ed82531..4a15f7e0f 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -19,7 +19,7 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {sendNonRetryableError, shutdownServer} from './grpc-endpoint'; -describe.only('runQuery', () => { +describe('runQuery', () => { it('should report an error to the user when it occurs', done => { const server = startServer( async () => { From 30a7838073f4b1cc84e441cdf9003c1590bd31d8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 13:00:11 -0400 Subject: [PATCH 10/43] Eliminate unused import --- mock-server/datastore-server.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index c2aa366fc..9dea22294 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import {ServiceError} from 'google-gax'; - const {dirname, resolve} = require('node:path'); const PROTO_PATH = __dirname + '/../protos/google/datastore/v1/datastore.proto'; From ae6f01c9d535fd5ba7f2d61f56288048dbd08bc8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 13:23:58 -0400 Subject: [PATCH 11/43] Complete reproduction of the timeout error --- test/grpc-endpoint.ts | 27 ++++++++++++++++++++++++++- test/with-runquery-mockserver.ts | 6 +++--- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/test/grpc-endpoint.ts b/test/grpc-endpoint.ts index 2f83a0fc2..093b2424d 100644 --- a/test/grpc-endpoint.ts +++ b/test/grpc-endpoint.ts @@ -2,7 +2,6 @@ const grpc = require('@grpc/grpc-js'); import {ServiceError} from 'google-gax'; export function sendNonRetryableError(call: any, callback: (arg1: any, arg2: {}) => {}) { - // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. const metadata = new grpc.Metadata(); metadata.set( 'grpc-server-stats-bin', @@ -15,6 +14,32 @@ export function sendNonRetryableError(call: any, callback: (arg1: any, arg2: {}) callback(error, {}); } +// TODO: Encapsulate this in a class: +let errorSeriesCount = 0; + +function generateError() { + // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. + errorSeriesCount++; + const metadata = new grpc.Metadata(); + metadata.set( + 'grpc-server-stats-bin', + Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), + ); + const error = new Error('error message') as ServiceError; + error.code = 4; + error.details = `error details: error count: ${errorSeriesCount}`; + error.metadata = metadata; + return error; +} + +export function sendErrorSeries( + call: any, + callback: (arg1: any, arg2: {}) => {}, +) { + const error = generateError(); + callback(error, {}); +} + export function shutdownServer(server: any) { return new Promise((resolve, reject) => { // Assuming 'server.tryShutdown' is a function that takes a callback. diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 4a15f7e0f..d64d6be1e 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -17,9 +17,9 @@ import {Datastore} from '../src'; import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; -import {sendNonRetryableError, shutdownServer} from './grpc-endpoint'; +import {sendErrorSeries, shutdownServer} from './grpc-endpoint'; -describe('runQuery', () => { +describe.only('runQuery', () => { it('should report an error to the user when it occurs', done => { const server = startServer( async () => { @@ -47,7 +47,7 @@ describe('runQuery', () => { done(e); } }, - {runQuery: sendNonRetryableError}, + {runQuery: sendErrorSeries}, ); }); }); From e36bd3d568614f2d24c1db8a8a930e927b41b9a5 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 15:09:30 -0400 Subject: [PATCH 12/43] Rename the file to more accurately describe its use --- test/{grpc-endpoint.ts => mock-server-tester.ts} | 0 test/with-createreadstream-mockserver.ts | 2 +- test/with-runquery-mockserver.ts | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename test/{grpc-endpoint.ts => mock-server-tester.ts} (100%) diff --git a/test/grpc-endpoint.ts b/test/mock-server-tester.ts similarity index 100% rename from test/grpc-endpoint.ts rename to test/mock-server-tester.ts diff --git a/test/with-createreadstream-mockserver.ts b/test/with-createreadstream-mockserver.ts index 104b606cd..6c3567931 100644 --- a/test/with-createreadstream-mockserver.ts +++ b/test/with-createreadstream-mockserver.ts @@ -17,7 +17,7 @@ import {Datastore} from '../src'; import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; -import {sendNonRetryableError} from './grpc-endpoint'; +import {sendNonRetryableError} from './mock-server-tester'; describe.skip('lookup', () => { it('should report an error to the user when it occurs', done => { diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index d64d6be1e..27d0a8ac3 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -17,7 +17,7 @@ import {Datastore} from '../src'; import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; -import {sendErrorSeries, shutdownServer} from './grpc-endpoint'; +import {sendErrorSeries, shutdownServer} from './mock-server-tester'; describe.only('runQuery', () => { it('should report an error to the user when it occurs', done => { From 60303d6f0b7584647522318411f49e737000c26e Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 15:11:28 -0400 Subject: [PATCH 13/43] Change the error message --- test/with-runquery-mockserver.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 27d0a8ac3..fb87c6aa6 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -36,9 +36,11 @@ describe.only('runQuery', () => { assert.fail('The call should not have succeeded'); } catch (e) { // The test should produce the right error message here for the user. + // TODO: Later on we are going to decide on what the error message should be + // The error message is based on client library behavior. assert.strictEqual( (e as Error).message, - '5 NOT_FOUND: error details', + '4 DEADLINE_EXCEEDED: error details', ); await shutdownServer(server); done(); From 0005a22703f36c010440ad5eb6fa6342fda533fc Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 15:22:22 -0400 Subject: [PATCH 14/43] Add a class for containing the error series --- test/mock-server-tester.ts | 45 ++++++++++++++++---------------- test/with-runquery-mockserver.ts | 7 ++--- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index 093b2424d..8bc3dc7de 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -14,30 +14,31 @@ export function sendNonRetryableError(call: any, callback: (arg1: any, arg2: {}) callback(error, {}); } -// TODO: Encapsulate this in a class: -let errorSeriesCount = 0; +export class ErrorGenerator { + // TODO: Encapsulate this in a class: + private errorSeriesCount = 0; -function generateError() { - // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. - errorSeriesCount++; - const metadata = new grpc.Metadata(); - metadata.set( - 'grpc-server-stats-bin', - Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), - ); - const error = new Error('error message') as ServiceError; - error.code = 4; - error.details = `error details: error count: ${errorSeriesCount}`; - error.metadata = metadata; - return error; -} + generateError() { + // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. + this.errorSeriesCount++; + const metadata = new grpc.Metadata(); + metadata.set( + 'grpc-server-stats-bin', + Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), + ); + const error = new Error('error message') as ServiceError; + error.code = 4; + error.details = `error details: error count: ${this.errorSeriesCount}`; + error.metadata = metadata; + return error; + } -export function sendErrorSeries( - call: any, - callback: (arg1: any, arg2: {}) => {}, -) { - const error = generateError(); - callback(error, {}); + sendErrorSeries() { + return (call: any, callback: (arg1: any, arg2: {}) => {}) => { + const error = this.generateError(); + callback(error, {}); + }; + } } export function shutdownServer(server: any) { diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index fb87c6aa6..4cf228109 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -17,10 +17,11 @@ import {Datastore} from '../src'; import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; -import {sendErrorSeries, shutdownServer} from './mock-server-tester'; +import {ErrorGenerator, shutdownServer} from './mock-server-tester'; describe.only('runQuery', () => { it('should report an error to the user when it occurs', done => { + const errorGenerator = new ErrorGenerator(); const server = startServer( async () => { try { @@ -40,7 +41,7 @@ describe.only('runQuery', () => { // The error message is based on client library behavior. assert.strictEqual( (e as Error).message, - '4 DEADLINE_EXCEEDED: error details', + '4 DEADLINE_EXCEEDED: error details: error count: 1', ); await shutdownServer(server); done(); @@ -49,7 +50,7 @@ describe.only('runQuery', () => { done(e); } }, - {runQuery: sendErrorSeries}, + {runQuery: errorGenerator.sendErrorSeries}, ); }); }); From a9bf0df95fd292844f82317109c6e7cf05cb2f55 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 15:27:02 -0400 Subject: [PATCH 15/43] More clear intent in test --- test/with-runquery-mockserver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 4cf228109..bde7cf792 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -50,7 +50,7 @@ describe.only('runQuery', () => { done(e); } }, - {runQuery: errorGenerator.sendErrorSeries}, + {runQuery: errorGenerator.sendErrorSeries()}, ); }); }); From 941c6eb447ee82af6af99e95e1f964509d2b8944 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 15:31:17 -0400 Subject: [PATCH 16/43] skip the test --- test/with-runquery-mockserver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index bde7cf792..6b1d2ee5a 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -19,7 +19,7 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {ErrorGenerator, shutdownServer} from './mock-server-tester'; -describe.only('runQuery', () => { +describe.skip('runQuery', () => { it('should report an error to the user when it occurs', done => { const errorGenerator = new ErrorGenerator(); const server = startServer( From 2b4691df9946444b0a94987dc4a38ff9b058b79d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 15:35:39 -0400 Subject: [PATCH 17/43] Update tests with more accurate descriptions of what they do --- test/with-createreadstream-mockserver.ts | 4 ++-- test/with-runquery-mockserver.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/with-createreadstream-mockserver.ts b/test/with-createreadstream-mockserver.ts index 6c3567931..163dbf78a 100644 --- a/test/with-createreadstream-mockserver.ts +++ b/test/with-createreadstream-mockserver.ts @@ -19,8 +19,8 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {sendNonRetryableError} from './mock-server-tester'; -describe.skip('lookup', () => { - it('should report an error to the user when it occurs', done => { +describe.skip('Should make calls to lookup', () => { + it('should report a NOT_FOUND error to the user when it occurs', done => { startServer( async () => { try { diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 6b1d2ee5a..5c38b9f07 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -19,8 +19,8 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {ErrorGenerator, shutdownServer} from './mock-server-tester'; -describe.skip('runQuery', () => { - it('should report an error to the user when it occurs', done => { +describe.skip('Should make calls to runQuery', () => { + it('should report a DEADLINE_EXCEEDED error to the user when it occurs with the original error details', done => { const errorGenerator = new ErrorGenerator(); const server = startServer( async () => { From 1eacc705dd3220b220ee1fafdeb94742c94659b4 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Tue, 6 May 2025 15:56:11 -0400 Subject: [PATCH 18/43] Cleanup the types everywhere --- mock-server/datastore-server.ts | 27 +++++++----- test/mock-server-tester.ts | 73 ++++++++++++++++++++++++++++++--- 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index 9dea22294..343247d41 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +import {ServiceError} from 'google-gax'; +import {Server} from '@grpc/grpc-js'; + const {dirname, resolve} = require('node:path'); const PROTO_PATH = __dirname + '/../protos/google/datastore/v1/datastore.proto'; @@ -37,18 +40,14 @@ const packageDefinition = protoLoader.loadSync(PROTO_PATH, { }); const descriptor = grpc.loadPackageDefinition(packageDefinition); -/** - * Implements the runQuery RPC method. - */ -function grpcEndpoint(call: any, callback: MockServiceCallback) { - // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. - callback(null, {message: 'Hello'}); -} +export type CallType = any; +export type SuccessType = any; +export type GrpcErrorType = ServiceError | null; -type MockServiceCallback = (arg1: any, arg2: {}) => {}; +type MockServiceCallback = (arg1: GrpcErrorType, arg2: SuccessType) => {}; interface MockServiceConfiguration { - [endpoint: string]: (call: any, callback: MockServiceCallback) => void; + [endpoint: string]: (call: CallType, callback: MockServiceCallback) => void; } /** @@ -57,7 +56,15 @@ interface MockServiceConfiguration { export function startServer( cb: () => void, serviceConfigurationOverride?: MockServiceConfiguration, -) { +): Server { + /** + * Implements the runQuery RPC method. + */ + function grpcEndpoint(call: CallType, callback: MockServiceCallback) { + // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. + callback(null, {message: 'Hello'}); + } + const server = new grpc.Server(); const service = descriptor.google.datastore.v1.Datastore.service; // On the next line, change runQuery to the grpc method you want to investigate diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index 8bc3dc7de..b6a8cc3af 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -1,7 +1,44 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + CallType, + GrpcErrorType, + SuccessType, +} from '../mock-server/datastore-server'; + const grpc = require('@grpc/grpc-js'); import {ServiceError} from 'google-gax'; +import {Server} from '@grpc/grpc-js'; -export function sendNonRetryableError(call: any, callback: (arg1: any, arg2: {}) => {}) { +/** + * Simulates a non-retryable error response from a gRPC service. + * + * This function is designed for use in mock server setups during testing. + * When invoked as a gRPC endpoint handler, it responds with a pre-configured + * error object representing a non-retryable error. The error includes a + * NOT_FOUND code (5), a fixed details message, and some metadata. + * + * @param {any} call - The gRPC call object, representing the incoming request. + * @param {function} callback - The callback function to be invoked with the + * simulated error response. It expects two arguments: an error object and an + * empty object (representing no response data). + */ +export function sendNonRetryableError( + call: CallType, + callback: (arg1: GrpcErrorType, arg2: SuccessType) => {}, +) { const metadata = new grpc.Metadata(); metadata.set( 'grpc-server-stats-bin', @@ -15,9 +52,19 @@ export function sendNonRetryableError(call: any, callback: (arg1: any, arg2: {}) } export class ErrorGenerator { - // TODO: Encapsulate this in a class: private errorSeriesCount = 0; + /** + * Generates an error object for testing purposes. + * + * This method creates a `ServiceError` object, simulating an error + * that might be returned by a gRPC service. The error includes a + * `DEADLINE_EXCEEDED` code (4), a details message indicating the number of + * errors generated so far by this instance, and some metadata. + * + * @returns {ServiceError} A `ServiceError` object representing a simulated + * gRPC error. + */ generateError() { // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. this.errorSeriesCount++; @@ -33,22 +80,38 @@ export class ErrorGenerator { return error; } + /** + * Returns a function that simulates an error response from a gRPC service. + * + * This method is designed to be used in mock server setups for testing purposes. + * It returns a function that, when called, will invoke a callback with a + * pre-configured error object, simulating a gRPC service responding with an error. + * The error includes a DEADLINE_EXCEEDED code (4), a details message indicating the + * number of errors generated so far by this instance, and some metadata. + * + * @returns {function} A function that takes a `call` object (representing the + * gRPC call) and a `callback` function, and responds to the call with a + * simulated error. + */ sendErrorSeries() { - return (call: any, callback: (arg1: any, arg2: {}) => {}) => { + return ( + call: CallType, + callback: (arg1: GrpcErrorType, arg2: SuccessType) => {}, + ) => { const error = this.generateError(); callback(error, {}); }; } } -export function shutdownServer(server: any) { +export function shutdownServer(server: Server) { return new Promise((resolve, reject) => { // Assuming 'server.tryShutdown' is a function that takes a callback. // The callback is expected to be called when the shutdown attempt is complete. // If 'tryShutdown' itself can throw an error or indicate an immediate failure // (without calling the callback), you might need to wrap this call in a try...catch block. - server.tryShutdown((error: any) => { + server.tryShutdown((error: Error | undefined) => { if (error) { // If the callback is called with an error, reject the promise. console.error('Server shutdown failed:', error); From 8683a7775de1eb8e7bc326adaaa0be819c31a44d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 13:38:50 -0400 Subject: [PATCH 19/43] Modify the test to use UNAVAILABLE --- test/mock-server-tester.ts | 8 +- test/with-runquery-mockserver.ts | 7 +- test/with-transaction-commit-mockserver.ts | 90 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 test/with-transaction-commit-mockserver.ts diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index b6a8cc3af..6d4017e7b 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -65,7 +65,7 @@ export class ErrorGenerator { * @returns {ServiceError} A `ServiceError` object representing a simulated * gRPC error. */ - generateError() { + generateError(code: number) { // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. this.errorSeriesCount++; const metadata = new grpc.Metadata(); @@ -74,7 +74,7 @@ export class ErrorGenerator { Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), ); const error = new Error('error message') as ServiceError; - error.code = 4; + error.code = code; error.details = `error details: error count: ${this.errorSeriesCount}`; error.metadata = metadata; return error; @@ -93,12 +93,12 @@ export class ErrorGenerator { * gRPC call) and a `callback` function, and responds to the call with a * simulated error. */ - sendErrorSeries() { + sendErrorSeries(code: number) { return ( call: CallType, callback: (arg1: GrpcErrorType, arg2: SuccessType) => {}, ) => { - const error = this.generateError(); + const error = this.generateError(code); callback(error, {}); }; } diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 5c38b9f07..17215e8cc 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -18,9 +18,10 @@ import * as assert from 'assert'; import {startServer} from '../mock-server/datastore-server'; import {ErrorGenerator, shutdownServer} from './mock-server-tester'; +import {grpc} from 'google-gax'; -describe.skip('Should make calls to runQuery', () => { - it('should report a DEADLINE_EXCEEDED error to the user when it occurs with the original error details', done => { +describe.only('Should make calls to runQuery', () => { + it('should report an UNAVAILABLE error to the user when it occurs with the original error details', done => { const errorGenerator = new ErrorGenerator(); const server = startServer( async () => { @@ -50,7 +51,7 @@ describe.skip('Should make calls to runQuery', () => { done(e); } }, - {runQuery: errorGenerator.sendErrorSeries()}, + {runQuery: errorGenerator.sendErrorSeries(grpc.status.UNAVAILABLE)}, ); }); }); diff --git a/test/with-transaction-commit-mockserver.ts b/test/with-transaction-commit-mockserver.ts new file mode 100644 index 000000000..8c12b660d --- /dev/null +++ b/test/with-transaction-commit-mockserver.ts @@ -0,0 +1,90 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {describe, it} from 'mocha'; +import {Datastore} from '../src'; +import * as assert from 'assert'; + +import {startServer} from '../mock-server/datastore-server'; +import {ErrorGenerator, shutdownServer} from './mock-server-tester'; + +describe('Should make calls to commit', () => { + it('should report a DEADLINE_EXCEEDED error to the user when it occurs with the original error details', done => { + async function write( + docKey: string, + // docToWrite: any // sync.ISyncDocument | undefined + ) { + const datastore = new Datastore({ + namespace: `${Date.now()}`, + apiEndpoint: 'localhost:50051', + }); + const key = datastore.key(['sync_document', docKey]); + + const transaction = datastore.transaction(); + + try { + await transaction.run(); + + const [datastoreDoc] = await transaction.get(key, {}); + + transaction.save({ + key, + data: { + metadata: [ + { + name: 'some-string', + value: 'some-value', + }, + ], + }, + + excludeFromIndexes: ['instance', 'instance.*'], + }); + + await transaction.commit(); + + // return toWrite; + } catch (e) { + await transaction.rollback(); + + throw e; + } + } + const errorGenerator = new ErrorGenerator(); + const server = startServer( + async () => { + try { + try { + await write('key'); + } catch (e) { + // The test should produce the right error message here for the user. + // TODO: Later on we are going to decide on what the error message should be + // The error message is based on client library behavior. + assert.strictEqual( + (e as Error).message, + '4 DEADLINE_EXCEEDED: error details: error count: 1', + ); + await shutdownServer(server); + done(); + } + } catch (e) { + done(e); + } + }, + { + commit: errorGenerator.sendErrorSeries(4), + }, + ); + }); +}); From 341e37e4b609b2374a9a2dde20b3e48b67e669c8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 13:41:47 -0400 Subject: [PATCH 20/43] rename test file --- ...h-createreadstream-mockserver.ts => with-lookup-mockserver.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{with-createreadstream-mockserver.ts => with-lookup-mockserver.ts} (100%) diff --git a/test/with-createreadstream-mockserver.ts b/test/with-lookup-mockserver.ts similarity index 100% rename from test/with-createreadstream-mockserver.ts rename to test/with-lookup-mockserver.ts From e8b9ddc558af0ada9551b138e4eb89e3ad3b3f7d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 13:50:59 -0400 Subject: [PATCH 21/43] Delete files to be left out of the PR for now --- test/with-lookup-mockserver.ts | 51 ------------ test/with-transaction-commit-mockserver.ts | 90 ---------------------- 2 files changed, 141 deletions(-) delete mode 100644 test/with-lookup-mockserver.ts delete mode 100644 test/with-transaction-commit-mockserver.ts diff --git a/test/with-lookup-mockserver.ts b/test/with-lookup-mockserver.ts deleted file mode 100644 index 163dbf78a..000000000 --- a/test/with-lookup-mockserver.ts +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import {describe, it} from 'mocha'; -import {Datastore} from '../src'; -import * as assert from 'assert'; - -import {startServer} from '../mock-server/datastore-server'; -import {sendNonRetryableError} from './mock-server-tester'; - -describe.skip('Should make calls to lookup', () => { - it('should report a NOT_FOUND error to the user when it occurs', done => { - startServer( - async () => { - try { - try { - const datastore = new Datastore({ - namespace: `${Date.now()}`, - apiEndpoint: 'localhost:50051', - }); - const postKey = datastore.key(['Post', 'post1']); - await datastore.get(postKey); - console.log('call failed'); - assert.fail('The call should not have succeeded'); - } catch (e) { - // The test should produce the right error message here for the user. - assert.strictEqual( - (e as Error).message, - '5 NOT_FOUND: error details', - ); - done(); - } - } catch (e) { - done(e); - } - }, - {lookup: sendNonRetryableError}, - ); - }); -}); diff --git a/test/with-transaction-commit-mockserver.ts b/test/with-transaction-commit-mockserver.ts deleted file mode 100644 index 8c12b660d..000000000 --- a/test/with-transaction-commit-mockserver.ts +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import {describe, it} from 'mocha'; -import {Datastore} from '../src'; -import * as assert from 'assert'; - -import {startServer} from '../mock-server/datastore-server'; -import {ErrorGenerator, shutdownServer} from './mock-server-tester'; - -describe('Should make calls to commit', () => { - it('should report a DEADLINE_EXCEEDED error to the user when it occurs with the original error details', done => { - async function write( - docKey: string, - // docToWrite: any // sync.ISyncDocument | undefined - ) { - const datastore = new Datastore({ - namespace: `${Date.now()}`, - apiEndpoint: 'localhost:50051', - }); - const key = datastore.key(['sync_document', docKey]); - - const transaction = datastore.transaction(); - - try { - await transaction.run(); - - const [datastoreDoc] = await transaction.get(key, {}); - - transaction.save({ - key, - data: { - metadata: [ - { - name: 'some-string', - value: 'some-value', - }, - ], - }, - - excludeFromIndexes: ['instance', 'instance.*'], - }); - - await transaction.commit(); - - // return toWrite; - } catch (e) { - await transaction.rollback(); - - throw e; - } - } - const errorGenerator = new ErrorGenerator(); - const server = startServer( - async () => { - try { - try { - await write('key'); - } catch (e) { - // The test should produce the right error message here for the user. - // TODO: Later on we are going to decide on what the error message should be - // The error message is based on client library behavior. - assert.strictEqual( - (e as Error).message, - '4 DEADLINE_EXCEEDED: error details: error count: 1', - ); - await shutdownServer(server); - done(); - } - } catch (e) { - done(e); - } - }, - { - commit: errorGenerator.sendErrorSeries(4), - }, - ); - }); -}); From ec849ca3fd4a61bf1955a62aa1c36dfc3b79a8aa Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 14:06:25 -0400 Subject: [PATCH 22/43] Fix the checks in the test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Number of errors won’t be deterministic before a timeout occurs --- test/with-runquery-mockserver.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 17215e8cc..80465f693 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -40,10 +40,17 @@ describe.only('Should make calls to runQuery', () => { // The test should produce the right error message here for the user. // TODO: Later on we are going to decide on what the error message should be // The error message is based on client library behavior. - assert.strictEqual( - (e as Error).message, - '4 DEADLINE_EXCEEDED: error details: error count: 1', + const message = (e as Error).message; + assert.match( + message, + /Total timeout of API google.datastore.v1.Datastore exceeded 60000 milliseconds retrying error Error: 14 UNAVAILABLE: error details: error count:/, ); + const substringToFind = + 'before any response was received. : Previous errors : [{message: 14 UNAVAILABLE: error details: error count: 1, code: 14, details: , note: },{message: 14 UNAVAILABLE: error details: error count: 2, code: 14, details: , note: },'; + const escapedSubstringRegex = new RegExp( + substringToFind.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), + ); + assert.match(message, escapedSubstringRegex); await shutdownServer(server); done(); } From 60311970748a09e6b63f9edea03b91d0b77d69a3 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 14:10:27 -0400 Subject: [PATCH 23/43] better documentation --- test/mock-server-tester.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index 6d4017e7b..e9e8f1ef1 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -89,6 +89,7 @@ export class ErrorGenerator { * The error includes a DEADLINE_EXCEEDED code (4), a details message indicating the * number of errors generated so far by this instance, and some metadata. * + * @param {number} code The grpc error code for the error sent back * @returns {function} A function that takes a `call` object (representing the * gRPC call) and a `callback` function, and responds to the call with a * simulated error. From 1dbbc020333ede4e0cba20d5e6bee99bb33bc42d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 14:15:41 -0400 Subject: [PATCH 24/43] cast as any --- src/request.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/request.ts b/src/request.ts index 132f9011c..259e4d76a 100644 --- a/src/request.ts +++ b/src/request.ts @@ -69,6 +69,7 @@ import * as protos from '../protos/protos'; import {serializer} from 'google-gax'; import * as gax from 'google-gax'; import {SaveDataValue} from './interfaces/save'; +import {Message} from 'protobufjs'; type JSONValue = | string | number @@ -84,7 +85,7 @@ const Struct = root.lookupType('Struct'); // This function decodes Struct proto values function decodeStruct(structValue: google.protobuf.IStruct): JSONValue { - return serializer.toProto3JSON(Struct.fromObject(structValue)); + return serializer.toProto3JSON(Struct.fromObject(structValue) as any); } // This function gets a RunQueryInfo object that contains explain metrics that From 6a64eaec87c3696b9a56a597932cad740d4eab9d Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 14:24:25 -0400 Subject: [PATCH 25/43] solve linting errors --- mock-server/datastore-server.ts | 1 + test/mock-server-tester.ts | 5 ++--- test/with-runquery-mockserver.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index 343247d41..4db8f1d11 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// eslint-disable-next-line n/no-extraneous-import import {ServiceError} from 'google-gax'; import {Server} from '@grpc/grpc-js'; diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index e9e8f1ef1..683ea0bfc 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -18,9 +18,8 @@ import { SuccessType, } from '../mock-server/datastore-server'; -const grpc = require('@grpc/grpc-js'); +import grpc = require('@grpc/grpc-js'); import {ServiceError} from 'google-gax'; -import {Server} from '@grpc/grpc-js'; /** * Simulates a non-retryable error response from a gRPC service. @@ -105,7 +104,7 @@ export class ErrorGenerator { } } -export function shutdownServer(server: Server) { +export function shutdownServer(server: grpc.Server) { return new Promise((resolve, reject) => { // Assuming 'server.tryShutdown' is a function that takes a callback. // The callback is expected to be called when the shutdown attempt is complete. diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 80465f693..2d608166c 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -20,7 +20,7 @@ import {startServer} from '../mock-server/datastore-server'; import {ErrorGenerator, shutdownServer} from './mock-server-tester'; import {grpc} from 'google-gax'; -describe.only('Should make calls to runQuery', () => { +describe('Should make calls to runQuery', () => { it('should report an UNAVAILABLE error to the user when it occurs with the original error details', done => { const errorGenerator = new ErrorGenerator(); const server = startServer( From 0f8fb7265fb820ea7c91256d31feec7c9f97c619 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Wed, 7 May 2025 14:38:52 -0400 Subject: [PATCH 26/43] Shorter timeout call --- test/with-runquery-mockserver.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 2d608166c..50fe6ede0 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -33,7 +33,8 @@ describe('Should make calls to runQuery', () => { }); const postKey = datastore.key(['Post', 'post1']); const query = datastore.createQuery('Post').hasAncestor(postKey); - await datastore.runQuery(query); + // Make the call with a shorter timeout: + await datastore.runQuery(query, {gaxOptions: {timeout: 10000}}); console.log('call failed'); assert.fail('The call should not have succeeded'); } catch (e) { From 5bdd6bcc0545dfc37bfba798ecf92fa6dace45f6 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 8 May 2025 10:35:46 -0400 Subject: [PATCH 27/43] Add a couple of console logs --- test/with-runquery-mockserver.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 50fe6ede0..7a678e4b8 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -42,6 +42,8 @@ describe('Should make calls to runQuery', () => { // TODO: Later on we are going to decide on what the error message should be // The error message is based on client library behavior. const message = (e as Error).message; + console.log('The error message:'); + console.log(message); assert.match( message, /Total timeout of API google.datastore.v1.Datastore exceeded 60000 milliseconds retrying error Error: 14 UNAVAILABLE: error details: error count:/, From e7c0f43fde3f62bcb70579cd64a0a96039ac691b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 11:34:13 -0400 Subject: [PATCH 28/43] Reduce timeout --- test/with-runquery-mockserver.ts | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 7a678e4b8..55ec2553f 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -34,7 +34,7 @@ describe('Should make calls to runQuery', () => { const postKey = datastore.key(['Post', 'post1']); const query = datastore.createQuery('Post').hasAncestor(postKey); // Make the call with a shorter timeout: - await datastore.runQuery(query, {gaxOptions: {timeout: 10000}}); + await datastore.runQuery(query, {gaxOptions: {timeout: 5000}}); console.log('call failed'); assert.fail('The call should not have succeeded'); } catch (e) { @@ -44,16 +44,18 @@ describe('Should make calls to runQuery', () => { const message = (e as Error).message; console.log('The error message:'); console.log(message); - assert.match( - message, - /Total timeout of API google.datastore.v1.Datastore exceeded 60000 milliseconds retrying error Error: 14 UNAVAILABLE: error details: error count:/, + const substringToFind1 = + 'Total timeout of API google.datastore.v1.Datastore exceeded 5000 milliseconds retrying error Error: 14 UNAVAILABLE: error details: error count:'; + const escapedSubstringRegex1 = new RegExp( + substringToFind1.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), ); - const substringToFind = + assert.match(message, escapedSubstringRegex1); + const substringToFind2 = 'before any response was received. : Previous errors : [{message: 14 UNAVAILABLE: error details: error count: 1, code: 14, details: , note: },{message: 14 UNAVAILABLE: error details: error count: 2, code: 14, details: , note: },'; - const escapedSubstringRegex = new RegExp( - substringToFind.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), + const escapedSubstringRegex2 = new RegExp( + substringToFind2.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), ); - assert.match(message, escapedSubstringRegex); + assert.match(message, escapedSubstringRegex2); await shutdownServer(server); done(); } From c8312af177ed5095eea8dad1206e6ae8c88bac90 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 11:35:55 -0400 Subject: [PATCH 29/43] Refactor regex pattern --- test/with-runquery-mockserver.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 55ec2553f..db0667d4e 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -44,16 +44,17 @@ describe('Should make calls to runQuery', () => { const message = (e as Error).message; console.log('The error message:'); console.log(message); + const regexPattern = /[.*+?^${}()|[\]\\]/g; const substringToFind1 = 'Total timeout of API google.datastore.v1.Datastore exceeded 5000 milliseconds retrying error Error: 14 UNAVAILABLE: error details: error count:'; const escapedSubstringRegex1 = new RegExp( - substringToFind1.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), + substringToFind1.replace(regexPattern, '\\$&'), ); assert.match(message, escapedSubstringRegex1); const substringToFind2 = 'before any response was received. : Previous errors : [{message: 14 UNAVAILABLE: error details: error count: 1, code: 14, details: , note: },{message: 14 UNAVAILABLE: error details: error count: 2, code: 14, details: , note: },'; const escapedSubstringRegex2 = new RegExp( - substringToFind2.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), + substringToFind2.replace(regexPattern, '\\$&'), ); assert.match(message, escapedSubstringRegex2); await shutdownServer(server); From 5b1bcaf465c2d457f5c7335a227a6e057e3fd7de Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 11:40:44 -0400 Subject: [PATCH 30/43] Factor replace value --- test/with-runquery-mockserver.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index db0667d4e..f36d3241d 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -44,17 +44,18 @@ describe('Should make calls to runQuery', () => { const message = (e as Error).message; console.log('The error message:'); console.log(message); - const regexPattern = /[.*+?^${}()|[\]\\]/g; + const searchValue = /[.*+?^${}()|[\]\\]/g; + const replaceValue = '\\$&'; const substringToFind1 = 'Total timeout of API google.datastore.v1.Datastore exceeded 5000 milliseconds retrying error Error: 14 UNAVAILABLE: error details: error count:'; const escapedSubstringRegex1 = new RegExp( - substringToFind1.replace(regexPattern, '\\$&'), + substringToFind1.replace(searchValue, replaceValue), ); assert.match(message, escapedSubstringRegex1); const substringToFind2 = 'before any response was received. : Previous errors : [{message: 14 UNAVAILABLE: error details: error count: 1, code: 14, details: , note: },{message: 14 UNAVAILABLE: error details: error count: 2, code: 14, details: , note: },'; const escapedSubstringRegex2 = new RegExp( - substringToFind2.replace(regexPattern, '\\$&'), + substringToFind2.replace(searchValue, replaceValue), ); assert.match(message, escapedSubstringRegex2); await shutdownServer(server); From d5b3e639c00d27f77f3d2cca13bbd09a46695394 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 12:55:48 -0400 Subject: [PATCH 31/43] Provide a projectId --- test/with-runquery-mockserver.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index f36d3241d..ebfd49595 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -30,6 +30,7 @@ describe('Should make calls to runQuery', () => { const datastore = new Datastore({ namespace: `${Date.now()}`, apiEndpoint: 'localhost:50051', + projectId: 'test-project', // Provided to avoid 'Unable to detect a Project Id in the current environment.' }); const postKey = datastore.key(['Post', 'post1']); const query = datastore.createQuery('Post').hasAncestor(postKey); @@ -58,12 +59,12 @@ describe('Should make calls to runQuery', () => { substringToFind2.replace(searchValue, replaceValue), ); assert.match(message, escapedSubstringRegex2); - await shutdownServer(server); done(); } } catch (e) { done(e); } + await shutdownServer(server); }, {runQuery: errorGenerator.sendErrorSeries(grpc.status.UNAVAILABLE)}, ); From 7609fc013268d915f9c8f8696a2036ce79fa016b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 13:54:49 -0400 Subject: [PATCH 32/43] Revert import --- src/request.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/request.ts b/src/request.ts index 259e4d76a..dee2a09b2 100644 --- a/src/request.ts +++ b/src/request.ts @@ -69,7 +69,7 @@ import * as protos from '../protos/protos'; import {serializer} from 'google-gax'; import * as gax from 'google-gax'; import {SaveDataValue} from './interfaces/save'; -import {Message} from 'protobufjs'; + type JSONValue = | string | number From 676d561d67414c6023399e21d804cae5f4220ec0 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 13:56:57 -0400 Subject: [PATCH 33/43] move grpc service definition back to where it was --- mock-server/datastore-server.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index 4db8f1d11..74b262d0c 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -41,6 +41,14 @@ const packageDefinition = protoLoader.loadSync(PROTO_PATH, { }); const descriptor = grpc.loadPackageDefinition(packageDefinition); +/** + * Implements the runQuery RPC method. + */ +function grpcEndpoint(call: CallType, callback: MockServiceCallback) { + // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. + callback(null, {message: 'Hello'}); +} + export type CallType = any; export type SuccessType = any; export type GrpcErrorType = ServiceError | null; @@ -58,14 +66,6 @@ export function startServer( cb: () => void, serviceConfigurationOverride?: MockServiceConfiguration, ): Server { - /** - * Implements the runQuery RPC method. - */ - function grpcEndpoint(call: CallType, callback: MockServiceCallback) { - // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. - callback(null, {message: 'Hello'}); - } - const server = new grpc.Server(); const service = descriptor.google.datastore.v1.Datastore.service; // On the next line, change runQuery to the grpc method you want to investigate From 9c2f9a7f9da41e064d32aa5f229a456aef6e725c Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:12:19 -0400 Subject: [PATCH 34/43] Thread ResponseType parameter through server confg --- mock-server/datastore-server.ts | 23 ++++++++++++++++------- test/mock-server-tester.ts | 12 ++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index 74b262d0c..4886414ee 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -44,27 +44,36 @@ const descriptor = grpc.loadPackageDefinition(packageDefinition); /** * Implements the runQuery RPC method. */ -function grpcEndpoint(call: CallType, callback: MockServiceCallback) { +function grpcEndpoint( + call: CallType, + callback: MockServiceCallback, +) { // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. - callback(null, {message: 'Hello'}); + callback(null, {message: 'Hello'} as ResponseType); } export type CallType = any; export type SuccessType = any; export type GrpcErrorType = ServiceError | null; -type MockServiceCallback = (arg1: GrpcErrorType, arg2: SuccessType) => {}; +type MockServiceCallback = ( + arg1: GrpcErrorType, + arg2: ResponseType, +) => {}; -interface MockServiceConfiguration { - [endpoint: string]: (call: CallType, callback: MockServiceCallback) => void; +interface MockServiceConfiguration { + [endpoint: string]: ( + call: CallType, + callback: MockServiceCallback, + ) => void; } /** * Starts an RPC server that receives requests for datastore */ -export function startServer( +export function startServer( cb: () => void, - serviceConfigurationOverride?: MockServiceConfiguration, + serviceConfigurationOverride?: MockServiceConfiguration, ): Server { const server = new grpc.Server(); const service = descriptor.google.datastore.v1.Datastore.service; diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index 683ea0bfc..5d5da2171 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -34,9 +34,9 @@ import {ServiceError} from 'google-gax'; * simulated error response. It expects two arguments: an error object and an * empty object (representing no response data). */ -export function sendNonRetryableError( +export function sendNonRetryableError( call: CallType, - callback: (arg1: GrpcErrorType, arg2: SuccessType) => {}, + callback: (arg1: GrpcErrorType, arg2: ResponseType) => {}, ) { const metadata = new grpc.Metadata(); metadata.set( @@ -47,7 +47,7 @@ export function sendNonRetryableError( error.code = 5; error.details = 'error details'; error.metadata = metadata; - callback(error, {}); + callback(error, {} as ResponseType); } export class ErrorGenerator { @@ -93,13 +93,13 @@ export class ErrorGenerator { * gRPC call) and a `callback` function, and responds to the call with a * simulated error. */ - sendErrorSeries(code: number) { + sendErrorSeries(code: number) { return ( call: CallType, - callback: (arg1: GrpcErrorType, arg2: SuccessType) => {}, + callback: (arg1: GrpcErrorType, arg2: ResponseType) => {}, ) => { const error = this.generateError(code); - callback(error, {}); + callback(error, {} as ResponseType); }; } } From 015e5a653f8743848217289fea8038fef916e8b9 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:13:21 -0400 Subject: [PATCH 35/43] Remove SuccessType --- mock-server/datastore-server.ts | 1 - test/mock-server-tester.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index 4886414ee..8dc6f4923 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -53,7 +53,6 @@ function grpcEndpoint( } export type CallType = any; -export type SuccessType = any; export type GrpcErrorType = ServiceError | null; type MockServiceCallback = ( diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index 5d5da2171..0f0a63404 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -15,7 +15,6 @@ import { CallType, GrpcErrorType, - SuccessType, } from '../mock-server/datastore-server'; import grpc = require('@grpc/grpc-js'); From 69dbb30ecb7c45921fb82dc6567b260e184c8c40 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:28:11 -0400 Subject: [PATCH 36/43] delete file - not used --- test/mock-server-tester.ts | 125 ------------------------------------- 1 file changed, 125 deletions(-) delete mode 100644 test/mock-server-tester.ts diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts deleted file mode 100644 index 0f0a63404..000000000 --- a/test/mock-server-tester.ts +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2025 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import { - CallType, - GrpcErrorType, -} from '../mock-server/datastore-server'; - -import grpc = require('@grpc/grpc-js'); -import {ServiceError} from 'google-gax'; - -/** - * Simulates a non-retryable error response from a gRPC service. - * - * This function is designed for use in mock server setups during testing. - * When invoked as a gRPC endpoint handler, it responds with a pre-configured - * error object representing a non-retryable error. The error includes a - * NOT_FOUND code (5), a fixed details message, and some metadata. - * - * @param {any} call - The gRPC call object, representing the incoming request. - * @param {function} callback - The callback function to be invoked with the - * simulated error response. It expects two arguments: an error object and an - * empty object (representing no response data). - */ -export function sendNonRetryableError( - call: CallType, - callback: (arg1: GrpcErrorType, arg2: ResponseType) => {}, -) { - const metadata = new grpc.Metadata(); - metadata.set( - 'grpc-server-stats-bin', - Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), - ); - const error = new Error('error message') as ServiceError; - error.code = 5; - error.details = 'error details'; - error.metadata = metadata; - callback(error, {} as ResponseType); -} - -export class ErrorGenerator { - private errorSeriesCount = 0; - - /** - * Generates an error object for testing purposes. - * - * This method creates a `ServiceError` object, simulating an error - * that might be returned by a gRPC service. The error includes a - * `DEADLINE_EXCEEDED` code (4), a details message indicating the number of - * errors generated so far by this instance, and some metadata. - * - * @returns {ServiceError} A `ServiceError` object representing a simulated - * gRPC error. - */ - generateError(code: number) { - // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. - this.errorSeriesCount++; - const metadata = new grpc.Metadata(); - metadata.set( - 'grpc-server-stats-bin', - Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), - ); - const error = new Error('error message') as ServiceError; - error.code = code; - error.details = `error details: error count: ${this.errorSeriesCount}`; - error.metadata = metadata; - return error; - } - - /** - * Returns a function that simulates an error response from a gRPC service. - * - * This method is designed to be used in mock server setups for testing purposes. - * It returns a function that, when called, will invoke a callback with a - * pre-configured error object, simulating a gRPC service responding with an error. - * The error includes a DEADLINE_EXCEEDED code (4), a details message indicating the - * number of errors generated so far by this instance, and some metadata. - * - * @param {number} code The grpc error code for the error sent back - * @returns {function} A function that takes a `call` object (representing the - * gRPC call) and a `callback` function, and responds to the call with a - * simulated error. - */ - sendErrorSeries(code: number) { - return ( - call: CallType, - callback: (arg1: GrpcErrorType, arg2: ResponseType) => {}, - ) => { - const error = this.generateError(code); - callback(error, {} as ResponseType); - }; - } -} - -export function shutdownServer(server: grpc.Server) { - return new Promise((resolve, reject) => { - // Assuming 'server.tryShutdown' is a function that takes a callback. - // The callback is expected to be called when the shutdown attempt is complete. - // If 'tryShutdown' itself can throw an error or indicate an immediate failure - // (without calling the callback), you might need to wrap this call in a try...catch block. - - server.tryShutdown((error: Error | undefined) => { - if (error) { - // If the callback is called with an error, reject the promise. - console.error('Server shutdown failed:', error); - reject(error); - } else { - // If the callback is called without an error, resolve the promise. - console.log('Server has been shut down successfully.'); - resolve('done'); - } - }); - }); -} From a5f9730d4fd8ba12a2ceb74dd0e6e5eb68d06420 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:28:49 -0400 Subject: [PATCH 37/43] Revert "delete file - not used" This reverts commit 69dbb30ecb7c45921fb82dc6567b260e184c8c40. --- test/mock-server-tester.ts | 125 +++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 test/mock-server-tester.ts diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts new file mode 100644 index 000000000..0f0a63404 --- /dev/null +++ b/test/mock-server-tester.ts @@ -0,0 +1,125 @@ +// Copyright 2025 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { + CallType, + GrpcErrorType, +} from '../mock-server/datastore-server'; + +import grpc = require('@grpc/grpc-js'); +import {ServiceError} from 'google-gax'; + +/** + * Simulates a non-retryable error response from a gRPC service. + * + * This function is designed for use in mock server setups during testing. + * When invoked as a gRPC endpoint handler, it responds with a pre-configured + * error object representing a non-retryable error. The error includes a + * NOT_FOUND code (5), a fixed details message, and some metadata. + * + * @param {any} call - The gRPC call object, representing the incoming request. + * @param {function} callback - The callback function to be invoked with the + * simulated error response. It expects two arguments: an error object and an + * empty object (representing no response data). + */ +export function sendNonRetryableError( + call: CallType, + callback: (arg1: GrpcErrorType, arg2: ResponseType) => {}, +) { + const metadata = new grpc.Metadata(); + metadata.set( + 'grpc-server-stats-bin', + Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), + ); + const error = new Error('error message') as ServiceError; + error.code = 5; + error.details = 'error details'; + error.metadata = metadata; + callback(error, {} as ResponseType); +} + +export class ErrorGenerator { + private errorSeriesCount = 0; + + /** + * Generates an error object for testing purposes. + * + * This method creates a `ServiceError` object, simulating an error + * that might be returned by a gRPC service. The error includes a + * `DEADLINE_EXCEEDED` code (4), a details message indicating the number of + * errors generated so far by this instance, and some metadata. + * + * @returns {ServiceError} A `ServiceError` object representing a simulated + * gRPC error. + */ + generateError(code: number) { + // SET A BREAKPOINT HERE AND EXPLORE `call` TO SEE THE REQUEST. + this.errorSeriesCount++; + const metadata = new grpc.Metadata(); + metadata.set( + 'grpc-server-stats-bin', + Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), + ); + const error = new Error('error message') as ServiceError; + error.code = code; + error.details = `error details: error count: ${this.errorSeriesCount}`; + error.metadata = metadata; + return error; + } + + /** + * Returns a function that simulates an error response from a gRPC service. + * + * This method is designed to be used in mock server setups for testing purposes. + * It returns a function that, when called, will invoke a callback with a + * pre-configured error object, simulating a gRPC service responding with an error. + * The error includes a DEADLINE_EXCEEDED code (4), a details message indicating the + * number of errors generated so far by this instance, and some metadata. + * + * @param {number} code The grpc error code for the error sent back + * @returns {function} A function that takes a `call` object (representing the + * gRPC call) and a `callback` function, and responds to the call with a + * simulated error. + */ + sendErrorSeries(code: number) { + return ( + call: CallType, + callback: (arg1: GrpcErrorType, arg2: ResponseType) => {}, + ) => { + const error = this.generateError(code); + callback(error, {} as ResponseType); + }; + } +} + +export function shutdownServer(server: grpc.Server) { + return new Promise((resolve, reject) => { + // Assuming 'server.tryShutdown' is a function that takes a callback. + // The callback is expected to be called when the shutdown attempt is complete. + // If 'tryShutdown' itself can throw an error or indicate an immediate failure + // (without calling the callback), you might need to wrap this call in a try...catch block. + + server.tryShutdown((error: Error | undefined) => { + if (error) { + // If the callback is called with an error, reject the promise. + console.error('Server shutdown failed:', error); + reject(error); + } else { + // If the callback is called without an error, resolve the promise. + console.log('Server has been shut down successfully.'); + resolve('done'); + } + }); + }); +} From 5dbbefc13ea3368f2eb20dcbdad1ccddfbdd318b Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:30:24 -0400 Subject: [PATCH 38/43] linting fix --- test/mock-server-tester.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index 0f0a63404..c839773d3 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -12,10 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { - CallType, - GrpcErrorType, -} from '../mock-server/datastore-server'; +import {CallType, GrpcErrorType} from '../mock-server/datastore-server'; import grpc = require('@grpc/grpc-js'); import {ServiceError} from 'google-gax'; From efa763159a30badc46fe53da273cfec8428949d2 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:33:39 -0400 Subject: [PATCH 39/43] Replace any in call type --- mock-server/datastore-server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mock-server/datastore-server.ts b/mock-server/datastore-server.ts index 8dc6f4923..0c0f5fb42 100644 --- a/mock-server/datastore-server.ts +++ b/mock-server/datastore-server.ts @@ -52,7 +52,7 @@ function grpcEndpoint( callback(null, {message: 'Hello'} as ResponseType); } -export type CallType = any; +export type CallType = () => {}; export type GrpcErrorType = ServiceError | null; type MockServiceCallback = ( From e1e65998b3aee513699d230e0d08b4c487e9b140 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:38:21 -0400 Subject: [PATCH 40/43] Eliminate sendNonRetryableError, it is not used --- test/mock-server-tester.ts | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index c839773d3..2e6d3a60e 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -17,35 +17,6 @@ import {CallType, GrpcErrorType} from '../mock-server/datastore-server'; import grpc = require('@grpc/grpc-js'); import {ServiceError} from 'google-gax'; -/** - * Simulates a non-retryable error response from a gRPC service. - * - * This function is designed for use in mock server setups during testing. - * When invoked as a gRPC endpoint handler, it responds with a pre-configured - * error object representing a non-retryable error. The error includes a - * NOT_FOUND code (5), a fixed details message, and some metadata. - * - * @param {any} call - The gRPC call object, representing the incoming request. - * @param {function} callback - The callback function to be invoked with the - * simulated error response. It expects two arguments: an error object and an - * empty object (representing no response data). - */ -export function sendNonRetryableError( - call: CallType, - callback: (arg1: GrpcErrorType, arg2: ResponseType) => {}, -) { - const metadata = new grpc.Metadata(); - metadata.set( - 'grpc-server-stats-bin', - Buffer.from([0, 0, 116, 73, 159, 3, 0, 0, 0, 0]), - ); - const error = new Error('error message') as ServiceError; - error.code = 5; - error.details = 'error details'; - error.metadata = metadata; - callback(error, {} as ResponseType); -} - export class ErrorGenerator { private errorSeriesCount = 0; From 8ffcea1607c7b7c0c78c84b2d52bc350a01de27a Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:41:33 -0400 Subject: [PATCH 41/43] Revert try-server changes --- test/try-server.ts | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/test/try-server.ts b/test/try-server.ts index 458a18c07..28f67d0d7 100644 --- a/test/try-server.ts +++ b/test/try-server.ts @@ -17,21 +17,17 @@ import {Datastore} from '../src'; import {startServer} from '../mock-server/datastore-server'; -describe.skip('Try server', () => { - it('should try to connect to the running server', done => { +describe('Try server', () => { + it.skip('should try to connect to the running server', done => { startServer(async () => { - try { - const datastore = new Datastore({ - namespace: `${Date.now()}`, - apiEndpoint: 'localhost:50051', - }); - const postKey = datastore.key(['Post', 'post1']); - const query = datastore.createQuery('Post').hasAncestor(postKey); - const allResults = await datastore.runQuery(query); - done(); - } catch (e) { - done(e); - } + const datastore = new Datastore({ + namespace: `${Date.now()}`, + apiEndpoint: 'localhost:50051', + }); + const postKey = datastore.key(['Post', 'post1']); + const query = datastore.createQuery('Post').hasAncestor(postKey); + const allResults = await datastore.runQuery(query); + done(); }); }); }); From e36357dd29eb7915d0c00c474103467eb247e1e8 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 15:48:11 -0400 Subject: [PATCH 42/43] remove console errors and console logs --- test/mock-server-tester.ts | 2 -- test/with-runquery-mockserver.ts | 3 --- 2 files changed, 5 deletions(-) diff --git a/test/mock-server-tester.ts b/test/mock-server-tester.ts index 2e6d3a60e..257066739 100644 --- a/test/mock-server-tester.ts +++ b/test/mock-server-tester.ts @@ -81,11 +81,9 @@ export function shutdownServer(server: grpc.Server) { server.tryShutdown((error: Error | undefined) => { if (error) { // If the callback is called with an error, reject the promise. - console.error('Server shutdown failed:', error); reject(error); } else { // If the callback is called without an error, resolve the promise. - console.log('Server has been shut down successfully.'); resolve('done'); } }); diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index ebfd49595..910de202d 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -36,15 +36,12 @@ describe('Should make calls to runQuery', () => { const query = datastore.createQuery('Post').hasAncestor(postKey); // Make the call with a shorter timeout: await datastore.runQuery(query, {gaxOptions: {timeout: 5000}}); - console.log('call failed'); assert.fail('The call should not have succeeded'); } catch (e) { // The test should produce the right error message here for the user. // TODO: Later on we are going to decide on what the error message should be // The error message is based on client library behavior. const message = (e as Error).message; - console.log('The error message:'); - console.log(message); const searchValue = /[.*+?^${}()|[\]\\]/g; const replaceValue = '\\$&'; const substringToFind1 = From a36a7deb4e2ba7251b633eb36e7d498ebe090f50 Mon Sep 17 00:00:00 2001 From: Daniel Bruce Date: Thu, 22 May 2025 16:54:50 -0400 Subject: [PATCH 43/43] Eliminate some comments --- test/with-runquery-mockserver.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/with-runquery-mockserver.ts b/test/with-runquery-mockserver.ts index 910de202d..f1dc087bc 100644 --- a/test/with-runquery-mockserver.ts +++ b/test/with-runquery-mockserver.ts @@ -38,9 +38,6 @@ describe('Should make calls to runQuery', () => { await datastore.runQuery(query, {gaxOptions: {timeout: 5000}}); assert.fail('The call should not have succeeded'); } catch (e) { - // The test should produce the right error message here for the user. - // TODO: Later on we are going to decide on what the error message should be - // The error message is based on client library behavior. const message = (e as Error).message; const searchValue = /[.*+?^${}()|[\]\\]/g; const replaceValue = '\\$&';