feat: introduce import/export + Index class (#734)

Author: stephenplusplus

package.json

@@ -24,7 +24,7 @@
   ],
   "scripts": {
     "clean": "gts clean",
-    "compile": "tsc -p . && cp -r proto* build/",
+    "compile": "tsc -p . && cp -r proto* build/ && cp -r system-test/data build/system-test",
     "compile-protos": "compileProtos src",
     "docs": "jsdoc -c .jsdoc.js",
     "predocs-test": "npm run docs",
@@ -49,34 +49,40 @@
     "extend": "^3.0.2",
     "google-gax": "^2.2.0",
     "is": "^3.3.0",
+    "pumpify": "^2.0.1",
     "split-array-stream": "^2.0.0",
     "stream-events": "^1.0.5"
   },
   "devDependencies": {
+    "@google-cloud/storage": "^5.3.0",
+    "@microsoft/api-documenter": "^7.8.10",
+    "@microsoft/api-extractor": "^7.8.10",
     "@types/extend": "^3.0.1",
     "@types/is": "0.0.21",
+    "@types/js-yaml": "^3.12.5",
     "@types/mocha": "^8.0.0",
     "@types/node": "^13.9.0",
     "@types/proxyquire": "^1.3.28",
+    "@types/pumpify": "^1.4.1",
     "@types/sinon": "^9.0.0",
     "c8": "^7.1.0",
     "codecov": "^3.6.5",
     "gts": "^2.0.0",
+    "js-yaml": "^3.14.0",
     "jsdoc": "^3.6.3",
     "jsdoc-fresh": "^1.0.2",
     "jsdoc-region-tag": "^1.0.4",
     "linkinator": "^2.0.3",
     "mocha": "^8.0.0",
     "null-loader": "^4.0.0",
+    "p-queue": "^6.6.1",
     "pack-n-play": "^1.0.0-2",
     "proxyquire": "^2.1.3",
     "sinon": "^9.0.1",
     "ts-loader": "^8.0.0",
     "typescript": "^3.8.3",
     "webpack": "^4.42.0",
-    "webpack-cli": "^3.3.11",
-    "@microsoft/api-documenter": "^7.8.10",
-    "@microsoft/api-extractor": "^7.8.10"
+    "webpack-cli": "^3.3.11"
   },
   "engines": {
     "node": ">=10"

samples/export.js

@@ -0,0 +1,48 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+async function main(bucket = 'YOUR_BUCKET_NAME') {
+  // [START datastore_admin_entities_export]
+  const {Datastore} = require('@google-cloud/datastore');
+  const datastore = new Datastore();
+
+  async function exportEntities() {
+    /**
+     * TODO(developer): Uncomment these variables before running the sample.
+     */
+    // const bucket = 'YOUR_BUCKET_NAME';
+
+    const [exportOperation] = await datastore.export({bucket});
+    await exportOperation.promise();
+
+    // The export operation has created a new file in your bucket, e.g.
+    // gs://{YOUR_BUCKET_NAME}/{timestamp}/{timestamp}.overall_export.metadata
+    console.log(`Export file created: ${exportOperation.result.outputUrl}`);
+
+    // You may also choose to include only specific kinds and namespaces.
+    const [specificExportOperation] = await datastore.export({
+      bucket,
+      kinds: ['Employee', 'Task'],
+      namespaces: ['Company'],
+    });
+    await specificExportOperation.promise();
+    console.log(specificExportOperation.result.outputUrl);
+  }
+
+  await exportEntities();
+  // [END datastore_admin_entities_export]
+}
+
+const args = process.argv.slice(2);
+main(...args).catch(console.error);

samples/import.js

@@ -0,0 +1,53 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+async function main(file = 'YOUR_FILE_NAME') {
+  // [START datastore_admin_entities_import]
+  const {Datastore} = require('@google-cloud/datastore');
+  const datastore = new Datastore();
+
+  async function importEntities() {
+    /**
+     * TODO(developer): Uncomment these variables before running the sample.
+     */
+    // const file = 'YOUR_FILE_NAME';
+
+    const [importOperation] = await datastore.import({file});
+
+    // Uncomment to await the results of the operation.
+    // await importOperation.promise();
+
+    // Or cancel the operation.
+    await importOperation.cancel();
+
+    // You may also choose to include only specific kinds and namespaces.
+    const [specificImportOperation] = await datastore.import({
+      file,
+      kinds: ['Employee', 'Task'],
+      namespaces: ['Company'],
+    });
+
+    // Uncomment to await the results of the operation.
+    // await specificImportOperation.promise();
+
+    // Or cancel the operation.
+    await specificImportOperation.cancel();
+  }
+
+  await importEntities();
+  // [END datastore_admin_entities_import]
+}
+
+const args = process.argv.slice(2);
+main(...args).catch(console.error);

samples/indexes.get.js

@@ -0,0 +1,39 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+async function main(indexId = 'YOUR_INDEX_ID') {
+  // [START datastore_admin_index_get]
+  const {Datastore} = require('@google-cloud/datastore');
+  const datastore = new Datastore();
+
+  async function getIndex() {
+    /**
+     * TODO(developer): Uncomment these variables before running the sample.
+     */
+    // const indexId = 'YOUR_INDEX_ID';
+
+    // Create a reference to the index before performing operations on it.
+    const index = datastore.index(indexId);
+
+    // Get the index's metadata, including its state, properties, and more.
+    const [metadata] = await index.getMetadata();
+    console.log('Properties:', metadata.properties);
+  }
+
+  await getIndex();
+  // [END datastore_admin_index_get]
+}
+
+const args = process.argv.slice(2);
+main(...args).catch(console.error);

samples/indexes.list.js

@@ -0,0 +1,45 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+async function main() {
+  // [START datastore_admin_index_list]
+  const {Datastore} = require('@google-cloud/datastore');
+  const datastore = new Datastore();
+
+  async function listIndexes() {
+    const [indexes] = await datastore.getIndexes();
+
+    console.log(`${indexes.length} indexes returned.`);
+
+    // Each returned Index object includes metadata about the index.
+    const [firstIndex] = indexes;
+    console.log('Properties:', firstIndex.metadata.properties);
+
+    // You may also get Index references as a readable object stream.
+    datastore
+      .getIndexesStream()
+      .on('data', index => {
+        console.log('Properties:', index.metadata.properties);
+      })
+      .on('end', () => {
+        // All matching Index objects have been returned.
+      });
+  }
+
+  await listIndexes();
+  // [END datastore_admin_index_list]
+}
+
+const args = process.argv.slice(2);
+main(...args).catch(console.error);

samples/test/import-export.test.js

@@ -0,0 +1,46 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {Datastore} = require('@google-cloud/datastore');
+const {assert} = require('chai');
+const cp = require('child_process');
+const {before, describe, it} = require('mocha');
+
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+describe('import/export entities', async () => {
+  const datastore = new Datastore();
+
+  const BUCKET_ID = 'nodejs-datastore-system-tests';
+  let EXPORTED_FILE;
+
+  before(async () => {
+    const [exportOperation] = await datastore.export({
+      bucket: BUCKET_ID,
+    });
+    await exportOperation.promise();
+    EXPORTED_FILE = exportOperation.result.outputUrl;
+  });
+
+  it('should export entities', async () => {
+    const stdout = execSync(`node ./export.js ${BUCKET_ID}`);
+    assert.include(stdout, 'Export file created:');
+  });
+
+  it('should import entities', () => {
+    execSync(`node ./import.js ${EXPORTED_FILE}`);
+  });
+});

samples/test/indexes.test.js

@@ -0,0 +1,41 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+'use strict';
+
+const {Datastore} = require('@google-cloud/datastore');
+const {assert} = require('chai');
+const cp = require('child_process');
+const {describe, it} = require('mocha');
+
+const execSync = cmd => cp.execSync(cmd, {encoding: 'utf-8'});
+
+describe('indexes', async () => {
+  const datastore = new Datastore();
+
+  // @TODO: Until the protos support creating indexes, these tests depend on
+  // the remote state of declared indexes. Could be flaky!
+
+  it('should list indexes', async () => {
+    const [indexes] = await datastore.getIndexes();
+    const stdout = execSync('node ./indexes.list.js');
+    assert.include(stdout, `${indexes.length} indexes returned.`);
+  });
+
+  it('should get a specific index', async () => {
+    const [indexes] = await datastore.getIndexes();
+    const stdout = execSync(`node ./indexes.get.js ${indexes[0].id}`);
+    assert.include(stdout, 'Properties:');
+  });
+});