Scheduled backups for Firestore (#19)

Author: samtstern

.eslintrc.json

@@ -8,6 +8,7 @@
     },
     "extends": "eslint:recommended",
     "parserOptions": {
+        "ecmaVersion": 8,
         "sourceType": "module"
     },
     "rules": {

firestore/solution-scheduled-backups/.gcloudignore

@@ -0,0 +1,17 @@
+# This file specifies files that are *not* uploaded to Google Cloud Platform
+# using gcloud. It follows the same syntax as .gitignore, with the addition of
+# "#!include" directives (which insert the entries of the given .gitignore-style
+# file at that point).
+#
+# For more information, run:
+#   $ gcloud topic gcloudignore
+#
+.gcloudignore
+# If you would like to upload your .git directory, .gitignore file or files
+# from your .gitignore file, remove the corresponding line
+# below:
+.git
+.gitignore
+
+# Node.js dependencies:
+node_modules/

firestore/solution-scheduled-backups/README.md

@@ -0,0 +1,74 @@
+# Cloud Firestore Scheduled Backups
+
+This sample demonstrates using AppEngine cron jobs to run nightly backups
+of data in Cloud Firestore.
+
+## Setup
+
+### 1 - Create a Project
+If you haven't already, create a new Firebase project and create a Cloud
+Firestore database within the project.
+
+### 2 - Configure IAM
+This sample will use the AppEngine default service account to perform
+backups of your Cloud Firestore data. To do this, you will need to give
+the service account permission to access your data and save it to
+google cloud storage.
+
+Make the service account a Datastore Import/Export Admin:
+
+```shell
+$ gcloud projects add-iam-policy-binding YOUR_PROJECT_ID \
+    --member serviceAccount:[email protected] --role roles/datastore.importExportAdmin
+```
+
+Give the service account permission to write to the GCS bucket you
+are going to use. Here we will use the default bucket:
+
+```shell
+$ sutil iam ch \
+  serviceAccount:[email protected]:objectCreator \
+  gs://YOUR_PROJECT_ID.appspot.com
+```
+
+### 3 - Configure Cron
+Open `cron.yaml` and edit this line:
+
+```
+/cloud-firestore-export?outputUriPrefix=gs://BUCKET_NAME[/PATH]&collections=test1,test2
+```
+
+You shoudl change `gs://BUCKET_NAME[/PATH]` to the Google Cloud Storage
+path where your data should be backed up. If you only want to back up certain
+collections, change `test1,test2` to a comma-separated list of those collections.
+Otherwise, delete the `&collections=test1,test2` param.
+
+## Deploy
+
+To deploy the project, run:
+
+```
+$ gcloud app deploy app.yaml cron.yaml
+```
+
+To make sure it deployed correctly, navigate to https://YOUR_PROJECT_ID.appspot.com/
+
+## Test
+
+To test the backup, navigate to the following URL:
+https://YOUR_PROJECT_ID.appspot.com/cloud-firestore-export?outputUriPrefix=gs://YOUR_PROJECT_ID.appspot.com
+
+You should see some output like this, letting you know that a backup
+was started:
+
+```js
+{
+  name: "projects/YOUR_PROJECT_ID/databases/(default)/operations/ASA2NDIwNjI3ODQJGnRsdWFmZWQHEmxhcnRuZWNzdS1zYm9qLW5pbWRhFAosEg",
+  metadata: {
+    @type: "type.googleapis.com/google.firestore.admin.v1beta1.ExportDocumentsMetadata",
+    startTime: "2018-10-11T22:47:15.473517Z",
+    operationState: "PROCESSING",
+    outputUriPrefix: "gs://YOUR_PROJECT_ID.appspot.com/2018-10-11-22-47-15"
+  }
+}
+```

firestore/solution-scheduled-backups/app.js

@@ -0,0 +1,80 @@
+const axios = require('axios');
+const dateformat = require('dateformat');
+const express = require('express');
+const { google } = require('googleapis');
+
+const app = express();
+
+// Trigger a backup
+app.get('/cloud-firestore-export', async (req, res) => {
+  const auth = await google.auth.getClient({
+    scopes: ['https://www.googleapis.com/auth/datastore']
+  });
+
+  const accessTokenResponse = await auth.getAccessToken();
+  const accessToken = accessTokenResponse.token;
+
+  const headers = {
+    'Content-Type': 'application/json',
+    Authorization: 'Bearer ' + accessToken
+  };
+
+  const outputUriPrefix = req.param('outputUriPrefix');
+  if (!(outputUriPrefix && outputUriPrefix.indexOf('gs://') == 0)) {
+    res.status(500).send(`Malformed outputUriPrefix: ${outputUriPrefix}`);
+  }
+
+  // Construct a backup path folder based on the timestamp
+  const timestamp = dateformat(Date.now(), 'yyyy-mm-dd-HH-MM-ss');
+  let path = outputUriPrefix;
+  if (path.endsWith('/')) {
+    path += timestamp;
+  } else {
+    path += '/' + timestamp;
+  }
+
+  const body = {
+    outputUriPrefix: path
+  };
+
+  // If specified, mark specific collections for backup
+  const collectionParam = req.param('collections');
+  if (collectionParam) {
+    body.collectionIds = collectionParam.split(',');
+  }
+
+  const projectId = process.env.GOOGLE_CLOUD_PROJECT;
+  const url = `https://firestore.googleapis.com/v1beta1/projects/${projectId}/databases/(default):exportDocuments`;
+
+  try {
+    const response = await axios.post(url, body, { headers: headers });
+    res
+      .status(200)
+      .send(response.data)
+      .end();
+  } catch (e) {
+    if (e.response) {
+      console.warn(e.response.data);
+    }
+
+    res
+      .status(500)
+      .send('Could not start backup: ' + e)
+      .end();
+  }
+});
+
+// Index page, just to make it easy to see if the app is working.
+app.get('/', (req, res) => {
+  res
+    .status(200)
+    .send('[scheduled-backups]: Hello, world!')
+    .end();
+});
+
+// Start the server
+const PORT = process.env.PORT || 6060;
+app.listen(PORT, () => {
+  console.log(`App listening on port ${PORT}`);
+  console.log('Press Ctrl+C to quit.');
+});

firestore/solution-scheduled-backups/app.yaml

@@ -0,0 +1 @@
+runtime: nodejs8

firestore/solution-scheduled-backups/cron.yaml

@@ -0,0 +1,5 @@
+cron:
+- description: "Daily Cloud Firestore Export"
+  url: /cloud-firestore-export?outputUriPrefix=gs://BUCKET_NAME[/PATH]&collections=test1,test2
+  target: cloud-firestore-admin
+  schedule: every 24 hours

firestore/solution-scheduled-backups/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "solution-scheduled-backups",
+  "version": "1.0.0",
+  "description": "Scheduled Cloud Firestore backups via AppEngine cron",
+  "main": "app.js",
+  "engines": {
+    "node": "8.x.x"
+  },
+  "scripts": {
+    "deploy": "gcloud app deploy --quiet app.yaml cron.yaml",
+    "start": "node app.js"
+  },
+  "author": "Google, Inc.",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "axios": "^0.18.0",
+    "dateformat": "^3.0.3",
+    "express": "^4.16.4",
+    "googleapis": "^34.0.0"
+  },
+  "devDependencies": {
+    "prettier": "^1.14.3"
+  }
+}