Skip to content

Commit 610c8ed

Browse files
rwhoggrwhogg
committed
Firestore clone support
1 parent d00e11f commit 610c8ed

File tree

7 files changed

+264
-0
lines changed

7 files changed

+264
-0
lines changed

‎src/commands/firestore-databases-clone.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
import * as clc from "colorette";
2+
3+
import { Command } from "../command";
4+
import * as fsi from "../firestore/api";
5+
import * as types from "../firestore/api-types";
6+
import { getCurrentMinuteAsIsoString, parseDatabaseName } from "../firestore/util";
7+
import { logger } from "../logger";
8+
import { requirePermissions } from "../requirePermissions";
9+
import { Emulators } from "../emulator/types";
10+
import { warnEmulatorNotSupported } from "../emulator/commandUtils";
11+
import { EncryptionType, FirestoreOptions } from "../firestore/options";
12+
import { PrettyPrint } from "../firestore/pretty-print";
13+
import { FirebaseError } from "../error";
14+
15+
export const command = new Command("firestore:databases:clone <sourceDatabase> <targetDatabase>")
16+
.description("clone one Firestore database to another")
17+
.option(
18+
"-e, --encryption-type <encryptionType>",
19+
`encryption method of the cloned database; one of ${EncryptionType.USE_SOURCE_ENCRYPTION} (default), ` +
20+
`${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}, ${EncryptionType.GOOGLE_DEFAULT_ENCRYPTION}`,
21+
)
22+
// TODO(b/356137854): Remove allowlist only message once feature is public GA.
23+
.option(
24+
"-k, --kms-key-name <kmsKeyName>",
25+
"resource ID of the Cloud KMS key to encrypt the cloned database. This " +
26+
"feature is allowlist only in initial launch",
27+
)
28+
.option(
29+
"-s, --snapshot-time <snapshotTime>",
30+
"snapshot time of the source database to use, in ISO 8601 format. Can be any minutely snapshot after the database's earliest version time. If unspecified, takes the most recent available snapshot",
31+
)
32+
.before(requirePermissions, ["datastore.databases.clone"])
33+
.before(warnEmulatorNotSupported, Emulators.FIRESTORE)
34+
.action(async (sourceDatabase: string, targetDatabase: string, options: FirestoreOptions) => {
35+
const api = new fsi.FirestoreApi();
36+
const printer = new PrettyPrint();
37+
const helpCommandText = "See firebase firestore:databases:clone --help for more info.";
38+
39+
if (options.database) {
40+
throw new FirebaseError(`Please do not use --database for this command. ${helpCommandText}`);
41+
}
42+
43+
let snapshotTime: string;
44+
if (options.snapshotTime) {
45+
snapshotTime = options.snapshotTime;
46+
} else {
47+
snapshotTime = getCurrentMinuteAsIsoString();
48+
}
49+
50+
let encryptionConfig: types.EncryptionConfig | undefined = undefined;
51+
switch (options.encryptionType) {
52+
case EncryptionType.GOOGLE_DEFAULT_ENCRYPTION:
53+
throwIfKmsKeyNameIsSet(options.kmsKeyName);
54+
encryptionConfig = { googleDefaultEncryption: {} };
55+
break;
56+
case EncryptionType.USE_SOURCE_ENCRYPTION:
57+
throwIfKmsKeyNameIsSet(options.kmsKeyName);
58+
encryptionConfig = { useSourceEncryption: {} };
59+
break;
60+
case EncryptionType.CUSTOMER_MANAGED_ENCRYPTION:
61+
encryptionConfig = {
62+
customerManagedEncryption: { kmsKeyName: getKmsKeyOrThrow(options.kmsKeyName) },
63+
};
64+
break;
65+
case undefined:
66+
throwIfKmsKeyNameIsSet(options.kmsKeyName);
67+
break;
68+
default:
69+
throw new FirebaseError(`Invalid value for flag --encryption-type. ${helpCommandText}`);
70+
}
71+
72+
// projects must be the same
73+
const targetDatabaseName = parseDatabaseName(targetDatabase);
74+
const parentProject = targetDatabaseName.projectId;
75+
const targetDatabaseId = targetDatabaseName.databaseId;
76+
const sourceProject = parseDatabaseName(sourceDatabase).projectId;
77+
if (parentProject !== sourceProject) {
78+
throw new FirebaseError(`Source and target projects must match.`);
79+
}
80+
const lro: types.Operation = await api.cloneDatabase(
81+
sourceProject,
82+
{
83+
database: sourceDatabase,
84+
snapshotTime,
85+
},
86+
targetDatabaseId,
87+
encryptionConfig,
88+
);
89+
90+
if (options.json) {
91+
logger.info(JSON.stringify(lro, undefined, 2));
92+
} else if (lro.error) {
93+
logger.error(
94+
clc.bold(
95+
`Clone to ${printer.prettyDatabaseString(targetDatabase)} failed. See below for details.`,
96+
),
97+
);
98+
printer.prettyPrintOperation(lro);
99+
} else {
100+
logger.info(
101+
clc.bold(`Successfully initiated clone to ${printer.prettyDatabaseString(targetDatabase)}`),
102+
);
103+
logger.info(
104+
"Please be sure to configure Firebase rules in your Firebase config file for\n" +
105+
"the new database. By default, created databases will have closed rules that\n" +
106+
"block any incoming third-party traffic.",
107+
);
108+
logger.info();
109+
logger.info(`You can monitor the progress of this clone by executing this command:`);
110+
logger.info();
111+
logger.info(
112+
`firebase firestore:operations:describe --database="${targetDatabaseId}" ${lro.name}`,
113+
);
114+
logger.info();
115+
logger.info(
116+
`Once the clone is complete, your database may be viewed at ${printer.firebaseConsoleDatabaseUrl(options.project, targetDatabaseId)}`,
117+
);
118+
}
119+
120+
return lro;
121+
122+
function throwIfKmsKeyNameIsSet(kmsKeyName: string | undefined): void {
123+
if (kmsKeyName) {
124+
throw new FirebaseError(
125+
"--kms-key-name can only be set when specifying an --encryption-type " +
126+
`of ${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}.`,
127+
);
128+
}
129+
}
130+
131+
function getKmsKeyOrThrow(kmsKeyName: string | undefined): string {
132+
if (kmsKeyName) return kmsKeyName;
133+
134+
throw new FirebaseError(
135+
"--kms-key-name must be provided when specifying an --encryption-type " +
136+
`of ${EncryptionType.CUSTOMER_MANAGED_ENCRYPTION}.`,
137+
);
138+
}
139+
});

‎src/commands/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ export function load(client: any): any {
115115
client.firestore.databases.update = loadCommand("firestore-databases-update");
116116
client.firestore.databases.delete = loadCommand("firestore-databases-delete");
117117
client.firestore.databases.restore = loadCommand("firestore-databases-restore");
118+
client.firestore.databases.clone = loadCommand("firestore-databases-clone");
118119
client.firestore.backups = {};
119120
client.firestore.backups.schedules = {};
120121
client.firestore.backups.list = loadCommand("firestore-backups-list");

‎src/firestore/api-types.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,12 @@ export interface RestoreDatabaseReq {
219219
encryptionConfig?: EncryptionConfig;
220220
}
221221

222+
export interface CloneDatabaseReq {
223+
databaseId: string;
224+
pitrSnapshot: PitrSnapshot;
225+
encryptionConfig?: EncryptionConfig;
226+
}
227+
222228
export enum RecurrenceType {
223229
DAILY = "DAILY",
224230
WEEKLY = "WEEKLY",
@@ -239,3 +245,8 @@ export type EncryptionConfig =
239245
| UseCustomerManagedEncryption
240246
| UseSourceEncryption
241247
| UseGoogleDefaultEncryption;
248+
249+
export interface PitrSnapshot {
250+
database: string;
251+
snapshotTime: string;
252+
}

‎src/firestore/api.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -902,6 +902,39 @@ export class FirestoreApi {
902902
return database;
903903
}
904904

905+
/**
906+
* Clone one Firestore Database to another.
907+
* @param project the source project ID
908+
* @param pitrSnapshot Source database PITR snapshot specification
909+
* @param databaseId ID of the target database
910+
* @param encryptionConfig the encryption configuration of the new database
911+
*/
912+
async cloneDatabase(
913+
project: string,
914+
pitrSnapshot: types.PitrSnapshot,
915+
databaseId: string,
916+
encryptionConfig?: types.EncryptionConfig,
917+
): Promise<types.Operation> {
918+
const url = `/projects/${project}/databases:clone`;
919+
const payload: types.CloneDatabaseReq = {
920+
databaseId,
921+
pitrSnapshot,
922+
encryptionConfig,
923+
};
924+
const options = { queryParams: { databaseId: databaseId } };
925+
const res = await this.apiClient.post<types.CloneDatabaseReq, types.Operation>(
926+
url,
927+
payload,
928+
options,
929+
);
930+
const lro = res.body;
931+
if (!lro) {
932+
throw new FirebaseError("Not found");
933+
}
934+
935+
return lro;
936+
}
937+
905938
/**
906939
* List the long-running Firestore operations.
907940
* @param project the Firebase project id.

‎src/firestore/options.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ export interface FirestoreOptions extends Options {
3232
// CMEK
3333
encryptionType?: EncryptionType;
3434
kmsKeyName?: string;
35+
36+
// Clone
37+
snapshotTime?: string;
3538
}
3639

3740
export enum EncryptionType {

‎src/firestore/util.spec.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,32 @@ import { expect } from "chai";
22

33
import * as util from "./util";
44

5+
describe("Database name parsing", () => {
6+
it("should parse a database other than (default) correctly", () => {
7+
const name = "projects/myproject/databases/named-db";
8+
expect(util.parseDatabaseName(name)).to.eql({
9+
projectId: "myproject",
10+
databaseId: "named-db",
11+
});
12+
});
13+
14+
it("should parse the (default) database name correctly", () => {
15+
const name = "projects/myproject/databases/(default)";
16+
expect(util.parseDatabaseName(name)).to.eql({
17+
projectId: "myproject",
18+
databaseId: "(default)",
19+
});
20+
});
21+
22+
it("should work even if the name has a trailing slash", () => {
23+
const name = "projects/myproject/databases/with-trailing-slash/";
24+
expect(util.parseDatabaseName(name)).to.eql({
25+
projectId: "myproject",
26+
databaseId: "with-trailing-slash",
27+
});
28+
});
29+
});
30+
531
describe("IndexNameParsing", () => {
632
it("should parse an index name correctly", () => {
733
const name =
@@ -47,3 +73,13 @@ describe("IndexNameParsing", () => {
4773
});
4874
});
4975
});
76+
77+
describe("Get current minute", () => {
78+
it("should be a string in ISO 8601 format with no second or millisecond component", () => {
79+
const currentMinuteString = util.getCurrentMinuteAsIsoString();
80+
expect(currentMinuteString.endsWith("Z")).to.eql(true);
81+
const asDate = new Date(Date.parse(currentMinuteString));
82+
expect(asDate.getSeconds()).to.eql(0);
83+
expect(asDate.getMilliseconds()).to.eql(0);
84+
});
85+
});

‎src/firestore/util.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
import { FirebaseError } from "../error";
22

3+
interface DatabaseName {
4+
projectId: string;
5+
databaseId: string;
6+
}
7+
38
interface IndexName {
49
projectId: string;
510
databaseId: string;
@@ -14,6 +19,9 @@ interface FieldName {
1419
fieldPath: string;
1520
}
1621

22+
// projects/$PROJECT_ID/databases/$DATABASE_ID
23+
const DATABASE_NAME_REGEX = /projects\/([^\/]+?)\/databases\/([^\/]+)/;
24+
1725
// projects/$PROJECT_ID/databases/$DATABASE_ID/collectionGroups/$COLLECTION_GROUP_ID/indexes/$INDEX_ID
1826
const INDEX_NAME_REGEX =
1927
/projects\/([^\/]+?)\/databases\/([^\/]+?)\/collectionGroups\/([^\/]+?)\/indexes\/([^\/]*)/;
@@ -22,6 +30,25 @@ const INDEX_NAME_REGEX =
2230
const FIELD_NAME_REGEX =
2331
/projects\/([^\/]+?)\/databases\/([^\/]+?)\/collectionGroups\/([^\/]+?)\/fields\/([^\/]*)/;
2432

33+
/**
34+
* Parse a Database name into useful pieces.
35+
*/
36+
export function parseDatabaseName(name?: string): DatabaseName {
37+
if (!name) {
38+
throw new FirebaseError(`Cannot parse undefined database name.`);
39+
}
40+
41+
const m = name.match(DATABASE_NAME_REGEX);
42+
if (!m || m.length < 3) {
43+
throw new FirebaseError(`Error parsing database name: ${name}`);
44+
}
45+
46+
return {
47+
projectId: m[1],
48+
databaseId: m[2],
49+
};
50+
}
51+
2552
/**
2653
* Parse an Index name into useful pieces.
2754
*/
@@ -66,3 +93,17 @@ export function parseFieldName(name: string): FieldName {
6693
export function booleanXOR(a: boolean, b: boolean): boolean {
6794
return !!(Number(a) - Number(b));
6895
}
96+
97+
/**
98+
* Get the current time truncated to minutes.
99+
*
100+
* For some tasks, eg. database clone, this will be the most recent available snapshot time.
101+
*
102+
* @returns A Protobuf-friendly ISO string of the current time in the UTC timezone
103+
*/
104+
export function getCurrentMinuteAsIsoString(): string {
105+
// Note that JS Dates support millisecond precision at max and that toISOString forces the UTC timezone, which will be represented by the (Protobuf-friendly) "Z"
106+
const mostRecentTimestamp = new Date(Date.now());
107+
mostRecentTimestamp.setSeconds(0, 0);
108+
return mostRecentTimestamp.toISOString();
109+
}

0 commit comments

Comments
 (0)