Skip to content

Commit b8af18d

Browse files
joehantaeold
joehan
and
taeold
authored
Add caching for API enablement (#8766)
* Add caching for API enablement * Update src/ensureApiEnabled.ts Co-authored-by: Daniel Lee <[email protected]> * Use a constant --------- Co-authored-by: Daniel Lee <[email protected]>
1 parent 39145d7 commit b8af18d

File tree

3 files changed

+90
-0
lines changed

3 files changed

+90
-0
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
- Updated the Data Connect emulator to use pglite 0.3.x and Postgres 17, which fixes some crashes related to wire protocol inconsistencies. (#8679, #8658)
22
- Fixed an issue where the IAM enablement for GenKit monitoring would try to change an invalid service account. (#8756)
3+
- Added caching to API enablement checks to reduce burn of `serviceusage.googleapis.com` quota.

‎src/ensureApiEnabled.spec.ts

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,42 @@
11
import { expect } from "chai";
22
import * as nock from "nock";
3+
import * as sinon from "sinon";
4+
import { configstore } from "./configstore";
35

46
import { check, ensure, POLL_SETTINGS } from "./ensureApiEnabled";
57

68
const FAKE_PROJECT_ID = "my_project";
79
const FAKE_API = "myapi.googleapis.com";
10+
const FAKE_CACHE: Record<string, Record<string, boolean>> = {
11+
my_project: { "myapi.googleapis.com": true },
12+
};
813

914
describe("ensureApiEnabled", () => {
1015
describe("check", () => {
16+
const sandbox = sinon.createSandbox();
17+
let configstoreGetMock: sinon.SinonStub;
18+
let configstoreSetMock: sinon.SinonStub;
1119
before(() => {
1220
nock.disableNetConnect();
1321
});
1422

1523
after(() => {
1624
nock.enableNetConnect();
1725
});
26+
27+
beforeEach(() => {
28+
configstoreGetMock = sandbox.stub(configstore, "get");
29+
configstoreSetMock = sandbox.stub(configstore, "set");
30+
});
31+
32+
afterEach(() => {
33+
sandbox.restore();
34+
});
35+
1836
for (const prefix of ["", "https://", "http://"]) {
1937
it("should call the API to check if it's enabled", async () => {
38+
configstoreGetMock.returns(undefined);
39+
configstoreSetMock.returns(undefined);
2040
nock("https://serviceusage.googleapis.com")
2141
.get(`/v1/projects/${FAKE_PROJECT_ID}/services/${FAKE_API}`)
2242
.matchHeader("x-goog-quota-user", `projects/${FAKE_PROJECT_ID}`)
@@ -25,9 +45,12 @@ describe("ensureApiEnabled", () => {
2545
await check(FAKE_PROJECT_ID, prefix + FAKE_API, "", true);
2646

2747
expect(nock.isDone()).to.be.true;
48+
expect(configstoreSetMock.calledWith(FAKE_PROJECT_ID, FAKE_API));
2849
});
2950

3051
it("should return the value from the API", async () => {
52+
configstoreGetMock.returns(undefined);
53+
configstoreSetMock.returns(undefined);
3154
nock("https://serviceusage.googleapis.com")
3255
.get(`/v1/projects/${FAKE_PROJECT_ID}/services/${FAKE_API}`)
3356
.matchHeader("x-goog-quota-user", `projects/${FAKE_PROJECT_ID}`)
@@ -44,26 +67,41 @@ describe("ensureApiEnabled", () => {
4467

4568
await expect(check(FAKE_PROJECT_ID, prefix + FAKE_API, "", true)).to.eventually.be.false;
4669
});
70+
it("should skip the API call if the enablement is saved in the cache", async () => {
71+
configstoreGetMock.returns(FAKE_CACHE);
72+
configstoreSetMock.returns(undefined);
73+
74+
await expect(check(FAKE_PROJECT_ID, prefix + FAKE_API, "", true)).to.eventually.be.true;
75+
});
4776
}
4877
});
4978

5079
describe("ensure", () => {
80+
const sandbox = sinon.createSandbox();
81+
let configstoreGetMock: sinon.SinonStub;
82+
let configstoreSetMock: sinon.SinonStub;
5183
const originalPollInterval = POLL_SETTINGS.pollInterval;
5284
const originalPollsBeforeRetry = POLL_SETTINGS.pollsBeforeRetry;
5385
beforeEach(() => {
5486
nock.disableNetConnect();
5587
POLL_SETTINGS.pollInterval = 0;
5688
POLL_SETTINGS.pollsBeforeRetry = 0; // Zero means "one check".
89+
90+
configstoreGetMock = sandbox.stub(configstore, "get");
91+
configstoreSetMock = sandbox.stub(configstore, "set");
5792
});
5893

5994
afterEach(() => {
6095
nock.enableNetConnect();
6196
POLL_SETTINGS.pollInterval = originalPollInterval;
6297
POLL_SETTINGS.pollsBeforeRetry = originalPollsBeforeRetry;
98+
sandbox.restore();
6399
});
64100

65101
for (const prefix of ["", "https://", "http://"]) {
66102
it("should verify that the API is enabled, and stop if it is", async () => {
103+
configstoreGetMock.returns(undefined);
104+
configstoreSetMock.returns(undefined);
67105
nock("https://serviceusage.googleapis.com")
68106
.get(`/v1/projects/${FAKE_PROJECT_ID}/services/${FAKE_API}`)
69107
.matchHeader("x-goog-quota-user", `projects/${FAKE_PROJECT_ID}`)
@@ -73,7 +111,16 @@ describe("ensureApiEnabled", () => {
73111
await expect(ensure(FAKE_PROJECT_ID, prefix + FAKE_API, "", true)).to.not.be.rejected;
74112
});
75113

114+
it("should verify that the API is enabled (in the cache), and stop if it is", async () => {
115+
configstoreGetMock.returns(FAKE_CACHE);
116+
configstoreSetMock.returns(undefined);
117+
118+
await expect(ensure(FAKE_PROJECT_ID, prefix + FAKE_API, "", true)).to.not.be.rejected;
119+
});
120+
76121
it("should attempt to enable the API if it is not enabled", async () => {
122+
configstoreGetMock.returns(undefined);
123+
configstoreSetMock.returns(undefined);
77124
nock("https://serviceusage.googleapis.com")
78125
.get(`/v1/projects/${FAKE_PROJECT_ID}/services/${FAKE_API}`)
79126
.matchHeader("x-goog-quota-user", `projects/${FAKE_PROJECT_ID}`)
@@ -94,9 +141,12 @@ describe("ensureApiEnabled", () => {
94141
await expect(ensure(FAKE_PROJECT_ID, prefix + FAKE_API, "", true)).to.not.be.rejected;
95142

96143
expect(nock.isDone()).to.be.true;
144+
expect(configstoreSetMock.calledWith(FAKE_PROJECT_ID, FAKE_API));
97145
});
98146

99147
it("should retry enabling the API if it does not enable in time", async () => {
148+
configstoreGetMock.returns(undefined);
149+
configstoreSetMock.returns(undefined);
100150
nock("https://serviceusage.googleapis.com")
101151
.get(`/v1/projects/${FAKE_PROJECT_ID}/services/${FAKE_API}`)
102152
.matchHeader("x-goog-quota-user", `projects/${FAKE_PROJECT_ID}`)

‎src/ensureApiEnabled.ts

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { Client } from "./apiv2";
66
import * as utils from "./utils";
77
import { FirebaseError, isBillingError } from "./error";
88
import { logger } from "./logger";
9+
import { configstore } from "./configstore";
910

1011
export const POLL_SETTINGS = {
1112
pollInterval: 10000,
@@ -31,6 +32,9 @@ export async function check(
3132
silent = false,
3233
): Promise<boolean> {
3334
const apiName = apiUri.startsWith("http") ? new URL(apiUri).hostname : apiUri;
35+
if (checkAPIEnablementCache(projectId, apiName)) {
36+
return true;
37+
}
3438
const res = await apiClient.get<{ state: string }>(`/projects/${projectId}/services/${apiName}`, {
3539
headers: { "x-goog-quota-user": `projects/${projectId}` },
3640
skipLog: { resBody: true },
@@ -39,6 +43,9 @@ export async function check(
3943
if (isEnabled && !silent) {
4044
utils.logLabeledSuccess(prefix, `required API ${bold(apiName)} is enabled`);
4145
}
46+
if (isEnabled) {
47+
cacheEnabledAPI(projectId, apiName);
48+
}
4249
return isEnabled;
4350
}
4451

@@ -65,6 +72,7 @@ async function enable(projectId: string, apiName: string): Promise<void> {
6572
skipLog: { resBody: true },
6673
},
6774
);
75+
cacheEnabledAPI(projectId, apiName);
6876
} catch (err: any) {
6977
if (isBillingError(err)) {
7078
throw new FirebaseError(`Your project ${bold(
@@ -200,3 +208,34 @@ export async function bestEffortEnsure(
200208
export function enableApiURI(projectId: string, apiName: string): string {
201209
return `https://console.cloud.google.com/apis/library/${apiName}?project=${projectId}`;
202210
}
211+
212+
/**
213+
* To reduce serviceusage quota burn, we cache API enablement status in configstore.
214+
* Once we see that an API is enabled, we skip future checks. This is safe, because:
215+
* A - It's rare to disable APIs
216+
* B - If the API actually is disabled, the user gets a clear error message with a link to enable it.
217+
*
218+
* We intentionally do not cache when we see an API is not enabled - some users need to have admins enable APIS,
219+
* so we expect APIs to get enabled out of band frequently.
220+
*/
221+
222+
const API_ENABLEMENT_CACHE_KEY = "apiEnablementCache";
223+
function checkAPIEnablementCache(projectId: string, apiName: string): boolean {
224+
const cache = configstore.get(API_ENABLEMENT_CACHE_KEY) as Record<
225+
string,
226+
Record<string, boolean>
227+
>;
228+
return !!cache?.[projectId]?.[apiName];
229+
}
230+
231+
function cacheEnabledAPI(projectId: string, apiName: string) {
232+
const cache = (configstore.get(API_ENABLEMENT_CACHE_KEY) || {}) as Record<
233+
string,
234+
Record<string, true>
235+
>;
236+
if (!cache[projectId]) {
237+
cache[projectId] = {};
238+
}
239+
cache[projectId][apiName] = true;
240+
configstore.set(API_ENABLEMENT_CACHE_KEY, cache);
241+
}

0 commit comments

Comments
 (0)