Skip to content

Commit cb315f7

Browse files
dumbmorondumbmoron
committed
api: dynamic env reloading from path/url
1 parent 6389dec commit cb315f7

File tree

5 files changed

+131
-21
lines changed

5 files changed

+131
-21
lines changed

‎api/src/config.js‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { getVersion } from "@imput/version-info";
2-
import { loadEnvs, validateEnvs } from "./core/env.js";
2+
import { loadEnvs, validateEnvs, setupEnvWatcher } from "./core/env.js";
3+
import * as cluster from "./misc/cluster.js";
34

45
const version = await getVersion();
56

@@ -8,11 +9,22 @@ let env = loadEnvs();
89
const genericUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36";
910
const cobaltUserAgent = `cobalt/${version} (+https://github.com/imputnet/cobalt)`;
1011

12+
export const canonicalEnv = Object.freeze(structuredClone(process.env));
1113
export const setTunnelPort = (port) => env.tunnelPort = port;
1214
export const isCluster = env.instanceCount > 1;
15+
export const updateEnv = (newEnv) => {
16+
// tunnelPort is special and needs to get carried over here
17+
newEnv.tunnelPort = env.tunnelPort;
18+
env = newEnv;
19+
cluster.broadcast({ env_update: newEnv });
20+
}
1321

1422
await validateEnvs(env);
1523

24+
if (env.envFile) {
25+
setupEnvWatcher();
26+
}
27+
1628
export {
1729
env,
1830
genericUserAgent,

‎api/src/core/api.js‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -48,19 +48,23 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
4848
const startTime = new Date();
4949
const startTimestamp = startTime.getTime();
5050

51-
const serverInfo = JSON.stringify({
52-
cobalt: {
53-
version: version,
54-
url: env.apiURL,
55-
startTime: `${startTimestamp}`,
56-
durationLimit: env.durationLimit,
57-
turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
58-
services: [...env.enabledServices].map(e => {
59-
return friendlyServiceName(e);
60-
}),
61-
},
62-
git,
63-
})
51+
const getServerInfo = () => {
52+
return JSON.stringify({
53+
cobalt: {
54+
version: version,
55+
url: env.apiURL,
56+
startTime: `${startTimestamp}`,
57+
durationLimit: env.durationLimit,
58+
turnstileSitekey: env.sessionEnabled ? env.turnstileSitekey : undefined,
59+
services: [...env.enabledServices].map(e => {
60+
return friendlyServiceName(e);
61+
}),
62+
},
63+
git,
64+
});
65+
}
66+
67+
const serverInfo = getServerInfo();
6468

6569
const handleRateExceeded = (_, res) => {
6670
const { body } = createResponse("error", {
@@ -311,7 +315,7 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
311315

312316
app.get('/', (_, res) => {
313317
res.type('json');
314-
res.status(200).send(serverInfo);
318+
res.status(200).send(env.envFile ? getServerInfo() : serverInfo);
315319
})
316320

317321
app.get('/favicon.ico', (req, res) => {
@@ -331,10 +335,6 @@ export const runAPI = async (express, app, __dirname, isPrimary = true) => {
331335
setInterval(randomizeCiphers, 1000 * 60 * 30); // shuffle ciphers every 30 minutes
332336

333337
if (env.externalProxy) {
334-
if (env.freebindCIDR) {
335-
throw new Error('Freebind is not available when external proxy is enabled')
336-
}
337-
338338
setGlobalDispatcher(new ProxyAgent(env.externalProxy))
339339
}
340340

‎api/src/core/env.js‎

Lines changed: 87 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import { Constants } from "youtubei.js";
2-
import { supportsReusePort } from "../misc/cluster.js";
32
import { services } from "../processing/service-config.js";
3+
import { updateEnv, canonicalEnv, env as currentEnv } from "../config.js";
4+
5+
import { FileWatcher } from "../misc/file-watcher.js";
6+
import { isURL } from "../misc/utils.js";
7+
import * as cluster from "../misc/cluster.js";
8+
import { Yellow } from "../misc/console-text.js";
49

510
const forceLocalProcessingOptions = ["never", "session", "always"];
611

@@ -68,6 +73,9 @@ export const loadEnvs = (env = process.env) => {
6873

6974
// "never" | "session" | "always"
7075
forceLocalProcessing: env.FORCE_LOCAL_PROCESSING ?? "never",
76+
77+
envFile: env.API_ENV_FILE,
78+
envRemoteReloadInterval: 300,
7179
};
7280
}
7381

@@ -78,7 +86,7 @@ export const validateEnvs = async (env) => {
7886

7987
if (env.instanceCount > 1 && !env.redisURL) {
8088
throw new Error("API_REDIS_URL is required when API_INSTANCE_COUNT is >= 2");
81-
} else if (env.instanceCount > 1 && !await supportsReusePort()) {
89+
} else if (env.instanceCount > 1 && !await cluster.supportsReusePort()) {
8290
console.error('API_INSTANCE_COUNT is not supported in your environment. to use this env, your node.js');
8391
console.error('version must be >= 23.1.0, and you must be running a recent enough version of linux');
8492
console.error('(or other OS that supports it). for more info, see `reusePort` option on');
@@ -97,4 +105,81 @@ export const validateEnvs = async (env) => {
97105
console.error(`Supported options are are: ${forceLocalProcessingOptions.join(', ')}\n`);
98106
throw new Error("Invalid FORCE_LOCAL_PROCESSING");
99107
}
108+
109+
if (env.externalProxy && env.freebindCIDR) {
110+
throw new Error('freebind is not available when external proxy is enabled')
111+
}
112+
}
113+
114+
const reloadEnvs = async (contents) => {
115+
const newEnvs = {};
116+
117+
for (let line of (await contents).split('\n')) {
118+
line = line.trim();
119+
if (line === '') {
120+
continue;
121+
}
122+
123+
const [ key, value ] = line.split(/=(.+)?/);
124+
if (key) {
125+
newEnvs[key] = value || '';
126+
}
127+
}
128+
129+
const candidate = {
130+
...canonicalEnv,
131+
...newEnvs,
132+
};
133+
134+
const parsed = loadEnvs(candidate);
135+
await validateEnvs(parsed);
136+
updateEnv(parsed);
137+
}
138+
139+
const wrapReload = (contents) => {
140+
reloadEnvs(contents)
141+
.catch((e) => {
142+
console.error(`${Yellow('[!]')} Failed reloading environment variables at ${new Date().toISOString()}.`);
143+
console.error('Error:', e);
144+
});
145+
}
146+
147+
let watcher;
148+
const setupWatcherFromFile = (path) => {
149+
const load = () => wrapReload(watcher.read());
150+
151+
if (isURL(path)) {
152+
watcher = FileWatcher.fromFileProtocol(path);
153+
} else {
154+
watcher = new FileWatcher({ path });
155+
}
156+
157+
watcher.on('file-updated', load);
158+
load();
159+
}
160+
161+
const setupWatcherFromFetch = (url) => {
162+
const load = () => wrapReload(fetch(url).then(r => r.text()));
163+
setInterval(load, currentEnv.envRemoteReloadInterval);
164+
load();
165+
}
166+
167+
export const setupEnvWatcher = () => {
168+
if (cluster.isPrimary) {
169+
const envFile = currentEnv.envFile;
170+
const isFile = !isURL(envFile)
171+
|| new URL(envFile).protocol === 'file:';
172+
173+
if (isFile) {
174+
setupWatcherFromFile(envFile);
175+
} else {
176+
setupWatcherFromFetch(envFile);
177+
}
178+
} else if (cluster.isWorker) {
179+
process.on('message', (message) => {
180+
if ('env_update' in message) {
181+
updateEnv(message.env_update);
182+
}
183+
});
184+
}
100185
}

‎api/src/misc/utils.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,12 @@ export function splitFilenameExtension(filename) {
5252
export function zip(a, b) {
5353
return a.map((value, i) => [ value, b[i] ]);
5454
}
55+
56+
export function isURL(input) {
57+
try {
58+
new URL(input);
59+
return true;
60+
} catch {
61+
return false;
62+
}
63+
}

‎docs/api-env-variables.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ this document is not final and will expand over time. feel free to improve it!
1313
| API_REDIS_URL | | `redis://localhost:6379` |
1414
| DISABLED_SERVICES | | `bilibili,youtube` |
1515
| FORCE_LOCAL_PROCESSING | `never` | `always` |
16+
| API_ENV_FILE | | `/.env` |
1617

1718
[*view details*](#general)
1819

@@ -111,6 +112,9 @@ when set to `session`, only requests from session (Bearer token) clients will be
111112

112113
when set to `always`, all requests will be forced to use on-device processing, no matter the preference.
113114

115+
### API_ENV_FILE
116+
the URL or local path to a `key=value`-style environment variable file. this is used for dynamically reloading environment variables. **not all environment variables are able to be updated by this.** (e.g. the ratelimiters are instantiated when starting cobalt, and cannot be changed)
117+
114118
## networking
115119
[*jump to the table*](#networking-vars)
116120

0 commit comments

Comments
 (0)