diff --git a/src/commands/actors/push.ts b/src/commands/actors/push.ts index f0c2fc781..7cfc67114 100644 --- a/src/commands/actors/push.ts +++ b/src/commands/actors/push.ts @@ -79,6 +79,11 @@ export class ActorsPushCommand extends ApifyCommand { description: 'Directory where the Actor is located', required: false, }), + 'ignore-missing-secrets': Flags.boolean({ + description: 'Ignore missing secrets and show warnings instead of failing. Environment variables referencing missing secrets will be omitted.', + default: false, + required: false, + }), }; static override args = { @@ -280,7 +285,11 @@ Skipping push. Use --force to override.`, // Update Actor version const actorCurrentVersion = await actorClient.version(version).get(); const envVars = actorConfig!.environmentVariables - ? transformEnvToEnvVars(actorConfig!.environmentVariables as Record) + ? transformEnvToEnvVars( + actorConfig!.environmentVariables as Record, + undefined, + this.flags.ignoreMissingSecrets + ) : undefined; if (actorCurrentVersion) { diff --git a/src/commands/run.ts b/src/commands/run.ts index e72436e02..b9ffad48b 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -98,6 +98,11 @@ export class RunCommand extends ApifyCommand { stdin: StdinMode.Stringified, exclusive: ['input'], }), + 'ignore-missing-secrets': Flags.boolean({ + description: 'Ignore missing secrets and show warnings instead of failing. Environment variables referencing missing secrets will be omitted.', + default: false, + required: false, + }), }; async run() { @@ -259,7 +264,11 @@ export class RunCommand extends ApifyCommand { if (userId) localEnvVars[APIFY_ENV_VARS.USER_ID] = userId; if (token) localEnvVars[APIFY_ENV_VARS.TOKEN] = token; if (localConfig!.environmentVariables) { - const updatedEnv = replaceSecretsValue(localConfig!.environmentVariables as Record); + const updatedEnv = replaceSecretsValue( + localConfig!.environmentVariables as Record, + undefined, + this.flags.ignoreMissingSecrets + ); Object.assign(localEnvVars, updatedEnv); } diff --git a/src/lib/secrets.ts b/src/lib/secrets.ts index a373d8470..3fdd1507e 100644 --- a/src/lib/secrets.ts +++ b/src/lib/secrets.ts @@ -54,10 +54,13 @@ const isSecretKey = (envValue: string) => { * Replaces secure values in env with proper values from local secrets file. * @param env * @param secrets - Object with secrets, if not set, will be load from secrets file. + * @param ignoreMissingSecrets - If true, emit warnings for missing secrets instead of throwing errors */ -export const replaceSecretsValue = (env: Record, secrets?: Record) => { +export const replaceSecretsValue = (env: Record, secrets?: Record, ignoreMissingSecrets?: boolean) => { secrets = secrets || getSecretsFile(); const updatedEnv = {}; + const missingSecrets: string[] = []; + Object.keys(env).forEach((key) => { if (isSecretKey(env[key])) { const secretKey = env[key].replace(new RegExp(`^${SECRET_KEY_PREFIX}`), ''); @@ -65,15 +68,30 @@ export const replaceSecretsValue = (env: Record, secrets?: Recor // @ts-expect-error - we are replacing the value updatedEnv[key] = secrets[secretKey]; } else { - warning({ - message: `Value for ${secretKey} not found in local secrets. Set it by calling "apify secrets add ${secretKey} [SECRET_VALUE]"`, - }); + missingSecrets.push(secretKey); } } else { // @ts-expect-error - we are replacing the value updatedEnv[key] = env[key]; } }); + + if (missingSecrets.length > 0) { + if (ignoreMissingSecrets) { + // Emit warnings for each missing secret, keeping original behavior + missingSecrets.forEach(secretKey => { + warning({ + message: `Value for ${secretKey} not found in local secrets. Set it by calling "apify secrets add ${secretKey} [SECRET_VALUE]"`, + }); + }); + } else { + throw new Error( + `Missing secrets: ${missingSecrets.join(', ')}. ` + + `Set them by calling "apify secrets add ".` + ); + } + } + return updatedEnv; }; @@ -86,11 +104,15 @@ interface EnvVar { /** * Transform env to envVars format attribute, which uses Apify API * It replaces secrets to values from secrets file. + * @param env * @param secrets - Object with secrets, if not set, will be load from secrets file. + * @param ignoreMissingSecrets - If true, emit warnings for missing secrets instead of throwing errors */ -export const transformEnvToEnvVars = (env: Record, secrets?: Record) => { +export const transformEnvToEnvVars = (env: Record, secrets?: Record, ignoreMissingSecrets?: boolean) => { secrets = secrets || getSecretsFile(); const envVars: EnvVar[] = []; + const missingSecrets: string[] = []; + Object.keys(env).forEach((key) => { if (isSecretKey(env[key])) { const secretKey = env[key].replace(new RegExp(`^${SECRET_KEY_PREFIX}`), ''); @@ -101,9 +123,7 @@ export const transformEnvToEnvVars = (env: Record, secrets?: Rec isSecret: true, }); } else { - warning({ - message: `Value for ${secretKey} not found in local secrets. Set it by calling "apify secrets add ${secretKey} [SECRET_VALUE]"`, - }); + missingSecrets.push(secretKey); } } else { envVars.push({ @@ -112,5 +132,22 @@ export const transformEnvToEnvVars = (env: Record, secrets?: Rec }); } }); + + if (missingSecrets.length > 0) { + if (ignoreMissingSecrets) { + // Emit warnings for each missing secret, keeping original behavior + missingSecrets.forEach(secretKey => { + warning({ + message: `Value for ${secretKey} not found in local secrets. Set it by calling "apify secrets add ${secretKey} [SECRET_VALUE]"`, + }); + }); + } else { + throw new Error( + `Missing secrets: ${missingSecrets.join(', ')}. ` + + `Set them by calling "apify secrets add ".` + ); + } + } + return envVars; }; diff --git a/test/local/lib/secrets.test.ts b/test/local/lib/secrets.test.ts index cbfea832f..625b38339 100644 --- a/test/local/lib/secrets.test.ts +++ b/test/local/lib/secrets.test.ts @@ -1,10 +1,8 @@ -import { replaceSecretsValue } from '../../../src/lib/secrets.js'; +import { replaceSecretsValue, transformEnvToEnvVars } from '../../../src/lib/secrets.js'; describe('Secrets', () => { describe('replaceSecretsValue()', () => { - it('should work', () => { - const spy = vitest.spyOn(console, 'error'); - + it('should work with valid secrets', () => { const secrets = { myProdToken: 'mySecretToken', mongoUrl: 'mongo://bla@bla:supermongo.com:27017', @@ -13,7 +11,6 @@ describe('Secrets', () => { TOKEN: '@myProdToken', USER: 'jakub.drobnik@apify.com', MONGO_URL: '@mongoUrl', - WARNING: '@doesNotExist', }; const updatedEnv = replaceSecretsValue(env, secrets); @@ -22,9 +19,242 @@ describe('Secrets', () => { USER: 'jakub.drobnik@apify.com', MONGO_URL: secrets.mongoUrl, }); + }); + + it('should throw error when secret is missing', () => { + const secrets = { + myProdToken: 'mySecretToken', + }; + const env = { + TOKEN: '@myProdToken', + WARNING: '@doesNotExist', + }; + expect(() => replaceSecretsValue(env, secrets)).toThrow( + 'Missing secrets: doesNotExist. Set them by calling "apify secrets add ".' + ); + }); + + it('should throw error with multiple missing secrets', () => { + const secrets = {}; + const env = { + TOKEN: '@missingSecret1', + API_KEY: '@missingSecret2', + }; + + expect(() => replaceSecretsValue(env, secrets)).toThrow( + 'Missing secrets: missingSecret1, missingSecret2. Set them by calling "apify secrets add ".' + ); + }); + }); + + describe('transformEnvToEnvVars()', () => { + it('should work with valid secrets', () => { + const secrets = { + myProdToken: 'mySecretToken', + mongoUrl: 'mongo://bla@bla:supermongo.com:27017', + }; + const env = { + TOKEN: '@myProdToken', + USER: 'jakub.drobnik@apify.com', + MONGO_URL: '@mongoUrl', + }; + const envVars = transformEnvToEnvVars(env, secrets); + + expect(envVars).toStrictEqual([ + { + name: 'TOKEN', + value: secrets.myProdToken, + isSecret: true, + }, + { + name: 'USER', + value: 'jakub.drobnik@apify.com', + }, + { + name: 'MONGO_URL', + value: secrets.mongoUrl, + isSecret: true, + }, + ]); + }); + + it('should throw error when secret is missing', () => { + const secrets = { + myProdToken: 'mySecretToken', + }; + const env = { + TOKEN: '@myProdToken', + WARNING: '@doesNotExist', + }; + + expect(() => transformEnvToEnvVars(env, secrets)).toThrow( + 'Missing secrets: doesNotExist. Set them by calling "apify secrets add ".' + ); + }); + + it('should throw error with multiple missing secrets', () => { + const secrets = {}; + const env = { + TOKEN: '@missingSecret1', + API_KEY: '@missingSecret2', + }; + + expect(() => transformEnvToEnvVars(env, secrets)).toThrow( + 'Missing secrets: missingSecret1, missingSecret2. Set them by calling "apify secrets add ".' + ); + }); + }); + + describe('Integration scenarios', () => { + it('should handle mixed environment variables correctly', () => { + const secrets = { + validSecret: 'validValue', + }; + const env = { + VALID_SECRET: '@validSecret', + NORMAL_VAR: 'normalValue', + INVALID_SECRET: '@missingSecret', + }; + + // Should fail because of missing secret + expect(() => transformEnvToEnvVars(env, secrets)).toThrow('Missing secrets: missingSecret'); + }); + + it('should work with empty environment variables', () => { + const result = transformEnvToEnvVars({}); + expect(result).toEqual([]); + }); + + it('should work with environment variables that contain no secrets', () => { + const env = { + NORMAL_VAR1: 'value1', + NORMAL_VAR2: 'value2', + }; + const result = transformEnvToEnvVars(env); + + expect(result).toEqual([ + { + name: 'NORMAL_VAR1', + value: 'value1', + }, + { + name: 'NORMAL_VAR2', + value: 'value2', + }, + ]); + }); + + it('should maintain order of environment variables', () => { + const secrets = { + secret1: 'value1', + secret2: 'value2', + }; + const env = { + THIRD: 'third', + FIRST: '@secret1', + SECOND: '@secret2', + }; + const result = transformEnvToEnvVars(env, secrets); + + expect(result.map(r => r.name)).toEqual(['THIRD', 'FIRST', 'SECOND']); + }); + }); + + describe('ignoreMissingSecrets flag behavior', () => { + it('transformEnvToEnvVars should emit warnings when ignoreMissingSecrets is true', () => { + const spy = vitest.spyOn(console, 'error'); + + const secrets = { + validSecret: 'validValue', + }; + const env = { + VALID_SECRET: '@validSecret', + INVALID_SECRET: '@missingSecret', + NORMAL_VAR: 'normalValue', + }; + + const result = transformEnvToEnvVars(env, secrets, true); + + // Should include valid secret and normal var, but omit missing secret + expect(result).toEqual([ + { + name: 'VALID_SECRET', + value: 'validValue', + isSecret: true, + }, + { + name: 'NORMAL_VAR', + value: 'normalValue', + }, + ]); + + // Should have emitted a warning expect(spy).toHaveBeenCalled(); expect(spy.mock.calls[0][0]).to.include('Warning:'); }); + + it('replaceSecretsValue should emit warnings when ignoreMissingSecrets is true', () => { + const spy = vitest.spyOn(console, 'error'); + + const secrets = { + validSecret: 'validValue', + }; + const env = { + VALID_SECRET: '@validSecret', + INVALID_SECRET: '@missingSecret', + NORMAL_VAR: 'normalValue', + }; + + const result = replaceSecretsValue(env, secrets, true); + + // Should include valid secret and normal var, but omit missing secret + expect(result).toEqual({ + VALID_SECRET: 'validValue', + NORMAL_VAR: 'normalValue', + }); + + // Should have emitted a warning + expect(spy).toHaveBeenCalled(); + expect(spy.mock.calls[0][0]).to.include('Warning:'); + }); + + it('should still throw error when ignoreMissingSecrets is false (default behavior)', () => { + const secrets = {}; + const env = { + INVALID_SECRET: '@missingSecret', + }; + + // Should throw when ignoreMissingSecrets is false (default) + expect(() => transformEnvToEnvVars(env, secrets, false)).toThrow('Missing secrets: missingSecret'); + + // Should also throw when parameter is not provided (default) + expect(() => transformEnvToEnvVars(env, secrets)).toThrow('Missing secrets: missingSecret'); + }); + + it('should handle multiple missing secrets with ignoreMissingSecrets', () => { + const spy = vitest.spyOn(console, 'error'); + + const secrets = {}; + const env = { + SECRET1: '@missing1', + SECRET2: '@missing2', + NORMAL_VAR: 'value', + }; + + const result = transformEnvToEnvVars(env, secrets, true); + + expect(result).toEqual([ + { + name: 'NORMAL_VAR', + value: 'value', + }, + ]); + + // Should have emitted warnings for both missing secrets + expect(spy).toHaveBeenCalledTimes(2); + expect(spy.mock.calls[0][0]).to.include('Warning:'); + expect(spy.mock.calls[1][0]).to.include('Warning:'); + }); }); });