diff --git a/package-lock.json b/package-lock.json index 4fed2395c..fb8f45594 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19049,6 +19049,7 @@ "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.43.0", + "@apify/input_secrets": "^1.1.76", "acorn-loose": "^8.4.0", "countries-list": "^3.0.0" }, diff --git a/packages/input_schema/package.json b/packages/input_schema/package.json index 807f532f8..218349653 100644 --- a/packages/input_schema/package.json +++ b/packages/input_schema/package.json @@ -49,6 +49,7 @@ }, "dependencies": { "@apify/consts": "^2.43.0", + "@apify/input_secrets": "^1.1.76", "acorn-loose": "^8.4.0", "countries-list": "^3.0.0" }, diff --git a/packages/input_schema/src/intl.ts b/packages/input_schema/src/intl.ts index 1759c655d..22e584a1c 100644 --- a/packages/input_schema/src/intl.ts +++ b/packages/input_schema/src/intl.ts @@ -33,6 +33,8 @@ const intlStrings = { 'Field schema.properties.{fieldKey} does not exist, but it is specified in schema.required. Either define the field or remove it from schema.required.', 'inputSchema.validation.proxyGroupMustBeArrayOfStrings': 'Field {rootName}.{fieldKey}.apifyProxyGroups must be an array of strings.', + 'inputSchema.validation.secretFieldSchemaChanged': + 'The field schema.properties.{fieldKey} is a secret field, but its schema has changed. Please update the value in the input editor.', }; /** diff --git a/packages/input_schema/src/schema.json b/packages/input_schema/src/schema.json index 34d1a6d30..2343a9bab 100644 --- a/packages/input_schema/src/schema.json +++ b/packages/input_schema/src/schema.json @@ -152,6 +152,7 @@ "type": { "enum": ["string"] }, "title": { "type": "string" }, "description": { "type": "string" }, + "prefill": { "type": "string" }, "example": { "type": "string" }, "nullable": { "type": "boolean" }, "editor": { "enum": ["textfield", "textarea", "hidden"] }, @@ -166,48 +167,84 @@ "type": "object", "properties": { "type": { "enum": ["array"] }, - "editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "select", "hidden"] } + "editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "select", "hidden"] }, + "isSecret": { "type": "boolean" } }, "additionalProperties": true, "required": ["type", "title", "description", "editor"], "if": { "properties": { - "editor": { "const": "select" } + "isSecret": { + "not": { + "const": true + } + } } }, "then": { - "additionalProperties": false, - "required": ["items"], - "properties": { - "type": { "enum": ["array"] }, - "editor": { "enum": ["select"] }, - "title": { "type": "string" }, - "description": { "type": "string" }, - "default": { "type": "array" }, - "prefill": { "type": "array" }, - "example": { "type": "array" }, - "nullable": { "type": "boolean" }, - "minItems": { "type": "integer" }, - "maxItems": { "type": "integer" }, - "uniqueItems": { "type": "boolean" }, - "sectionCaption": { "type": "string" }, - "sectionDescription": { "type": "string" }, - "items": { - "type": "object", - "additionalProperties": false, - "properties": { - "type": { "enum": ["string"] }, - "enum": { - "type": "array", - "items": { "type": "string" }, - "uniqueItems": true + "if": { + "properties": { + "editor": { "const": "select" } + } + }, + "then": { + "additionalProperties": false, + "required": ["items"], + "properties": { + "type": { "enum": ["array"] }, + "editor": { "enum": ["select"] }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "default": { "type": "array" }, + "prefill": { "type": "array" }, + "example": { "type": "array" }, + "nullable": { "type": "boolean" }, + "minItems": { "type": "integer" }, + "maxItems": { "type": "integer" }, + "uniqueItems": { "type": "boolean" }, + "sectionCaption": { "type": "string" }, + "sectionDescription": { "type": "string" }, + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { "enum": ["string"] }, + "enum": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true + }, + "enumTitles": { + "type": "array", + "items": { "type": "string" } + } }, - "enumTitles": { - "type": "array", - "items": { "type": "string" } - } + "required": ["type", "enum"] }, - "required": ["type", "enum"] + "isSecret": { "enum": [false] } + } + }, + "else": { + "additionalProperties": false, + "properties": { + "type": { "enum": ["array"] }, + "editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "hidden"] }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "default": { "type": "array" }, + "prefill": { "type": "array" }, + "example": { "type": "array" }, + "nullable": { "type": "boolean" }, + "minItems": { "type": "integer" }, + "maxItems": { "type": "integer" }, + "uniqueItems": { "type": "boolean" }, + "sectionCaption": { "type": "string" }, + "sectionDescription": { "type": "string" }, + "placeholderKey": { "type": "string" }, + "placeholderValue": { "type": "string" }, + "patternKey": { "type": "string" }, + "patternValue": { "type": "string" }, + "isSecret": { "enum": [false] } } } }, @@ -215,10 +252,9 @@ "additionalProperties": false, "properties": { "type": { "enum": ["array"] }, - "editor": { "enum": ["json", "requestListSources", "pseudoUrls", "globs", "keyValue", "stringList", "hidden"] }, + "editor": { "enum": ["json", "hidden"] }, "title": { "type": "string" }, "description": { "type": "string" }, - "default": { "type": "array" }, "prefill": { "type": "array" }, "example": { "type": "array" }, "nullable": { "type": "boolean" }, @@ -227,35 +263,70 @@ "uniqueItems": { "type": "boolean" }, "sectionCaption": { "type": "string" }, "sectionDescription": { "type": "string" }, - "placeholderKey": { "type": "string" }, - "placeholderValue": { "type": "string" }, - "patternKey": { "type": "string" }, - "patternValue": { "type": "string" } + "isSecret": { "enum": [true] } } } }, "objectProperty": { "title": "Object property", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "type": { "enum": ["object"] }, "title": { "type": "string" }, "description": { "type": "string" }, - "default": { "type": "object" }, - "prefill": { "type": "object" }, - "example": { "type": "object" }, - "patternKey": { "type": "string" }, - "patternValue": { "type": "string" }, - "nullable": { "type": "boolean" }, - "minProperties": { "type": "integer" }, - "maxProperties": { "type": "integer" }, - "editor": { "enum": ["json", "proxy", "hidden"] }, - "sectionCaption": { "type": "string" }, - "sectionDescription": { "type": "string" } + "isSecret": { "type": "boolean" } }, - "required": ["type", "title", "description", "editor"] + "required": ["type", "title", "description", "editor"], + "if": { + "properties": { + "isSecret": { + "not": { + "const": true + } + } + } + }, + "then": { + "additionalProperties": false, + "properties": { + "type": { "enum": ["object"] }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "default": { "type": "object" }, + "prefill": { "type": "object" }, + "example": { "type": "object" }, + "patternKey": { "type": "string" }, + "patternValue": { "type": "string" }, + "nullable": { "type": "boolean" }, + "minProperties": { "type": "integer" }, + "maxProperties": { "type": "integer" }, + "editor": { "enum": ["json", "proxy", "hidden"] }, + "sectionCaption": { "type": "string" }, + "sectionDescription": { "type": "string" }, + "isSecret": { "enum": [false] } + } + }, + "else": { + "additionalProperties": false, + "properties": { + "type": { "enum": ["object"] }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "prefill": { "type": "object" }, + "example": { "type": "object" }, + "patternKey": { "type": "string" }, + "patternValue": { "type": "string" }, + "nullable": { "type": "boolean" }, + "minProperties": { "type": "integer" }, + "maxProperties": { "type": "integer" }, + "editor": { "enum": ["json", "hidden"] }, + "sectionCaption": { "type": "string" }, + "sectionDescription": { "type": "string" }, + "isSecret": { "enum": [true] } + } + } }, "integerProperty": { "title": "Integer property", diff --git a/packages/input_schema/src/utilities.ts b/packages/input_schema/src/utilities.ts index 84bf8c1f4..b573b5ba3 100644 --- a/packages/input_schema/src/utilities.ts +++ b/packages/input_schema/src/utilities.ts @@ -3,6 +3,7 @@ import type { ValidateFunction } from 'ajv'; import { countries } from 'countries-list'; import { PROXY_URL_REGEX, URL_REGEX } from '@apify/consts'; +import { isEncryptedValueForFieldSchema, isEncryptedValueForFieldType } from '@apify/input_secrets'; import { parseAjvError } from './input_schema'; import { m } from './intl'; @@ -133,13 +134,34 @@ export function validateInputUsingValidator( // Process AJV validation errors if (!isValid) { errors = validator.errors! + .filter((error) => { + // We are storing encrypted objects/arrays as strings, so AJV will throw type the error here. + // So we need to skip these errors. + if (error.keyword === 'type' && error.instancePath) { + const path = error.instancePath.replace(/^\//, '').split('/')[0]; + const propSchema = inputSchema.properties?.[path]; + const value = input[path]; + + // Check if the property is a secret and if the value is an encrypted value. + // We do additional validation of the field schema in the later part of this function + if ( + propSchema?.isSecret + && typeof value === 'string' + && (propSchema.type === 'object' || propSchema.type === 'array') + && isEncryptedValueForFieldType(value, propSchema.type) + ) { + return false; + } + } + return true; + }) .map((error) => parseAjvError(error, 'input', properties, input)) .filter((error) => !!error) as any[]; } Object.keys(properties).forEach((property) => { const value = input[property]; - const { type, editor, patternKey, patternValue } = properties[property]; + const { type, editor, patternKey, patternValue, isSecret } = properties[property]; const fieldErrors = []; // Check that proxy is required, if yes, valides that it's correctly setup if (type === 'object' && editor === 'proxy') { @@ -215,7 +237,7 @@ export function validateInputUsingValidator( } } // Check that object items fit patternKey and patternValue - if (type === 'object' && value) { + if (type === 'object' && value && typeof value === 'object') { if (patternKey) { const check = new RegExp(patternKey); const invalidKeys: any[] = []; @@ -249,6 +271,16 @@ export function validateInputUsingValidator( } } + // Additional validation for secret fields + if (isSecret && value && typeof value === 'string') { + // If the value is a valid encrypted string for the field type, + // we check if the field schema is likely to be still valid (is unchanged from the time of encryption). + if (isEncryptedValueForFieldType(value, type) && !isEncryptedValueForFieldSchema(value, properties[property])) { + // If not, we add an error message to the field errors and user needs to update the value in the input editor. + fieldErrors.push(m('inputSchema.validation.secretFieldSchemaChanged', { fieldKey: property })); + } + } + if (fieldErrors.length > 0) { const message = fieldErrors.join(', '); errors.push({ fieldKey: property, message }); diff --git a/packages/input_secrets/src/field_schema_utils.ts b/packages/input_secrets/src/field_schema_utils.ts new file mode 100644 index 000000000..f0bdd982a --- /dev/null +++ b/packages/input_secrets/src/field_schema_utils.ts @@ -0,0 +1,43 @@ +import crypto from 'node:crypto'; + +/** + * These keys are omitted from the field schema normalization process + * because they are not relevant for validation of values against the schema. + */ +const OMIT_KEYS = new Set(['title', 'description', 'sectionCaption', 'sectionDescription', 'nullable', 'example', 'prefill', 'editor']); + +/** + * Normalizes the field schema by removing irrelevant keys and sorting the remaining keys. + */ +function normalizeFieldSchema(value: any): any { + if (Array.isArray(value)) { + return value.map(normalizeFieldSchema); + } + + if (value && typeof value === 'object') { + const result: Record = {}; + Object.keys(value) + .filter((key) => !OMIT_KEYS.has(key)) + .sort() + .forEach((key) => { + result[key] = normalizeFieldSchema(value[key]); + }); + return result; + } + + return value; +} + +/** + * Generates a stable hash for the field schema. + * @param fieldSchema + */ +export function getFieldSchemaHash(fieldSchema: Record): string { + try { + const stringifiedSchema = JSON.stringify(normalizeFieldSchema(fieldSchema)); + // Create a SHA-256 hash of the stringified schema and return the first 10 characters in hex. + return crypto.createHash('sha256').update(stringifiedSchema).digest('hex').slice(0, 10); + } catch (err) { + throw new Error(`The field schema could not be stringified for hash: ${err}`); + } +} diff --git a/packages/input_secrets/src/input_secrets.ts b/packages/input_secrets/src/input_secrets.ts index 9e8a1fb4a..063a94995 100644 --- a/packages/input_secrets/src/input_secrets.ts +++ b/packages/input_secrets/src/input_secrets.ts @@ -4,14 +4,28 @@ import _testOw, { type Ow } from 'ow'; import { privateDecrypt, publicEncrypt } from '@apify/utilities'; +import { getFieldSchemaHash } from './field_schema_utils'; + // eslint-disable-next-line no-underscore-dangle declare const __injectedOw: Ow; const ow: Ow = typeof __injectedOw === 'undefined' ? _testOw : __injectedOw || _testOw; const BASE64_REGEXP = /[-A-Za-z0-9+/]*={0,3}/; -const ENCRYPTED_INPUT_VALUE_PREFIX = 'ENCRYPTED_VALUE'; -const ENCRYPTED_INPUT_VALUE_REGEXP = new RegExp(`^${ENCRYPTED_INPUT_VALUE_PREFIX}:(${BASE64_REGEXP.source}):(${BASE64_REGEXP.source})$`); + +// The encrypted value has a prefix, optional schema hash, encrypted password and encrypted value. +// - The prefix tells if the value is a string or a JSON object and needs to be parsed back after decryption. +// - The schema hash is optional and is used to verify if the schema has changed since the value was encrypted. +// - The encrypted password is used to decrypt the value. +// - The encrypted value is the actual encrypted data. + +// used for backward compatibility with old encrypted string values +const ENCRYPTED_STRING_VALUE_PREFIX = 'ENCRYPTED_VALUE'; +// we use this for all types of encrypted values (string, object, array) +const ENCRYPTED_JSON_VALUE_PREFIX = 'ENCRYPTED_JSON'; + +// All encrypted values must match this regular expression. +const ENCRYPTED_VALUE_REGEXP = new RegExp(`^(${ENCRYPTED_STRING_VALUE_PREFIX}|${ENCRYPTED_JSON_VALUE_PREFIX}):(?:(${BASE64_REGEXP.source}):)?(${BASE64_REGEXP.source}):(${BASE64_REGEXP.source})$`); /** * Get keys of secret fields from input schema @@ -23,13 +37,74 @@ export function getInputSchemaSecretFieldKeys(inputSchema: any): string[] { /** * Encrypts input secret value + * Depending on the type of value, it returns either a string (for strings) or an object (for objects) with the `secret` key. */ -export function encryptInputSecretValue({ value, publicKey }: { value: string, publicKey: KeyObject }): string { - ow(value, ow.string); +export function encryptInputSecretValue( + { value, publicKey, schema }: { value: T, publicKey: KeyObject, schema?: Record }, +): string { + ow(value, ow.any(ow.string, ow.object)); ow(publicKey, ow.object.instanceOf(KeyObject)); + ow(schema, ow.optional.object); + + const schemaHash = schema ? getFieldSchemaHash(schema) : null; + + // We are encrypting the value as a JSON string, so we need to stringify it first. + let valueStr: string; + try { + valueStr = JSON.stringify(value); + } catch (err) { + throw new Error(`The input value could not be stringified for encryption: ${err}`); + } + + const { encryptedValue, encryptedPassword } = publicEncrypt({ value: valueStr, publicKey }); + return `${ENCRYPTED_JSON_VALUE_PREFIX}:${schemaHash ? `${schemaHash}:` : ''}${encryptedPassword}:${encryptedValue}`; +} + +/** + * Checks if the value is an encrypted value for a specific field type. + * It validates the string value against the regular expression and checks the prefix. + * @param value - encrypted value to check + * @param fieldType - type of the field, can be 'string', 'object' or 'array' + */ +export function isEncryptedValueForFieldType(value: string, fieldType: 'string' | 'object' | 'array') { + ow(value, ow.string); + ow(fieldType, ow.string.oneOf(['string', 'object', 'array'])); + + const match = value.match(ENCRYPTED_VALUE_REGEXP); + if (!match) return false; + + const [, prefix] = match; - const { encryptedValue, encryptedPassword } = publicEncrypt({ value, publicKey }); - return `${ENCRYPTED_INPUT_VALUE_PREFIX}:${encryptedPassword}:${encryptedValue}`; + // For backward compatibility, we allow the old prefix only for string values. + if (prefix === ENCRYPTED_STRING_VALUE_PREFIX && fieldType !== 'string') return false; + + return true; +} + +/** + * Checks if the value is an encrypted value for a specific field schema. + * It validates the string value against the regular expression and checks the schema hash in + * the encrypted value against the hash of the field schema. + * @param value - encrypted value to check + * @param fieldSchema - schema of the field, used to get the hash + */ +export function isEncryptedValueForFieldSchema(value: string, fieldSchema: Record) { + ow(value, ow.string); + ow(fieldSchema, ow.object); + + const match = value.match(ENCRYPTED_VALUE_REGEXP); + if (!match) return false; + + const [, prefix, schemaHash] = match; + + if (prefix !== ENCRYPTED_STRING_VALUE_PREFIX && prefix !== ENCRYPTED_JSON_VALUE_PREFIX) return false; + + if (schemaHash) { + const fieldSchemaHash = getFieldSchemaHash(fieldSchema); + return schemaHash === fieldSchemaHash; + } + + return true; } /** @@ -50,8 +125,13 @@ export function encryptInputSecrets>( const value = input[key]; // NOTE: Skips already encrypted values. It can happens in case client already encrypted values, before // sending them using API. Or input was takes from task, run console or scheduler, where input is stored encrypted. - if (value && ow.isValid(value, ow.string) && !ENCRYPTED_INPUT_VALUE_REGEXP.test(value)) { - encryptedInput[key] = encryptInputSecretValue({ value: input[key], publicKey }); + if (value && !(ow.isValid(value, ow.string) && ENCRYPTED_VALUE_REGEXP.test(value))) { + try { + encryptedInput[key] = encryptInputSecretValue({ value: input[key], publicKey, schema: (inputSchema as any).properties[key] }); + } catch (err) { + throw new Error(`The input field "${key}" could not be encrypted. Try updating the field's value in the input editor. ` + + `Encryption error: ${err}`); + } } } @@ -72,12 +152,19 @@ export function decryptInputSecrets( const decryptedInput = {} as Record; for (const [key, value] of Object.entries(input)) { - if (ow.isValid(value, ow.string) && ENCRYPTED_INPUT_VALUE_REGEXP.test(value)) { - const match = value.match(ENCRYPTED_INPUT_VALUE_REGEXP); + if (typeof value === 'string' && ENCRYPTED_VALUE_REGEXP.test(value)) { + const match = value.match(ENCRYPTED_VALUE_REGEXP); if (!match) continue; - const [, encryptedPassword, encryptedValue] = match; + const [, prefix, , encryptedPassword, encryptedValue] = match; try { - decryptedInput[key] = privateDecrypt({ privateKey, encryptedPassword, encryptedValue }); + const decryptedValue = privateDecrypt({ privateKey, encryptedPassword, encryptedValue }); + + if (prefix === ENCRYPTED_STRING_VALUE_PREFIX) { + decryptedInput[key] = decryptedValue; + } else if (prefix === ENCRYPTED_JSON_VALUE_PREFIX) { + // For JSON values, we need to parse the decrypted string into an object. + decryptedInput[key] = JSON.parse(decryptedValue); + } } catch (err) { throw new Error(`The input field "${key}" could not be decrypted. Try updating the field's value in the input editor. ` + `Decryption error: ${err}`); diff --git a/test/input_schema.test.ts b/test/input_schema.test.ts index 9d756df61..4cf8b5995 100644 --- a/test/input_schema.test.ts +++ b/test/input_schema.test.ts @@ -200,13 +200,13 @@ describe('input_schema.json', () => { isSecret: true, description: 'Some description ...', editor: 'textfield', - maxLength: true, + default: true, }, }, }; expect(() => validateInputSchema(validator, schema)).toThrow( - 'Input schema is not valid (Property schema.properties.myField.maxLength is not allowed.)', + 'Input schema is not valid (Property schema.properties.myField.default is not allowed.)', ); }); diff --git a/test/input_schema_definition.test.ts b/test/input_schema_definition.test.ts index ee5efab98..64d60fc28 100644 --- a/test/input_schema_definition.test.ts +++ b/test/input_schema_definition.test.ts @@ -209,7 +209,7 @@ describe('input_schema.json', () => { turnOffConsoleWarnErrors(); }); - describe('special cases for isSecret string type', () => { + describe('special cases for isSecret property', () => { const isSchemaValid = (fields: object, isSecret?: boolean) => { return ajv.validate(inputSchema, { title: 'Test input schema', @@ -220,6 +220,7 @@ describe('input_schema.json', () => { title: 'Field title', description: 'My test field', type: 'string', + editor: 'textfield', isSecret, ...fields, }, @@ -227,7 +228,7 @@ describe('input_schema.json', () => { }); }; - it('should not allow all editors', () => { + it('string field should not allow all editors', () => { ['textfield', 'textarea', 'hidden'].forEach((editor) => { expect(isSchemaValid({ editor }, true)).toBe(true); }); @@ -236,13 +237,24 @@ describe('input_schema.json', () => { }); }); - it('should allow only string type', () => { - [{ type: 'string', editor: 'textfield' }].forEach((fields) => { - expect(isSchemaValid(fields, true)).toBe(true); + it('string field should not allow some fields', () => { + ['minLength', 'maxLength'].forEach((intField) => { + expect(isSchemaValid({ [intField]: 10 }, true)).toBe(false); + }); + ['default', 'pattern'].forEach((stringField) => { + expect(isSchemaValid({ [stringField]: 'bla' }, true)).toBe(false); }); + }); + + it('should allow only string, object and array type', () => { [ - { type: 'array', editor: 'stringList' }, + { type: 'string', editor: 'textfield' }, { type: 'object', editor: 'json' }, + { type: 'array', editor: 'json' }, + ].forEach((fields) => { + expect(isSchemaValid(fields, true)).toBe(true); + }); + [ { type: 'boolean' }, { type: 'integer' }, ].forEach((fields) => { @@ -250,15 +262,6 @@ describe('input_schema.json', () => { }); }); - it('should not allow some fields', () => { - ['minLength', 'maxLength'].forEach((intField) => { - expect(isSchemaValid({ [intField]: 10 }, true)).toBe(false); - }); - ['default', 'prefill', 'pattern'].forEach((stringField) => { - expect(isSchemaValid({ [stringField]: 'bla' }, true)).toBe(false); - }); - }); - it('should work without isSecret with all editors and properties', () => { expect(ajv.validate(inputSchema, { title: 'Test input schema', @@ -301,6 +304,84 @@ describe('input_schema.json', () => { }); }); + describe('special cases for isSecret object type', () => { + const isSchemaValid = (fields: object, isSecret?: boolean) => { + return ajv.validate(inputSchema, { + title: 'Test input schema', + type: 'object', + schemaVersion: 1, + properties: { + myField: { + title: 'Field title', + description: 'My test field', + type: 'object', + isSecret, + ...fields, + }, + }, + }); + }; + + it('should not allow all editors', () => { + ['json', 'hidden'].forEach((editor) => { + expect(isSchemaValid({ editor }, true)).toBe(true); + }); + ['proxy'].forEach((editor) => { + expect(isSchemaValid({ editor }, true)).toBe(false); + }); + }); + + it('should not allow some fields', () => { + ['minProperties', 'maxProperties'].forEach((intField) => { + expect(isSchemaValid({ [intField]: 10 }, true)).toBe(false); + }); + ['patternKey', 'patternValue', 'prefill', 'example'].forEach((stringField) => { + expect(isSchemaValid({ [stringField]: 'bla' }, true)).toBe(false); + }); + }); + + it('should work without isSecret with all editors and properties', () => { + expect(ajv.validate(inputSchema, { + title: 'Test input schema', + type: 'object', + schemaVersion: 1, + properties: { + myField: { + title: 'Field title', + description: 'My test field', + type: 'object', + editor: 'json', + isSecret: false, + minProperties: 2, + maxProperties: 100, + default: { key: 'value' }, + prefill: { key: 'value', key2: 'value2' }, + }, + }, + })).toBe(true); + + expect(ajv.validate(inputSchema, { + title: 'Test input schema', + type: 'object', + schemaVersion: 1, + properties: { + myField: { + title: 'Field title', + description: 'My test field', + type: 'object', + editor: 'json', + isSecret: false, + minProperties: 2, + maxProperties: 100, + default: { key: 'value' }, + prefill: { key: 'value', key2: 'value2' }, + bla: 'bla', // Validation failed because additional property + }, + }, + })).toBe(false); + }); + }); + describe('special cases for datepicker editor type', () => { it('should accept dateType field omitted', () => { expect(ajv.validate(inputSchema, { diff --git a/test/input_secrets.test.ts b/test/input_secrets.test.ts index 6748a3cb8..c55abe5be 100644 --- a/test/input_secrets.test.ts +++ b/test/input_secrets.test.ts @@ -24,6 +24,20 @@ const inputSchema = { isSecret: true, description: 'Description', }, + secureObject: { + title: 'Secure Object', + type: 'object', + editor: 'json', + isSecret: true, + description: 'Description', + }, + secureArray: { + title: 'Secure Array', + type: 'array', + editor: 'json', + isSecret: true, + description: 'Description', + }, customString: { title: 'String', type: 'string', @@ -36,15 +50,35 @@ const inputSchema = { describe('input secrets', () => { it('should decrypt encrypted values correctly', () => { - const testInput = { secure: 'my secret string', customString: 'just string' }; + const testInput = { + secure: 'my secret string', + secureObject: { + key1: 'value1', + key2: 'value2', + }, + secureArray: ['value1', 'value2'], + customString: 'just string', + }; const encryptedInput = encryptInputSecrets({ input: testInput, inputSchema, publicKey }); expect(encryptedInput.secure).not.toEqual(testInput.secure); + expect(encryptedInput.secureObject).not.toEqual(testInput.secureObject); + expect(typeof encryptedInput.secureObject).toEqual('string'); + expect(encryptedInput.secureArray).not.toEqual(testInput.secureArray); + expect(typeof encryptedInput.secureArray).toEqual('string'); expect(encryptedInput.customString).toEqual(testInput.customString); expect(testInput).toStrictEqual(decryptInputSecrets({ input: encryptedInput, privateKey })); }); it('should not decrypt already decrypted values', () => { - const testInput = { secure: 'my secret', customString: 'just string' }; + const testInput = { + secure: 'my secret string', + secureObject: { + key1: 'value1', + key2: 'value2', + }, + secureArray: ['value1', 'value2'], + customString: 'just string', + }; const encrypted1 = encryptInputSecrets({ input: testInput, inputSchema, publicKey }); const encrypted2 = encryptInputSecrets({ input: encrypted1, inputSchema, publicKey }); expect(testInput).toStrictEqual(decryptInputSecrets({ input: encrypted2, privateKey })); @@ -58,4 +92,22 @@ describe('input secrets', () => { expect(() => decryptInputSecrets({ input: encryptedInput, privateKey: publicKey })) .toThrow(`The input field "secure" could not be decrypted. Try updating the field's value in the input editor.`); }); + + it('should throw if secret object is not valid json', () => { + // eslint-disable-next-line max-len + const secure = 'ENCRYPTED_VALUE:M8QcrS+opESY1KTi4bLvAx0Czxa+idIBq3XKD6gbzb7/CpK9soZrFhqgUIWsFKHMxbISUQu/Btex+WmakhDJFRA/vLLBp4Mit9JY+hwfnfQcBfwuI+ajqYyary6YqQth6gHKF5TZqhu2S1lc+O5t4oRRTCm+Qyk2dYY5nP0muCixatFT3Fu5UzpbFhElH8QiEbySy5jtjZLHZmFe9oPdk3Z8fV0nug9QlEuvYwR1eWK7e0A72zklgfBVNvjsA7OJ2rkaHHef6x6s36k4nI8uIvEHMOZJfuTBjail8xW00BrsKiecuTuRsREYinAMUszunqg0uJthhJFk+3GsrJEkIg==:LX2wyg1xhv94GQf7GRnR8ySbNrdlGrN0icw55a5H3kXhZ2SdOriLcjyPAU9GJob/NlFjzNkf'; + // This is an example of an encrypted object that is not valid JSON: + // { "key1": "value1", "key2" } + // This should never happen in practice, but we want to test that the decryption function handles it gracefully. + // eslint-disable-next-line max-len + const secureObject = 'ENCRYPTED_JSON:kGUk2YdlMZGKdycmBUUZMSbZh/GMB+wvXkWDuI6G9cIzBnKQEqngpCb/lJSSdM4Gd1Xy6rwBVMxGm6ntnYaOyx6lgZqBs5hQqMe3Q0rK2ToW279ZNVNdMmeQDjPKKPpYEpz6p9yAmrRvWu7+1fW6UmazSYj1ErLI9WVJnG3MXb3CsSfQa3HHZ7Qtmgx5AXGT19z24cVSMqWsQOyJW2UwB83jcKcxqAS4w0YV9GsLgMX0K01BR1sXP303Om8c28h6EW6+Ad02pGWwANWjszwY/cWjCNXd44BqJxssLZ3rfk1EG8MkosdK0Zem9/8O4TCbxEAr7hQ2qVwNf43h4si05w==:ry21ohthwOdgBIR9TN0kxpSBe+h7rwhIxvSe4carBWYQWHSiYptLceQ55F8='; + + const encryptedInput = { + secure, + secureObject, + customString: 'just string', + }; + expect(() => decryptInputSecrets({ input: encryptedInput, privateKey })) + .toThrow(`The input field "secureObject" could not be decrypted.`); + }); }); diff --git a/test/utilities.client.test.ts b/test/utilities.client.test.ts index 2681d194e..354861a0d 100644 --- a/test/utilities.client.test.ts +++ b/test/utilities.client.test.ts @@ -1,8 +1,11 @@ +import { createPublicKey } from 'node:crypto'; + import Ajv from 'ajv'; import brokenClone from 'clone-deep'; import _ from 'underscore'; import { validateInputUsingValidator } from '@apify/input_schema'; +import { encryptInputSecrets } from '@apify/input_secrets'; import { buildOrVersionNumberIntToStr, escapeForBson, @@ -1088,6 +1091,186 @@ describe('utilities.client', () => { }); }); }); + + describe('special cases for isSecret properties', () => { + const publicKey = createPublicKey({ + // eslint-disable-next-line max-len + key: Buffer.from('LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF0dis3NlNXbklhOFFKWC94RUQxRQpYdnBBQmE3ajBnQnVYenJNUU5adjhtTW1RU0t2VUF0TmpOL2xacUZpQ0haZUQxU2VDcGV1MnFHTm5XbGRxNkhUCnh5cXJpTVZEbFNKaFBNT09QSENISVNVdFI4Tk5lR1Y1MU0wYkxJcENabHcyTU9GUjdqdENWejVqZFRpZ1NvYTIKQWxrRUlRZWQ4UVlDKzk1aGJoOHk5bGcwQ0JxdEdWN1FvMFZQR2xKQ0hGaWNuaWxLVFFZay9MZzkwWVFnUElPbwozbUppeFl5bWFGNmlMZTVXNzg1M0VHWUVFVWdlWmNaZFNjaGVBMEdBMGpRSFVTdnYvMEZjay9adkZNZURJOTVsCmJVQ0JoQjFDbFg4OG4wZUhzUmdWZE5vK0NLMDI4T2IvZTZTK1JLK09VaHlFRVdPTi90alVMdGhJdTJkQWtGcmkKOFFJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0tCg==', 'base64'), + }); + + it('should allow encrypted/raw input for secret string', () => { + const { inputSchema, validator } = buildInputSchema({ + field: { + title: 'Field title', + description: 'My test field', + type: 'string', + editor: 'json', + nullable: true, + isSecret: true, + }, + }); + const rawInput = { field: 'value' }; + const encryptedInput = encryptInputSecrets({ input: rawInput, inputSchema, publicKey }); + const validInputs = [ + rawInput, + encryptedInput, + { field: null }, + ]; + + const invalidInputs = [ + { field: {} }, + { field: [] }, + ]; + + let errorResults = validInputs + .map((input) => validateInputUsingValidator(validator, inputSchema, input)) + .filter((errors) => errors.length > 0); + expect(errorResults.length).toEqual(0); + + errorResults = invalidInputs + .map((input) => validateInputUsingValidator(validator, inputSchema, input)) + .filter((errors) => errors.length > 0); + expect(errorResults.length).toEqual(2); + + errorResults.forEach((result) => { + // Only one error should be thrown + expect(result.length).toEqual(1); + expect(result[0].fieldKey).toEqual('field'); + }); + }); + + it('should allow encrypted/raw input for secret object', () => { + const { inputSchema, validator } = buildInputSchema({ + field: { + title: 'Field title', + description: 'My test field', + type: 'object', + editor: 'json', + nullable: true, + isSecret: true, + }, + }); + const rawInput = { field: { key1: 'value1', key2: 'value2' } }; + const encryptedInput = encryptInputSecrets({ input: rawInput, inputSchema, publicKey }); + const validInputs = [ + rawInput, + encryptedInput, + { field: null }, + ]; + const invalidInputs = [ + { field: 'DATASET_ID' }, + { field: [] }, + ]; + + let errorResults = validInputs + .map((input) => validateInputUsingValidator(validator, inputSchema, input)) + .filter((errors) => errors.length > 0); + expect(errorResults.length).toEqual(0); + + errorResults = invalidInputs + .map((input) => validateInputUsingValidator(validator, inputSchema, input)) + .filter((errors) => errors.length > 0); + expect(errorResults.length).toEqual(2); + + errorResults.forEach((result) => { + // Only one error should be thrown + expect(result.length).toEqual(1); + expect(result[0].fieldKey).toEqual('field'); + }); + }); + + it('should allow encrypted/raw input for secret array', () => { + const { inputSchema, validator } = buildInputSchema({ + field: { + title: 'Field title', + description: 'My test field', + type: 'array', + editor: 'json', + nullable: true, + isSecret: true, + }, + }); + const rawInput = { field: ['value1', 'value2'] }; + const encryptedInput = encryptInputSecrets({ input: rawInput, inputSchema, publicKey }); + const validInputs = [ + rawInput, + encryptedInput, + { field: null }, + ]; + const invalidInputs = [ + { field: 'DATASET_ID' }, + { field: {} }, + ]; + + let errorResults = validInputs + .map((input) => validateInputUsingValidator(validator, inputSchema, input)) + .filter((errors) => errors.length > 0); + expect(errorResults.length).toEqual(0); + + errorResults = invalidInputs + .map((input) => validateInputUsingValidator(validator, inputSchema, input)) + .filter((errors) => errors.length > 0); + expect(errorResults.length).toEqual(2); + + errorResults.forEach((result) => { + // Only one error should be thrown + expect(result.length).toEqual(1); + expect(result[0].fieldKey).toEqual('field'); + }); + }); + + it('should throw error if field schema changed', () => { + const { inputSchema: originalSchema, validator } = buildInputSchema({ + field: { + title: 'Field title', + description: 'My test field', + type: 'object', + editor: 'json', + maxProperties: 5, + nullable: true, + isSecret: true, + }, + }); + const rawInput = { field: { key1: 'value1', key2: 'value2' } }; + const encryptedInput = encryptInputSecrets({ input: rawInput, inputSchema: originalSchema, publicKey }); + expect(validateInputUsingValidator(validator, originalSchema, rawInput)).toEqual([]); + expect(validateInputUsingValidator(validator, originalSchema, encryptedInput)).toEqual([]); + + const { inputSchema: modifiedTitleSchema, validator: modifiedTitleValidator } = buildInputSchema({ + field: { + title: 'Field new title', + description: 'My new field', + type: 'object', + editor: 'json', + maxProperties: 5, + nullable: true, + isSecret: true, + }, + }); + + expect(validateInputUsingValidator(modifiedTitleValidator, modifiedTitleSchema, rawInput)).toEqual([]); + expect(validateInputUsingValidator(modifiedTitleValidator, modifiedTitleSchema, encryptedInput)).toEqual([]); + + const { inputSchema: modifiedSchema, validator: modifiedValidator } = buildInputSchema({ + field: { + title: 'Field new title', + description: 'My new field', + type: 'object', + editor: 'json', + maxProperties: 8, + minProperties: 1, + nullable: true, + isSecret: true, + }, + }); + + expect(validateInputUsingValidator(modifiedTitleValidator, modifiedTitleSchema, rawInput)).toEqual([]); + const errors = validateInputUsingValidator(modifiedValidator, modifiedSchema, encryptedInput); + expect(errors).not.toEqual([]); + // eslint-disable-next-line max-len + expect(errors[0].message).toEqual('The field schema.properties.field is a secret field, but its schema has changed. Please update the value in the input editor.'); + }); + }); }); describe('#jsonStringifyExtended()', () => {