feat: respect input schema defaults in Actor.getInput() (#409)

Way too overdue, closes #287

---------

Co-authored-by: Martin Adámek <[email protected]>

Author: vladfrangu

package.json

@@ -61,7 +61,7 @@
         "test/**/*": [
             "eslint --fix"
         ],
-        "*": "prettier --write"
+        "*": "prettier --write --ignore-unknown"
     },
     "devDependencies": {
         "@apify/consts": "^2.29.0",

packages/apify/src/actor.ts

@@ -47,6 +47,11 @@ import { addTimeoutToPromise } from '@apify/timeout';
 import type { ChargeOptions, ChargeResult } from './charging.js';
 import { ChargingManager } from './charging.js';
 import { Configuration } from './configuration.js';
+import {
+    getDefaultsFromInputSchema,
+    noActorInputSchemaDefinedMarker,
+    readInputSchema,
+} from './input-schemas.js';
 import { KeyValueStore } from './key_value_store.js';
 import { PlatformEventManager } from './platform_event_manager.js';
 import type { ProxyConfigurationOptions } from './proxy_configuration.js';
@@ -1235,18 +1240,27 @@ export class Actor<Data extends Dictionary = Dictionary> {
         const inputSecretsPrivateKeyPassphrase = this.config.get(
             'inputSecretsPrivateKeyPassphrase',
         );
-        const input = await this.getValue<T>(this.config.get('inputKey'));
+        const rawInput = await this.getValue<T>(this.config.get('inputKey'));
+
+        let input = rawInput as T;
+
         if (
-            ow.isValid(input, ow.object.nonEmpty) &&
+            ow.isValid(rawInput, ow.object.nonEmpty) &&
             inputSecretsPrivateKeyFile &&
             inputSecretsPrivateKeyPassphrase
         ) {
             const privateKey = createPrivateKey({
                 key: Buffer.from(inputSecretsPrivateKeyFile, 'base64'),
                 passphrase: inputSecretsPrivateKeyPassphrase,
             });
-            return decryptInputSecrets<T>({ input, privateKey });
+
+            input = decryptInputSecrets({ input: rawInput, privateKey });
+        }
+
+        if (ow.isValid(input, ow.object.nonEmpty) && !Buffer.isBuffer(input)) {
+            input = await this.inferDefaultsFromInputSchema(input);
         }
+
         return input;
     }
 
@@ -2299,4 +2313,35 @@ export class Actor<Data extends Dictionary = Dictionary> {
             ].join('\n'),
         );
     }
+
+    private async inferDefaultsFromInputSchema<T extends Dictionary>(
+        input: T,
+    ): Promise<T> {
+        // TODO: https://github.com/apify/apify-shared-js/issues/547
+
+        // On platform, this is already handled
+        if (this.isAtHome()) {
+            return input;
+        }
+
+        // On local, we can get the input schema from the local config
+        const inputSchema = readInputSchema();
+
+        // Don't emit warning if there is no input schema defined
+        if (inputSchema === noActorInputSchemaDefinedMarker) {
+            return input;
+        }
+
+        if (!inputSchema) {
+            log.warning(
+                'Failed to find the input schema for the local run of this Actor. Your input will be missing fields that have default values set if they are missing from the input you are using.',
+            );
+
+            return input;
+        }
+
+        const defaults = getDefaultsFromInputSchema(inputSchema);
+
+        return { ...defaults, ...input };
+    }
 }

packages/apify/src/input-schemas.ts

@@ -0,0 +1,92 @@
+// TODO: https://github.com/apify/apify-shared-js/issues/547
+
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import process from 'node:process';
+
+import type { Dictionary } from '@crawlee/utils';
+
+// These paths are used *if* there is no `input` field in the actor.json configuration file!
+const DEFAULT_INPUT_SCHEMA_PATHS = [
+    ['.actor', 'INPUT_SCHEMA.json'],
+    ['INPUT_SCHEMA.json'],
+    ['.actor', 'input_schema.json'],
+    ['input_schema.json'],
+];
+
+const ACTOR_SPECIFICATION_FOLDER = '.actor';
+
+const LOCAL_CONFIG_NAME = 'actor.json';
+
+const readJSONIfExists = (path: string): Dictionary | null => {
+    if (existsSync(path)) {
+        const content = readFileSync(path, 'utf8');
+        return JSON.parse(content);
+    }
+
+    return null;
+};
+
+/**
+ * @ignore
+ */
+export const noActorInputSchemaDefinedMarker = Symbol.for(
+    'apify.noActorInputSchemaDefined',
+);
+
+export const readInputSchema = ():
+    | Dictionary
+    | null
+    | typeof noActorInputSchemaDefinedMarker => {
+    const localConfig = readJSONIfExists(
+        join(process.cwd(), ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_NAME),
+    );
+
+    // Input schema nested in the actor config
+    if (typeof localConfig?.input === 'object') {
+        return localConfig.input;
+    }
+
+    // Input schema path from the actor config
+    if (typeof localConfig?.input === 'string') {
+        const fullPath = join(
+            process.cwd(),
+            ACTOR_SPECIFICATION_FOLDER,
+            localConfig.input,
+        );
+
+        return readJSONIfExists(fullPath);
+    }
+
+    // Try to find it from possible default paths
+    for (const path of DEFAULT_INPUT_SCHEMA_PATHS) {
+        const fullPath = join(process.cwd(), ...path);
+
+        const result = readJSONIfExists(fullPath);
+
+        if (result) {
+            return result;
+        }
+    }
+
+    // If we are in an Actor context, BUT we do not have an input schema defined, we want to skip the warning
+    if (!localConfig?.input) {
+        return noActorInputSchemaDefinedMarker;
+    }
+
+    return null;
+};
+
+export const getDefaultsFromInputSchema = (inputSchema: any) => {
+    const defaults: Record<string, unknown> = {};
+
+    for (const [key, fieldSchema] of Object.entries<any>(
+        inputSchema.properties,
+    )) {
+        if (fieldSchema.default !== undefined) {
+            defaults[key] = fieldSchema.default;
+        }
+    }
+
+    return defaults;
+};

test/e2e/runSdkTests.mjs

@@ -16,6 +16,11 @@ const basePath = join(rootPath, 'sdk');
 const actorBasePath = join(basePath, 'actorBase');
 
 async function run() {
+    if (!process.env.APIFY_TOKEN) {
+        log.error('APIFY_TOKEN is not set in the environment variables.');
+        return;
+    }
+
     log.info(`Running E2E SDK tests`);
 
     const paths = await readdir(basePath, { withFileTypes: true });

test/e2e/sdk/actorInput/.actor/Dockerfile

@@ -0,0 +1,26 @@
+FROM node:22 AS builder
+
+COPY /package*.json ./
+RUN npm --quiet set progress=false \
+    && npm install --only=prod --no-optional --no-audit \
+    && npm update
+
+COPY /apify.tgz /apify.tgz
+RUN npm --quiet install /apify.tgz
+
+FROM apify/actor-node:22
+
+RUN rm -r node_modules
+COPY --from=builder /node_modules ./node_modules
+COPY --from=builder /package*.json ./
+COPY /.actor ./.actor
+COPY /src ./src
+
+RUN echo "Installed NPM packages:" \
+    && (npm list --only=prod --no-optional --all || true) \
+    && echo "Node.js version:" \
+    && node --version \
+    && echo "NPM version:" \
+    && npm --version
+
+CMD npm start --silent

test/e2e/sdk/actorInput/.actor/actor.json

@@ -0,0 +1,20 @@
+{
+    "actorSpecification": 1,
+    "name": "apify-sdk-js-test-input",
+    "version": "0.0",
+    "input": {
+        "title": "Actor Input",
+        "description": "Test input",
+        "type": "object",
+        "schemaVersion": 1,
+        "properties": {
+            "foo": {
+                "title": "Foo",
+                "type": "string",
+                "description": "Foo",
+                "default": "bar",
+                "editor": "textfield"
+            }
+        }
+    }
+}

test/e2e/sdk/actorInput/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "apify-sdk-js-test-harness",
+    "version": "0.0.1",
+    "type": "module",
+    "description": "This is an example of an Apify actor.",
+    "engines": {
+        "node": ">=22.0.0"
+    },
+    "dependencies": {
+        "apify": "*"
+    },
+    "scripts": {
+        "start": "node ./src/main.mjs"
+    },
+    "author": "It's not you it's me",
+    "license": "ISC"
+}

test/e2e/sdk/actorInput/src/main.mjs

@@ -0,0 +1,11 @@
+import { Actor, log } from 'apify';
+
+await Actor.init();
+
+const input = await Actor.getInput();
+
+log.info(`Input: ${JSON.stringify(input)}`);
+
+await Actor.setValue('RECEIVED_INPUT', input);
+
+await Actor.exit();

test/e2e/sdk/actorInput/test.mjs

@@ -0,0 +1,44 @@
+import assert from 'node:assert/strict';
+import test from 'node:test';
+
+import { ApifyClient, KeyValueStore } from 'apify';
+import { sleep } from 'crawlee';
+
+const client = new ApifyClient({
+    token: process.env.APIFY_TOKEN,
+});
+
+const actor = client.actor(process.argv[2]);
+
+const runActor = async (input, options) => {
+    const { id: runId } = await actor.call(input, options);
+    await client.run(runId).waitForFinish();
+    await sleep(6000); // wait for updates to propagate to MongoDB
+    return await client.run(runId).get();
+};
+
+test('defaults work', async () => {
+    const run = await runActor({}, {});
+
+    assert.strictEqual(run.status, 'SUCCEEDED');
+
+    const store = await KeyValueStore.open(run.defaultKeyValueStoreId, {
+        storageClient: client,
+    });
+
+    const receivedInput = await store.getValue('RECEIVED_INPUT');
+    assert.deepEqual(receivedInput, { foo: 'bar' });
+});
+
+test('input is passed through', async () => {
+    const run = await runActor({ foo: 'baz' }, {});
+
+    assert.strictEqual(run.status, 'SUCCEEDED');
+
+    const store = await KeyValueStore.open(run.defaultKeyValueStoreId, {
+        storageClient: client,
+    });
+
+    const receivedInput = await store.getValue('RECEIVED_INPUT');
+    assert.deepEqual(receivedInput, { foo: 'baz' });
+});