From f4e17546b9852f580a900fc3717e513b228b6486 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 7 Oct 2025 16:23:50 +0200 Subject: [PATCH 1/6] refactor: Use Apify-provided environment variables to obtain PPE pricing information --- .../src/internals/crawler_setup.ts | 4 +- packages/apify/src/charging.ts | 103 ++++++++++++------ packages/apify/src/configuration.ts | 4 + 3 files changed, 78 insertions(+), 33 deletions(-) diff --git a/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts b/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts index 7f662b56e3..a62424d4ea 100644 --- a/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts +++ b/packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts @@ -885,9 +885,9 @@ export class CrawlerSetup implements CrawlerSetupOptions { skipLinksP, globalStoreP, logP, - // eslint-disable-next-line @typescript-eslint/await-thenable + requestQueueP, - // eslint-disable-next-line @typescript-eslint/await-thenable + keyValueStoreP, ]); diff --git a/packages/apify/src/charging.ts b/packages/apify/src/charging.ts index 5ac7777846..9f49818521 100644 --- a/packages/apify/src/charging.ts +++ b/packages/apify/src/charging.ts @@ -94,6 +94,24 @@ export class ChargingManager { this.purgeChargingLogDataset = configuration.get('purgeOnStart'); this.useChargingLogDataset = configuration.get('useChargingLogDataset'); + if ( + configuration.get('actorPricingInfo') && + configuration.get('chargedEventCounts') + ) { + this.loadPricingInfo( + JSON.parse( + configuration.get('actorPricingInfo'), + ) as ActorRunPricingInfo, + configuration.get('maxTotalChargeUsd'), + ); + this.loadChargedEventCounts( + JSON.parse(configuration.get('chargedEventCounts')) as Record< + string, + number + >, + ); + } + if (this.useChargingLogDataset && this.isAtHome) { throw new Error( 'Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment', @@ -117,12 +135,48 @@ export class ChargingManager { return this.pricingModel === 'PAY_PER_EVENT'; } + private loadPricingInfo( + pricingInfo: ActorRunPricingInfo | undefined, + maxTotalChargeUsd: number | undefined, + ) { + this.pricingModel = pricingInfo?.pricingModel; + + // Load per-event pricing information + if (pricingInfo?.pricingModel === 'PAY_PER_EVENT') { + for (const [eventName, eventPricing] of Object.entries( + pricingInfo.pricingPerEvent.actorChargeEvents, + )) { + this.pricingInfo[eventName] = { + price: eventPricing.eventPriceUsd, + title: eventPricing.eventTitle, + }; + } + + this.maxTotalChargeUsd = + maxTotalChargeUsd ?? this.maxTotalChargeUsd; + } + } + + private loadChargedEventCounts( + chargedEventCounts: Record | undefined, + ) { + this.chargingState = {}; + + for (const [eventName, chargeCount] of Object.entries( + chargedEventCounts ?? {}, + )) { + this.chargingState[eventName] = { + chargeCount, + totalChargedAmount: + chargeCount * (this.pricingInfo[eventName]?.price ?? 0), + }; + } + } + /** * Initialize the ChargingManager by loading pricing information and charging state via Apify API. */ async init(): Promise { - this.chargingState = {}; - // Retrieve pricing information if (this.isAtHome) { if (this.actorRunId === undefined) { @@ -131,37 +185,20 @@ export class ChargingManager { ); } - const run = await this.apifyClient.run(this.actorRunId).get(); - if (run === undefined) { - throw new Error('Actor run not found'); - } - - this.pricingModel = run.pricingInfo?.pricingModel; - - // Load per-event pricing information - if (run.pricingInfo?.pricingModel === 'PAY_PER_EVENT') { - for (const [eventName, eventPricing] of Object.entries( - run.pricingInfo.pricingPerEvent.actorChargeEvents, - )) { - this.pricingInfo[eventName] = { - price: eventPricing.eventPriceUsd, - title: eventPricing.eventTitle, - }; + if ( + this.chargingState === undefined || + this.pricingModel === undefined + ) { + const run = await this.apifyClient.run(this.actorRunId).get(); + if (run === undefined) { + throw new Error('Actor run not found'); } - this.maxTotalChargeUsd = - run.options.maxTotalChargeUsd ?? this.maxTotalChargeUsd; - } - - // Load charged event counts - for (const [eventName, chargeCount] of Object.entries( - run.chargedEventCounts ?? {}, - )) { - this.chargingState[eventName] = { - chargeCount, - totalChargedAmount: - chargeCount * (this.pricingInfo[eventName]?.price ?? 0), - }; + this.loadPricingInfo( + run.pricingInfo, + run.options.maxTotalChargeUsd, + ); + this.loadChargedEventCounts(run.chargedEventCounts); } } @@ -186,6 +223,10 @@ export class ChargingManager { this.LOCAL_CHARGING_LOG_DATASET_NAME, ); } + + if (this.chargingState === undefined) { + throw new Error('init() method left `chargingState` uninitialized'); + } } private async ensureChargingLogDatasetOnPlatform(): Promise { diff --git a/packages/apify/src/configuration.ts b/packages/apify/src/configuration.ts index 209ffeabfa..537b865c02 100644 --- a/packages/apify/src/configuration.ts +++ b/packages/apify/src/configuration.ts @@ -39,6 +39,8 @@ export interface ConfigurationOptions extends CoreConfigurationOptions { metaOrigin?: (typeof META_ORIGINS)[keyof typeof META_ORIGINS]; testPayPerEvent?: boolean; useChargingLogDataset?: boolean; + actorPricingInfo?: string; + chargedEventCounts?: string; } /** @@ -179,6 +181,8 @@ export class Configuration extends CoreConfiguration { ACTOR_MAX_TOTAL_CHARGE_USD: 'maxTotalChargeUsd', ACTOR_TEST_PAY_PER_EVENT: 'testPayPerEvent', ACTOR_USE_CHARGING_LOG_DATASET: 'useChargingLogDataset', + APIFY_ACTOR_PRICING_INFO: 'actorPricingInfo', + APIFY_CHARGED_ACTOR_EVENT_COUNTS: 'chargedEventCounts', }; protected static override INTEGER_VARS = [ From 85046dd77283f465e8a8fcd7802fa0f9bd4c07bc Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 7 Oct 2025 16:34:48 +0200 Subject: [PATCH 2/6] Fix init --- packages/apify/src/charging.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/apify/src/charging.ts b/packages/apify/src/charging.ts index 9f49818521..3a7ba73d4b 100644 --- a/packages/apify/src/charging.ts +++ b/packages/apify/src/charging.ts @@ -202,6 +202,8 @@ export class ChargingManager { } } + this.chargingState ??= {}; + if (!this.isPayPerEvent || !this.useChargingLogDataset) { return; } @@ -223,10 +225,6 @@ export class ChargingManager { this.LOCAL_CHARGING_LOG_DATASET_NAME, ); } - - if (this.chargingState === undefined) { - throw new Error('init() method left `chargingState` uninitialized'); - } } private async ensureChargingLogDatasetOnPlatform(): Promise { From c5b7fee0943c790446072cac51023eb3333a367a Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 8 Oct 2025 14:21:56 +0200 Subject: [PATCH 3/6] print environment in failing e2e test --- test/e2e/sdk/actorCharge/test.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/sdk/actorCharge/test.mjs b/test/e2e/sdk/actorCharge/test.mjs index f67c190d4b..a1278a1446 100644 --- a/test/e2e/sdk/actorCharge/test.mjs +++ b/test/e2e/sdk/actorCharge/test.mjs @@ -59,6 +59,7 @@ test('charge limit', async () => { }); test('default options start cost-unlimited runs', async () => { + console.log(process.env) const run = await runActor({}, {}); assert.strictEqual(run.status, 'SUCCEEDED'); From e2cd42491b29a710a48cdec59a06754d6c096834 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 8 Oct 2025 15:51:34 +0200 Subject: [PATCH 4/6] Fix unlimited budget case --- packages/apify/src/charging.ts | 9 ++++---- test/apify/charging.test.ts | 35 +++++++++++++++++++++++++++++++ test/e2e/sdk/actorCharge/test.mjs | 1 - 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/apify/src/charging.ts b/packages/apify/src/charging.ts index 3a7ba73d4b..87d55f5bcd 100644 --- a/packages/apify/src/charging.ts +++ b/packages/apify/src/charging.ts @@ -102,7 +102,7 @@ export class ChargingManager { JSON.parse( configuration.get('actorPricingInfo'), ) as ActorRunPricingInfo, - configuration.get('maxTotalChargeUsd'), + configuration.get('maxTotalChargeUsd') || Infinity, ); this.loadChargedEventCounts( JSON.parse(configuration.get('chargedEventCounts')) as Record< @@ -137,7 +137,7 @@ export class ChargingManager { private loadPricingInfo( pricingInfo: ActorRunPricingInfo | undefined, - maxTotalChargeUsd: number | undefined, + maxTotalChargeUsd: number, ) { this.pricingModel = pricingInfo?.pricingModel; @@ -152,8 +152,7 @@ export class ChargingManager { }; } - this.maxTotalChargeUsd = - maxTotalChargeUsd ?? this.maxTotalChargeUsd; + this.maxTotalChargeUsd = maxTotalChargeUsd; } } @@ -196,7 +195,7 @@ export class ChargingManager { this.loadPricingInfo( run.pricingInfo, - run.options.maxTotalChargeUsd, + run.options.maxTotalChargeUsd || Infinity, ); this.loadChargedEventCounts(run.chargedEventCounts); } diff --git a/test/apify/charging.test.ts b/test/apify/charging.test.ts index b6cc3c6a0b..3dc99df2a7 100644 --- a/test/apify/charging.test.ts +++ b/test/apify/charging.test.ts @@ -14,10 +14,16 @@ describe('ChargingManager', () => { afterEach(async () => { await Actor.exit({ exit: false }); + + // @ts-expect-error + Actor._instance = undefined; // eslint-disable-line no-underscore-dangle + await localStorageEmulator.destroy(); delete process.env.ACTOR_TEST_PAY_PER_EVENT; delete process.env.ACTOR_MAX_TOTAL_CHARGE_USD; + delete process.env.APIFY_ACTOR_PRICING_INFO; + delete process.env.APIFY_CHARGED_ACTOR_EVENT_COUNTS; }); describe('charge()', () => { @@ -40,5 +46,34 @@ describe('ChargingManager', () => { expect(zeroResult.eventChargeLimitReached).toBe(false); expect(zeroResult.chargedCount).toBe(0); }); + + test('should charge events when ACTOR_MAX_TOTAL_CHARGE_USD is set to "" (cost-unlimited)', async () => { + process.env.ACTOR_MAX_TOTAL_CHARGE_USD = ''; + + // Set pricing info via env vars (as if coming from platform) + process.env.APIFY_ACTOR_PRICING_INFO = JSON.stringify({ + pricingModel: 'PAY_PER_EVENT', + pricingPerEvent: { + actorChargeEvents: { + foobar: { + eventTitle: 'Foo bar', + eventPriceUsd: 0.1, + eventDescription: 'Foo foo bar bar', + }, + }, + }, + }); + process.env.APIFY_CHARGED_ACTOR_EVENT_COUNTS = JSON.stringify({}); + + await Actor.init(); + + const chargeResult = await Actor.charge({ + eventName: 'foobar', + count: 4, + }); + + expect(chargeResult.chargedCount).toBe(4); + expect(chargeResult.eventChargeLimitReached).toBe(false); + }); }); }); diff --git a/test/e2e/sdk/actorCharge/test.mjs b/test/e2e/sdk/actorCharge/test.mjs index a1278a1446..f67c190d4b 100644 --- a/test/e2e/sdk/actorCharge/test.mjs +++ b/test/e2e/sdk/actorCharge/test.mjs @@ -59,7 +59,6 @@ test('charge limit', async () => { }); test('default options start cost-unlimited runs', async () => { - console.log(process.env) const run = await runActor({}, {}); assert.strictEqual(run.status, 'SUCCEEDED'); From 51a708b77401d0f5d004987eb878a998965c4cd3 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 9 Oct 2025 13:37:18 +0200 Subject: [PATCH 5/6] Improve readability --- packages/apify/src/charging.ts | 127 +++++++++++++++++---------------- 1 file changed, 65 insertions(+), 62 deletions(-) diff --git a/packages/apify/src/charging.ts b/packages/apify/src/charging.ts index 87d55f5bcd..a26d68e884 100644 --- a/packages/apify/src/charging.ts +++ b/packages/apify/src/charging.ts @@ -86,7 +86,10 @@ export class ChargingManager { private apifyClient: ApifyClient; - constructor(configuration: Configuration, apifyClient: ApifyClient) { + constructor( + private configuration: Configuration, + apifyClient: ApifyClient, + ) { this.maxTotalChargeUsd = configuration.get('maxTotalChargeUsd') || Infinity; // convert `0` to `Infinity` in case the value is an empty string this.isAtHome = configuration.get('isAtHome'); @@ -94,24 +97,6 @@ export class ChargingManager { this.purgeChargingLogDataset = configuration.get('purgeOnStart'); this.useChargingLogDataset = configuration.get('useChargingLogDataset'); - if ( - configuration.get('actorPricingInfo') && - configuration.get('chargedEventCounts') - ) { - this.loadPricingInfo( - JSON.parse( - configuration.get('actorPricingInfo'), - ) as ActorRunPricingInfo, - configuration.get('maxTotalChargeUsd') || Infinity, - ); - this.loadChargedEventCounts( - JSON.parse(configuration.get('chargedEventCounts')) as Record< - string, - number - >, - ); - } - if (this.useChargingLogDataset && this.isAtHome) { throw new Error( 'Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment', @@ -124,8 +109,6 @@ export class ChargingManager { 'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment', ); } - - this.pricingModel = 'PAY_PER_EVENT'; } this.apifyClient = apifyClient; @@ -135,11 +118,67 @@ export class ChargingManager { return this.pricingModel === 'PAY_PER_EVENT'; } - private loadPricingInfo( - pricingInfo: ActorRunPricingInfo | undefined, - maxTotalChargeUsd: number, - ) { - this.pricingModel = pricingInfo?.pricingModel; + private async fetchPricingInfo(): Promise<{ + pricingInfo?: ActorRunPricingInfo; + chargedEventCounts?: Record; + maxTotalChargeUsd: number; + }> { + if ( + this.configuration.get('actorPricingInfo') && + this.configuration.get('chargedEventCounts') + ) { + return { + pricingInfo: JSON.parse( + this.configuration.get('actorPricingInfo'), + ) as ActorRunPricingInfo, + chargedEventCounts: JSON.parse( + this.configuration.get('chargedEventCounts'), + ) as Record, + maxTotalChargeUsd: + this.configuration.get('maxTotalChargeUsd') || Infinity, + }; + } + + if (this.isAtHome) { + if (this.actorRunId === undefined) { + throw new Error( + 'Actor run ID not found even though the Actor is running on Apify', + ); + } + + const run = await this.apifyClient.run(this.actorRunId).get(); + if (run === undefined) { + throw new Error('Actor run not found'); + } + + return { + pricingInfo: run.pricingInfo, + chargedEventCounts: run.chargedEventCounts, + maxTotalChargeUsd: run.options.maxTotalChargeUsd || Infinity, + }; + } + + return { + pricingInfo: undefined, + chargedEventCounts: {}, + maxTotalChargeUsd: + this.configuration.get('maxTotalChargeUsd') || Infinity, + }; + } + + /** + * Initialize the ChargingManager by loading pricing information and charging state via Apify API. + */ + async init(): Promise { + // Retrieve pricing information + const { pricingInfo, chargedEventCounts, maxTotalChargeUsd } = + await this.fetchPricingInfo(); + + if (this.configuration.get('testPayPerEvent')) { + this.pricingModel = 'PAY_PER_EVENT'; + } else { + this.pricingModel ??= pricingInfo?.pricingModel; + } // Load per-event pricing information if (pricingInfo?.pricingModel === 'PAY_PER_EVENT') { @@ -154,11 +193,7 @@ export class ChargingManager { this.maxTotalChargeUsd = maxTotalChargeUsd; } - } - private loadChargedEventCounts( - chargedEventCounts: Record | undefined, - ) { this.chargingState = {}; for (const [eventName, chargeCount] of Object.entries( @@ -170,38 +205,6 @@ export class ChargingManager { chargeCount * (this.pricingInfo[eventName]?.price ?? 0), }; } - } - - /** - * Initialize the ChargingManager by loading pricing information and charging state via Apify API. - */ - async init(): Promise { - // Retrieve pricing information - if (this.isAtHome) { - if (this.actorRunId === undefined) { - throw new Error( - 'Actor run ID not found even though the Actor is running on Apify', - ); - } - - if ( - this.chargingState === undefined || - this.pricingModel === undefined - ) { - const run = await this.apifyClient.run(this.actorRunId).get(); - if (run === undefined) { - throw new Error('Actor run not found'); - } - - this.loadPricingInfo( - run.pricingInfo, - run.options.maxTotalChargeUsd || Infinity, - ); - this.loadChargedEventCounts(run.chargedEventCounts); - } - } - - this.chargingState ??= {}; if (!this.isPayPerEvent || !this.useChargingLogDataset) { return; From f73f9b6235fa0fd02096ac26fa46022b2237e5e1 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 9 Oct 2025 14:40:40 +0200 Subject: [PATCH 6/6] Move check --- packages/apify/src/charging.ts | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/packages/apify/src/charging.ts b/packages/apify/src/charging.ts index a26d68e884..d94fdea17f 100644 --- a/packages/apify/src/charging.ts +++ b/packages/apify/src/charging.ts @@ -97,20 +97,6 @@ export class ChargingManager { this.purgeChargingLogDataset = configuration.get('purgeOnStart'); this.useChargingLogDataset = configuration.get('useChargingLogDataset'); - if (this.useChargingLogDataset && this.isAtHome) { - throw new Error( - 'Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment', - ); - } - - if (configuration.get('testPayPerEvent')) { - if (this.isAtHome) { - throw new Error( - 'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment', - ); - } - } - this.apifyClient = apifyClient; } @@ -170,6 +156,21 @@ export class ChargingManager { * Initialize the ChargingManager by loading pricing information and charging state via Apify API. */ async init(): Promise { + // Validate config - it may have changed since the instantiation + if (this.useChargingLogDataset && this.isAtHome) { + throw new Error( + 'Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment', + ); + } + + if (this.configuration.get('testPayPerEvent')) { + if (this.isAtHome) { + throw new Error( + 'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment', + ); + } + } + // Retrieve pricing information const { pricingInfo, chargedEventCounts, maxTotalChargeUsd } = await this.fetchPricingInfo();