perf: Use Apify-provided environment variables to obtain PPE pricing information (#483)

- closes #481

Author: janbuchar

packages/actor-scraper/web-scraper/src/internals/crawler_setup.ts

@@ -885,9 +885,9 @@ export class CrawlerSetup implements CrawlerSetupOptions {
             skipLinksP,
             globalStoreP,
             logP,
-            // eslint-disable-next-line @typescript-eslint/await-thenable
+
             requestQueueP,
-            // eslint-disable-next-line @typescript-eslint/await-thenable
+
             keyValueStoreP,
         ]);
 

packages/apify/src/charging.ts

@@ -86,44 +86,45 @@ export class ChargingManager {
 
     private apifyClient: ApifyClient;
 
-    constructor(configuration: Configuration, apifyClient: ApifyClient) {
+    constructor(
+        private configuration: Configuration,
+        apifyClient: ApifyClient,
+    ) {
         this.maxTotalChargeUsd =
             configuration.get('maxTotalChargeUsd') || Infinity; // convert `0` to `Infinity` in case the value is an empty string
         this.isAtHome = configuration.get('isAtHome');
         this.actorRunId = configuration.get('actorRunId');
         this.purgeChargingLogDataset = configuration.get('purgeOnStart');
         this.useChargingLogDataset = configuration.get('useChargingLogDataset');
 
-        if (this.useChargingLogDataset && this.isAtHome) {
-            throw new Error(
-                'Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment',
-            );
-        }
-
-        if (configuration.get('testPayPerEvent')) {
-            if (this.isAtHome) {
-                throw new Error(
-                    'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment',
-                );
-            }
-
-            this.pricingModel = 'PAY_PER_EVENT';
-        }
-
         this.apifyClient = apifyClient;
     }
 
     private get isPayPerEvent() {
         return this.pricingModel === 'PAY_PER_EVENT';
     }
 
-    /**
-     * Initialize the ChargingManager by loading pricing information and charging state via Apify API.
-     */
-    async init(): Promise<void> {
-        this.chargingState = {};
+    private async fetchPricingInfo(): Promise<{
+        pricingInfo?: ActorRunPricingInfo;
+        chargedEventCounts?: Record<string, number>;
+        maxTotalChargeUsd: number;
+    }> {
+        if (
+            this.configuration.get('actorPricingInfo') &&
+            this.configuration.get('chargedEventCounts')
+        ) {
+            return {
+                pricingInfo: JSON.parse(
+                    this.configuration.get('actorPricingInfo'),
+                ) as ActorRunPricingInfo,
+                chargedEventCounts: JSON.parse(
+                    this.configuration.get('chargedEventCounts'),
+                ) as Record<string, number>,
+                maxTotalChargeUsd:
+                    this.configuration.get('maxTotalChargeUsd') || Infinity,
+            };
+        }
 
-        // Retrieve pricing information
         if (this.isAtHome) {
             if (this.actorRunId === undefined) {
                 throw new Error(
@@ -136,33 +137,74 @@ export class ChargingManager {
                 throw new Error('Actor run not found');
             }
 
-            this.pricingModel = run.pricingInfo?.pricingModel;
-
-            // Load per-event pricing information
-            if (run.pricingInfo?.pricingModel === 'PAY_PER_EVENT') {
-                for (const [eventName, eventPricing] of Object.entries(
-                    run.pricingInfo.pricingPerEvent.actorChargeEvents,
-                )) {
-                    this.pricingInfo[eventName] = {
-                        price: eventPricing.eventPriceUsd,
-                        title: eventPricing.eventTitle,
-                    };
-                }
-
-                this.maxTotalChargeUsd =
-                    run.options.maxTotalChargeUsd ?? this.maxTotalChargeUsd;
+            return {
+                pricingInfo: run.pricingInfo,
+                chargedEventCounts: run.chargedEventCounts,
+                maxTotalChargeUsd: run.options.maxTotalChargeUsd || Infinity,
+            };
+        }
+
+        return {
+            pricingInfo: undefined,
+            chargedEventCounts: {},
+            maxTotalChargeUsd:
+                this.configuration.get('maxTotalChargeUsd') || Infinity,
+        };
+    }
+
+    /**
+     * Initialize the ChargingManager by loading pricing information and charging state via Apify API.
+     */
+    async init(): Promise<void> {
+        // Validate config - it may have changed since the instantiation
+        if (this.useChargingLogDataset && this.isAtHome) {
+            throw new Error(
+                'Using the ACTOR_USE_CHARGING_LOG_DATASET environment variable is only supported in a local development environment',
+            );
+        }
+
+        if (this.configuration.get('testPayPerEvent')) {
+            if (this.isAtHome) {
+                throw new Error(
+                    'Using the ACTOR_TEST_PAY_PER_EVENT environment variable is only supported in a local development environment',
+                );
             }
+        }
+
+        // Retrieve pricing information
+        const { pricingInfo, chargedEventCounts, maxTotalChargeUsd } =
+            await this.fetchPricingInfo();
+
+        if (this.configuration.get('testPayPerEvent')) {
+            this.pricingModel = 'PAY_PER_EVENT';
+        } else {
+            this.pricingModel ??= pricingInfo?.pricingModel;
+        }
 
-            // Load charged event counts
-            for (const [eventName, chargeCount] of Object.entries(
-                run.chargedEventCounts ?? {},
+        // Load per-event pricing information
+        if (pricingInfo?.pricingModel === 'PAY_PER_EVENT') {
+            for (const [eventName, eventPricing] of Object.entries(
+                pricingInfo.pricingPerEvent.actorChargeEvents,
             )) {
-                this.chargingState[eventName] = {
-                    chargeCount,
-                    totalChargedAmount:
-                        chargeCount * (this.pricingInfo[eventName]?.price ?? 0),
+                this.pricingInfo[eventName] = {
+                    price: eventPricing.eventPriceUsd,
+                    title: eventPricing.eventTitle,
                 };
             }
+
+            this.maxTotalChargeUsd = maxTotalChargeUsd;
+        }
+
+        this.chargingState = {};
+
+        for (const [eventName, chargeCount] of Object.entries(
+            chargedEventCounts ?? {},
+        )) {
+            this.chargingState[eventName] = {
+                chargeCount,
+                totalChargedAmount:
+                    chargeCount * (this.pricingInfo[eventName]?.price ?? 0),
+            };
         }
 
         if (!this.isPayPerEvent || !this.useChargingLogDataset) {

packages/apify/src/configuration.ts

@@ -39,6 +39,8 @@ export interface ConfigurationOptions extends CoreConfigurationOptions {
     metaOrigin?: (typeof META_ORIGINS)[keyof typeof META_ORIGINS];
     testPayPerEvent?: boolean;
     useChargingLogDataset?: boolean;
+    actorPricingInfo?: string;
+    chargedEventCounts?: string;
 }
 
 /**
@@ -179,6 +181,8 @@ export class Configuration extends CoreConfiguration {
         ACTOR_MAX_TOTAL_CHARGE_USD: 'maxTotalChargeUsd',
         ACTOR_TEST_PAY_PER_EVENT: 'testPayPerEvent',
         ACTOR_USE_CHARGING_LOG_DATASET: 'useChargingLogDataset',
+        APIFY_ACTOR_PRICING_INFO: 'actorPricingInfo',
+        APIFY_CHARGED_ACTOR_EVENT_COUNTS: 'chargedEventCounts',
     };
 
     protected static override INTEGER_VARS = [

test/apify/charging.test.ts

@@ -14,10 +14,16 @@ describe('ChargingManager', () => {
 
     afterEach(async () => {
         await Actor.exit({ exit: false });
+
+        // @ts-expect-error
+        Actor._instance = undefined; // eslint-disable-line no-underscore-dangle
+
         await localStorageEmulator.destroy();
 
         delete process.env.ACTOR_TEST_PAY_PER_EVENT;
         delete process.env.ACTOR_MAX_TOTAL_CHARGE_USD;
+        delete process.env.APIFY_ACTOR_PRICING_INFO;
+        delete process.env.APIFY_CHARGED_ACTOR_EVENT_COUNTS;
     });
 
     describe('charge()', () => {
@@ -40,5 +46,34 @@ describe('ChargingManager', () => {
             expect(zeroResult.eventChargeLimitReached).toBe(false);
             expect(zeroResult.chargedCount).toBe(0);
         });
+
+        test('should charge events when ACTOR_MAX_TOTAL_CHARGE_USD is set to "" (cost-unlimited)', async () => {
+            process.env.ACTOR_MAX_TOTAL_CHARGE_USD = '';
+
+            // Set pricing info via env vars (as if coming from platform)
+            process.env.APIFY_ACTOR_PRICING_INFO = JSON.stringify({
+                pricingModel: 'PAY_PER_EVENT',
+                pricingPerEvent: {
+                    actorChargeEvents: {
+                        foobar: {
+                            eventTitle: 'Foo bar',
+                            eventPriceUsd: 0.1,
+                            eventDescription: 'Foo foo bar bar',
+                        },
+                    },
+                },
+            });
+            process.env.APIFY_CHARGED_ACTOR_EVENT_COUNTS = JSON.stringify({});
+
+            await Actor.init();
+
+            const chargeResult = await Actor.charge({
+                eventName: 'foobar',
+                count: 4,
+            });
+
+            expect(chargeResult.chargedCount).toBe(4);
+            expect(chargeResult.eventChargeLimitReached).toBe(false);
+        });
     });
 });