Skip to content

Commit b5b19f0

Browse files
PijukatelPijukatel
committed
Draft.
TODO: Add test
1 parent e976265 commit b5b19f0

16 files changed

+47
-36
lines changed

‎packages/core/src/autoscaling/snapshotter.ts‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -299,21 +299,34 @@ export class Snapshotter {
299299
protected _snapshotMemory(systemInfo: SystemInfo) {
300300
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
301301
this._pruneSnapshots(this.memorySnapshots, createdAt);
302-
const { memCurrentBytes } = systemInfo;
302+
const { memCurrentBytes, memTotalBytes } = systemInfo;
303+
304+
let maxMemoryBytes;
305+
if (this.maxMemoryBytes > 0 && this.maxMemoryBytes <= 1) {
306+
// Treat it as ratio of the total actual memory
307+
if (!memTotalBytes)
308+
throw new Error(
309+
'Incorrect memory info provided by the systemInfo event, total memory is required to be able to use maxMemoryBytes as a ratio.',
310+
);
311+
maxMemoryBytes = this.maxMemoryBytes! * memTotalBytes;
312+
} else {
313+
maxMemoryBytes = this.maxMemoryBytes!;
314+
}
315+
303316
const snapshot: MemorySnapshot = {
304317
createdAt,
305-
isOverloaded: memCurrentBytes! / this.maxMemoryBytes! > this.maxUsedMemoryRatio,
318+
isOverloaded: memCurrentBytes! / maxMemoryBytes > this.maxUsedMemoryRatio,
306319
usedBytes: memCurrentBytes,
307320
};
308321

309322
this.memorySnapshots.push(snapshot);
310-
this._memoryOverloadWarning(systemInfo);
323+
this._memoryOverloadWarning(systemInfo, maxMemoryBytes);
311324
}
312325

313326
/**
314327
* Checks for critical memory overload and logs it to the console.
315328
*/
316-
protected _memoryOverloadWarning(systemInfo: SystemInfo) {
329+
protected _memoryOverloadWarning(systemInfo: SystemInfo, maxMemoryBytes: number) {
317330
const { memCurrentBytes } = systemInfo;
318331
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
319332
if (
@@ -322,18 +335,18 @@ export class Snapshotter {
322335
)
323336
return;
324337

325-
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes!;
326-
const reserveMemory = this.maxMemoryBytes! * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
338+
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * maxMemoryBytes;
339+
const reserveMemory = maxMemoryBytes * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
327340
const criticalOverloadBytes = maxDesiredMemoryBytes + reserveMemory;
328341
const isCriticalOverload = memCurrentBytes! > criticalOverloadBytes;
329342

330343
if (isCriticalOverload) {
331-
const usedPercentage = Math.round((memCurrentBytes! / this.maxMemoryBytes!) * 100);
344+
const usedPercentage = Math.round((memCurrentBytes! / maxMemoryBytes) * 100);
332345
const toMb = (bytes: number) => Math.round(bytes / 1024 ** 2);
333346
this.log.warning(
334347
'Memory is critically overloaded. ' +
335348
`Using ${toMb(memCurrentBytes!)} MB of ${toMb(
336-
this.maxMemoryBytes!,
349+
maxMemoryBytes,
337350
)} MB (${usedPercentage}%). Consider increasing available memory.`,
338351
);
339352
this.lastLoggedCriticalMemoryOverloadAt = createdAt;

‎packages/core/src/autoscaling/system_status.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export interface SystemInfo {
1414
eventLoopInfo: ClientInfo;
1515
cpuInfo: ClientInfo;
1616
clientInfo: ClientInfo;
17+
memTotalBytes?: number;
1718
memCurrentBytes?: number;
1819
/**
1920
* Platform only property

‎packages/core/src/crawlers/error_snapshotter.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import crypto from 'node:crypto';
22

3-
import type { CrawlingContext } from '../crawlers/crawler_commons';
43
import type { KeyValueStore } from '../storages';
4+
import type { CrawlingContext } from './crawler_commons';
55
import type { ErrnoException } from './error_tracker';
66

77
// Define the following types as we cannot import the complete types from the respective packages

‎packages/core/src/crawlers/error_tracker.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { inspect } from 'node:util';
22

3-
import type { CrawlingContext } from '../crawlers/crawler_commons';
3+
import type { CrawlingContext } from './crawler_commons';
44
import { ErrorSnapshotter } from './error_snapshotter';
55

66
/**

‎packages/core/src/events/local_event_manager.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,11 +107,13 @@ export class LocalEventManager extends EventManager {
107107
if (this.config.get('systemInfoV2')) {
108108
const memInfo = await getMemoryInfoV2(await this.isContainerizedWrapper());
109109
return {
110+
totalBytes: memInfo.totalBytes,
110111
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
111112
};
112113
}
113114
const memInfo = await getMemoryInfo();
114115
return {
116+
totalBytes: memInfo.totalBytes,
115117
memCurrentBytes: memInfo.mainProcessBytes + memInfo.childProcessesBytes,
116118
};
117119
} catch (err) {

‎packages/utils/test/non-error-objects-working.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { ErrorTracker } from '../../core/src/crawlers/error_tracker';
1+
import { ErrorTracker } from '@crawlee/core/src/crawlers/error_tracker';
22

33
describe('ErrorTracker', () => {
44
test('processing a non-error error should not crash', () => {

‎test/browser-pool/anonymize-proxy-sugar.test.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1+
import { anonymizeProxySugar } from '@crawlee/browser-pool/src/anonymize-proxy';
12
import { anonymizeProxy } from 'proxy-chain';
23
import { vi } from 'vitest';
34

4-
import { anonymizeProxySugar } from '../../packages/browser-pool/src/anonymize-proxy';
5-
65
describe('anonymizeProxySugar', () => {
76
// Mock the anonymizeProxy function from proxy-chain
87
beforeEach(() => {

‎test/browser-pool/browser-pool.test.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
import http from 'node:http';
33
import { promisify } from 'node:util';
44

5+
import { BrowserPool } from '@crawlee/browser-pool/src';
6+
import type { BrowserController } from '@crawlee/browser-pool/src/abstract-classes/browser-controller';
7+
import { BROWSER_POOL_EVENTS } from '@crawlee/browser-pool/src/events';
8+
import { BrowserName, OperatingSystemsName } from '@crawlee/browser-pool/src/fingerprinting/types';
9+
import { PlaywrightPlugin } from '@crawlee/browser-pool/src/playwright/playwright-plugin';
10+
import { PuppeteerPlugin } from '@crawlee/browser-pool/src/puppeteer/puppeteer-plugin';
511
import { sleep } from 'crawlee';
612
import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
713
import playwright from 'playwright';
@@ -11,12 +17,6 @@ import puppeteer from 'puppeteer';
1117

1218
import { addTimeoutToPromise } from '@apify/timeout';
1319

14-
import type { BrowserController } from '../../packages/browser-pool/src/abstract-classes/browser-controller';
15-
import { BrowserPool } from '../../packages/browser-pool/src/browser-pool';
16-
import { BROWSER_POOL_EVENTS } from '../../packages/browser-pool/src/events';
17-
import { BrowserName, OperatingSystemsName } from '../../packages/browser-pool/src/fingerprinting/types';
18-
import { PlaywrightPlugin } from '../../packages/browser-pool/src/playwright/playwright-plugin';
19-
import { PuppeteerPlugin } from '../../packages/browser-pool/src/puppeteer/puppeteer-plugin';
2020
import { createProxyServer } from './browser-plugins/create-proxy-server';
2121

2222
const fingerprintingMatrix: [string, PlaywrightPlugin | PuppeteerPlugin][] = [

‎test/browser-pool/index.test.ts‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
11
import * as modules from '@crawlee/browser-pool';
2-
3-
import { BrowserPool } from '../../packages/browser-pool/src/browser-pool';
4-
import { PlaywrightPlugin } from '../../packages/browser-pool/src/playwright/playwright-plugin';
5-
import { PuppeteerPlugin } from '../../packages/browser-pool/src/puppeteer/puppeteer-plugin';
2+
import { BrowserPool } from '@crawlee/browser-pool/src';
3+
import { PlaywrightPlugin } from '@crawlee/browser-pool/src/playwright/playwright-plugin';
4+
import { PuppeteerPlugin } from '@crawlee/browser-pool/src/puppeteer/puppeteer-plugin';
65

76
describe('Exports', () => {
87
test('Modules', () => {

‎test/core/error_tracker.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { ErrorTracker } from '../../packages/core/src/crawlers/error_tracker';
1+
import { ErrorTracker } from '@crawlee/core/src/crawlers/error_tracker';
22

33
const random = () => Math.random().toString(36).slice(2);
44

0 commit comments

Comments
 (0)