diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45cf7a1..445310e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,19 +22,19 @@ jobs: - name: Generate docs run: npm run generate-docs - name: Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "build" path: "dist" - name: Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "docs" path: "docs" release: name: Release - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/beta' runs-on: ubuntu-latest concurrency: release-${{ github.ref }} environment: @@ -52,10 +52,10 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: "20" + node-version: "22" - name: Install modules run: npm ci --ignore-scripts - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts - name: Move artifacts @@ -73,16 +73,16 @@ jobs: id: set-npm-url run: | if [ -f .semanticRelease.npmPackage.deployedVersion.txt ]; then - echo "npm-url=https://www.npmjs.com/package/node-llama-cpp/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT + echo "npm-url=https://www.npmjs.com/package/ipull/v/$(cat .semanticRelease.npmPackage.deployedVersion.txt)" >> $GITHUB_OUTPUT fi - name: Upload docs to GitHub Pages if: steps.set-npm-url.outputs.npm-url != '' - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 with: name: pages-docs path: docs - name: Deploy docs to GitHub Pages - if: steps.set-npm-url.outputs.npm-url != '' - uses: actions/deploy-pages@v2 + if: steps.set-npm-url.outputs.npm-url != '' && github.ref == 'refs/heads/main' + uses: actions/deploy-pages@v4 with: artifact_name: pages-docs diff --git a/.releaserc.json b/.releaserc.json index a395035..ef8eaf5 100644 --- a/.releaserc.json +++ b/.releaserc.json @@ -1,6 +1,10 @@ { "branches": [ - "main" + "main", + { + "name": "beta", + "prerelease": true + } ], "ci": true, "plugins": [ diff --git a/README.md b/README.md index cc532b2..665f1d7 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ npx ipull http://example.com/file.large ## Features - Download using parallels connections +- Maximize download speed (automatic parallelization, 3+) - Pausing and resuming downloads - Node.js and browser support - Smart retry on fail @@ -73,8 +74,9 @@ import {downloadFileBrowser} from "ipull/dist/browser.js"; const downloader = await downloadFileBrowser({ url: 'https://example.com/file.large', - onWrite: (cursor: number, buffer: Uint8Array, options) => { - console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); + onWrite: (cursor: number, buffers: Uint8Array[], options) => { + const totalLength = buffers.reduce((acc, buffer) => acc + buffer.length, 0); + console.log(`Writing ${totalLength} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); } }); @@ -234,12 +236,12 @@ If the maximum reties was reached the download will fail and an error will be th ```ts import {downloadFile} from 'ipull'; -const downloader = await downloadFile({ - url: 'https://example.com/file.large', - directory: './this/path' -}); - try { + const downloader = await downloadFile({ + url: 'https://example.com/file.large', + directory: './this/path' + }); + await downloader.download(); } catch (error) { console.error(`Download failed: ${error.message}`); @@ -250,6 +252,8 @@ try { In some edge cases, the re-try mechanism may give the illusion that the download is stuck. +(You can see this in the progress object that "retrying" is true) + To debug this, disable the re-try mechanism: ```js @@ -258,6 +262,9 @@ const downloader = await downloadFile({ directory: './this/path', retry: { retries: 0 + }, + retryFetchDownloadInfo: { + retries: 0 } }); ``` @@ -286,6 +293,27 @@ downloader.on("progress", (progress) => { }); ``` +### Remote Download Listing + +If you want to show in the CLI the progress of a file downloading in remote. + +```ts +const originaldownloader = await downloadFile({ + url: 'https://example.com/file.large', + directory: './this/path' +}); + +const remoteDownloader = downloadFileRemote({ + cliProgress: true +}); + +originaldownloader.on("progress", (progress) => { + remoteDownloader.emitRemoteProgress(progress); +}); + +await originaldownloader.download(); +``` + ### Download multiple files If you want to download multiple files, you can use the `downloadSequence` function. diff --git a/examples/browser-log.ts b/examples/browser-log.ts index fbb0c77..75b40d0 100644 --- a/examples/browser-log.ts +++ b/examples/browser-log.ts @@ -1,11 +1,12 @@ -import {downloadFileBrowser} from "ipull/dist/browser.js"; +import {downloadFileBrowser} from "ipull/browser"; const BIG_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/9/9e/1_dubrovnik_pano_-_edit1.jpg"; // 40mb const downloader = await downloadFileBrowser({ url: BIG_IMAGE, - onWrite: (cursor: number, buffer: Uint8Array, options) => { - console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); + onWrite: (cursor: number, buffers: Uint8Array[], options: DownloadEngineWriteStreamOptionsBrowser) => { + const totalLength = buffers.reduce((acc, b) => acc + b.byteLength, 0); + console.log(`Writing ${totalLength} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`); } }); diff --git a/package-lock.json b/package-lock.json index 80da83d..4d76fa1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,7 +27,8 @@ "sleep-promise": "^9.1.0", "slice-ansi": "^7.1.0", "stdout-update": "^4.0.1", - "strip-ansi": "^7.1.0" + "strip-ansi": "^7.1.0", + "uid": "^2.0.2" }, "bin": { "ipull": "dist/cli/cli.js" @@ -1247,6 +1248,15 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@lukeed/csprng": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@lukeed/csprng/-/csprng-1.1.0.tgz", + "integrity": "sha512-Z7C/xXCiGWsg0KuKsHTKJxbWhpI3Vs5GwLfOean7MGyVFGqdRgBbAjOCh6u4bbjPc/8MJ2pZmK/0DLdCbivLDA==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@material/material-color-utilities": { "version": "0.2.7", "resolved": "https://registry.npmjs.org/@material/material-color-utilities/-/material-color-utilities-0.2.7.tgz", @@ -7307,24 +7317,6 @@ "thenify-all": "^1.0.0" } }, - "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -10530,6 +10522,25 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postcss/node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -12370,10 +12381,11 @@ } }, "node_modules/typescript": { - "version": "5.5.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", - "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "version": "5.5.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.4.tgz", + "integrity": "sha512-Mtq29sKDAEYP7aljRgtPOpTvOfbwRWlS6dPRzwjdE+C0R4brX/GUyhHSecbHMFLNBLcJIPt9nl9yG5TZ1weH+Q==", "dev": true, + "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12407,6 +12419,18 @@ "node": ">=0.8.0" } }, + "node_modules/uid": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/uid/-/uid-2.0.2.tgz", + "integrity": "sha512-u3xV3X7uzvi5b1MncmZo3i2Aw222Zk1keqLA1YkHldREkAhAqi65wuPfe7lHx8H/Wzy+8CE7S7uS3jekIM5s8g==", + "license": "MIT", + "dependencies": { + "@lukeed/csprng": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/unbox-primitive": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", diff --git a/package.json b/package.json index 4616151..5655c5e 100644 --- a/package.json +++ b/package.json @@ -147,6 +147,7 @@ "sleep-promise": "^9.1.0", "slice-ansi": "^7.1.0", "stdout-update": "^4.0.1", - "strip-ansi": "^7.1.0" + "strip-ansi": "^7.1.0", + "uid": "^2.0.2" } } diff --git a/src/browser.ts b/src/browser.ts index d81a030..2c5ae9e 100644 --- a/src/browser.ts +++ b/src/browser.ts @@ -1,4 +1,4 @@ -import {downloadFileBrowser, DownloadFileBrowserOptions, downloadSequenceBrowser} from "./download/browser-download.js"; +import {downloadFileBrowser, DownloadFileBrowserOptions, downloadFileRemoteBrowser, downloadSequenceBrowser, DownloadSequenceBrowserOptions} from "./download/browser-download.js"; import DownloadEngineBrowser from "./download/download-engine/engine/download-engine-browser.js"; import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js"; import StatusCodeError from "./download/download-engine/streams/download-engine-fetch-stream/errors/status-code-error.js"; @@ -9,17 +9,21 @@ import FetchStreamError from "./download/download-engine/streams/download-engine import IpullError from "./errors/ipull-error.js"; import EngineError from "./download/download-engine/engine/error/engine-error.js"; import {FormattedStatus} from "./download/transfer-visualize/format-transfer-status.js"; -import DownloadEngineMultiDownload from "./download/download-engine/engine/download-engine-multi-download.js"; +import DownloadEngineMultiDownload, {DownloadEngineMultiAllowedEngines} from "./download/download-engine/engine/download-engine-multi-download.js"; import HttpError from "./download/download-engine/streams/download-engine-fetch-stream/errors/http-error.js"; import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js"; import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js"; -import {NoDownloadEngineProvidedError} from "./download/download-engine/engine/error/no-download-engine-provided-error.js"; +import {DownloadEngineRemote} from "./download/download-engine/engine/DownloadEngineRemote.js"; +import { + DownloadEngineWriteStreamOptionsBrowser +} from "./download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.js"; export { DownloadFlags, DownloadStatus, downloadFileBrowser, + downloadFileRemoteBrowser, downloadSequenceBrowser, EmptyResponseError, HttpError, @@ -29,15 +33,18 @@ export { FetchStreamError, IpullError, EngineError, - InvalidOptionError, - NoDownloadEngineProvidedError + InvalidOptionError }; export type { + DownloadEngineRemote, BaseDownloadEngine, DownloadFileBrowserOptions, DownloadEngineBrowser, DownloadEngineMultiDownload, + DownloadEngineMultiAllowedEngines, FormattedStatus, - SaveProgressInfo + SaveProgressInfo, + DownloadSequenceBrowserOptions, + DownloadEngineWriteStreamOptionsBrowser }; diff --git a/src/download/browser-download.ts b/src/download/browser-download.ts index b4f4ebf..a6d61f0 100644 --- a/src/download/browser-download.ts +++ b/src/download/browser-download.ts @@ -1,34 +1,43 @@ import DownloadEngineBrowser, {DownloadEngineOptionsBrowser} from "./download-engine/engine/download-engine-browser.js"; -import DownloadEngineMultiDownload from "./download-engine/engine/download-engine-multi-download.js"; -import {NoDownloadEngineProvidedError} from "./download-engine/engine/error/no-download-engine-provided-error.js"; +import DownloadEngineMultiDownload, {DownloadEngineMultiDownloadOptions} from "./download-engine/engine/download-engine-multi-download.js"; +import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js"; +import {DownloadSequenceOptions} from "./node-download.js"; +import {DownloadEngineRemote} from "./download-engine/engine/DownloadEngineRemote.js"; const DEFAULT_PARALLEL_STREAMS_FOR_BROWSER = 3; -export type DownloadFileBrowserOptions = DownloadEngineOptionsBrowser & { - /** @deprecated use partURLs instead */ - partsURL?: string[]; -}; +export type DownloadFileBrowserOptions = DownloadEngineOptionsBrowser; + +export type DownloadSequenceBrowserOptions = DownloadEngineMultiDownloadOptions; /** * Download one file in the browser environment. */ export async function downloadFileBrowser(options: DownloadFileBrowserOptions) { - // TODO: Remove in the next major version - if (!("url" in options) && options.partsURL) { - options.partURLs ??= options.partsURL; - } - options.parallelStreams ??= DEFAULT_PARALLEL_STREAMS_FOR_BROWSER; return await DownloadEngineBrowser.createFromOptions(options); } +/** + * Stream events for a download from remote session, doing so by calling `emitRemoteProgress` with the progress info. + */ +export function downloadFileRemoteBrowser() { + return new DownloadEngineRemote(); +} + /** * Download multiple files in the browser environment. */ -export async function downloadSequenceBrowser(...downloads: (DownloadEngineBrowser | Promise)[]) { - if (downloads.length === 0) { - throw new NoDownloadEngineProvidedError(); +export async function downloadSequenceBrowser(options?: DownloadSequenceBrowserOptions | DownloadEngineBrowser | Promise, ...downloads: (DownloadEngineBrowser | Promise)[]) { + let downloadOptions: DownloadSequenceOptions = {}; + if (options instanceof BaseDownloadEngine || options instanceof Promise) { + downloads.unshift(options); + } else if (options) { + downloadOptions = options; } - return await DownloadEngineMultiDownload.fromEngines(downloads); + const downloader = new DownloadEngineMultiDownload(downloadOptions); + await downloader.addDownload(...downloads); + + return downloader; } diff --git a/src/download/download-engine/download-file/download-engine-file.ts b/src/download/download-engine/download-file/download-engine-file.ts index f7b3b5c..8963f71 100644 --- a/src/download/download-engine/download-file/download-engine-file.ts +++ b/src/download/download-engine/download-file/download-engine-file.ts @@ -8,6 +8,8 @@ import {withLock} from "lifecycle-utils"; import switchProgram, {AvailablePrograms} from "./download-programs/switch-program.js"; import BaseDownloadProgram from "./download-programs/base-download-program.js"; import {pushComment} from "./utils/push-comment.js"; +import {uid} from "uid"; +import {DownloaderProgramManager} from "./downloaderProgramManager.js"; export type DownloadEngineFileOptions = { chunkSize?: number; @@ -19,8 +21,10 @@ export type DownloadEngineFileOptions = { onFinishAsync?: () => Promise onStartedAsync?: () => Promise onCloseAsync?: () => Promise + onPausedAsync?: () => Promise onSaveProgressAsync?: (progress: SaveProgressInfo) => Promise programType?: AvailablePrograms + autoIncreaseParallelStreams?: boolean /** @internal */ skipExisting?: boolean; @@ -42,9 +46,13 @@ export type DownloadEngineFileEvents = { [key: string]: any }; +const DEFAULT_CHUNKS_SIZE_FOR_CHUNKS_PROGRAM = 1024 * 1024 * 5; // 5MB +const DEFAULT_CHUNKS_SIZE_FOR_STREAM_PROGRAM = 1024 * 1024; // 1MB + const DEFAULT_OPTIONS: Omit = { - chunkSize: 1024 * 1024 * 5, - parallelStreams: 1 + chunkSize: 0, + parallelStreams: 3, + autoIncreaseParallelStreams: true }; export default class DownloadEngineFile extends EventEmitter { @@ -52,6 +60,7 @@ export default class DownloadEngineFile extends EventEmitter c.isRetrying); + thisStatus.retryingTotalAttempts = Math.max(...streamContexts.map(x => x.retryingAttempts)); + thisStatus.streamsNotResponding = streamContexts.reduce((acc, cur) => acc + (cur.isStreamNotResponding ? 1 : 0), 0); + + return thisStatus; } protected get _activePart() { @@ -113,8 +153,8 @@ export default class DownloadEngineFile extends EventEmitter acc + bytes, 0); + const streamingBytes = Object.values(this._activeStreamContext) + .reduce((acc, cur) => acc + cur.streamBytes, 0); const streamBytes = this._activeDownloadedChunkSize + streamingBytes; const streamBytesMin = Math.min(streamBytes, this._activePart.size || streamBytes); @@ -123,6 +163,13 @@ export default class DownloadEngineFile extends EventEmitter 0 ? this._progress.chunks.length : Infinity; await this._downloadSlice(0, chunksToRead); @@ -198,7 +258,7 @@ export default class DownloadEngineFile extends EventEmitter this._activeStreamContext[startChunk] ??= {streamBytes: 0, retryingAttempts: 0}; + const fetchState = this.options.fetchStream.withSubState({ chunkSize: this._progress.chunkSize, startChunk, - endChunk, + endChunk: endChunk, + lastChunkEndsFile: endChunk === Infinity || endChunk === this._progress.chunks.length, totalSize: this._activePart.size, url: this._activePart.downloadURL!, rangeSupport: this._activePart.acceptRange, onProgress: (length: number) => { - this._activeStreamBytes[startChunk] = length; + getContext().streamBytes = length; this._sendProgressDownloadPart(); } }); + fetchState.addListener("retryingOn", () => { + const context = getContext(); + context.isRetrying = true; + context.retryingAttempts++; + this._sendProgressDownloadPart(); + }); + + fetchState.addListener("retryingOff", () => { + getContext().isRetrying = false; + }); + + fetchState.addListener("streamNotRespondingOn", () => { + getContext().isStreamNotResponding = true; + this._sendProgressDownloadPart(); + }); + + fetchState.addListener("streamNotRespondingOff", () => { + getContext().isStreamNotResponding = false; + }); const downloadedPartsSize = this._downloadedPartsSize; this._progress.chunks[startChunk] = ChunkStatus.IN_PROGRESS; - return (async () => { - const allWrites = new Set>(); + const allWrites = new Set>(); - let lastChunkSize = 0; - await fetchState.fetchChunks((chunks, writePosition, index) => { - if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) { - return; - } + let lastChunkSize = 0, lastInProgressIndex = startChunk; + await fetchState.fetchChunks((chunks, writePosition, index) => { + if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) { + return; + } - for (const chunk of chunks) { - const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunk); - writePosition += chunk.length; - if (writePromise) { - allWrites.add(writePromise); - writePromise.then(() => { - allWrites.delete(writePromise); - }); - } - } + const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunks); + if (writePromise) { + allWrites.add(writePromise); + writePromise.then(() => { + allWrites.delete(writePromise); + }); + } - // if content length is 0, we do not know how many chunks we should have - if (this._activePart.size === 0) { - this._progress.chunks.push(ChunkStatus.NOT_STARTED); - } + // if content length is 0, we do not know how many chunks we should have + if (this._activePart.size === 0) { + this._progress.chunks.push(ChunkStatus.NOT_STARTED); + } - this._progress.chunks[index] = ChunkStatus.COMPLETE; - lastChunkSize = chunks.reduce((last, current) => last + current.length, 0); - delete this._activeStreamBytes[startChunk]; - void this._saveProgress(); + this._progress.chunks[index] = ChunkStatus.COMPLETE; + lastChunkSize = chunks.reduce((last, current) => last + current.length, 0); + getContext().streamBytes = 0; + void this._saveProgress(); - const nextChunk = this._progress.chunks[index + 1]; - const shouldReadNext = endChunk - index > 1; // grater than 1, meaning there is a next chunk + const nextChunk = this._progress.chunks[index + 1]; + const shouldReadNext = fetchState.state.endChunk - index > 1; // grater than 1, meaning there is a next chunk - if (shouldReadNext) { - if (nextChunk == null || nextChunk != ChunkStatus.NOT_STARTED) { - return fetchState.close(); - } - this._progress.chunks[index + 1] = ChunkStatus.IN_PROGRESS; + if (shouldReadNext) { + if (nextChunk == null || nextChunk != ChunkStatus.NOT_STARTED) { + return fetchState.close(); } - }); - - // On dynamic content length, we need to adjust the last chunk size - if (this._activePart.size === 0) { - this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize; - this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE); + this._progress.chunks[lastInProgressIndex = index + 1] = ChunkStatus.IN_PROGRESS; } + }); + + if (this._progress.chunks[lastInProgressIndex] === ChunkStatus.IN_PROGRESS) { + this._progress.chunks[lastInProgressIndex] = ChunkStatus.NOT_STARTED; + } + + // On dynamic content length, we need to adjust the last chunk size + if (this._activePart.size === 0) { + this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize; + this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE); + } - delete this._activeStreamBytes[startChunk]; - await Promise.all(allWrites); - })(); + delete this._activeStreamContext[startChunk]; + await Promise.all(allWrites); } protected _saveProgress() { @@ -288,7 +369,7 @@ export default class DownloadEngineFile extends EventEmitter { - if (thisProgress === this._latestProgressDate) { + if (thisProgress === this._latestProgressDate && !this._closed && this._downloadStatus !== DownloadStatus.Finished) { await this.options.onSaveProgressAsync?.(this._progress); } }); @@ -299,14 +380,14 @@ export default class DownloadEngineFile extends EventEmitter void; + protected _activeDownloads: Promise[] = []; protected constructor(_savedProgress: SaveProgressInfo, _downloadSlice: DownloadSlice) { this._downloadSlice = _downloadSlice; this.savedProgress = _savedProgress; + this._parallelStreams = this.savedProgress.parallelStreams; + } + + get parallelStreams() { + return this._parallelStreams; + } + + set parallelStreams(value: number) { + const needReload = value > this._parallelStreams; + this._parallelStreams = value; + if (needReload) { + this._reload?.(); + } + } + + incParallelStreams() { + this.parallelStreams = this._activeDownloads.length + 1; + } + + decParallelStreams() { + this.parallelStreams = this._activeDownloads.length - 1; + } + + waitForStreamToEnd() { + return Promise.race(this._activeDownloads); } public async download(): Promise { - if (this.savedProgress.parallelStreams === 1) { + if (this._parallelStreams === 1) { return await this._downloadSlice(0, this.savedProgress.chunks.length); } - const activeDownloads: Promise[] = []; + this._createFirstSlices(); - // eslint-disable-next-line no-constant-condition - while (true) { - while (activeDownloads.length >= this.savedProgress.parallelStreams) { - if (this._aborted) return; - await Promise.race(activeDownloads); + while (!this._aborted) { + if (this._activeDownloads.length >= this._parallelStreams) { + await this._waitForStreamEndWithReload(); + continue; } const slice = this._createOneSlice(); - if (slice == null) break; - - if (this._aborted) return; - const promise = this._downloadSlice(slice.start, slice.end); - activeDownloads.push(promise); - promise.then(() => { - activeDownloads.splice(activeDownloads.indexOf(promise), 1); - }); + if (slice == null) { + if (this._activeDownloads.length === 0) { + break; + } + await this._waitForStreamEndWithReload(); + continue; + } + this._createDownload(slice); } + } + + private async _waitForStreamEndWithReload() { + const promiseResolvers = promiseWithResolvers(); + this._reload = promiseResolvers.resolve; + return await Promise.race(this._activeDownloads.concat([promiseResolvers.promise])); + } + + private _createDownload(slice: ProgramSlice) { + const promise = this._downloadSlice(slice.start, slice.end); + this._activeDownloads.push(promise); + promise.then(() => { + this._activeDownloads.splice(this._activeDownloads.indexOf(promise), 1); + }); + } - await Promise.all(activeDownloads); + /** + * Create all the first slices at one - make sure they will not overlap to reduce stream aborts at later stages + */ + private _createFirstSlices() { + const slices: ProgramSlice[] = []; + for (let i = 0; i < this.parallelStreams; i++) { + const slice = this._createOneSlice(); + if (slice) { + const lastSlice = slices.find(x => x.end > slice.start && x.start < slice.start); + if (lastSlice) { + lastSlice.end = slice.start; + } + this.savedProgress.chunks[slice.start] = ChunkStatus.IN_PROGRESS; + slices.push(slice); + } else { + break; + } + } + + for (const slice of slices) { + this._createDownload(slice); + } } protected abstract _createOneSlice(): ProgramSlice | null; diff --git a/src/download/download-engine/download-file/download-programs/download-program-stream.ts b/src/download/download-engine/download-file/download-programs/download-program-stream.ts index 5c90138..4d3e0b1 100644 --- a/src/download/download-engine/download-file/download-programs/download-program-stream.ts +++ b/src/download/download-engine/download-file/download-programs/download-program-stream.ts @@ -11,7 +11,8 @@ export default class DownloadProgramStream extends BaseDownloadProgram { const slice = this._findChunksSlices()[0]; if (!slice) return null; const length = slice.end - slice.start; - return {start: Math.floor(slice.start + length / 2), end: slice.end}; + const start = slice.start == 0 ? slice.start : Math.floor(slice.start + length / 2); + return {start, end: slice.end}; } private _findChunksSlices() { diff --git a/src/download/download-engine/download-file/downloaderProgramManager.ts b/src/download/download-engine/download-file/downloaderProgramManager.ts new file mode 100644 index 0000000..6f15b47 --- /dev/null +++ b/src/download/download-engine/download-file/downloaderProgramManager.ts @@ -0,0 +1,106 @@ +import BaseDownloadProgram from "./download-programs/base-download-program.js"; +import DownloadEngineFile from "./download-engine-file.js"; +import {DownloadStatus, ProgressStatus} from "./progress-status-file.js"; +import sleep from "sleep-promise"; + +const BASE_AVERAGE_SPEED_TIME = 1000; +const AVERAGE_SPEED_TIME = 1000 * 8; +const ALLOW_SPEED_DECREASE_PERCENTAGE = 10; +const ADD_MORE_PARALLEL_IF_SPEED_INCREASE_PERCENTAGE = 10; + +export class DownloaderProgramManager { + // date, speed + private _speedHistory: [number, number][] = []; + private _lastResumeDate = 0; + private _lastAverageSpeed = 0; + private _increasePaused = false; + protected _removeEvent?: () => void; + + constructor(protected _program: BaseDownloadProgram, protected _download: DownloadEngineFile) { + this._initEvents(); + } + + private _initEvents() { + let lastTransferredBytes = 0; + let lastTransferredBytesDate = 0; + + let watNotActive = true; + const progressEvent = (event: ProgressStatus) => { + const now = Date.now(); + + if (event.retrying || event.downloadStatus != DownloadStatus.Active) { + watNotActive = true; + } else { + if (watNotActive) { + this._lastResumeDate = now; + watNotActive = false; + } + + const isTimeToCalculate = lastTransferredBytesDate + BASE_AVERAGE_SPEED_TIME < now; + if (lastTransferredBytesDate === 0 || isTimeToCalculate) { + if (isTimeToCalculate) { + const speedPerSec = event.transferredBytes - lastTransferredBytes; + this._speedHistory.push([now, speedPerSec]); + } + + lastTransferredBytesDate = now; + lastTransferredBytes = event.transferredBytes; + } + + } + + const deleteAllBefore = now - AVERAGE_SPEED_TIME; + this._speedHistory = this._speedHistory.filter(([date]) => date > deleteAllBefore); + + + if (!watNotActive && now - this._lastResumeDate > AVERAGE_SPEED_TIME) { + this._checkAction(); + } + }; + + this._download.on("progress", progressEvent); + this._removeEvent = () => this._download.off("progress", progressEvent); + } + + private _calculateAverageSpeed() { + const totalSpeed = this._speedHistory.reduce((acc, [, speed]) => acc + speed, 0); + return totalSpeed / (this._speedHistory.length || 1); + } + + private async _checkAction() { + const lastAverageSpeed = this._lastAverageSpeed; + const newAverageSpeed = this._calculateAverageSpeed(); + const speedDecreasedOK = (lastAverageSpeed - newAverageSpeed) / newAverageSpeed * 100 > ALLOW_SPEED_DECREASE_PERCENTAGE; + + if (!speedDecreasedOK) { + this._lastAverageSpeed = newAverageSpeed; + } + + if (this._increasePaused || newAverageSpeed > lastAverageSpeed || speedDecreasedOK) { + return; + } + + this._increasePaused = true; + this._program.incParallelStreams(); + let sleepTime = AVERAGE_SPEED_TIME; + + while (sleepTime <= AVERAGE_SPEED_TIME) { + await sleep(sleepTime); + sleepTime = Date.now() - this._lastResumeDate; + } + + const newSpeed = this._calculateAverageSpeed(); + const bestLastSpeed = Math.max(newAverageSpeed, lastAverageSpeed); + const speedIncreasedOK = newSpeed > bestLastSpeed && (newSpeed - bestLastSpeed) / bestLastSpeed * 100 > ADD_MORE_PARALLEL_IF_SPEED_INCREASE_PERCENTAGE; + + if (speedIncreasedOK) { + this._increasePaused = false; + } else { + this._program.decParallelStreams(); + } + } + + close() { + this._removeEvent?.(); + } +} diff --git a/src/download/download-engine/download-file/progress-status-file.ts b/src/download/download-engine/download-file/progress-status-file.ts index a2d7e42..d07359d 100644 --- a/src/download/download-engine/download-file/progress-status-file.ts +++ b/src/download/download-engine/download-file/progress-status-file.ts @@ -1,4 +1,5 @@ export type ProgressStatus = { + downloadId: string, totalBytes: number, totalDownloadParts: number, fileName: string, @@ -10,6 +11,9 @@ export type ProgressStatus = { transferAction: string downloadStatus: DownloadStatus downloadFlags: DownloadFlags[] + retrying: boolean + retryingTotalAttempts: number + streamsNotResponding: number }; export enum DownloadStatus { @@ -39,6 +43,10 @@ export default class ProgressStatusFile { public totalBytes: number = 0; public startTime: number = 0; public endTime: number = 0; + public downloadId: string = ""; + public retrying = false; + public retryingTotalAttempts = 0; + public streamsNotResponding = 0; public constructor( totalDownloadParts: number, @@ -68,7 +76,13 @@ export default class ProgressStatusFile { this.endTime = Date.now(); } - public createStatus(downloadPart: number, transferredBytes: number, totalBytes = this.totalBytes, downloadStatus = DownloadStatus.Active, comment = this.comment): ProgressStatusFile { + public createStatus( + downloadPart: number, + transferredBytes: number, + totalBytes = this.totalBytes, + downloadStatus = DownloadStatus.Active, + comment = this.comment + ): ProgressStatusFile { const newStatus = new ProgressStatusFile( this.totalDownloadParts, this.fileName, @@ -83,6 +97,7 @@ export default class ProgressStatusFile { newStatus.totalBytes = totalBytes; newStatus.startTime = this.startTime; newStatus.endTime = this.endTime; + newStatus.downloadId = this.downloadId; return newStatus; } diff --git a/src/download/download-engine/engine/DownloadEngineRemote.ts b/src/download/download-engine/engine/DownloadEngineRemote.ts new file mode 100644 index 0000000..e9ff5c9 --- /dev/null +++ b/src/download/download-engine/engine/DownloadEngineRemote.ts @@ -0,0 +1,73 @@ +import {BaseDownloadEngineEvents} from "./base-download-engine.js"; +import {EventEmitter} from "eventemitter3"; +import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; +import {DownloadStatus} from "../download-file/progress-status-file.js"; +import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; +import {promiseWithResolvers} from "../utils/promiseWithResolvers.js"; + +export class DownloadEngineRemote extends EventEmitter { + /** + * @internal + */ + _downloadEndPromise = promiseWithResolvers(); + /** + * @internal + */ + _downloadStarted = false; + + private _latestStatus: FormattedStatus = ProgressStatisticsBuilder.loadingStatusEmptyStatistics(); + + public get status() { + return this._latestStatus!; + } + + public get downloadStatues() { + return [this.status]; + } + + public get downloadSize() { + return this._latestStatus?.totalBytes ?? 0; + } + + public get fileName() { + return this._latestStatus?.fileName ?? ""; + } + + public download() { + return this._downloadEndPromise.promise; + } + + public emitRemoteProgress(progress: FormattedStatus) { + this._latestStatus = progress; + this.emit("progress", progress); + const isStatusChanged = this._latestStatus?.downloadStatus !== progress.downloadStatus; + + if (!isStatusChanged) { + return; + } + + switch (progress.downloadStatus) { + case DownloadStatus.Active: + if (this._latestStatus?.downloadStatus === DownloadStatus.Paused) { + this.emit("resumed"); + } else { + this.emit("start"); + this._downloadStarted = true; + } + break; + case DownloadStatus.Finished: + case DownloadStatus.Cancelled: + this.emit("finished"); + this.emit("closed"); + this._downloadEndPromise.resolve(); + break; + case DownloadStatus.Paused: + this.emit("paused"); + break; + } + + if (progress.downloadStatus === DownloadStatus.Active) { + this.emit("start"); + } + } +} diff --git a/src/download/download-engine/engine/base-download-engine.ts b/src/download/download-engine/engine/base-download-engine.ts index a1c6303..4b9dc90 100644 --- a/src/download/download-engine/engine/base-download-engine.ts +++ b/src/download/download-engine/engine/base-download-engine.ts @@ -3,29 +3,35 @@ import DownloadEngineFile, {DownloadEngineFileOptions} from "../download-file/do import BaseDownloadEngineFetchStream, {BaseDownloadEngineFetchStreamOptions} from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js"; import UrlInputError from "./error/url-input-error.js"; import {EventEmitter} from "eventemitter3"; -import ProgressStatisticsBuilder, {ProgressStatusWithIndex} from "../../transfer-visualize/progress-statistics-builder.js"; -import DownloadAlreadyStartedError from "./error/download-already-started-error.js"; +import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; import retry from "async-retry"; import {AvailablePrograms} from "../download-file/download-programs/switch-program.js"; import StatusCodeError from "../streams/download-engine-fetch-stream/errors/status-code-error.js"; import {InvalidOptionError} from "./error/InvalidOptionError.js"; +import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; +import {promiseWithResolvers} from "../utils/promiseWithResolvers.js"; const IGNORE_HEAD_STATUS_CODES = [405, 501, 404]; export type InputURLOptions = { partURLs: string[] } | { url: string }; -export type BaseDownloadEngineOptions = InputURLOptions & BaseDownloadEngineFetchStreamOptions & { +export type CreateDownloadFileOptions = { + reuseRedirectURL?: boolean +}; + +export type BaseDownloadEngineOptions = CreateDownloadFileOptions & InputURLOptions & BaseDownloadEngineFetchStreamOptions & { chunkSize?: number; parallelStreams?: number; retry?: retry.Options comment?: string; - programType?: AvailablePrograms + programType?: AvailablePrograms, + autoIncreaseParallelStreams?: boolean }; export type BaseDownloadEngineEvents = { start: () => void paused: () => void resumed: () => void - progress: (progress: ProgressStatusWithIndex) => void + progress: (progress: FormattedStatus) => void save: (progress: SaveProgressInfo) => void finished: () => void closed: () => void @@ -36,8 +42,16 @@ export default class BaseDownloadEngine extends EventEmitter(); + /** + * @internal + */ + _downloadStarted = false; + protected _latestStatus?: FormattedStatus; protected constructor(engine: DownloadEngineFile, options: DownloadEngineFileOptions) { super(); @@ -96,18 +110,22 @@ export default class BaseDownloadEngine extends EventEmitter { this._latestStatus = status; - return this.emit("progress", status); + this.emit("progress", status); }); } async download() { if (this._downloadStarted) { - throw new DownloadAlreadyStartedError(); + return this._downloadEndPromise.promise; } try { this._downloadStarted = true; - await this._engine.download(); + const promise = this._engine.download(); + promise + .then(this._downloadEndPromise.resolve) + .catch(this._downloadEndPromise.reject); + await promise; } finally { await this.close(); } @@ -125,7 +143,7 @@ export default class BaseDownloadEngine extends EventEmitter { try { const {length, acceptRange, newURL, fileName} = await fetchStream.fetchDownloadInfo(part); - const downloadURL = newURL ?? part; + const downloadURL = reuseRedirectURL ? (newURL ?? part) : part; const size = length || 0; downloadFile.totalSize += size; - downloadFile.parts.push({ + if (index === 0 && fileName) { + downloadFile.localFileName = fileName; + } + + return { downloadURL, size, acceptRange: size > 0 && acceptRange - }); - - if (counter++ === 0 && fileName) { - downloadFile.localFileName = fileName; - } + }; } catch (error: any) { if (error instanceof StatusCodeError && IGNORE_HEAD_STATUS_CODES.includes(error.statusCode)) { // if the server does not support HEAD request, we will skip that step - downloadFile.parts.push({ + return { downloadURL: part, size: 0, acceptRange: false - }); - continue; + }; } throw error; } - } + })); return downloadFile; } diff --git a/src/download/download-engine/engine/download-engine-multi-download.ts b/src/download/download-engine/engine/download-engine-multi-download.ts index ebdbadf..110f667 100644 --- a/src/download/download-engine/engine/download-engine-multi-download.ts +++ b/src/download/download-engine/engine/download-engine-multi-download.ts @@ -1,51 +1,126 @@ import {EventEmitter} from "eventemitter3"; import {FormattedStatus} from "../../transfer-visualize/format-transfer-status.js"; -import ProgressStatisticsBuilder, {ProgressStatusWithIndex} from "../../transfer-visualize/progress-statistics-builder.js"; +import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js"; import BaseDownloadEngine, {BaseDownloadEngineEvents} from "./base-download-engine.js"; -import DownloadAlreadyStartedError from "./error/download-already-started-error.js"; -import {concurrency} from "./utils/concurrency.js"; +import {concurrency} from "../utils/concurrency.js"; import {DownloadFlags, DownloadStatus} from "../download-file/progress-status-file.js"; -import {NoDownloadEngineProvidedError} from "./error/no-download-engine-provided-error.js"; +import {DownloadEngineRemote} from "./DownloadEngineRemote.js"; +import {promiseWithResolvers} from "../utils/promiseWithResolvers.js"; -const DEFAULT_PARALLEL_DOWNLOADS = 1; - -type DownloadEngineMultiAllowedEngines = BaseDownloadEngine; +export type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineRemote | DownloadEngineMultiDownload; type DownloadEngineMultiDownloadEvents = BaseDownloadEngineEvents & { childDownloadStarted: (engine: Engine) => void childDownloadClosed: (engine: Engine) => void + downloadAdded: (engine: Engine) => void }; export type DownloadEngineMultiDownloadOptions = { parallelDownloads?: number + /** + * Unpack inner downloads statues to the main download statues, + * useful for showing CLI progress in separate downloads or tracking download progress separately + */ + unpackInnerMultiDownloadsStatues?: boolean + /** + * Finalize download (change .ipull file to original extension) after all downloads are settled + */ + finalizeDownloadAfterAllSettled?: boolean + + /** + * Do not start download automatically + * @internal + */ + naturalDownloadStart?: boolean + + downloadName?: string + downloadComment?: string }; +const DEFAULT_OPTIONS = { + parallelDownloads: 1, + unpackInnerMultiDownloadsStatues: true, + finalizeDownloadAfterAllSettled: true +} satisfies DownloadEngineMultiDownloadOptions; + export default class DownloadEngineMultiDownload extends EventEmitter { - public readonly downloads: Engine[]; - public readonly options: DownloadEngineMultiDownloadOptions; + public readonly downloads: Engine[] = []; + protected _options: DownloadEngineMultiDownloadOptions; protected _aborted = false; protected _activeEngines = new Set(); protected _progressStatisticsBuilder = new ProgressStatisticsBuilder(); - protected _downloadStatues: (ProgressStatusWithIndex | FormattedStatus)[] = []; + protected _downloadStatues: FormattedStatus[] | FormattedStatus[][] = []; protected _closeFiles: (() => Promise)[] = []; - protected _lastStatus?: ProgressStatusWithIndex; + protected _lastStatus: FormattedStatus = null!; protected _loadingDownloads = 0; - - protected constructor(engines: (DownloadEngineMultiAllowedEngines | DownloadEngineMultiDownload)[], options: DownloadEngineMultiDownloadOptions) { + protected _reloadDownloadParallelisms?: () => void; + protected _engineWaitPromises = new Set>(); + /** + * @internal + */ + _downloadEndPromise = promiseWithResolvers(); + /** + * @internal + */ + _downloadStarted = false; + + /** + * @internal + */ + constructor(options: DownloadEngineMultiDownloadOptions = {}) { super(); - this.downloads = DownloadEngineMultiDownload._extractEngines(engines); - this.options = options; + this._options = {...DEFAULT_OPTIONS, ...options}; this._init(); } + public get activeDownloads() { + return Array.from(this._activeEngines); + } + + public get parallelDownloads() { + return this._options.parallelDownloads; + } + + public get loadingDownloads() { + if (!this._options.unpackInnerMultiDownloadsStatues) { + return this._loadingDownloads; + } + + let totalLoading = this._loadingDownloads; + for (const download of this.downloads) { + if (download instanceof DownloadEngineMultiDownload) { + totalLoading += download.loadingDownloads; + } + } + + return totalLoading; + } + + /** + * @internal + */ + public get _flatEngines(): Engine[] { + return this.downloads.map(engine => { + if (engine instanceof DownloadEngineMultiDownload) { + return engine._flatEngines; + } + return engine; + }) + .flat(); + } + + public set parallelDownloads(value) { + if (this._options.parallelDownloads === value) return; + this._options.parallelDownloads = value; + this._reloadDownloadParallelisms?.(); + } + public get downloadStatues() { - return this._downloadStatues; + const statues = this._downloadStatues.flat(); + return statues.filter(((status, index) => statues.findIndex(x => x.downloadId === status.downloadId) === index)); } public get status() { - if (!this._lastStatus) { - throw new NoDownloadEngineProvidedError(); - } return this._lastStatus; } @@ -58,82 +133,129 @@ export default class DownloadEngineMultiDownload { progress = { ...progress, + fileName: this._options.downloadName ?? progress.fileName, + comment: this._options.downloadComment ?? progress.comment, downloadFlags: progress.downloadFlags.concat([DownloadFlags.DownloadSequence]) }; this._lastStatus = progress; this.emit("progress", progress); }); - let index = 0; - for (const engine of this.downloads) { - this._addEngine(engine, index++); - } - - // Prevent multiple progress events on adding engines - this._progressStatisticsBuilder.add(...this.downloads); + const originalProgress = this._progressStatisticsBuilder.status; + this._lastStatus = { + ...originalProgress, + downloadFlags: originalProgress.downloadFlags.concat([DownloadFlags.DownloadSequence]) + }; } private _addEngine(engine: Engine, index: number) { - this._downloadStatues[index] = engine.status; + this.emit("downloadAdded", engine); + const getStatus = (defaultProgress = engine.status) => + (this._options.unpackInnerMultiDownloadsStatues && engine instanceof DownloadEngineMultiDownload ? engine.downloadStatues : defaultProgress); + + this._downloadStatues[index] = getStatus(); engine.on("progress", (progress) => { - this._downloadStatues[index] = progress; + this._downloadStatues[index] = getStatus(progress); }); - this._changeEngineFinishDownload(engine); + if (this._options.finalizeDownloadAfterAllSettled) { + this._changeEngineFinishDownload(engine); + } + + this.downloads.push(engine); + this._reloadDownloadParallelisms?.(); } - public async addDownload(engine: Engine | DownloadEngineMultiDownload | Promise>) { + public async _addDownloadNoStatisticUpdate(engine: Engine | Promise) { const index = this.downloads.length + this._loadingDownloads; this._downloadStatues[index] = ProgressStatisticsBuilder.loadingStatusEmptyStatistics(); this._loadingDownloads++; this._progressStatisticsBuilder._totalDownloadParts++; - const awaitEngine = engine instanceof Promise ? await engine : engine; - this._progressStatisticsBuilder._totalDownloadParts--; - this._loadingDownloads--; + this._progressStatisticsBuilder._sendLatestProgress(); - if (awaitEngine instanceof DownloadEngineMultiDownload) { - let countEngines = 0; - for (const subEngine of awaitEngine.downloads) { - this._addEngine(subEngine, index + countEngines++); - this.downloads.push(subEngine); - } - this._progressStatisticsBuilder.add(...awaitEngine.downloads); - } else { - this._addEngine(awaitEngine, index); - this.downloads.push(awaitEngine); - this._progressStatisticsBuilder.add(awaitEngine); + const isPromise = engine instanceof Promise; + if (isPromise) { + this._engineWaitPromises.add(engine); } - } - - public async download(): Promise { - if (this._activeEngines.size) { - throw new DownloadAlreadyStartedError(); + const awaitEngine = isPromise ? await engine : engine; + if (isPromise) { + this._engineWaitPromises.delete(engine); } + this._progressStatisticsBuilder._totalDownloadParts--; + this._loadingDownloads--; - this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active; - this.emit("start"); + this._addEngine(awaitEngine, index); + this._progressStatisticsBuilder.add(awaitEngine, true); + return awaitEngine; + } - const concurrencyCount = this.options.parallelDownloads ?? DEFAULT_PARALLEL_DOWNLOADS; - await concurrency(this.downloads, concurrencyCount, async (engine) => { - if (this._aborted) return; - this._activeEngines.add(engine); + public async addDownload(...engines: (Engine | Promise)[]) { + await Promise.all(engines.map(this._addDownloadNoStatisticUpdate.bind(this))); + } - this.emit("childDownloadStarted", engine); - await engine.download(); - this.emit("childDownloadClosed", engine); + public async download(): Promise { + if (this._downloadStarted) { + return this._downloadEndPromise.promise; + } - this._activeEngines.delete(engine); - }); + try { + this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active; + this._downloadStarted = true; + this.emit("start"); + + const concurrencyCount = this._options.parallelDownloads || DEFAULT_OPTIONS.parallelDownloads; + let continueIteration = true; + while (this._loadingDownloads > 0 || continueIteration) { + continueIteration = false; + const {reload, promise} = concurrency(this.downloads, concurrencyCount, async (engine) => { + if (this._aborted) return; + this._activeEngines.add(engine); + + this.emit("childDownloadStarted", engine); + if (engine._downloadStarted || this._options.naturalDownloadStart) { + await engine._downloadEndPromise.promise; + } else { + await engine.download(); + } + this.emit("childDownloadClosed", engine); + + this._activeEngines.delete(engine); + }); + this._reloadDownloadParallelisms = reload; + await promise; + continueIteration = this._engineWaitPromises.size > 0; + if (continueIteration) { + await Promise.race(this._engineWaitPromises); + } + } - this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Finished; - this.emit("finished"); - await this._finishEnginesDownload(); - await this.close(); + this._downloadEndPromise = promiseWithResolvers(); + this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Finished; + this.emit("finished"); + await this._finishEnginesDownload(); + await this.close(); + this._downloadEndPromise.resolve(); + } catch (error) { + this._downloadEndPromise.reject(error); + throw error; + } } private _changeEngineFinishDownload(engine: Engine) { + if (engine instanceof DownloadEngineMultiDownload) { + const _finishEnginesDownload = engine._finishEnginesDownload.bind(engine); + engine._finishEnginesDownload = async () => { + }; + this._closeFiles.push(_finishEnginesDownload); + return; + } + + if (!(engine instanceof BaseDownloadEngine)) { + return; + } + const options = engine._fileEngineOptions; const onFinishAsync = options.onFinishAsync; const onCloseAsync = options.onCloseAsync; @@ -153,12 +275,16 @@ export default class DownloadEngineMultiDownload engine.pause()); + this._activeEngines.forEach(engine => { + if ("pause" in engine) engine.pause(); + }); } public resume(): void { this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active; - this._activeEngines.forEach(engine => engine.resume()); + this._activeEngines.forEach(engine => { + if ("resume" in engine) engine.resume(); + }); } public async close() { @@ -170,23 +296,14 @@ export default class DownloadEngineMultiDownload engine.close()); + .map(engine => { + if ("close" in engine) { + return engine.close(); + } + return Promise.resolve(); + }); await Promise.all(closePromises); this.emit("closed"); } - - protected static _extractEngines(engines: Engine[]) { - return engines.map(engine => { - if (engine instanceof DownloadEngineMultiDownload) { - return engine.downloads; - } - return engine; - }) - .flat(); - } - - public static async fromEngines(engines: (Engine | Promise)[], options: DownloadEngineMultiDownloadOptions = {}) { - return new DownloadEngineMultiDownload(await Promise.all(engines), options); - } } diff --git a/src/download/download-engine/engine/download-engine-nodejs.ts b/src/download/download-engine/engine/download-engine-nodejs.ts index 5acdd66..a27f1e3 100644 --- a/src/download/download-engine/engine/download-engine-nodejs.ts +++ b/src/download/download-engine/engine/download-engine-nodejs.ts @@ -16,8 +16,12 @@ export const PROGRESS_FILE_EXTENSION = ".ipull"; type PathOptions = { directory: string } | { savePath: string }; export type DownloadEngineOptionsNodejs = PathOptions & BaseDownloadEngineOptions & { fileName?: string; - fetchStrategy?: "localFile" | "fetch"; + fetchStrategy?: "local" | "remote"; skipExisting?: boolean; + debounceWrite?: { + maxTime: number + maxSize: number + } }; export type DownloadEngineOptionsNodejsCustomFetch = DownloadEngineOptionsNodejs & { @@ -47,12 +51,16 @@ export default class DownloadEngineNodejs { if (this.options.skipExisting) return; - await this.options.writeStream.saveMedataAfterFile(progress); + await this.options.writeStream.saveMetadataAfterFile(progress); + }; + + this._engine.options.onPausedAsync = async () => { + await this.options.writeStream.ensureBytesSynced(); }; // Try to clone the file if it's a single part download this._engine.options.onStartedAsync = async () => { - if (this.options.skipExisting || this.options.fetchStrategy !== "localFile" || this.options.partURLs.length !== 1) return; + if (this.options.skipExisting || this.options.fetchStrategy !== "local" || this.options.partURLs.length !== 1) return; try { const {reflinkFile} = await import("@reflink/reflink"); @@ -130,7 +138,7 @@ export default class DownloadEngineNodejs(array: Value[], concurrencyCount: number, callback: (value: Value) => Promise): Promise { - return new Promise((resolve, reject) => { - let index = 0; - let activeCount = 0; - - function next() { - if (index === array.length && activeCount === 0) { - resolve(); - return; - } - - while (activeCount < concurrencyCount && index < array.length) { - activeCount++; - callback(array[index++]) - .then(() => { - activeCount--; - next(); - }, reject); - } - } - - next(); - }); -} diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts index f0d290a..46f8593 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.ts @@ -6,10 +6,17 @@ import HttpError from "./errors/http-error.js"; import StatusCodeError from "./errors/status-code-error.js"; import sleep from "sleep-promise"; +export const STREAM_NOT_RESPONDING_TIMEOUT = 1000 * 3; + export const MIN_LENGTH_FOR_MORE_INFO_REQUEST = 1024 * 1024 * 3; // 3MB export type BaseDownloadEngineFetchStreamOptions = { retry?: retry.Options + retryFetchDownloadInfo?: retry.Options + /** + * Max wait for next data stream + */ + maxStreamWait?: number /** * If true, the engine will retry the request if the server returns a status code between 500 and 599 */ @@ -46,6 +53,7 @@ export type FetchSubState = { url: string, startChunk: number, endChunk: number, + lastChunkEndsFile: boolean, totalSize: number, chunkSize: number, rangeSupport?: boolean, @@ -57,14 +65,25 @@ export type BaseDownloadEngineFetchStreamEvents = { resumed: () => void aborted: () => void errorCountIncreased: (errorCount: number, error: Error) => void + retryingOn: (error: Error, attempt: number) => void + retryingOff: () => void + streamNotRespondingOn: () => void + streamNotRespondingOff: () => void }; export type WriteCallback = (data: Uint8Array[], position: number, index: number) => void; const DEFAULT_OPTIONS: BaseDownloadEngineFetchStreamOptions = { retryOnServerError: true, + maxStreamWait: 1000 * 15, retry: { - retries: 150, + retries: 50, + factor: 1.5, + minTimeout: 200, + maxTimeout: 5_000 + }, + retryFetchDownloadInfo: { + retries: 5, factor: 1.5, minTimeout: 200, maxTimeout: 5_000 @@ -73,8 +92,10 @@ const DEFAULT_OPTIONS: BaseDownloadEngineFetchStreamOptions = { }; export default abstract class BaseDownloadEngineFetchStream extends EventEmitter { - public readonly programType?: AvailablePrograms; + public readonly defaultProgramType?: AvailablePrograms; + public readonly availablePrograms: AvailablePrograms[] = ["chunks", "stream"]; public readonly abstract transferAction: string; + public readonly supportDynamicStreamLength: boolean = false; public readonly options: Partial = {}; public state: FetchSubState = null!; public paused?: Promise; @@ -133,14 +154,25 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter let throwErr: Error | null = null; const tryHeaders = "tryHeaders" in this.options && this.options.tryHeaders ? this.options.tryHeaders.slice() : []; + let retryingOn = false; const fetchDownloadInfoCallback = async (): Promise => { try { - return await this.fetchDownloadInfoWithoutRetry(url); + const response = await this.fetchDownloadInfoWithoutRetry(url); + if (retryingOn) { + retryingOn = false; + this.emit("retryingOff"); + } + return response; } catch (error: any) { + this.errorCount.value++; + this.emit("errorCountIncreased", this.errorCount.value, error); + if (error instanceof HttpError && !this.retryOnServerError(error)) { if ("tryHeaders" in this.options && tryHeaders.length) { this.options.headers = tryHeaders.shift(); + retryingOn = true; + this.emit("retryingOn", error, this.errorCount.value); await sleep(this.options.tryHeadersDelay ?? 0); return await fetchDownloadInfoCallback(); } @@ -149,10 +181,9 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter return null; } - this.errorCount.value++; - this.emit("errorCountIncreased", this.errorCount.value, error); - if (error instanceof StatusCodeError && error.retryAfter) { + retryingOn = true; + this.emit("retryingOn", error, this.errorCount.value); await sleep(error.retryAfter * 1000); return await fetchDownloadInfoCallback(); } @@ -161,8 +192,7 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter } }; - - const response = ("defaultFetchDownloadInfo" in this.options && this.options.defaultFetchDownloadInfo) || await retry(fetchDownloadInfoCallback, this.options.retry); + const response = ("defaultFetchDownloadInfo" in this.options && this.options.defaultFetchDownloadInfo) || await retry(fetchDownloadInfoCallback, this.options.retryFetchDownloadInfo); if (throwErr) { throw throwErr; } @@ -175,20 +205,29 @@ export default abstract class BaseDownloadEngineFetchStream extends EventEmitter public async fetchChunks(callback: WriteCallback) { let lastStartLocation = this.state.startChunk; let retryResolvers = retryAsyncStatementSimple(this.options.retry); + let retryingOn = false; // eslint-disable-next-line no-constant-condition while (true) { try { - return await this.fetchWithoutRetryChunks(callback); + return await this.fetchWithoutRetryChunks((...args) => { + if (retryingOn) { + retryingOn = false; + this.emit("retryingOff"); + } + callback(...args); + }); } catch (error: any) { if (error?.name === "AbortError") return; + this.errorCount.value++; + this.emit("errorCountIncreased", this.errorCount.value, error); + if (error instanceof HttpError && !this.retryOnServerError(error)) { throw error; } - this.errorCount.value++; - this.emit("errorCountIncreased", this.errorCount.value, error); - + retryingOn = true; + this.emit("retryingOn", error, this.errorCount.value); if (error instanceof StatusCodeError && error.retryAfter) { await sleep(error.retryAfter * 1000); continue; diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts index 7d0b10f..8888480 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts @@ -1,15 +1,25 @@ -import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, MIN_LENGTH_FOR_MORE_INFO_REQUEST, WriteCallback} from "./base-download-engine-fetch-stream.js"; +import BaseDownloadEngineFetchStream, { + DownloadInfoResponse, + FetchSubState, + MIN_LENGTH_FOR_MORE_INFO_REQUEST, + STREAM_NOT_RESPONDING_TIMEOUT, + WriteCallback +} from "./base-download-engine-fetch-stream.js"; import InvalidContentLengthError from "./errors/invalid-content-length-error.js"; import SmartChunkSplit from "./utils/smart-chunk-split.js"; import {parseContentDisposition} from "./utils/content-disposition.js"; import StatusCodeError from "./errors/status-code-error.js"; import {parseHttpContentRange} from "./utils/httpRange.js"; import {browserCheck} from "./utils/browserCheck.js"; +import {EmptyStreamTimeoutError} from "./errors/EmptyStreamTimeoutError.js"; +import prettyMilliseconds from "pretty-ms"; type GetNextChunk = () => Promise> | ReadableStreamReadResult; export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream { private _fetchDownloadInfoWithHEAD = false; + private _activeController?: AbortController; public override transferAction = "Downloading"; + public override readonly supportDynamicStreamLength = true; withSubState(state: FetchSubState): this { const fetchStream = new DownloadEngineFetchStreamFetch(this.options); @@ -26,10 +36,22 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe headers.range = `bytes=${this._startSize}-${this._endSize - 1}`; } - const controller = new AbortController(); - const response = await fetch(this.appendToURL(this.state.url), { + if (!this._activeController?.signal.aborted) { + this._activeController?.abort(); + } + + let response: Response | null = null; + this._activeController = new AbortController(); + this.on("aborted", () => { + if (!response) { + this._activeController?.abort(); + } + }); + + + response = await fetch(this.appendToURL(this.state.url), { headers, - signal: controller.signal + signal: this._activeController.signal }); if (response.status < 200 || response.status >= 300) { @@ -42,10 +64,6 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe throw new InvalidContentLengthError(expectedContentLength, contentLength); } - this.on("aborted", () => { - controller.abort(); - }); - const reader = response.body!.getReader(); return await this.chunkGenerator(callback, () => reader.read()); } @@ -114,17 +132,58 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe // eslint-disable-next-line no-constant-condition while (true) { - const {done, value} = await getNextChunk(); + const chunkInfo = await this._wrapperStreamNotResponding(getNextChunk()); await this.paused; - if (done || this.aborted) break; + if (!chunkInfo || this.aborted || chunkInfo.done) break; - smartSplit.addChunk(value); + smartSplit.addChunk(chunkInfo.value); this.state.onProgress?.(smartSplit.savedLength); } - smartSplit.sendLeftovers(); + smartSplit.closeAndSendLeftoversIfLengthIsUnknown(); + } + + protected _wrapperStreamNotResponding(promise: Promise | T): Promise | T | void { + if (!(promise instanceof Promise)) { + return promise; + } + + return new Promise((resolve, reject) => { + let streamNotRespondedInTime = false; + let timeoutMaxStreamWaitThrows = false; + const timeoutNotResponding = setTimeout(() => { + streamNotRespondedInTime = true; + this.emit("streamNotRespondingOn"); + }, STREAM_NOT_RESPONDING_TIMEOUT); + + const timeoutMaxStreamWait = setTimeout(() => { + timeoutMaxStreamWaitThrows = true; + reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(this.options.maxStreamWait!)}`)); + this._activeController?.abort(); + }, this.options.maxStreamWait); + + this.addListener("aborted", resolve); + + promise + .then(resolve) + .catch(error => { + if (timeoutMaxStreamWaitThrows || this.aborted) { + return; + } + reject(error); + }) + .finally(() => { + clearTimeout(timeoutNotResponding); + clearTimeout(timeoutMaxStreamWait); + if (streamNotRespondedInTime) { + this.emit("streamNotRespondingOff"); + } + this.removeListener("aborted", resolve); + }); + }); } + protected static convertHeadersToRecord(headers: Headers): { [key: string]: string } { const headerObj: { [key: string]: string } = {}; headers.forEach((value, key) => { @@ -132,4 +191,5 @@ export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFe }); return headerObj; } + } diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts index 370ea7c..322006e 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.ts @@ -1,4 +1,10 @@ -import BaseDownloadEngineFetchStream, {DownloadInfoResponse, FetchSubState, MIN_LENGTH_FOR_MORE_INFO_REQUEST, WriteCallback} from "./base-download-engine-fetch-stream.js"; +import BaseDownloadEngineFetchStream, { + DownloadInfoResponse, + FetchSubState, + MIN_LENGTH_FOR_MORE_INFO_REQUEST, + STREAM_NOT_RESPONDING_TIMEOUT, + WriteCallback +} from "./base-download-engine-fetch-stream.js"; import EmptyResponseError from "./errors/empty-response-error.js"; import StatusCodeError from "./errors/status-code-error.js"; import XhrError from "./errors/xhr-error.js"; @@ -7,11 +13,15 @@ import retry from "async-retry"; import {AvailablePrograms} from "../../download-file/download-programs/switch-program.js"; import {parseContentDisposition} from "./utils/content-disposition.js"; import {parseHttpContentRange} from "./utils/httpRange.js"; +import prettyMilliseconds from "pretty-ms"; +import {EmptyStreamTimeoutError} from "./errors/EmptyStreamTimeoutError.js"; export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream { private _fetchDownloadInfoWithHEAD = true; - public override readonly programType: AvailablePrograms = "chunks"; + public override readonly defaultProgramType: AvailablePrograms = "chunks"; + public override readonly availablePrograms: AvailablePrograms[] = ["chunks"]; + public override transferAction = "Downloading"; withSubState(state: FetchSubState): this { @@ -43,7 +53,41 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc xhr.setRequestHeader(key, value); } + let lastNotRespondingTimeoutIndex: any; + let lastMaxStreamWaitTimeoutIndex: any; + let streamNotResponding = false; + const clearStreamTimeout = () => { + if (streamNotResponding) { + this.emit("streamNotRespondingOff"); + streamNotResponding = false; + } + + if (lastNotRespondingTimeoutIndex) { + clearTimeout(lastNotRespondingTimeoutIndex); + } + + if (lastMaxStreamWaitTimeoutIndex) { + clearTimeout(lastMaxStreamWaitTimeoutIndex); + } + }; + + const createStreamTimeout = () => { + clearStreamTimeout(); + + lastNotRespondingTimeoutIndex = setTimeout(() => { + streamNotResponding = true; + this.emit("streamNotRespondingOn"); + }, STREAM_NOT_RESPONDING_TIMEOUT); + + lastMaxStreamWaitTimeoutIndex = setTimeout(() => { + reject(new EmptyStreamTimeoutError(`Stream timeout after ${prettyMilliseconds(this.options.maxStreamWait!)}`)); + xhr.abort(); + }, this.options.maxStreamWait); + }; + + xhr.onload = () => { + clearStreamTimeout(); const contentLength = parseInt(xhr.getResponseHeader("content-length")!); if (this.state.rangeSupport && contentLength !== end - start) { @@ -51,6 +95,10 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc } if (xhr.status >= 200 && xhr.status < 300) { + if (xhr.response.length != contentLength) { + throw new InvalidContentLengthError(contentLength, xhr.response.length); + } + const arrayBuffer = xhr.response; if (arrayBuffer) { resolve(new Uint8Array(arrayBuffer)); @@ -63,18 +111,22 @@ export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetc }; xhr.onerror = () => { + clearStreamTimeout(); reject(new XhrError(`Failed to fetch ${url}`)); }; xhr.onprogress = (event) => { + createStreamTimeout(); if (event.lengthComputable) { onProgress?.(event.loaded); } }; xhr.send(); + createStreamTimeout(); this.on("aborted", () => { + clearStreamTimeout(); xhr.abort(); }); }); diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/errors/EmptyStreamTimeoutError.ts b/src/download/download-engine/streams/download-engine-fetch-stream/errors/EmptyStreamTimeoutError.ts new file mode 100644 index 0000000..0d53a14 --- /dev/null +++ b/src/download/download-engine/streams/download-engine-fetch-stream/errors/EmptyStreamTimeoutError.ts @@ -0,0 +1,3 @@ +import FetchStreamError from "./fetch-stream-error.js"; + +export class EmptyStreamTimeoutError extends FetchStreamError {} diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts index 2f9a000..32e14b8 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/utils/smart-chunk-split.ts @@ -3,19 +3,29 @@ import {WriteCallback} from "../base-download-engine-fetch-stream.js"; export type SmartChunkSplitOptions = { chunkSize: number; startChunk: number; + endChunk: number; + lastChunkEndsFile: boolean; + totalSize: number }; export default class SmartChunkSplit { private readonly _callback: WriteCallback; private readonly _options: SmartChunkSplitOptions; + private readonly _lastChunkSize: number; private _bytesWriteLocation: number; - private _bytesLeftovers: number = 0; private _chunks: Uint8Array[] = []; + private _closed = false; public constructor(_callback: WriteCallback, _options: SmartChunkSplitOptions) { this._options = _options; this._callback = _callback; this._bytesWriteLocation = _options.startChunk * _options.chunkSize; + this._lastChunkSize = _options.lastChunkEndsFile ? + this.calcLastChunkSize() : this._options.chunkSize; + } + + public calcLastChunkSize() { + return this._options.totalSize - Math.max(this._options.endChunk - 1, 0) * this._options.chunkSize; } public addChunk(data: Uint8Array) { @@ -24,32 +34,45 @@ export default class SmartChunkSplit { } public get savedLength() { - return this._bytesLeftovers + this._chunks.reduce((acc, chunk) => acc + chunk.length, 0); + return this._chunks.reduce((acc, chunk) => acc + chunk.length, 0); } - public sendLeftovers() { - if (this.savedLength > 0) { + closeAndSendLeftoversIfLengthIsUnknown() { + if (this._chunks.length > 0 && this._options.endChunk === Infinity) { this._callback(this._chunks, this._bytesWriteLocation, this._options.startChunk++); } + this._closed = true; } private _sendChunk() { - while (this.savedLength >= this._options.chunkSize) { - if (this._chunks.length === 0) { - this._callback([], this._bytesWriteLocation, this._options.startChunk++); - this._bytesWriteLocation += this._options.chunkSize; - this._bytesLeftovers -= this._options.chunkSize; - } + if (this._closed) return; + + const checkThreshold = () => + (this._options.endChunk - this._options.startChunk === 1 ? + this._lastChunkSize : this._options.chunkSize); - let sendLength = this._bytesLeftovers; + let calcChunkThreshold = 0; + while (this.savedLength >= (calcChunkThreshold = checkThreshold())) { + let sendLength = 0; for (let i = 0; i < this._chunks.length; i++) { - sendLength += this._chunks[i].byteLength; - if (sendLength >= this._options.chunkSize) { + const currentChunk = this._chunks[i]; + sendLength += currentChunk.length; + if (sendLength >= calcChunkThreshold) { const sendChunks = this._chunks.splice(0, i + 1); + const diffLength = sendLength - calcChunkThreshold; + + if (diffLength > 0) { + const lastChunkEnd = currentChunk.length - diffLength; + const lastChunk = currentChunk.subarray(0, lastChunkEnd); + + sendChunks.pop(); + sendChunks.push(lastChunk); + + this._chunks.unshift(currentChunk.subarray(lastChunkEnd)); + } this._callback(sendChunks, this._bytesWriteLocation, this._options.startChunk++); - this._bytesWriteLocation += sendLength - this._bytesLeftovers; - this._bytesLeftovers = sendLength - this._options.chunkSize; + this._bytesWriteLocation += calcChunkThreshold; break; } } diff --git a/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts b/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts index b43f587..3ea036b 100644 --- a/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts +++ b/src/download/download-engine/streams/download-engine-fetch-stream/utils/stream-response.ts @@ -20,7 +20,7 @@ export default async function streamResponse(stream: IStreamResponse, downloadEn }); stream.on("close", () => { - smartSplit.sendLeftovers(); + smartSplit.closeAndSendLeftoversIfLengthIsUnknown(); resolve(); }); diff --git a/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts b/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts index cbacdcc..ddee247 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.ts @@ -1,5 +1,5 @@ export default abstract class BaseDownloadEngineWriteStream { - abstract write(cursor: number, buffer: Uint8Array): Promise | void; + abstract write(cursor: number, buffers: Uint8Array[]): Promise | void; close(): void | Promise { } diff --git a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts index daa7f9e..8099ac0 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.ts @@ -4,12 +4,12 @@ import BaseDownloadEngineWriteStream from "./base-download-engine-write-stream.j import WriterIsClosedError from "./errors/writer-is-closed-error.js"; import WriterNotDefineError from "./errors/writer-not-define-error.js"; -type DownloadEngineWriteStreamOptionsBrowser = { +export type DownloadEngineWriteStreamOptionsBrowser = { retry?: retry.Options file?: DownloadFile }; -export type DownloadEngineWriteStreamBrowserWriter = (cursor: number, buffer: Uint8Array, options: DownloadEngineWriteStreamOptionsBrowser) => Promise | void; +export type DownloadEngineWriteStreamBrowserWriter = (cursor: number, buffers: Uint8Array[], options: DownloadEngineWriteStreamOptionsBrowser) => Promise | void; export default class DownloadEngineWriteStreamBrowser extends BaseDownloadEngineWriteStream { protected readonly _writer?: DownloadEngineWriteStreamBrowserWriter; @@ -44,18 +44,26 @@ export default class DownloadEngineWriteStreamBrowser extends BaseDownloadEngine return this._memory = newMemory; } - public write(cursor: number, buffer: Uint8Array) { + public write(cursor: number, buffers: Uint8Array[]) { if (this.writerClosed) { throw new WriterIsClosedError(); } if (!this._writer) { - this._ensureBuffer(cursor + buffer.byteLength) - .set(buffer, cursor); - this._bytesWritten += buffer.byteLength; + const totalLength = buffers.reduce((sum, buffer) => sum + buffer.length, 0); + const bigBuffer = this._ensureBuffer(cursor + totalLength); + let writeLocation = cursor; + + for (const buffer of buffers) { + bigBuffer.set(buffer, writeLocation); + writeLocation += buffer.length; + } + + this._bytesWritten += totalLength; return; } - return this._writer(cursor, buffer, this.options); + + return this._writer(cursor, buffers, this.options); } public get result() { diff --git a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts index ebf6032..12bfa73 100644 --- a/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts +++ b/src/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.ts @@ -4,27 +4,71 @@ import retry from "async-retry"; import {withLock} from "lifecycle-utils"; import BaseDownloadEngineWriteStream from "./base-download-engine-write-stream.js"; import WriterIsClosedError from "./errors/writer-is-closed-error.js"; +import {BytesWriteDebounce} from "./utils/BytesWriteDebounce.js"; export type DownloadEngineWriteStreamOptionsNodeJS = { retry?: retry.Options mode: string; + debounceWrite?: { + maxTime?: number + maxSize?: number + } }; -const DEFAULT_OPTIONS: DownloadEngineWriteStreamOptionsNodeJS = { - mode: "r+" -}; +const DEFAULT_OPTIONS = { + mode: "r+", + debounceWrite: { + maxTime: 1000 * 5, // 5 seconds + maxSize: 1024 * 1024 * 2 // 2 MB + } +} satisfies DownloadEngineWriteStreamOptionsNodeJS; +const MAX_AUTO_DEBOUNCE_SIZE = 1024 * 1024 * 100; // 100 MB +const AUTO_DEBOUNCE_SIZE_PERCENT = 0.05; +const MAX_META_SIZE = 10485760; // 10 MB const NOT_ENOUGH_SPACE_ERROR_CODE = "ENOSPC"; export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineWriteStream { private _fd: FileHandle | null = null; private _fileWriteFinished = false; + private _writeDebounce: BytesWriteDebounce; + private _fileSize = 0; + public readonly options: DownloadEngineWriteStreamOptionsNodeJS; - public fileSize = 0; + public autoDebounceMaxSize = false; constructor(public path: string, public finalPath: string, options: Partial = {}) { super(); - this.options = {...DEFAULT_OPTIONS, ...options}; + + this.autoDebounceMaxSize = !options.debounceWrite?.maxSize; + const optionsWithDefaults = this.options = { + ...DEFAULT_OPTIONS, + ...options, + debounceWrite: { + ...DEFAULT_OPTIONS.debounceWrite, + ...options.debounceWrite + } + }; + + this._writeDebounce = new BytesWriteDebounce({ + ...optionsWithDefaults.debounceWrite, + writev: (cursor, buffers) => this._writeWithoutDebounce(cursor, buffers) + }); + } + + public get fileSize() { + return this._fileSize; + } + + public set fileSize(value) { + this._fileSize = value; + + if (this.autoDebounceMaxSize) { + this.options.debounceWrite!.maxSize = Math.max( + Math.min(value * AUTO_DEBOUNCE_SIZE_PERCENT, MAX_AUTO_DEBOUNCE_SIZE), + DEFAULT_OPTIONS.debounceWrite.maxSize + ); + } } private async _ensureFileOpen() { @@ -40,12 +84,16 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW }); } - async write(cursor: number, buffer: Uint8Array) { + async write(cursor: number, buffers: Uint8Array[]) { + await this._writeDebounce.addChunk(cursor, buffers); + } + + async _writeWithoutDebounce(cursor: number, buffers: Uint8Array[]) { let throwError: Error | false = false; await retry(async () => { try { - return await this._writeWithoutRetry(cursor, buffer); + return await this._writeWithoutRetry(cursor, buffers); } catch (error: any) { if (error?.code === NOT_ENOUGH_SPACE_ERROR_CODE) { throwError = error; @@ -60,7 +108,12 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW } } - async ftruncate(size = this.fileSize) { + async ensureBytesSynced() { + await this._writeDebounce.writeAll(); + } + + async ftruncate(size = this._fileSize) { + await this.ensureBytesSynced(); this._fileWriteFinished = true; await retry(async () => { const fd = await this._ensureFileOpen(); @@ -68,7 +121,7 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW }, this.options.retry); } - async saveMedataAfterFile(data: any) { + async saveMetadataAfterFile(data: any) { if (this._fileWriteFinished) { throw new WriterIsClosedError(); } @@ -78,7 +131,7 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW const encoder = new TextEncoder(); const uint8Array = encoder.encode(jsonString); - await this.write(this.fileSize, uint8Array); + await this.write(this._fileSize, [uint8Array]); } async loadMetadataAfterFileWithoutRetry() { @@ -89,13 +142,16 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW const fd = await this._ensureFileOpen(); try { const state = await fd.stat(); - const metadataSize = state.size - this.fileSize; - if (metadataSize <= 0) { + const metadataSize = state.size - this._fileSize; + if (metadataSize <= 0 || metadataSize >= MAX_META_SIZE) { + if (this._fileSize > 0 && state.size > this._fileSize) { + await this.ftruncate(); + } return; } const metadataBuffer = Buffer.alloc(metadataSize); - await fd.read(metadataBuffer, 0, metadataSize, this.fileSize); + await fd.read(metadataBuffer, 0, metadataSize, this._fileSize); const decoder = new TextDecoder(); const metadataString = decoder.decode(metadataBuffer); @@ -108,10 +164,12 @@ export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineW } } - private async _writeWithoutRetry(cursor: number, buffer: Uint8Array) { - const fd = await this._ensureFileOpen(); - const {bytesWritten} = await fd.write(buffer, 0, buffer.length, cursor); - return bytesWritten; + private async _writeWithoutRetry(cursor: number, buffers: Uint8Array[]) { + return await withLock(this, "lockWriteOperation", async () => { + const fd = await this._ensureFileOpen(); + const {bytesWritten} = await fd.writev(buffers, cursor); + return bytesWritten; + }); } override async close() { diff --git a/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts new file mode 100644 index 0000000..805dc36 --- /dev/null +++ b/src/download/download-engine/streams/download-engine-write-stream/utils/BytesWriteDebounce.ts @@ -0,0 +1,96 @@ +import sleep from "sleep-promise"; + +export type BytesWriteDebounceOptions = { + maxTime: number; + maxSize: number; + writev: (index: number, buffers: Uint8Array[]) => Promise; +}; + +export class BytesWriteDebounce { + private _writeChunks: { + index: number; + buffer: Uint8Array; + }[] = []; + private _lastWriteTime = Date.now(); + private _totalSizeOfChunks = 0; + private _checkWriteInterval = false; + + constructor(private _options: BytesWriteDebounceOptions) { + + } + + async addChunk(index: number, buffers: Uint8Array[]) { + let writeIndex = index; + for (const buffer of buffers) { + this._writeChunks.push({index: writeIndex, buffer}); + this._totalSizeOfChunks += buffer.length; + writeIndex += buffer.length; + } + + await this._writeIfNeeded(); + this.checkIfWriteNeededInterval(); + } + + private async _writeIfNeeded() { + if (this._totalSizeOfChunks >= this._options.maxSize || Date.now() - this._lastWriteTime >= this._options.maxTime) { + await this.writeAll(); + } + } + + private async checkIfWriteNeededInterval() { + if (this._checkWriteInterval) { + return; + } + this._checkWriteInterval = true; + + while (this._writeChunks.length > 0) { + await this._writeIfNeeded(); + const timeUntilMaxLimitAfterWrite = this._options.maxTime - (Date.now() - this._lastWriteTime); + await sleep(Math.max(timeUntilMaxLimitAfterWrite, 0)); + } + + this._checkWriteInterval = false; + } + + writeAll() { + if (this._writeChunks.length === 0) { + return; + } + + this._writeChunks = this._writeChunks.sort((a, b) => a.index - b.index); + const firstWrite = this._writeChunks[0]; + + let writeIndex = firstWrite.index; + let buffers: Uint8Array[] = [firstWrite.buffer]; + let buffersTotalLength = firstWrite.buffer.length; + + const writePromises: Promise[] = []; + + for (let i = 1; i < this._writeChunks.length; i++) { + const nextWriteLocation = writeIndex + buffersTotalLength; + const currentWrite = this._writeChunks[i]; + + if (currentWrite.index < nextWriteLocation) { // overlapping, prefer the last buffer (newer data) + const lastBuffer = buffers.pop()!; + buffers.push(currentWrite.buffer); + buffersTotalLength += currentWrite.buffer.length - lastBuffer.length; + } else if (nextWriteLocation === currentWrite.index) { + buffers.push(currentWrite.buffer); + buffersTotalLength += currentWrite.buffer.length; + } else { + writePromises.push(this._options.writev(writeIndex, buffers)); + + writeIndex = currentWrite.index; + buffers = [currentWrite.buffer]; + buffersTotalLength = currentWrite.buffer.length; + } + } + + writePromises.push(this._options.writev(writeIndex, buffers)); + + this._writeChunks = []; + this._totalSizeOfChunks = 0; + this._lastWriteTime = Date.now(); + return Promise.all(writePromises); + } +} diff --git a/src/download/download-engine/types.ts b/src/download/download-engine/types.ts index f1a96e0..c6ae82c 100644 --- a/src/download/download-engine/types.ts +++ b/src/download/download-engine/types.ts @@ -11,6 +11,7 @@ export enum ChunkStatus { } export type SaveProgressInfo = { + downloadId: string, part: number, chunks: ChunkStatus[], chunkSize: number, diff --git a/src/download/download-engine/utils/concurrency.ts b/src/download/download-engine/utils/concurrency.ts new file mode 100644 index 0000000..2da18d0 --- /dev/null +++ b/src/download/download-engine/utils/concurrency.ts @@ -0,0 +1,30 @@ +import {promiseWithResolvers} from "./promiseWithResolvers.js"; + +export function concurrency(array: Value[], concurrencyCount: number, callback: (value: Value) => Promise) { + const {resolve, reject, promise} = promiseWithResolvers(); + let index = 0; + let activeCount = 0; + + function reload() { + if (index === array.length && activeCount === 0) { + resolve(); + return; + } + + while (activeCount < concurrencyCount && index < array.length) { + activeCount++; + callback(array[index++]) + .then(() => { + activeCount--; + reload(); + }, reject); + } + } + + reload(); + + return { + promise, + reload + }; +} diff --git a/src/download/download-engine/utils/promiseWithResolvers.ts b/src/download/download-engine/utils/promiseWithResolvers.ts new file mode 100644 index 0000000..a1fb4a5 --- /dev/null +++ b/src/download/download-engine/utils/promiseWithResolvers.ts @@ -0,0 +1,9 @@ +export function promiseWithResolvers() { + let resolve: (value: T) => void; + let reject: (reason?: any) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return {promise, resolve: resolve!, reject: reject!}; +} diff --git a/src/download/node-download.ts b/src/download/node-download.ts index 69647fe..5b5a1f7 100644 --- a/src/download/node-download.ts +++ b/src/download/node-download.ts @@ -1,35 +1,35 @@ import DownloadEngineNodejs, {DownloadEngineOptionsNodejs} from "./download-engine/engine/download-engine-nodejs.js"; import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js"; import DownloadEngineMultiDownload, {DownloadEngineMultiDownloadOptions} from "./download-engine/engine/download-engine-multi-download.js"; -import CliAnimationWrapper, {CliProgressDownloadEngineOptions} from "./transfer-visualize/transfer-cli/cli-animation-wrapper.js"; -import {CLI_LEVEL} from "./transfer-visualize/transfer-cli/transfer-cli.js"; -import {NoDownloadEngineProvidedError} from "./download-engine/engine/error/no-download-engine-provided-error.js"; +import {CliProgressDownloadEngineOptions, globalCLI} from "./transfer-visualize/transfer-cli/GlobalCLI.js"; +import {DownloadEngineRemote} from "./download-engine/engine/DownloadEngineRemote.js"; const DEFAULT_PARALLEL_STREAMS_FOR_NODEJS = 3; -export type DownloadFileOptions = DownloadEngineOptionsNodejs & CliProgressDownloadEngineOptions & { - /** @deprecated use partURLs instead */ - partsURL?: string[]; -}; +export type DownloadFileOptions = DownloadEngineOptionsNodejs & CliProgressDownloadEngineOptions; /** * Download one file with CLI progress */ export async function downloadFile(options: DownloadFileOptions) { - // TODO: Remove in the next major version - if (!("url" in options) && options.partsURL) { - options.partURLs ??= options.partsURL; - } - - options.parallelStreams ??= DEFAULT_PARALLEL_STREAMS_FOR_NODEJS; const downloader = DownloadEngineNodejs.createFromOptions(options); - const wrapper = new CliAnimationWrapper(downloader, options); + globalCLI.addDownload(downloader, options); - await wrapper.attachAnimation(); return await downloader; } +/** + * Stream events for a download from remote session, doing so by calling `emitRemoteProgress` with the progress info. + * - Supports CLI progress. + */ +export function downloadFileRemote(options?: CliProgressDownloadEngineOptions) { + const downloader = new DownloadEngineRemote(); + globalCLI.addDownload(downloader, options); + + return downloader; +} + export type DownloadSequenceOptions = CliProgressDownloadEngineOptions & DownloadEngineMultiDownloadOptions & { fetchStrategy?: "localFile" | "fetch"; }; @@ -46,14 +46,9 @@ export async function downloadSequence(options?: DownloadSequenceOptions | Downl downloadOptions = options; } - if (downloads.length === 0) { - throw new NoDownloadEngineProvidedError(); - } - - downloadOptions.cliLevel = CLI_LEVEL.HIGH; - const downloader = DownloadEngineMultiDownload.fromEngines(downloads, downloadOptions); - const wrapper = new CliAnimationWrapper(downloader, downloadOptions); + const downloader = new DownloadEngineMultiDownload(downloadOptions); + globalCLI.addDownload(downloader, downloadOptions); + await downloader.addDownload(...downloads); - await wrapper.attachAnimation(); - return await downloader; + return downloader; } diff --git a/src/download/transfer-visualize/progress-statistics-builder.ts b/src/download/transfer-visualize/progress-statistics-builder.ts index db10ada..978fb2b 100644 --- a/src/download/transfer-visualize/progress-statistics-builder.ts +++ b/src/download/transfer-visualize/progress-statistics-builder.ts @@ -5,6 +5,7 @@ import {createFormattedStatus, FormattedStatus} from "./format-transfer-status.j import DownloadEngineFile from "../download-engine/download-file/download-engine-file.js"; import ProgressStatusFile, {DownloadStatus, ProgressStatus} from "../download-engine/download-file/progress-status-file.js"; import DownloadEngineMultiDownload from "../download-engine/engine/download-engine-multi-download.js"; +import {DownloadEngineRemote} from "../download-engine/engine/DownloadEngineRemote.js"; export type ProgressStatusWithIndex = FormattedStatus & { index: number, @@ -14,12 +15,13 @@ interface CliProgressBuilderEvents { progress: (progress: ProgressStatusWithIndex) => void; } -export type AnyEngine = DownloadEngineFile | BaseDownloadEngine | DownloadEngineMultiDownload; +export type AnyEngine = DownloadEngineFile | BaseDownloadEngine | DownloadEngineMultiDownload | DownloadEngineRemote; export default class ProgressStatisticsBuilder extends EventEmitter { - private _engines: AnyEngine[] = []; + private _engines = new Set(); private _activeTransfers: { [index: number]: number } = {}; private _totalBytes = 0; private _transferredBytes = 0; + private _latestEngine: AnyEngine | null = null; /** * @internal */ @@ -27,8 +29,38 @@ export default class ProgressStatisticsBuilder extends EventEmitter { + const retrying = Number(data.retrying); + this._retrying += retrying - lastRetrying; + lastRetrying = retrying; + + this._retryingTotalAttempts += data.retryingTotalAttempts - lastRetryingTotalAttempts; + lastRetryingTotalAttempts = data.retryingTotalAttempts; + + this._streamsNotResponding += data.streamsNotResponding - lastStreamsNotResponding; + lastStreamsNotResponding = data.streamsNotResponding; + this._sendProgress(data, index, downloadPartStart); }); @@ -71,10 +114,20 @@ export default class ProgressStatisticsBuilder extends EventEmitter 0, + retryingTotalAttempts: this._retryingTotalAttempts, + streamsNotResponding: this._streamsNotResponding }), index }; - - this.emit("progress", this._lastStatus); } static oneStatistics(engine: DownloadEngineFile) { diff --git a/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts new file mode 100644 index 0000000..e14e928 --- /dev/null +++ b/src/download/transfer-visualize/transfer-cli/GlobalCLI.ts @@ -0,0 +1,162 @@ +import DownloadEngineMultiDownload, {DownloadEngineMultiAllowedEngines} from "../../download-engine/engine/download-engine-multi-download.js"; +import TransferCli, {TransferCliOptions} from "./transfer-cli.js"; +import {BaseMultiProgressBar} from "./multiProgressBars/BaseMultiProgressBar.js"; +import switchCliProgressStyle, {AvailableCLIProgressStyle} from "./progress-bars/switch-cli-progress-style.js"; +import {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar.js"; +import cliSpinners from "cli-spinners"; +import {DownloadStatus} from "../../download-engine/download-file/progress-status-file.js"; +import BaseDownloadEngine from "../../download-engine/engine/base-download-engine.js"; +import {DownloadEngineRemote} from "../../download-engine/engine/DownloadEngineRemote.js"; + +type AllowedDownloadEngine = DownloadEngineMultiDownload | BaseDownloadEngine | DownloadEngineRemote; + +const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "auto"; + +export type CliProgressDownloadEngineOptions = { + truncateName?: boolean | number; + cliProgress?: boolean; + maxViewDownloads?: number; + createMultiProgressBar?: typeof BaseMultiProgressBar, + cliStyle?: AvailableCLIProgressStyle | ((status: CliFormattedStatus) => string) + cliName?: string; + loadingAnimation?: cliSpinners.SpinnerName; +}; + +class GlobalCLI { + private _multiDownloadEngine = this._createMultiDownloadEngine(); + private _eventsRegistered = new Set(); + private _transferCLI = GlobalCLI._createOptions({}); + private _cliActive = false; + private _downloadOptions = new WeakMap(); + + constructor() { + this._registerCLIEvents(); + } + + async addDownload(engine: AllowedDownloadEngine | Promise, cliOptions: CliProgressDownloadEngineOptions = {}) { + if (!this._cliActive && cliOptions.cliProgress) { + this._transferCLI = GlobalCLI._createOptions(cliOptions); + } + + if (engine instanceof Promise) { + engine.then((engine) => this._downloadOptions.set(engine, cliOptions)); + } else { + this._downloadOptions.set(engine, cliOptions); + } + + await this._multiDownloadEngine.addDownload(engine); + this._multiDownloadEngine.download(); + } + + private _createMultiDownloadEngine() { + return new DownloadEngineMultiDownload({ + unpackInnerMultiDownloadsStatues: true, + finalizeDownloadAfterAllSettled: false, + naturalDownloadStart: true, + parallelDownloads: Number.MAX_VALUE + }); + } + + private _registerCLIEvents() { + const isDownloadActive = (parentEngine: DownloadEngineMultiDownload = this._multiDownloadEngine) => { + if (parentEngine.loadingDownloads > 0) { + return true; + } + + for (const engine of parentEngine.activeDownloads) { + if (engine instanceof DownloadEngineMultiDownload) { + if (isDownloadActive(engine)) { + return true; + } + } + + if (engine.status.downloadStatus === DownloadStatus.Active || parentEngine.status.downloadStatus === DownloadStatus.Active && [DownloadStatus.Loading, DownloadStatus.NotStarted].includes(engine.status.downloadStatus)) { + return true; + } + } + + return false; + }; + + const checkPauseCLI = () => { + if (this._cliActive && !isDownloadActive()) { + this._transferCLI.stop(); + this._cliActive = false; + } + }; + + const checkCloseCLI = (engine: DownloadEngineMultiAllowedEngines) => { + this._eventsRegistered.delete(engine); + checkPauseCLI(); + }; + + const checkResumeCLI = (engine: DownloadEngineMultiAllowedEngines) => { + if (engine.status.downloadStatus === DownloadStatus.Active) { + this._transferCLI.start(); + this._cliActive = true; + } + }; + + this._multiDownloadEngine.on("finished", () => { + this._multiDownloadEngine = this._createMultiDownloadEngine(); + this._eventsRegistered = new Set(); + }); + + const eventsRegistered = this._eventsRegistered; + this._multiDownloadEngine.on("childDownloadStarted", function registerEngineStatus(engine) { + if (eventsRegistered.has(engine)) return; + eventsRegistered.add(engine); + + checkResumeCLI(engine); + engine.on("paused", checkPauseCLI); + engine.on("closed", () => checkCloseCLI(engine)); + engine.on("resumed", () => checkResumeCLI(engine)); + engine.on("start", () => checkResumeCLI(engine)); + + if (engine instanceof DownloadEngineMultiDownload) { + engine.on("childDownloadStarted", registerEngineStatus); + } + }); + + const getCLIEngines = (multiEngine: DownloadEngineMultiDownload) => { + const enginesToShow: AllowedDownloadEngine[] = []; + for (const engine of multiEngine.activeDownloads) { + const isShowEngine = this._downloadOptions.get(engine)?.cliProgress; + if (engine instanceof DownloadEngineMultiDownload) { + if (isShowEngine) { + enginesToShow.push(...engine._flatEngines); + continue; + } + enginesToShow.push(...getCLIEngines(engine)); + } else if (isShowEngine) { + enginesToShow.push(engine); + } + } + + return enginesToShow.filter((engine, index, self) => self.indexOf(engine) === index); + }; + + this._multiDownloadEngine.on("progress", (progress) => { + if (!this._cliActive) return; + const statues = getCLIEngines(this._multiDownloadEngine) + .map(x => x.status); + this._transferCLI.updateStatues(statues, progress, this._multiDownloadEngine.loadingDownloads); + }); + } + + private static _createOptions(options: CliProgressDownloadEngineOptions) { + const cliOptions: Partial = {...options}; + cliOptions.createProgressBar ??= typeof options.cliStyle === "function" ? + { + createStatusLine: options.cliStyle, + multiProgressBar: options.createMultiProgressBar ?? BaseMultiProgressBar + } : + switchCliProgressStyle(options.cliStyle ?? DEFAULT_CLI_STYLE, { + truncateName: options.truncateName, + loadingSpinner: options.loadingAnimation + }); + return new TransferCli(cliOptions); + } +} + +export const globalCLI = new GlobalCLI(); diff --git a/src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts b/src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts deleted file mode 100644 index d1a2c48..0000000 --- a/src/download/transfer-visualize/transfer-cli/cli-animation-wrapper.ts +++ /dev/null @@ -1,89 +0,0 @@ -import DownloadEngineNodejs from "../../download-engine/engine/download-engine-nodejs.js"; -import DownloadEngineMultiDownload from "../../download-engine/engine/download-engine-multi-download.js"; -import switchCliProgressStyle, {AvailableCLIProgressStyle} from "./progress-bars/switch-cli-progress-style.js"; -import {CliFormattedStatus} from "./progress-bars/base-transfer-cli-progress-bar.js"; -import TransferCli, {CLI_LEVEL, TransferCliOptions} from "./transfer-cli.js"; -import {BaseMultiProgressBar} from "./multiProgressBars/BaseMultiProgressBar.js"; -import cliSpinners from "cli-spinners"; - -const DEFAULT_CLI_STYLE: AvailableCLIProgressStyle = "auto"; -type AllowedDownloadEngines = DownloadEngineNodejs | DownloadEngineMultiDownload; - -export type CliProgressDownloadEngineOptions = { - truncateName?: boolean | number; - cliProgress?: boolean; - maxViewDownloads?: number; - createMultiProgressBar?: typeof BaseMultiProgressBar, - cliStyle?: AvailableCLIProgressStyle | ((status: CliFormattedStatus) => string) - cliName?: string; - cliAction?: string; - fetchStrategy?: "localFile" | "fetch"; - loadingAnimation?: cliSpinners.SpinnerName; - /** @internal */ - cliLevel?: CLI_LEVEL; -}; - -export default class CliAnimationWrapper { - private readonly _downloadEngine: Promise; - private readonly _options: CliProgressDownloadEngineOptions; - private _activeCLI?: TransferCli; - - public constructor(downloadEngine: Promise, _options: CliProgressDownloadEngineOptions) { - this._options = _options; - this._downloadEngine = downloadEngine; - this._init(); - } - - private _init() { - if (!this._options.cliProgress) { - return; - } - this._options.cliAction ??= this._options.fetchStrategy === "localFile" ? "Copying" : "Downloading"; - - const cliOptions: Partial = {...this._options}; - if (this._options.cliAction) { - cliOptions.action = this._options.cliAction; - } - if (this._options.cliName) { - cliOptions.name = this._options.cliName; - } - - cliOptions.createProgressBar = typeof this._options.cliStyle === "function" ? - { - createStatusLine: this._options.cliStyle, - multiProgressBar: this._options.createMultiProgressBar ?? BaseMultiProgressBar - } : - switchCliProgressStyle(this._options.cliStyle ?? DEFAULT_CLI_STYLE, { - truncateName: this._options.truncateName, - loadingSpinner: this._options.loadingAnimation - }); - - this._activeCLI = new TransferCli(cliOptions, this._options.cliLevel); - } - - public async attachAnimation() { - if (!this._activeCLI) { - return; - } - this._activeCLI.loadingAnimation.start(); - - try { - const engine = await this._downloadEngine; - this._activeCLI.loadingAnimation.stop(); - - engine.once("start", () => { - this._activeCLI?.start(); - - engine.on("progress", (progress) => { - this._activeCLI?.updateStatues(engine.downloadStatues, progress); - }); - - engine.on("closed", () => { - this._activeCLI?.stop(); - }); - }); - } finally { - this._activeCLI.loadingAnimation.stop(); - } - } -} diff --git a/src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts b/src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts deleted file mode 100644 index 588bee8..0000000 --- a/src/download/transfer-visualize/transfer-cli/loading-animation/base-loading-animation.ts +++ /dev/null @@ -1,74 +0,0 @@ -import UpdateManager from "stdout-update"; -import sleep from "sleep-promise"; -import {CLIProgressPrintType} from "../multiProgressBars/BaseMultiProgressBar.js"; - -export type BaseLoadingAnimationOptions = { - updateIntervalMs?: number | null; - loadingText?: string; - logType: CLIProgressPrintType -}; - -export const DEFAULT_LOADING_ANIMATION_OPTIONS: BaseLoadingAnimationOptions = { - loadingText: "Gathering information", - logType: "update" -}; - -const DEFAULT_UPDATE_INTERVAL_MS = 300; - -export default abstract class BaseLoadingAnimation { - protected options: BaseLoadingAnimationOptions; - protected stdoutManager = UpdateManager.getInstance(); - protected _animationActive = false; - - - protected constructor(options: BaseLoadingAnimationOptions = DEFAULT_LOADING_ANIMATION_OPTIONS) { - this.options = options; - this._processExit = this._processExit.bind(this); - } - - protected _render(): void { - const frame = this.createFrame(); - - if (this.options.logType === "update") { - this.stdoutManager.update([frame]); - } else { - console.log(frame); - } - } - - protected abstract createFrame(): string; - - async start() { - process.on("SIGINT", this._processExit); - - if (this.options.logType === "update") { - this.stdoutManager.hook(); - } - - this._animationActive = true; - while (this._animationActive) { - this._render(); - await sleep(this.options.updateIntervalMs || DEFAULT_UPDATE_INTERVAL_MS); - } - } - - stop(): void { - if (!this._animationActive) { - return; - } - - this._animationActive = false; - - if (this.options.logType === "update") { - this.stdoutManager.erase(); - this.stdoutManager.unhook(false); - } - - process.off("SIGINT", this._processExit); - } - - private _processExit() { - this.stop(); - process.exit(0); - } -} diff --git a/src/download/transfer-visualize/transfer-cli/loading-animation/cli-spinners-loading-animation.ts b/src/download/transfer-visualize/transfer-cli/loading-animation/cli-spinners-loading-animation.ts deleted file mode 100644 index 54d1fda..0000000 --- a/src/download/transfer-visualize/transfer-cli/loading-animation/cli-spinners-loading-animation.ts +++ /dev/null @@ -1,23 +0,0 @@ -import BaseLoadingAnimation, {BaseLoadingAnimationOptions, DEFAULT_LOADING_ANIMATION_OPTIONS} from "./base-loading-animation.js"; -import {Spinner} from "cli-spinners"; - -export default class CliSpinnersLoadingAnimation extends BaseLoadingAnimation { - private _spinner: Spinner; - private _frameIndex = 0; - - public constructor(spinner: Spinner, options: BaseLoadingAnimationOptions) { - options = {...DEFAULT_LOADING_ANIMATION_OPTIONS, ...options}; - options.updateIntervalMs ??= spinner.interval; - super(options); - this._spinner = spinner; - } - - protected createFrame(): string { - const frame = this._spinner.frames[this._frameIndex]; - this._frameIndex++; - if (this._frameIndex >= this._spinner.frames.length) { - this._frameIndex = 0; - } - return `${frame} ${this.options.loadingText}`; - } -} diff --git a/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts b/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts index d393167..d4c5086 100644 --- a/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts +++ b/src/download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.ts @@ -3,11 +3,12 @@ import {FormattedStatus} from "../../format-transfer-status.js"; import {DownloadStatus} from "../../../download-engine/download-file/progress-status-file.js"; import chalk from "chalk"; import prettyBytes from "pretty-bytes"; +import cliSpinners from "cli-spinners"; export type MultiProgressBarOptions = { maxViewDownloads: number; createProgressBar: TransferCliProgressBar - action?: string; + loadingAnimation: cliSpinners.SpinnerName, }; export type CLIProgressPrintType = "update" | "log"; @@ -16,14 +17,12 @@ export class BaseMultiProgressBar { public readonly updateIntervalMs: null | number = null; public readonly printType: CLIProgressPrintType = "update"; + public constructor(protected options: MultiProgressBarOptions) { } protected createProgresses(statuses: FormattedStatus[]): string { - return statuses.map((status) => { - status.transferAction = this.options.action ?? status.transferAction; - return this.options.createProgressBar.createStatusLine(status); - }) + return statuses.map((status) => this.options.createProgressBar.createStatusLine(status)) .join("\n"); } @@ -49,8 +48,8 @@ export class BaseMultiProgressBar { } // eslint-disable-next-line @typescript-eslint/no-unused-vars - createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus) { - if (statuses.length < this.options.maxViewDownloads) { + createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus, loadingDownloads = 0) { + if (statuses.length < this.options.maxViewDownloads - Math.min(loadingDownloads, 1)) { return this.createProgresses(statuses); } @@ -58,10 +57,10 @@ export class BaseMultiProgressBar { const tasksLogs = this.createProgresses(allStatusesSorted.slice(0, this.options.maxViewDownloads)); if (notFinished) { - return tasksLogs + `\nand ${chalk.gray(remaining)} more out of ${chalk.blueBright(statuses.length)} downloads.`; + return tasksLogs + `\nand ${chalk.gray((remaining + loadingDownloads).toLocaleString())} more out of ${chalk.blueBright(statuses.length.toLocaleString())} downloads.`; } const totalSize = allStatusesSorted.reduce((acc, status) => acc + status.totalBytes, 0); - return tasksLogs + `\n${chalk.green(`All ${statuses.length} downloads (${prettyBytes(totalSize)}) finished.`)}`; + return tasksLogs + `\n${chalk.green(`All ${statuses.length.toLocaleString()} downloads (${prettyBytes(totalSize)}) finished.`)}`; } } diff --git a/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts b/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts index 2bf59c0..b257edc 100644 --- a/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts +++ b/src/download/transfer-visualize/transfer-cli/multiProgressBars/SummaryMultiProgressBar.ts @@ -8,15 +8,15 @@ export class SummaryMultiProgressBar extends BaseMultiProgressBar { private _parallelDownloads = 0; private _lastStatuses: FormattedStatus[] = []; - override createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus) { + override createMultiProgressBar(statuses: FormattedStatus[], oneStatus: FormattedStatus, loadingDownloads = 0) { oneStatus = structuredClone(oneStatus); oneStatus.downloadFlags.push(DownloadFlags.DownloadSequence); const linesToPrint: FormattedStatus[] = []; - let index = 0; for (const status of statuses) { - const isStatusChanged = this._lastStatuses[index++]?.downloadStatus !== status.downloadStatus; + const lastStatus = this._lastStatuses.find(x => x.downloadId === status.downloadId); + const isStatusChanged = lastStatus?.downloadStatus !== status.downloadStatus; const copyStatus = structuredClone(status); if (isStatusChanged) { @@ -38,7 +38,7 @@ export class SummaryMultiProgressBar extends BaseMultiProgressBar { const activeDownloads = statuses.filter((status) => status.downloadStatus === DownloadStatus.Active).length; this._parallelDownloads ||= activeDownloads; const finishedDownloads = statuses.filter((status) => status.downloadStatus === DownloadStatus.Finished).length; - oneStatus.comment = `${finishedDownloads}/${statuses.length} files done${this._parallelDownloads > 1 ? ` (${activeDownloads} active)` : ""}`; + oneStatus.comment = `${finishedDownloads.toLocaleString()}/${(statuses.length + loadingDownloads).toLocaleString()} files done${this._parallelDownloads > 1 ? ` (${activeDownloads.toLocaleString()} active)` : ""}`; return this.createProgresses(filterStatusesSliced); } diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts index c8f356f..17b8284 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/base-transfer-cli-progress-bar.ts @@ -64,6 +64,16 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa return this.status.startTime < Date.now() - SKIP_ETA_START_TIME; } + protected get alertStatus() { + if (this.status.retrying) { + return `(retrying #${this.status.retryingTotalAttempts})`; + } else if (this.status.streamsNotResponding) { + return `(${this.status.streamsNotResponding} streams not responding)`; + } + + return ""; + } + protected getNameSize(fileName = this.status.fileName) { return this.options.truncateName === false ? fileName.length @@ -170,6 +180,7 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa const {formattedPercentage, formattedSpeed, formatTransferredOfTotal, formatTotal} = this.status; const status = this.switchTransferToShortText(); + const alertStatus = this.alertStatus; return renderDataLine([ { type: "status", @@ -177,6 +188,12 @@ export default class BaseTransferCliProgressBar implements TransferCliProgressBa size: status.length, formatter: (text) => chalk.cyan(text) }, + { + type: "status", + fullText: alertStatus, + size: alertStatus.length, + formatter: (text) => chalk.ansi256(196)(text) + }, { type: "spacer", fullText: " ", diff --git a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts index a9d0429..e54648e 100644 --- a/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts +++ b/src/download/transfer-visualize/transfer-cli/progress-bars/fancy-transfer-cli-progress-bar.ts @@ -19,47 +19,58 @@ export default class FancyTransferCliProgressBar extends BaseTransferCliProgress const progressBarText = ` ${formattedPercentageWithPadding} (${formatTransferred}/${formatTotal}) `; const dimEta: DataLine = this.getETA(" | ", text => chalk.dim(text)); + const alertStatus = this.alertStatus; - return renderDataLine([{ - type: "status", - fullText: "", - size: 1, - formatter: () => STATUS_ICONS.activeDownload - }, { - type: "spacer", - fullText: " ", - size: " ".length - }, ...this.getNameAndCommentDataParts(), { - type: "spacer", - fullText: " ", - size: " ".length - }, { - type: "progressBar", - fullText: progressBarText, - size: Math.max(progressBarText.length, `100.0% (1024.00MB/${formatTotal})`.length), - flex: 4, - addEndPadding: 4, - maxSize: 40, - formatter(_, size) { - const leftPad = " ".repeat(Math.floor((size - progressBarText.length) / 2)); - return renderProgressBar({ - barText: leftPad + ` ${chalk.black.bgWhiteBright(formattedPercentageWithPadding)} ${chalk.gray(`(${formatTransferred}/${formatTotal})`)} `, - backgroundText: leftPad + ` ${chalk.yellow.bgGray(formattedPercentageWithPadding)} ${chalk.white(`(${formatTransferred}/${formatTotal})`)} `, - length: size, - loadedPercentage: percentage / 100, - barStyle: chalk.black.bgWhiteBright, - backgroundStyle: chalk.bgGray - }); - } - }, { - type: "spacer", - fullText: " ", - size: " ".length - }, { - type: "speed", - fullText: formattedSpeed, - size: Math.max("00.00kB/s".length, formattedSpeed.length) - }, ...dimEta]); + return renderDataLine([ + { + type: "status", + fullText: "", + size: 1, + formatter: () => STATUS_ICONS.activeDownload + }, + { + type: "status", + fullText: alertStatus, + size: alertStatus.length, + formatter: (text) => chalk.ansi256(196)(text) + }, + { + type: "spacer", + fullText: " ", + size: " ".length + }, ...this.getNameAndCommentDataParts(), { + type: "spacer", + fullText: " ", + size: " ".length + }, { + type: "progressBar", + fullText: progressBarText, + size: Math.max(progressBarText.length, `100.0% (1024.00MB/${formatTotal})`.length), + flex: 4, + addEndPadding: 4, + maxSize: 40, + formatter(_, size) { + const leftPad = " ".repeat(Math.floor((size - progressBarText.length) / 2)); + return renderProgressBar({ + barText: leftPad + ` ${chalk.black.bgWhiteBright(formattedPercentageWithPadding)} ${chalk.gray(`(${formatTransferred}/${formatTotal})`)} `, + backgroundText: leftPad + ` ${chalk.yellow.bgGray(formattedPercentageWithPadding)} ${chalk.white(`(${formatTransferred}/${formatTotal})`)} `, + length: size, + loadedPercentage: percentage / 100, + barStyle: chalk.black.bgWhiteBright, + backgroundStyle: chalk.bgGray + }); + } + }, { + type: "spacer", + fullText: " ", + size: " ".length + }, { + type: "speed", + fullText: formattedSpeed, + size: Math.max("00.00kB/s".length, formattedSpeed.length) + }, + ...dimEta + ]); } protected override renderFinishedLine() { diff --git a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts index 9e64af8..a4e8052 100644 --- a/src/download/transfer-visualize/transfer-cli/transfer-cli.ts +++ b/src/download/transfer-visualize/transfer-cli/transfer-cli.ts @@ -2,13 +2,11 @@ import UpdateManager from "stdout-update"; import debounce from "lodash.debounce"; import {TransferCliProgressBar} from "./progress-bars/base-transfer-cli-progress-bar.js"; import cliSpinners from "cli-spinners"; -import CliSpinnersLoadingAnimation from "./loading-animation/cli-spinners-loading-animation.js"; import {FormattedStatus} from "../format-transfer-status.js"; import switchCliProgressStyle from "./progress-bars/switch-cli-progress-style.js"; import {BaseMultiProgressBar} from "./multiProgressBars/BaseMultiProgressBar.js"; export type TransferCliOptions = { - action?: string, name?: string, maxViewDownloads: number; truncateName: boolean | number; @@ -17,7 +15,6 @@ export type TransferCliOptions = { createProgressBar: TransferCliProgressBar; createMultiProgressBar: typeof BaseMultiProgressBar, loadingAnimation: cliSpinners.SpinnerName, - loadingText?: string; }; export const DEFAULT_TRANSFER_CLI_OPTIONS: TransferCliOptions = { @@ -27,30 +24,20 @@ export const DEFAULT_TRANSFER_CLI_OPTIONS: TransferCliOptions = { maxDebounceWait: process.platform === "win32" ? 500 : 100, createProgressBar: switchCliProgressStyle("auto", {truncateName: true}), loadingAnimation: "dots", - loadingText: "Gathering information", createMultiProgressBar: BaseMultiProgressBar }; -export enum CLI_LEVEL { - LOW = 0, - HIGH = 2 -} - - export default class TransferCli { - public static activeCLILevel = CLI_LEVEL.LOW; - public readonly loadingAnimation: CliSpinnersLoadingAnimation; protected options: TransferCliOptions; protected stdoutManager = UpdateManager.getInstance(); - protected myCLILevel: number; - protected latestProgress: [FormattedStatus[], FormattedStatus] = null!; + protected latestProgress: [FormattedStatus[], FormattedStatus, number] = null!; private _cliStopped = true; private readonly _updateStatuesDebounce: () => void; private _multiProgressBar: BaseMultiProgressBar; private _isFirstPrint = true; + private _lastProgressLong = ""; - public constructor(options: Partial, myCLILevel = CLI_LEVEL.LOW) { - TransferCli.activeCLILevel = this.myCLILevel = myCLILevel; + public constructor(options: Partial) { this.options = {...DEFAULT_TRANSFER_CLI_OPTIONS, ...options}; this._multiProgressBar = new this.options.createProgressBar.multiProgressBar(this.options); @@ -59,18 +46,11 @@ export default class TransferCli { maxWait: maxDebounceWait }); - this.loadingAnimation = new CliSpinnersLoadingAnimation(cliSpinners[this.options.loadingAnimation], { - loadingText: this.options.loadingText, - updateIntervalMs: this._multiProgressBar.updateIntervalMs, - logType: this._multiProgressBar.printType - }); this._processExit = this._processExit.bind(this); - - } start() { - if (this.myCLILevel !== TransferCli.activeCLILevel) return; + if (!this._cliStopped) return; this._cliStopped = false; if (this._multiProgressBar.printType === "update") { this.stdoutManager.hook(); @@ -79,9 +59,9 @@ export default class TransferCli { } stop() { - if (this._cliStopped || this.myCLILevel !== TransferCli.activeCLILevel) return; - this._updateStatues(); + if (this._cliStopped) return; this._cliStopped = true; + this._updateStatues(); if (this._multiProgressBar.printType === "update") { this.stdoutManager.unhook(false); } @@ -93,8 +73,8 @@ export default class TransferCli { process.exit(0); } - updateStatues(statues: FormattedStatus[], oneStatus: FormattedStatus) { - this.latestProgress = [statues, oneStatus]; + updateStatues(statues: FormattedStatus[], oneStatus: FormattedStatus, loadingDownloads = 0) { + this.latestProgress = [statues, oneStatus, loadingDownloads]; if (this._isFirstPrint) { this._isFirstPrint = false; @@ -105,12 +85,10 @@ export default class TransferCli { } private _updateStatues() { - if (this._cliStopped || this.myCLILevel !== TransferCli.activeCLILevel) { - return; // Do not update if there is a higher level CLI, meaning that this CLI is sub-CLI - } - + if (!this.latestProgress) return; const printLog = this._multiProgressBar.createMultiProgressBar(...this.latestProgress); - if (printLog) { + if (printLog && this._lastProgressLong != printLog) { + this._lastProgressLong = printLog; this._logUpdate(printLog); } } diff --git a/src/index.ts b/src/index.ts index dbc4183..b98a0c4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,5 @@ import DownloadEngineNodejs from "./download/download-engine/engine/download-engine-nodejs.js"; -import {downloadFile, DownloadFileOptions, downloadSequence, DownloadSequenceOptions} from "./download/node-download.js"; +import {downloadFile, DownloadFileOptions, downloadFileRemote, downloadSequence, DownloadSequenceOptions} from "./download/node-download.js"; import {SaveProgressInfo} from "./download/download-engine/types.js"; import PathNotAFileError from "./download/download-engine/streams/download-engine-fetch-stream/errors/path-not-a-file-error.js"; import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js"; @@ -10,17 +10,19 @@ import FetchStreamError from "./download/download-engine/streams/download-engine import IpullError from "./errors/ipull-error.js"; import EngineError from "./download/download-engine/engine/error/engine-error.js"; import {FormattedStatus} from "./download/transfer-visualize/format-transfer-status.js"; -import DownloadEngineMultiDownload from "./download/download-engine/engine/download-engine-multi-download.js"; +import DownloadEngineMultiDownload, {DownloadEngineMultiAllowedEngines} from "./download/download-engine/engine/download-engine-multi-download.js"; import HttpError from "./download/download-engine/streams/download-engine-fetch-stream/errors/http-error.js"; import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js"; import {InvalidOptionError} from "./download/download-engine/engine/error/InvalidOptionError.js"; import {BaseMultiProgressBar, MultiProgressBarOptions} from "./download/transfer-visualize/transfer-cli/multiProgressBars/BaseMultiProgressBar.js"; import {DownloadFlags, DownloadStatus} from "./download/download-engine/download-file/progress-status-file.js"; -import {NoDownloadEngineProvidedError} from "./download/download-engine/engine/error/no-download-engine-provided-error.js"; +import {DownloadEngineRemote} from "./download/download-engine/engine/DownloadEngineRemote.js"; +import {CliProgressDownloadEngineOptions} from "./download/transfer-visualize/transfer-cli/GlobalCLI.js"; export { DownloadFlags, DownloadStatus, + downloadFileRemote, downloadFile, downloadSequence, BaseMultiProgressBar, @@ -33,20 +35,22 @@ export { FetchStreamError, IpullError, EngineError, - InvalidOptionError, - NoDownloadEngineProvidedError + InvalidOptionError }; export type { BaseDownloadEngine, + DownloadEngineRemote, DownloadFileOptions, DownloadSequenceOptions, DownloadEngineNodejs, DownloadEngineMultiDownload, + DownloadEngineMultiAllowedEngines, SaveProgressInfo, FormattedStatus, - MultiProgressBarOptions + MultiProgressBarOptions, + CliProgressDownloadEngineOptions }; diff --git a/test/browser.test.ts b/test/browser.test.ts index 58208d7..251e4b5 100644 --- a/test/browser.test.ts +++ b/test/browser.test.ts @@ -11,7 +11,9 @@ globalThis.XMLHttpRequest = await import("xmlhttprequest-ssl").then(m => m.XMLHt describe("Browser Fetch API", () => { test.concurrent("Download file browser - memory", async (context) => { const downloader = await downloadFileBrowser({ - url: BIG_FILE + url: BIG_FILE, + parallelStreams: 2, + autoIncreaseParallelStreams: false }); await downloader.download(); @@ -21,26 +23,40 @@ describe("Browser Fetch API", () => { }); test.concurrent("Download file browser", async (context) => { - let buffer = Buffer.alloc(0); + const response = await ensureLocalFile(BIG_FILE, BIG_FILE_EXAMPLE); + const bufferIsCorrect = Buffer.from(await fs.readFile(response)); + + let bigBuffer = Buffer.alloc(0); let lastWrite = 0; const downloader = await downloadFileBrowser({ url: BIG_FILE, + parallelStreams: 2, + autoIncreaseParallelStreams: false, onWrite(cursor, data) { - buffer.set(data, cursor); - if (cursor + data.length > lastWrite) { - lastWrite = cursor + data.length; + let writeLocation = cursor; + for (const buffer of data) { + bigBuffer.set(buffer, writeLocation); + writeLocation += buffer.length; + } + + if (writeLocation > lastWrite) { + lastWrite = writeLocation; } } }); - buffer = Buffer.alloc(downloader.file.totalSize); - + bigBuffer = Buffer.alloc(downloader.file.totalSize); await downloader.download(); - context.expect(hashBuffer(buffer)) - .toMatchInlineSnapshot("\"9ae3ff19ee04fc02e9c60ce34e42858d16b46eeb88634d2035693c1ae9dbcbc9\""); + + const diff = bigBuffer.findIndex((value, index) => value !== bufferIsCorrect[index]); + context.expect(diff) + .toBe(-1); + context.expect(lastWrite) .toBe(downloader.file.totalSize); - }); + context.expect(hashBuffer(bigBuffer)) + .toMatchInlineSnapshot("\"9ae3ff19ee04fc02e9c60ce34e42858d16b46eeb88634d2035693c1ae9dbcbc9\""); + }, {repeats: 4, concurrent: true}); }, {timeout: 1000 * 60 * 3}); describe("Browser Fetch memory", () => { diff --git a/test/copy-file.test.ts b/test/copy-file.test.ts index 0115dda..b867404 100644 --- a/test/copy-file.test.ts +++ b/test/copy-file.test.ts @@ -17,7 +17,7 @@ describe("File Copy", async () => { fileName: copyFileToName, chunkSize: 4, parallelStreams: 1, - fetchStrategy: "localFile", + fetchStrategy: "local", cliProgress: false }); await engine.download(); @@ -37,7 +37,7 @@ describe("File Copy", async () => { url: fileToCopy, directory: ".", fileName: copyFileToName, - fetchStrategy: "localFile", + fetchStrategy: "local", cliProgress: false, parallelStreams: 1 }); diff --git a/test/daynamic-content-length.test.ts b/test/daynamic-content-length.test.ts index 3625c69..e821d38 100644 --- a/test/daynamic-content-length.test.ts +++ b/test/daynamic-content-length.test.ts @@ -15,10 +15,12 @@ describe("Dynamic content download", async () => { beforeAll(async () => { regularDownload = await ensureLocalFile(DYNAMIC_DOWNLOAD_FILE, ORIGINAL_FILE); originalFileHash = await fileHash(regularDownload); - }); + }, 1000 * 30); afterAll(async () => { - await fs.remove(regularDownload); + if (regularDownload) { + await fs.remove(regularDownload); + } }); @@ -26,7 +28,11 @@ describe("Dynamic content download", async () => { const downloader = await downloadFile({ url: DYNAMIC_DOWNLOAD_FILE, directory: ".", - fileName: IPUll_FILE + fileName: IPUll_FILE, + defaultFetchDownloadInfo: { + acceptRange: false, + length: 0 + } }); await downloader.download(); @@ -39,7 +45,11 @@ describe("Dynamic content download", async () => { test.concurrent("Browser Download", async (context) => { const downloader = await downloadFileBrowser({ - url: DYNAMIC_DOWNLOAD_FILE + url: DYNAMIC_DOWNLOAD_FILE, + defaultFetchDownloadInfo: { + acceptRange: false, + length: 0 + } }); await downloader.download(); diff --git a/test/download.test.ts b/test/download.test.ts index 2bd3b86..53099c3 100644 --- a/test/download.test.ts +++ b/test/download.test.ts @@ -23,6 +23,7 @@ describe("File Download", () => { const downloader = new DownloadEngineFile(file, { parallelStreams: randomNumber, chunkSize: 1024 ** 2, + autoIncreaseParallelStreams: false, fetchStream, writeStream }); @@ -46,7 +47,7 @@ describe("File Download", () => { let totalBytesWritten = 0; const fetchStream = new DownloadEngineFetchStreamFetch(); const writeStream = new DownloadEngineWriteStreamBrowser((cursor, data) => { - totalBytesWritten += data.byteLength; + totalBytesWritten += data.reduce((sum, buffer) => sum + buffer.length, 0); }); const file = await createDownloadFile(BIG_FILE); diff --git a/tsconfig.json b/tsconfig.json index 82085a0..d8117e1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,11 +1,11 @@ { "compilerOptions": { "lib": [ - "esNext", + "es2022", "DOM" ], - "module": "nodeNext", - "target": "esNext", + "module": "NodeNext", + "target": "es2022", "esModuleInterop": true, "noImplicitAny": true, "noImplicitReturns": true,