diff --git a/.github/workflows/blob-publish.yml b/.github/workflows/blob-publish.yml new file mode 100644 index 0000000000..a19e395ccb --- /dev/null +++ b/.github/workflows/blob-publish.yml @@ -0,0 +1,70 @@ +name: Blob - Version and Release + +on: + workflow_dispatch: + inputs: + newversion: + type: choice + description: "Semantic Version Bump Type" + default: patch + options: + - patch + - minor + - major + +concurrency: + group: "push-to-main" + +defaults: + run: + working-directory: packages/blob + +jobs: + version_and_release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + # Needed to push the tag and the commit on the main branch, otherwise we get: + # > Run git push --follow-tags + # remote: error: GH006: Protected branch update failed for refs/heads/main. + # remote: error: Changes must be made through a pull request. Required status check "lint" is expected. + token: ${{ secrets.BOT_ACCESS_TOKEN }} + - run: corepack enable + - uses: actions/setup-node@v3 + with: + node-version: "20" + cache: "pnpm" + cache-dependency-path: | + packages/blob/pnpm-lock.yaml + # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED + registry-url: "https://registry.npmjs.org" + - run: pnpm install + - run: git config --global user.name machineuser + - run: git config --global user.email infra+machineuser@huggingface.co + - run: | + PACKAGE_VERSION=$(node -p "require('./package.json').version") + BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')") + # Update package.json with the new version + node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" + git commit . -m "🔖 @huggingface/blob $BUMPED_VERSION" + git tag "blob-v$BUMPED_VERSION" + + - run: pnpm publish --no-git-checks . + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - run: git pull --rebase && git push --follow-tags + # hack - reuse actions/setup-node@v3 just to set a new registry + - uses: actions/setup-node@v3 + with: + node-version: "20" + registry-url: "https://npm.pkg.github.com" + # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558) + # - run: pnpm publish --no-git-checks . + # env: + # NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Update Doc" + uses: peter-evans/repository-dispatch@v2 + with: + event-type: doc-build + token: ${{ secrets.BOT_ACCESS_TOKEN }} diff --git a/.github/workflows/dduf-publish.yml b/.github/workflows/dduf-publish.yml new file mode 100644 index 0000000000..8d543d03e1 --- /dev/null +++ b/.github/workflows/dduf-publish.yml @@ -0,0 +1,73 @@ +name: DDUF - Version and Release + +on: + workflow_dispatch: + inputs: + newversion: + type: choice + description: "Semantic Version Bump Type" + default: patch + options: + - patch + - minor + - major + +concurrency: + group: "push-to-main" + +defaults: + run: + working-directory: packages/dduf + +jobs: + version_and_release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + # Needed to push the tag and the commit on the main branch, otherwise we get: + # > Run git push --follow-tags + # remote: error: GH006: Protected branch update failed for refs/heads/main. + # remote: error: Changes must be made through a pull request. Required status check "lint" is expected. + token: ${{ secrets.BOT_ACCESS_TOKEN }} + - run: corepack enable + - uses: actions/setup-node@v3 + with: + node-version: "20" + cache: "pnpm" + cache-dependency-path: | + packages/dduf/pnpm-lock.yaml + # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED + registry-url: "https://registry.npmjs.org" + - run: pnpm install + - run: git config --global user.name machineuser + - run: git config --global user.email infra+machineuser@huggingface.co + - run: | + PACKAGE_VERSION=$(node -p "require('./package.json').version") + BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')") + # Update package.json with the new version + node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" + git commit . -m "🔖 @huggingface/dduf $BUMPED_VERSION" + git tag "dduf-v$BUMPED_VERSION" + + - name: "Check Deps are published before publishing this package" + run: pnpm -w check-deps blob + + - run: pnpm publish --no-git-checks . + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - run: git pull --rebase && git push --follow-tags + # hack - reuse actions/setup-node@v3 just to set a new registry + - uses: actions/setup-node@v3 + with: + node-version: "20" + registry-url: "https://npm.pkg.github.com" + # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558) + # - run: pnpm publish --no-git-checks . + # env: + # NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Update Doc" + uses: peter-evans/repository-dispatch@v2 + with: + event-type: doc-build + token: ${{ secrets.BOT_ACCESS_TOKEN }} diff --git a/packages/blob/.eslintignore b/packages/blob/.eslintignore new file mode 100644 index 0000000000..67c4f0d9d6 --- /dev/null +++ b/packages/blob/.eslintignore @@ -0,0 +1,2 @@ +dist +sha256.js diff --git a/packages/blob/.prettierignore b/packages/blob/.prettierignore new file mode 100644 index 0000000000..4fafcf634d --- /dev/null +++ b/packages/blob/.prettierignore @@ -0,0 +1,5 @@ +pnpm-lock.yaml +# In order to avoid code samples to have tabs, they don't display well on npm +README.md +dist +sha256.js \ No newline at end of file diff --git a/packages/blob/LICENSE b/packages/blob/LICENSE new file mode 100644 index 0000000000..5e9693e35e --- /dev/null +++ b/packages/blob/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Hugging Face + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/blob/README.md b/packages/blob/README.md new file mode 100644 index 0000000000..84ebce19c8 --- /dev/null +++ b/packages/blob/README.md @@ -0,0 +1,90 @@ +# 🤗 Hugging Face Blobs + +Utilities to convert a string or URL to a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob) object, whether it represents a local file or a remote URL. + +`fetch` already returns a `Blob` object for remote URLs, but it loads the entire file in memory. This utility makes ad-hoc http range requests when calling `.slice()` on the blob. + +## Install + +```console +pnpm add @huggingface/blob + +npm add @huggingface/blob + +yarn add @huggingface/blob +``` + +### Deno + +```ts +// esm.sh +import { FileBlob } from "https://esm.sh/@huggingface/blob/FileBlob"; +import { WebBlob } from "https://esm.sh/@huggingface/blob/WebBlob"; +import { createBlob } from "https://esm.sh/@huggingface/blob"; +// or npm: +import { FileBlob } from "npm:@huggingface/blob/FileBlob"; +import { WebBlob } from "npm:@huggingface/blob/WebBlob"; +import { createBlob } from "npm:@huggingface/blob"; +``` + +## Usage + + +```ts +import { FileBlob } from "@huggingface/blob/FileBlob"; +import { WebBlob } from "@huggingface/blob/WebBlob"; +import { createBlob } from "@huggingface/blob"; + +const fileBlob = await FileBlob.create("path/to/file"); +const webBlob = await WebBlob.create("https://url/to/file"); + +const blob = await createBlob("..."); // Automatically detects if it's a file or web URL +``` + +## API + +### createBlob + +Creates a Blob object from a string or URL. Automatically detects if it's a file or web URL. + +```ts +await createBlob("...", { + /** + * Custom fetch function to use, in case it resolves to a Web Blob. + * + * Useful for adding headers, etc. + */ + fetch: ..., +}); + +### FileBlob + +```ts +await FileBlob.create("path/to/file"); +await FileBlob.create(new URL("file:///path/to/file")); +``` + +### WebBlob + +Creates a Blob object from a URL. If the file is less than 1MB (as indicated by the Content-Length header), by default it will be cached in memory in entirety upon blob creation. + +This class is useful for large files that do not need to be loaded all at once in memory, as it makes range requests for the data. + +```ts +await WebBlob.create("https://url/to/file"); +await WebBlob.create(new URL("https://url/to/file")); + +await WebBlob.create("https://url/to/file", { + /** + * Custom fetch function to use. Useful for adding headers, etc. + */ + fetch: ..., + /** + * If the file is less than the specified size, it will be cached in memory in entirety upon blob creation, + * instead of doing range requests for the data. + * + * @default 1_000_000 + */ + cacheBelow: ... +}) +``` \ No newline at end of file diff --git a/packages/blob/index.ts b/packages/blob/index.ts new file mode 100644 index 0000000000..e910bb060c --- /dev/null +++ b/packages/blob/index.ts @@ -0,0 +1 @@ +export * from "./src/index"; diff --git a/packages/blob/package.json b/packages/blob/package.json new file mode 100644 index 0000000000..26c989be28 --- /dev/null +++ b/packages/blob/package.json @@ -0,0 +1,64 @@ +{ + "name": "@huggingface/blob", + "packageManager": "pnpm@8.10.5", + "version": "0.0.1", + "description": "Utilities to convert URLs and files to Blobs, internally used by Hugging Face libs", + "repository": "https://github.com/huggingface/huggingface.js.git", + "publishConfig": { + "access": "public" + }, + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "require": "./dist/index.js", + "import": "./dist/index.mjs" + }, + "./package.json": "./package.json", + "./WebBlob": { + "require": "./dist/src/utils/WebBlob.js", + "import": "./dist/src/utils/WebBlob.mjs" + }, + "./FileBlob": { + "require": "./dist/src/utils/FileBlob.js", + "import": "./dist/src/utils/FileBlob.mjs" + } + }, + "browser": { + "./src/utils/FileBlob.ts": false, + "./dist/index.js": "./dist/browser/index.js", + "./dist/index.mjs": "./dist/browser/index.mjs" + }, + "source": "index.ts", + "scripts": { + "lint": "eslint --quiet --fix --ext .cjs,.ts .", + "lint:check": "eslint --ext .cjs,.ts .", + "format": "prettier --write .", + "format:check": "prettier --check .", + "prepublishOnly": "pnpm run build", + "build": "tsup && tsc --emitDeclarationOnly --declaration && cp dist/index.d.ts dist/index.m.d.ts && cp dist/src/utils/FileBlob.d.ts dist/src/utils/FileBlob.m.d.ts && cp dist/src/utils/WebBlob.d.ts dist/src/utils/WebBlob.m.d.ts", + "prepare": "pnpm run build", + "test": "vitest run", + "test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts", + "check": "tsc" + }, + "files": [ + "src", + "dist", + "index.ts", + "tsconfig.json" + ], + "keywords": [ + "huggingface", + "hugging", + "face", + "blob", + "lazy" + ], + "author": "Hugging Face", + "license": "MIT", + "devDependencies": { + "@types/node": "^20.11.28" + } +} diff --git a/packages/blob/pnpm-lock.yaml b/packages/blob/pnpm-lock.yaml new file mode 100644 index 0000000000..f8939bbbb9 --- /dev/null +++ b/packages/blob/pnpm-lock.yaml @@ -0,0 +1,22 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +devDependencies: + '@types/node': + specifier: ^20.11.28 + version: 20.11.28 + +packages: + + /@types/node@20.11.28: + resolution: {integrity: sha512-M/GPWVS2wLkSkNHVeLkrF2fD5Lx5UC4PxA0uZcKc6QqbIQUJyW1jVjueJYi1z8n0I5PxYrtpnPnWglE+y9A0KA==} + dependencies: + undici-types: 5.26.5 + dev: true + + /undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + dev: true diff --git a/packages/blob/src/index.ts b/packages/blob/src/index.ts new file mode 100644 index 0000000000..eadd56ab84 --- /dev/null +++ b/packages/blob/src/index.ts @@ -0,0 +1 @@ +export { createBlob } from "./utils/createBlob"; diff --git a/packages/blob/src/utils/FileBlob.spec.ts b/packages/blob/src/utils/FileBlob.spec.ts new file mode 100644 index 0000000000..2ed51d8e38 --- /dev/null +++ b/packages/blob/src/utils/FileBlob.spec.ts @@ -0,0 +1,45 @@ +import { open, stat } from "node:fs/promises"; +import { TextDecoder } from "node:util"; +import { describe, expect, it } from "vitest"; +import { FileBlob } from "./FileBlob"; + +describe("FileBlob", () => { + it("should create a FileBlob with a slice on the entire file", async () => { + const file = await open("package.json", "r"); + const { size } = await stat("package.json"); + + const fileBlob = await FileBlob.create("package.json"); + + expect(fileBlob).toMatchObject({ + path: "package.json", + start: 0, + end: size, + }); + expect(fileBlob.size).toBe(size); + expect(fileBlob.type).toBe(""); + const text = await fileBlob.text(); + const expectedText = (await file.read(Buffer.alloc(size), 0, size)).buffer.toString("utf8"); + expect(text).toBe(expectedText); + const result = await fileBlob.stream().getReader().read(); + expect(new TextDecoder().decode(result.value)).toBe(expectedText); + }); + + it("should create a slice on the file", async () => { + const file = await open("package.json", "r"); + const fileBlob = await FileBlob.create("package.json"); + + const slice = fileBlob.slice(10, 20); + + expect(slice).toMatchObject({ + path: "package.json", + start: 10, + end: 20, + }); + expect(slice.size).toBe(10); + const sliceText = await slice.text(); + const expectedText = (await file.read(Buffer.alloc(10), 0, 10, 10)).buffer.toString("utf8"); + expect(sliceText).toBe(expectedText); + const result = await slice.stream().getReader().read(); + expect(new TextDecoder().decode(result.value)).toBe(expectedText); + }); +}); diff --git a/packages/blob/src/utils/FileBlob.ts b/packages/blob/src/utils/FileBlob.ts new file mode 100644 index 0000000000..e783ca6fa6 --- /dev/null +++ b/packages/blob/src/utils/FileBlob.ts @@ -0,0 +1,118 @@ +import { createReadStream } from "node:fs"; +import { open, stat } from "node:fs/promises"; +import { Readable } from "node:stream"; +import type { FileHandle } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +/** + * @internal + * + * A FileBlob is a replacement for the Blob class that allows to lazy read files + * in order to preserve memory. + * + * It is a drop-in replacement for the Blob class, so you can use it as a Blob. + * + * The main difference is the instantiation, which is done asynchronously using the `FileBlob.create` method. + * + * @example + * const fileBlob = await FileBlob.create("path/to/package.json"); + * + * await fetch("https://aschen.tech", { method: "POST", body: fileBlob }); + */ +export class FileBlob extends Blob { + /** + * Creates a new FileBlob on the provided file. + * + * @param path Path to the file to be lazy readed + */ + static async create(path: string | URL): Promise { + path = path instanceof URL ? fileURLToPath(path) : path; + + const { size } = await stat(path); + + const fileBlob = new FileBlob(path, 0, size); + + return fileBlob; + } + + private path: string; + private start: number; + private end: number; + + private constructor(path: string, start: number, end: number) { + super(); + + this.path = path; + this.start = start; + this.end = end; + } + + /** + * Returns the size of the blob. + */ + override get size(): number { + return this.end - this.start; + } + + /** + * Returns a new instance of FileBlob that is a slice of the current one. + * + * The slice is inclusive of the start and exclusive of the end. + * + * The slice method does not supports negative start/end. + * + * @param start beginning of the slice + * @param end end of the slice + */ + override slice(start = 0, end = this.size): FileBlob { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on FileBlob.slice"); + } + + const slice = new FileBlob(this.path, this.start + start, Math.min(this.start + end, this.end)); + + return slice; + } + + /** + * Read the part of the file delimited by the FileBlob and returns it as an ArrayBuffer. + */ + override async arrayBuffer(): Promise { + const slice = await this.execute((file) => file.read(Buffer.alloc(this.size), 0, this.size, this.start)); + + return slice.buffer; + } + + /** + * Read the part of the file delimited by the FileBlob and returns it as a string. + */ + override async text(): Promise { + const buffer = (await this.arrayBuffer()) as Buffer; + + return buffer.toString("utf8"); + } + + /** + * Returns a stream around the part of the file delimited by the FileBlob. + */ + override stream(): ReturnType { + return Readable.toWeb(createReadStream(this.path, { start: this.start, end: this.end - 1 })) as ReturnType< + Blob["stream"] + >; + } + + /** + * We are opening and closing the file for each action to prevent file descriptor leaks. + * + * It is an intended choice of developer experience over performances. + */ + private async execute(action: (file: FileHandle) => Promise) { + const file = await open(this.path, "r"); + + try { + return await action(file); + } finally { + await file.close(); + } + } +} diff --git a/packages/blob/src/utils/WebBlob.spec.ts b/packages/blob/src/utils/WebBlob.spec.ts new file mode 100644 index 0000000000..919c8f0199 --- /dev/null +++ b/packages/blob/src/utils/WebBlob.spec.ts @@ -0,0 +1,83 @@ +import { describe, expect, it, beforeAll } from "vitest"; +import { WebBlob } from "./WebBlob"; + +describe("WebBlob", () => { + const resourceUrl = new URL("https://huggingface.co/spaces/aschen/push-model-from-web/raw/main/mobilenet/model.json"); + let fullText: string; + let size: number; + let contentType: string; + + beforeAll(async () => { + const response = await fetch(resourceUrl, { method: "HEAD" }); + size = Number(response.headers.get("content-length")); + contentType = response.headers.get("content-type") || ""; + fullText = await (await fetch(resourceUrl)).text(); + }); + + it("should create a WebBlob with a slice on the entire resource", async () => { + const webBlob = await WebBlob.create(resourceUrl, { cacheBelow: 0 }); + + expect(webBlob).toMatchObject({ + url: resourceUrl, + start: 0, + end: size, + contentType, + }); + expect(webBlob).toBeInstanceOf(WebBlob); + expect(webBlob.size).toBe(size); + expect(webBlob.type).toBe(contentType); + + const text = await webBlob.text(); + expect(text).toBe(fullText); + + const streamText = await new Response(webBlob.stream()).text(); + expect(streamText).toBe(fullText); + }); + + it("should create a WebBlob with a slice on the entire resource, cached", async () => { + const webBlob = await WebBlob.create(resourceUrl, { cacheBelow: 1_000_000 }); + + expect(webBlob).not.toBeInstanceOf(WebBlob); + expect(webBlob.size).toBe(size); + expect(webBlob.type.replace(/;\s*charset=utf-8/, "")).toBe(contentType.replace(/;\s*charset=utf-8/, "")); + + const text = await webBlob.text(); + expect(text).toBe(fullText); + + const streamText = await new Response(webBlob.stream()).text(); + expect(streamText).toBe(fullText); + }); + + it("should lazy load a LFS file hosted on Hugging Face", async () => { + const stableDiffusionUrl = + "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/resolve/main/unet/diffusion_pytorch_model.fp16.safetensors"; + const url = new URL(stableDiffusionUrl); + const webBlob = await WebBlob.create(url); + + expect(webBlob.size).toBe(5_135_149_760); + expect(webBlob).toBeInstanceOf(WebBlob); + expect(webBlob).toMatchObject({ url }); + expect(await webBlob.slice(10, 22).text()).toBe("__metadata__"); + }); + + it("should create a slice on the file", async () => { + const expectedText = fullText.slice(10, 20); + + const slice = (await WebBlob.create(resourceUrl, { cacheBelow: 0 })).slice(10, 20); + + expect(slice).toMatchObject({ + url: resourceUrl, + start: 10, + end: 20, + contentType, + }); + expect(slice.size).toBe(10); + expect(slice.type).toBe(contentType); + + const sliceText = await slice.text(); + expect(sliceText).toBe(expectedText); + + const streamText = await new Response(slice.stream()).text(); + expect(streamText).toBe(expectedText); + }); +}); diff --git a/packages/blob/src/utils/WebBlob.ts b/packages/blob/src/utils/WebBlob.ts new file mode 100644 index 0000000000..fe35813fe0 --- /dev/null +++ b/packages/blob/src/utils/WebBlob.ts @@ -0,0 +1,111 @@ +/** + * WebBlob is a Blob implementation for web resources that supports range requests. + */ + +interface WebBlobCreateOptions { + /** + * @default 1_000_000 + * + * Objects below that size will immediately be fetched and put in RAM, rather + * than streamed ad-hoc + */ + cacheBelow?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; +} + +export class WebBlob extends Blob { + static async create(url: URL, opts?: WebBlobCreateOptions): Promise { + const customFetch = opts?.fetch ?? fetch; + const response = await customFetch(url, { method: "HEAD" }); + + const size = Number(response.headers.get("content-length")); + const contentType = response.headers.get("content-type") || ""; + const supportRange = response.headers.get("accept-ranges") === "bytes"; + + if (!supportRange || size < (opts?.cacheBelow ?? 1_000_000)) { + return await (await customFetch(url)).blob(); + } + + return new WebBlob(url, 0, size, contentType, true, customFetch); + } + + private url: URL; + private start: number; + private end: number; + private contentType: string; + private full: boolean; + private fetch: typeof fetch; + + constructor(url: URL, start: number, end: number, contentType: string, full: boolean, customFetch: typeof fetch) { + super([]); + + this.url = url; + this.start = start; + this.end = end; + this.contentType = contentType; + this.full = full; + this.fetch = customFetch; + } + + override get size(): number { + return this.end - this.start; + } + + override get type(): string { + return this.contentType; + } + + override slice(start = 0, end = this.size): WebBlob { + if (start < 0 || end < 0) { + new TypeError("Unsupported negative start/end on FileBlob.slice"); + } + + const slice = new WebBlob( + this.url, + this.start + start, + Math.min(this.start + end, this.end), + this.contentType, + start === 0 && end === this.size ? this.full : false, + this.fetch + ); + + return slice; + } + + override async arrayBuffer(): Promise { + const result = await this.fetchRange(); + + return result.arrayBuffer(); + } + + override async text(): Promise { + const result = await this.fetchRange(); + + return result.text(); + } + + override stream(): ReturnType { + const stream = new TransformStream(); + + this.fetchRange() + .then((response) => response.body?.pipeThrough(stream)) + .catch((error) => stream.writable.abort(error.message)); + + return stream.readable; + } + + private fetchRange(): Promise { + const fetch = this.fetch; // to avoid this.fetch() which is bound to the instance instead of globalThis + if (this.full) { + return fetch(this.url); + } + return fetch(this.url, { + headers: { + Range: `bytes=${this.start}-${this.end - 1}`, + }, + }); + } +} diff --git a/packages/blob/src/utils/createBlob.ts b/packages/blob/src/utils/createBlob.ts new file mode 100644 index 0000000000..0cf54206da --- /dev/null +++ b/packages/blob/src/utils/createBlob.ts @@ -0,0 +1,30 @@ +import { WebBlob } from "./WebBlob"; +import { isFrontend } from "./isFrontend"; + +/** + * This function allow to retrieve either a FileBlob or a WebBlob from a URL. + * + * From the backend: + * - support local files + * - support http resources with absolute URLs + * + * From the frontend: + * - support http resources with absolute or relative URLs + */ +export async function createBlob(url: URL, opts?: { fetch?: typeof fetch }): Promise { + if (url.protocol === "http:" || url.protocol === "https:") { + return WebBlob.create(url, { fetch: opts?.fetch }); + } + + if (isFrontend) { + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); + } + + if (url.protocol === "file:") { + const { FileBlob } = await import("./FileBlob"); + + return FileBlob.create(url); + } + + throw new TypeError(`Unsupported URL protocol "${url.protocol}"`); +} diff --git a/packages/blob/src/utils/isBackend.ts b/packages/blob/src/utils/isBackend.ts new file mode 100644 index 0000000000..1e6f279986 --- /dev/null +++ b/packages/blob/src/utils/isBackend.ts @@ -0,0 +1,6 @@ +const isBrowser = typeof window !== "undefined" && typeof window.document !== "undefined"; + +const isWebWorker = + typeof self === "object" && self.constructor && self.constructor.name === "DedicatedWorkerGlobalScope"; + +export const isBackend = !isBrowser && !isWebWorker; diff --git a/packages/blob/src/utils/isFrontend.ts b/packages/blob/src/utils/isFrontend.ts new file mode 100644 index 0000000000..0b9bab392e --- /dev/null +++ b/packages/blob/src/utils/isFrontend.ts @@ -0,0 +1,3 @@ +import { isBackend } from "./isBackend"; + +export const isFrontend = !isBackend; diff --git a/packages/blob/tsconfig.json b/packages/blob/tsconfig.json new file mode 100644 index 0000000000..254606a30e --- /dev/null +++ b/packages/blob/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "lib": ["ES2022", "DOM"], + "module": "CommonJS", + "moduleResolution": "node", + "target": "ES2022", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "skipLibCheck": true, + "noImplicitOverride": true, + "outDir": "./dist", + "declaration": true, + "declarationMap": true + }, + "include": ["src", "index.ts"], + "exclude": ["dist"] +} diff --git a/packages/blob/tsup.config.ts b/packages/blob/tsup.config.ts new file mode 100644 index 0000000000..c08bf2a125 --- /dev/null +++ b/packages/blob/tsup.config.ts @@ -0,0 +1,25 @@ +import type { Options } from "tsup"; + +const baseConfig: Options = { + entry: ["./index.ts"], + format: ["cjs", "esm"], + outDir: "dist", + clean: true, +}; + +const nodeConfig: Options = { + ...baseConfig, + entry: ["./index.ts", "./src/utils/WebBlob.ts", "./src/utils/FileBlob.ts"], + platform: "node", +}; + +const browserConfig: Options = { + ...baseConfig, + entry: ["./index.ts", "./src/utils/WebBlob.ts"], + platform: "browser", + target: "es2018", + splitting: true, + outDir: "dist/browser", +}; + +export default [nodeConfig, browserConfig]; diff --git a/packages/blob/vitest-browser.config.mts b/packages/blob/vitest-browser.config.mts new file mode 100644 index 0000000000..65be77c7ac --- /dev/null +++ b/packages/blob/vitest-browser.config.mts @@ -0,0 +1,7 @@ +import { configDefaults, defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + exclude: [...configDefaults.exclude, "src/utils/FileBlob.spec.ts"], + }, +}); diff --git a/packages/dduf/.eslintignore b/packages/dduf/.eslintignore new file mode 100644 index 0000000000..67c4f0d9d6 --- /dev/null +++ b/packages/dduf/.eslintignore @@ -0,0 +1,2 @@ +dist +sha256.js diff --git a/packages/dduf/.prettierignore b/packages/dduf/.prettierignore new file mode 100644 index 0000000000..cac0c69496 --- /dev/null +++ b/packages/dduf/.prettierignore @@ -0,0 +1,4 @@ +pnpm-lock.yaml +# In order to avoid code samples to have tabs, they don't display well on npm +README.md +dist \ No newline at end of file diff --git a/packages/dduf/LICENSE b/packages/dduf/LICENSE new file mode 100644 index 0000000000..5e9693e35e --- /dev/null +++ b/packages/dduf/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Hugging Face + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/dduf/README.md b/packages/dduf/README.md new file mode 100644 index 0000000000..993333965c --- /dev/null +++ b/packages/dduf/README.md @@ -0,0 +1,33 @@ +# 🤗 Hugging Face DDUF + +Very alpha version of a DDUF checker / parser. + +## Install + +```console +pnpm add @huggingface/dduf + +npm add @huggingface/dduf + +yarn add @huggingface/dduf +``` + +### Deno + +```ts +// esm.sh +import { checkDDUF } from "https://esm.sh/@huggingface/dduf"; +// or npm: +import { checkDDUF } from "npm:@huggingface/dduf"; +``` + +## Usage + + +```ts +import { checkDDUF } from "@huggingface/dduf"; + +for await (const entry of checkDDUF(URL | Blob, { log: console.log })) { + console.log("file", entry); +} +``` diff --git a/packages/dduf/index.ts b/packages/dduf/index.ts new file mode 100644 index 0000000000..3bd16e178a --- /dev/null +++ b/packages/dduf/index.ts @@ -0,0 +1 @@ +export * from "./src"; diff --git a/packages/dduf/package.json b/packages/dduf/package.json new file mode 100644 index 0000000000..61b6330a85 --- /dev/null +++ b/packages/dduf/package.json @@ -0,0 +1,58 @@ +{ + "name": "@huggingface/dduf", + "packageManager": "pnpm@8.10.5", + "version": "0.0.1", + "description": "Very alpha lib to check DDUF compliance", + "repository": "https://github.com/huggingface/huggingface.js.git", + "publishConfig": { + "access": "public" + }, + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "require": "./dist/index.js", + "import": "./dist/index.mjs" + }, + "./package.json": "./package.json" + }, + "browser": { + "./dist/index.js": "./dist/browser/index.js", + "./dist/index.mjs": "./dist/browser/index.mjs" + }, + "source": "index.ts", + "scripts": { + "lint": "eslint --quiet --fix --ext .cjs,.ts .", + "lint:check": "eslint --ext .cjs,.ts .", + "format": "prettier --write .", + "format:check": "prettier --check .", + "prepublishOnly": "pnpm run build", + "build": "tsup && tsc --emitDeclarationOnly --declaration", + "prepare": "pnpm run build", + "test": "vitest run", + "test:browser": "vitest run --browser.name=chrome --browser.headless", + "check": "tsc" + }, + "files": [ + "src", + "dist", + "index.ts", + "tsconfig.json" + ], + "keywords": [ + "huggingface", + "hugging", + "face", + "dduf" + ], + "author": "Hugging Face", + "license": "MIT", + "devDependencies": { + "@types/node": "^20.11.28" + }, + "dependencies": { + "@huggingface/blob": "workspace:^" + } +} diff --git a/packages/dduf/pnpm-lock.yaml b/packages/dduf/pnpm-lock.yaml new file mode 100644 index 0000000000..0c7f7196a3 --- /dev/null +++ b/packages/dduf/pnpm-lock.yaml @@ -0,0 +1,27 @@ +lockfileVersion: '6.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +dependencies: + '@huggingface/blob': + specifier: workspace:^ + version: link:../blob + +devDependencies: + '@types/node': + specifier: ^20.11.28 + version: 20.17.9 + +packages: + + /@types/node@20.17.9: + resolution: {integrity: sha512-0JOXkRyLanfGPE2QRCwgxhzlBAvaRdCNMcvbd7jFfpmD4eEXll7LRwy5ymJmyeZqk7Nh7eD2LeUyQ68BbndmXw==} + dependencies: + undici-types: 6.19.8 + dev: true + + /undici-types@6.19.8: + resolution: {integrity: sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==} + dev: true diff --git a/packages/dduf/src/check-dduf.spec.ts b/packages/dduf/src/check-dduf.spec.ts new file mode 100644 index 0000000000..ad98b98234 --- /dev/null +++ b/packages/dduf/src/check-dduf.spec.ts @@ -0,0 +1,178 @@ +import { describe, expect, it } from "vitest"; +import { checkDDUF, type DDUFFileEntry } from "./check-dduf"; + +describe("check-dduf", () => { + it("should work", async () => { + const files: DDUFFileEntry[] = []; + for await (const file of checkDDUF( + new URL("https://huggingface.co/spaces/coyotte508/dduf-check/resolve/main/file-64.dduf") + )) { + files.push(file); + } + + expect(files).toEqual([ + { + fileHeaderOffset: 0, + name: "vae/", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 82, + name: "vae/config.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 178, + name: "vae/diffusion_pytorch_model.safetensors", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 295, + name: "text_encoder_2/", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 388, + name: "text_encoder_2/config.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 495, + name: "text_encoder_2/model-00002-of-00002.safetensors", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 620, + name: "text_encoder_2/models.saftensors.index.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 744, + name: "text_encoder_2/model-00001-of-00002.safetensors", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 869, + name: "transformer/", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 959, + name: "transformer/config.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 1063, + name: "transformer/diffusion_pytorch_model.safetensors", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 1188, + name: "tokenizer_2/", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 1278, + name: "tokenizer_2/vocab.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 1381, + name: "tokenizer_2/special_tokens_map.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 1497, + name: "tokenizer_2/tokenizer_config.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 1611, + name: "tokenizer_2/spiece.gguf", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 1712, + name: "tokenizer/", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 1800, + name: "tokenizer/vocab.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 1901, + name: "tokenizer/special_tokens_map.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 2015, + name: "tokenizer/tokenizer_config.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 2127, + name: "scheduler/", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 2215, + name: "scheduler/scheduler-config.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 2327, + name: "text_encoder/", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 2418, + name: "text_encoder/config.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 2523, + name: "text_encoder/model-00002-of-00002.safetensors", + size: 0, + type: "file", + }, + { + fileHeaderOffset: 2646, + name: "text_encoder/models.saftensors.index.json", + size: 3, + type: "file", + }, + { + fileHeaderOffset: 2768, + name: "text_encoder/model-00001-of-00002.safetensors", + size: 0, + type: "file", + }, + ]); + }); +}); diff --git a/packages/dduf/src/check-dduf.ts b/packages/dduf/src/check-dduf.ts new file mode 100644 index 0000000000..a4f17a9593 --- /dev/null +++ b/packages/dduf/src/check-dduf.ts @@ -0,0 +1,216 @@ +import { checkFilename } from "./check-filename"; +import { createBlob } from "@huggingface/blob"; + +export interface DDUFFileEntry { + type: "file"; + name: string; + size: number; + fileHeaderOffset: number; +} + +export async function* checkDDUF(url: Blob | URL, opts?: { log?: (x: string) => void }): AsyncGenerator { + const blob = url instanceof Blob ? url : await createBlob(url); + + opts?.log?.("File size: " + blob.size); + + // DDUF is a zip file, uncompressed. + + const last100kB = await blob.slice(blob.size - 100_000, blob.size).arrayBuffer(); + + const view = new DataView(last100kB); + + let index = view.byteLength - 22; + let found = false; + + while (index >= 0) { + if (view.getUint32(index, true) === 0x06054b50) { + found = true; + break; + } + + index--; + } + + if (!found) { + throw new Error("DDUF footer not found in last 100kB of file"); + } + + opts?.log?.("DDUF footer found at offset " + (blob.size - last100kB.byteLength + index)); + + const diskNumber = view.getUint16(index + 4, true); + + opts?.log?.("Disk number: " + diskNumber); + + if (diskNumber !== 0 && diskNumber !== 0xffff) { + throw new Error("Multi-disk archives not supported"); + } + + let fileCount = view.getUint16(index + 10, true); + let centralDirSize = view.getUint32(index + 12, true); + let centralDirOffset = view.getUint32(index + 16, true); + const isZip64 = centralDirOffset === 0xffffffff; + + opts?.log?.("File count: " + fileCount); + + if (isZip64) { + opts?.log?.("Zip64 format detected"); + + index -= 20; + found = false; + while (index >= 0) { + if (view.getUint32(index, true) === 0x07064b50) { + found = true; + break; + } + + index--; + } + + if (!found) { + throw new Error("Zip64 footer not found in last 100kB of file"); + } + + opts?.log?.("Zip64 footer found at offset " + (blob.size - last100kB.byteLength + index)); + + const diskWithCentralDir = view.getUint32(index + 4, true); + + if (diskWithCentralDir !== 0) { + throw new Error("Multi-disk archives not supported"); + } + + const endCentralDirOffset = Number(view.getBigUint64(index + 8, true)); + + index = endCentralDirOffset - (blob.size - last100kB.byteLength); + + if (index < 0) { + throw new Error("Central directory offset is outside the last 100kB of the file"); + } + + if (view.getUint32(index, true) !== 0x06064b50) { + throw new Error("Invalid central directory header"); + } + + const thisDisk = view.getUint16(index + 16, true); + const centralDirDisk = view.getUint16(index + 20, true); + + if (thisDisk !== 0) { + throw new Error("Multi-disk archives not supported"); + } + + if (centralDirDisk !== 0) { + throw new Error("Multi-disk archives not supported"); + } + + centralDirSize = Number(view.getBigUint64(index + 40, true)); + centralDirOffset = Number(view.getBigUint64(index + 48, true)); + fileCount = Number(view.getBigUint64(index + 32, true)); + + opts?.log?.("File count zip 64: " + fileCount); + } + + opts?.log?.("Central directory size: " + centralDirSize); + opts?.log?.("Central directory offset: " + centralDirOffset); + + const centralDir = + centralDirOffset > blob.size - last100kB.byteLength + ? last100kB.slice( + centralDirOffset - (blob.size - last100kB.byteLength), + centralDirOffset - (blob.size - last100kB.byteLength) + centralDirSize + ) + : await blob.slice(centralDirOffset, centralDirOffset + centralDirSize).arrayBuffer(); + + const centralDirView = new DataView(centralDir); + let offset = 0; + + for (let i = 0; i < fileCount; i++) { + if (centralDirView.getUint32(offset + 0, true) !== 0x02014b50) { + throw new Error("Invalid central directory file header"); + } + + if (offset + 46 > centralDir.byteLength) { + throw new Error("Unexpected end of central directory"); + } + + const compressionMethod = centralDirView.getUint16(offset + 10, true); + + if (compressionMethod !== 0) { + throw new Error("Unsupported compression method: " + compressionMethod); + } + + const filenameLength = centralDirView.getUint16(offset + 28, true); + const fileName = new TextDecoder().decode(new Uint8Array(centralDir, offset + 46, filenameLength)); + + opts?.log?.("File " + i); + opts?.log?.("File name: " + fileName); + + checkFilename(fileName); + + const fileDiskNumber = centralDirView.getUint16(34, true); + + if (fileDiskNumber !== 0 && fileDiskNumber !== 0xffff) { + throw new Error("Multi-disk archives not supported"); + } + + let size = centralDirView.getUint32(offset + 24, true); + let compressedSize = centralDirView.getUint32(offset + 20, true); + let filePosition = centralDirView.getUint32(offset + 42, true); + + const extraFieldLength = centralDirView.getUint16(offset + 30, true); + + if (size === 0xffffffff || compressedSize === 0xffffffff || filePosition === 0xffffffff) { + opts?.log?.("File size is in zip64 format"); + + const extraFields = new DataView(centralDir, offset + 46 + filenameLength, extraFieldLength); + + let extraFieldOffset = 0; + + while (extraFieldOffset < extraFieldLength) { + const headerId = extraFields.getUint16(extraFieldOffset, true); + const extraFieldSize = extraFields.getUint16(extraFieldOffset + 2, true); + if (headerId !== 0x0001) { + extraFieldOffset += 4 + extraFieldSize; + continue; + } + + const zip64ExtraField = new DataView( + centralDir, + offset + 46 + filenameLength + extraFieldOffset + 4, + extraFieldSize + ); + let zip64ExtraFieldOffset = 0; + + if (size === 0xffffffff) { + size = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true)); + zip64ExtraFieldOffset += 8; + } + + if (compressedSize === 0xffffffff) { + compressedSize = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true)); + zip64ExtraFieldOffset += 8; + } + + if (filePosition === 0xffffffff) { + filePosition = Number(zip64ExtraField.getBigUint64(zip64ExtraFieldOffset, true)); + zip64ExtraFieldOffset += 8; + } + + break; + } + } + + if (size !== compressedSize) { + throw new Error("Compressed size and size differ: " + compressedSize + " vs " + size); + } + opts?.log?.("File size: " + size); + + const commentLength = centralDirView.getUint16(offset + 32, true); + + opts?.log?.("File header position in archive: " + filePosition); + + offset += 46 + filenameLength + extraFieldLength + commentLength; + + yield { type: "file", name: fileName, size, fileHeaderOffset: filePosition }; + } + + opts?.log?.("All files checked"); +} diff --git a/packages/dduf/src/check-filename.ts b/packages/dduf/src/check-filename.ts new file mode 100644 index 0000000000..5eba2548c9 --- /dev/null +++ b/packages/dduf/src/check-filename.ts @@ -0,0 +1,17 @@ +export function checkFilename(filename: string): void { + if ( + !filename.endsWith(".safetensors") && + !filename.endsWith(".json") && + !filename.endsWith(".gguf") && + !filename.endsWith(".txt") && + !filename.endsWith("/") + ) { + throw new Error("Files must have a .safetensors, .txt, .gguf or .json extension"); + } + + const split = filename.split("/"); + + if (split.length > 2) { + throw new Error("Files must be only one level deep, not more"); + } +} diff --git a/packages/dduf/src/index.ts b/packages/dduf/src/index.ts new file mode 100644 index 0000000000..ab8badb8e4 --- /dev/null +++ b/packages/dduf/src/index.ts @@ -0,0 +1 @@ +export * from "./check-dduf"; diff --git a/packages/dduf/tsconfig.json b/packages/dduf/tsconfig.json new file mode 100644 index 0000000000..254606a30e --- /dev/null +++ b/packages/dduf/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "allowSyntheticDefaultImports": true, + "lib": ["ES2022", "DOM"], + "module": "CommonJS", + "moduleResolution": "node", + "target": "ES2022", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "skipLibCheck": true, + "noImplicitOverride": true, + "outDir": "./dist", + "declaration": true, + "declarationMap": true + }, + "include": ["src", "index.ts"], + "exclude": ["dist"] +} diff --git a/packages/dduf/tsup.config.ts b/packages/dduf/tsup.config.ts new file mode 100644 index 0000000000..8306dcbfcd --- /dev/null +++ b/packages/dduf/tsup.config.ts @@ -0,0 +1,25 @@ +import type { Options } from "tsup"; + +const baseConfig: Options = { + entry: ["./index.ts"], + format: ["cjs", "esm"], + outDir: "dist", + clean: true, +}; + +const nodeConfig: Options = { + ...baseConfig, + entry: ["./index.ts"], + platform: "node", +}; + +const browserConfig: Options = { + ...baseConfig, + entry: ["./index.ts"], + platform: "browser", + target: "es2018", + splitting: true, + outDir: "dist/browser", +}; + +export default [nodeConfig, browserConfig]; diff --git a/packages/hub/src/lib/cache-management.ts b/packages/hub/src/lib/cache-management.ts index 98c3be5b40..84b6640779 100644 --- a/packages/hub/src/lib/cache-management.ts +++ b/packages/hub/src/lib/cache-management.ts @@ -25,7 +25,7 @@ const FILES_TO_IGNORE: string[] = [".DS_Store"]; export const REPO_ID_SEPARATOR: string = "--"; export function getRepoFolderName({ name, type }: RepoId): string { - const parts = [`${type}s`, ...name.split("/")] + const parts = [`${type}s`, ...name.split("/")]; return parts.join(REPO_ID_SEPARATOR); } diff --git a/packages/hub/src/lib/dataset-info.spec.ts b/packages/hub/src/lib/dataset-info.spec.ts index 982bdcf40a..ae235e5e83 100644 --- a/packages/hub/src/lib/dataset-info.spec.ts +++ b/packages/hub/src/lib/dataset-info.spec.ts @@ -20,9 +20,9 @@ describe("datasetInfo", () => { }); it("should return the dataset info with author", async () => { - const info: DatasetEntry & Pick = await datasetInfo({ + const info: DatasetEntry & Pick = await datasetInfo({ name: "nyu-mll/glue", - additionalFields: ['author'], + additionalFields: ["author"], }); expect(info).toEqual({ id: "621ffdd236468d709f181e3f", @@ -32,12 +32,12 @@ describe("datasetInfo", () => { updatedAt: expect.any(Date), likes: expect.any(Number), private: false, - author: 'nyu-mll' + author: "nyu-mll", }); }); it("should return the dataset info for a specific revision", async () => { - const info: DatasetEntry & Pick = await datasetInfo({ + const info: DatasetEntry & Pick = await datasetInfo({ name: "nyu-mll/glue", revision: "cb2099c76426ff97da7aa591cbd317d91fb5fcb7", additionalFields: ["sha"], @@ -50,7 +50,7 @@ describe("datasetInfo", () => { updatedAt: expect.any(Date), likes: expect.any(Number), private: false, - sha: 'cb2099c76426ff97da7aa591cbd317d91fb5fcb7' + sha: "cb2099c76426ff97da7aa591cbd317d91fb5fcb7", }); }); }); diff --git a/packages/hub/src/lib/dataset-info.ts b/packages/hub/src/lib/dataset-info.ts index bb62df7f87..542b5aa0f4 100644 --- a/packages/hub/src/lib/dataset-info.ts +++ b/packages/hub/src/lib/dataset-info.ts @@ -31,7 +31,9 @@ export async function datasetInfo< ]).toString(); const response = await (params.fetch || fetch)( - `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent(params.revision ?? "HEAD")}?${search.toString()}`, + `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, { headers: { ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), diff --git a/packages/hub/src/lib/download-file-to-cache-dir.spec.ts b/packages/hub/src/lib/download-file-to-cache-dir.spec.ts index 05e2e6de9e..879a4d4069 100644 --- a/packages/hub/src/lib/download-file-to-cache-dir.spec.ts +++ b/packages/hub/src/lib/download-file-to-cache-dir.spec.ts @@ -8,46 +8,52 @@ import { getHFHubCachePath, getRepoFolderName } from "./cache-management"; import { toRepoId } from "../utils/toRepoId"; import { downloadFileToCacheDir } from "./download-file-to-cache-dir"; -vi.mock('node:fs/promises', () => ({ +vi.mock("node:fs/promises", () => ({ writeFile: vi.fn(), rename: vi.fn(), symlink: vi.fn(), lstat: vi.fn(), mkdir: vi.fn(), - stat: vi.fn() + stat: vi.fn(), })); -vi.mock('./paths-info', () => ({ +vi.mock("./paths-info", () => ({ pathsInfo: vi.fn(), })); const DUMMY_REPO: RepoId = { - name: 'hello-world', - type: 'model', + name: "hello-world", + type: "model", }; const DUMMY_ETAG = "dummy-etag"; // utility test method to get blob file path -function _getBlobFile(params: { +function _getBlobFile(params: { repo: RepoDesignation; etag: string; - cacheDir?: string, // default to {@link getHFHubCache} + cacheDir?: string; // default to {@link getHFHubCache} }) { return join(params.cacheDir ?? getHFHubCachePath(), getRepoFolderName(toRepoId(params.repo)), "blobs", params.etag); } // utility test method to get snapshot file path -function _getSnapshotFile(params: { +function _getSnapshotFile(params: { repo: RepoDesignation; path: string; - revision : string; - cacheDir?: string, // default to {@link getHFHubCache} + revision: string; + cacheDir?: string; // default to {@link getHFHubCache} }) { - return join(params.cacheDir ?? getHFHubCachePath(), getRepoFolderName(toRepoId(params.repo)), "snapshots", params.revision, params.path); + return join( + params.cacheDir ?? getHFHubCachePath(), + getRepoFolderName(toRepoId(params.repo)), + "snapshots", + params.revision, + params.path + ); } -describe('downloadFileToCacheDir', () => { +describe("downloadFileToCacheDir", () => { const fetchMock: typeof fetch = vi.fn(); beforeEach(() => { vi.resetAllMocks(); @@ -55,43 +61,45 @@ describe('downloadFileToCacheDir', () => { vi.mocked(fetchMock).mockResolvedValue({ status: 200, ok: true, - body: 'dummy-body' + body: "dummy-body", } as unknown as Response); // prevent to use caching - vi.mocked(stat).mockRejectedValue(new Error('Do not exists')); - vi.mocked(lstat).mockRejectedValue(new Error('Do not exists')); + vi.mocked(stat).mockRejectedValue(new Error("Do not exists")); + vi.mocked(lstat).mockRejectedValue(new Error("Do not exists")); }); - test('should throw an error if fileDownloadInfo return nothing', async () => { + test("should throw an error if fileDownloadInfo return nothing", async () => { await expect(async () => { await downloadFileToCacheDir({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", fetch: fetchMock, }); - }).rejects.toThrowError('cannot get path info for /README.md'); + }).rejects.toThrowError("cannot get path info for /README.md"); - expect(pathsInfo).toHaveBeenCalledWith(expect.objectContaining({ - repo: DUMMY_REPO, - paths: ['/README.md'], - fetch: fetchMock, - })); + expect(pathsInfo).toHaveBeenCalledWith( + expect.objectContaining({ + repo: DUMMY_REPO, + paths: ["/README.md"], + fetch: fetchMock, + }) + ); }); - test('existing symlinked and blob should not re-download it', async () => { + test("existing symlinked and blob should not re-download it", async () => { // ///snapshots/README.md const expectPointer = _getSnapshotFile({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", }); // stat ensure a symlink and the pointed file exists - vi.mocked(stat).mockResolvedValue({} as Stats) // prevent default mocked reject + vi.mocked(stat).mockResolvedValue({} as Stats); // prevent default mocked reject const output = await downloadFileToCacheDir({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", fetch: fetchMock, revision: "dd4bc8b21efa05ec961e3efc4ee5e3832a3679c7", }); @@ -100,17 +108,17 @@ describe('downloadFileToCacheDir', () => { // Get call argument for stat const starArg = vi.mocked(stat).mock.calls[0][0]; - expect(starArg).toBe(expectPointer) + expect(starArg).toBe(expectPointer); expect(fetchMock).not.toHaveBeenCalledWith(); expect(output).toBe(expectPointer); }); - test('existing blob should only create the symlink', async () => { + test("existing blob should only create the symlink", async () => { // ///snapshots/README.md const expectPointer = _getSnapshotFile({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", revision: "dummy-commit-hash", }); // //blobs/ @@ -122,21 +130,23 @@ describe('downloadFileToCacheDir', () => { // mock existing blob only no symlink vi.mocked(lstat).mockResolvedValue({} as Stats); // mock pathsInfo resolve content - vi.mocked(pathsInfo).mockResolvedValue([{ - oid: DUMMY_ETAG, - size: 55, - path: 'README.md', - type: 'file', - lastCommit: { - date: new Date(), - id: 'dummy-commit-hash', - title: 'Commit msg', + vi.mocked(pathsInfo).mockResolvedValue([ + { + oid: DUMMY_ETAG, + size: 55, + path: "README.md", + type: "file", + lastCommit: { + date: new Date(), + id: "dummy-commit-hash", + title: "Commit msg", + }, }, - }]); + ]); const output = await downloadFileToCacheDir({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", fetch: fetchMock, }); @@ -153,11 +163,11 @@ describe('downloadFileToCacheDir', () => { expect(output).toBe(expectPointer); }); - test('expect resolve value to be the pointer path of downloaded file', async () => { + test("expect resolve value to be the pointer path of downloaded file", async () => { // ///snapshots/README.md const expectPointer = _getSnapshotFile({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", revision: "dummy-commit-hash", }); // //blobs/ @@ -166,21 +176,23 @@ describe('downloadFileToCacheDir', () => { etag: DUMMY_ETAG, }); - vi.mocked(pathsInfo).mockResolvedValue([{ - oid: DUMMY_ETAG, - size: 55, - path: 'README.md', - type: 'file', - lastCommit: { - date: new Date(), - id: 'dummy-commit-hash', - title: 'Commit msg', + vi.mocked(pathsInfo).mockResolvedValue([ + { + oid: DUMMY_ETAG, + size: 55, + path: "README.md", + type: "file", + lastCommit: { + date: new Date(), + id: "dummy-commit-hash", + title: "Commit msg", + }, }, - }]); + ]); const output = await downloadFileToCacheDir({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", fetch: fetchMock, }); @@ -191,11 +203,11 @@ describe('downloadFileToCacheDir', () => { expect(output).toBe(expectPointer); }); - test('should write fetch response to blob', async () => { + test("should write fetch response to blob", async () => { // ///snapshots/README.md const expectPointer = _getSnapshotFile({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", revision: "dummy-commit-hash", }); // //blobs/ @@ -205,30 +217,32 @@ describe('downloadFileToCacheDir', () => { }); // mock pathsInfo resolve content - vi.mocked(pathsInfo).mockResolvedValue([{ - oid: DUMMY_ETAG, - size: 55, - path: 'README.md', - type: 'file', - lastCommit: { - date: new Date(), - id: 'dummy-commit-hash', - title: 'Commit msg', + vi.mocked(pathsInfo).mockResolvedValue([ + { + oid: DUMMY_ETAG, + size: 55, + path: "README.md", + type: "file", + lastCommit: { + date: new Date(), + id: "dummy-commit-hash", + title: "Commit msg", + }, }, - }]); + ]); await downloadFileToCacheDir({ repo: DUMMY_REPO, - path: '/README.md', + path: "/README.md", fetch: fetchMock, }); const incomplete = `${expectedBlob}.incomplete`; // 1. should write fetch#response#body to incomplete file - expect(writeFile).toHaveBeenCalledWith(incomplete, 'dummy-body'); + expect(writeFile).toHaveBeenCalledWith(incomplete, "dummy-body"); // 2. should rename the incomplete to the blob expected name expect(rename).toHaveBeenCalledWith(incomplete, expectedBlob); // 3. should create symlink pointing to blob expect(symlink).toHaveBeenCalledWith(expectedBlob, expectPointer); }); -}); \ No newline at end of file +}); diff --git a/packages/hub/src/lib/download-file-to-cache-dir.ts b/packages/hub/src/lib/download-file-to-cache-dir.ts index 72869f3075..92c4edbf0d 100644 --- a/packages/hub/src/lib/download-file-to-cache-dir.ts +++ b/packages/hub/src/lib/download-file-to-cache-dir.ts @@ -21,7 +21,7 @@ function getFilePointer(storageFolder: string, revision: string, relativeFilenam */ async function exists(path: string, followSymlinks?: boolean): Promise { try { - if(followSymlinks) { + if (followSymlinks) { await stat(path); } else { await lstat(path); @@ -54,7 +54,7 @@ export async function downloadFileToCacheDir( */ revision?: string; hubUrl?: string; - cacheDir?: string, + cacheDir?: string; /** * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. */ diff --git a/packages/hub/src/lib/download-file.spec.ts b/packages/hub/src/lib/download-file.spec.ts index f442f152a1..01fc64c945 100644 --- a/packages/hub/src/lib/download-file.spec.ts +++ b/packages/hub/src/lib/download-file.spec.ts @@ -3,8 +3,8 @@ import { downloadFile } from "./download-file"; import type { RepoId } from "../types/public"; const DUMMY_REPO: RepoId = { - name: 'hello-world', - type: 'model', + name: "hello-world", + type: "model", }; describe("downloadFile", () => { @@ -17,12 +17,12 @@ describe("downloadFile", () => { await downloadFile({ repo: DUMMY_REPO, - path: '/README.md', - hubUrl: 'http://dummy-hub', + path: "/README.md", + hubUrl: "http://dummy-hub", fetch: fetchMock, }); - expect(fetchMock).toHaveBeenCalledWith('http://dummy-hub/hello-world/resolve/main//README.md', expect.anything()); + expect(fetchMock).toHaveBeenCalledWith("http://dummy-hub/hello-world/resolve/main//README.md", expect.anything()); }); test("raw params should use raw url", async () => { @@ -34,12 +34,12 @@ describe("downloadFile", () => { await downloadFile({ repo: DUMMY_REPO, - path: 'README.md', + path: "README.md", raw: true, fetch: fetchMock, }); - expect(fetchMock).toHaveBeenCalledWith('https://huggingface.co/hello-world/raw/main/README.md', expect.anything()); + expect(fetchMock).toHaveBeenCalledWith("https://huggingface.co/hello-world/raw/main/README.md", expect.anything()); }); test("internal server error should propagate the error", async () => { @@ -49,17 +49,17 @@ describe("downloadFile", () => { ok: false, headers: new Map([["Content-Type", "application/json"]]), json: () => ({ - error: 'Dummy internal error', + error: "Dummy internal error", }), } as unknown as Response); await expect(async () => { await downloadFile({ repo: DUMMY_REPO, - path: 'README.md', + path: "README.md", raw: true, fetch: fetchMock, }); - }).rejects.toThrowError('Dummy internal error'); + }).rejects.toThrowError("Dummy internal error"); }); -}); \ No newline at end of file +}); diff --git a/packages/hub/src/lib/model-info.spec.ts b/packages/hub/src/lib/model-info.spec.ts index 3886f30391..3657e964d7 100644 --- a/packages/hub/src/lib/model-info.spec.ts +++ b/packages/hub/src/lib/model-info.spec.ts @@ -21,7 +21,7 @@ describe("modelInfo", () => { }); it("should return the model info with author", async () => { - const info: ModelEntry & Pick = await modelInfo({ + const info: ModelEntry & Pick = await modelInfo({ name: "openai-community/gpt2", additionalFields: ["author"], }); @@ -39,10 +39,10 @@ describe("modelInfo", () => { }); it("should return the model info for a specific revision", async () => { - const info: ModelEntry & Pick = await modelInfo({ + const info: ModelEntry & Pick = await modelInfo({ name: "openai-community/gpt2", additionalFields: ["sha"], - revision: 'f27b190eeac4c2302d24068eabf5e9d6044389ae', + revision: "f27b190eeac4c2302d24068eabf5e9d6044389ae", }); expect(info).toEqual({ id: "621ffdc036468d709f17434d", diff --git a/packages/hub/src/lib/model-info.ts b/packages/hub/src/lib/model-info.ts index 828322e673..4e4291c3b4 100644 --- a/packages/hub/src/lib/model-info.ts +++ b/packages/hub/src/lib/model-info.ts @@ -31,7 +31,9 @@ export async function modelInfo< ]).toString(); const response = await (params.fetch || fetch)( - `${params?.hubUrl || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent(params.revision ?? "HEAD")}?${search.toString()}`, + `${params?.hubUrl || HUB_URL}/api/models/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, { headers: { ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), diff --git a/packages/hub/src/lib/paths-info.spec.ts b/packages/hub/src/lib/paths-info.spec.ts index 994d623aef..837f4a1924 100644 --- a/packages/hub/src/lib/paths-info.spec.ts +++ b/packages/hub/src/lib/paths-info.spec.ts @@ -16,8 +16,8 @@ describe("pathsInfo", () => { expect(result).toHaveLength(1); const modelPathInfo = result[0]; - expect(modelPathInfo.path).toBe('tf_model.h5'); - expect(modelPathInfo.type).toBe('file'); + expect(modelPathInfo.path).toBe("tf_model.h5"); + expect(modelPathInfo.type).toBe("file"); // lfs pointer, therefore lfs should be defined expect(modelPathInfo?.lfs).toBeDefined(); expect(modelPathInfo?.lfs?.oid).toBe("a7a17d6d844b5de815ccab5f42cad6d24496db3850a2a43d8258221018ce87d2"); @@ -31,8 +31,8 @@ describe("pathsInfo", () => { it("expand parmas should fetch lastCommit and securityFileStatus", async () => { const result: (PathInfo & { - lastCommit: CommitInfo, - securityFileStatus: SecurityFileStatus, + lastCommit: CommitInfo; + securityFileStatus: SecurityFileStatus; })[] = await pathsInfo({ repo: { name: "bert-base-uncased", @@ -57,7 +57,7 @@ describe("pathsInfo", () => { }); it("non-LFS pointer should have lfs undefined", async () => { - const result: (PathInfo)[] = await pathsInfo({ + const result: PathInfo[] = await pathsInfo({ repo: { name: "bert-base-uncased", type: "model", diff --git a/packages/hub/src/lib/paths-info.ts b/packages/hub/src/lib/paths-info.ts index 4c9a1de20f..c706768f18 100644 --- a/packages/hub/src/lib/paths-info.ts +++ b/packages/hub/src/lib/paths-info.ts @@ -5,32 +5,32 @@ import { HUB_URL } from "../consts"; import { createApiError } from "../error"; export interface LfsPathInfo { - "oid": string, - "size": number, - "pointerSize": number + oid: string; + size: number; + pointerSize: number; } -export interface CommitInfo { - "id": string, - "title": string, - "date": Date, +export interface CommitInfo { + id: string; + title: string; + date: Date; } export interface SecurityFileStatus { - "status": string, + status: string; } export interface PathInfo { - path: string, - type: string, - oid: string, - size: number, + path: string; + type: string; + oid: string; + size: number; /** * Only defined when path is LFS pointer */ - lfs?: LfsPathInfo, - lastCommit?: CommitInfo, - securityFileStatus?: SecurityFileStatus + lfs?: LfsPathInfo; + lastCommit?: CommitInfo; + securityFileStatus?: SecurityFileStatus; } // Define the overloaded signatures @@ -45,8 +45,8 @@ export function pathsInfo( * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. */ fetch?: typeof fetch; - } & Partial -): Promise<(PathInfo & {lastCommit: CommitInfo, securityFileStatus: SecurityFileStatus })[]>; + } & Partial +): Promise<(PathInfo & { lastCommit: CommitInfo; securityFileStatus: SecurityFileStatus })[]>; export function pathsInfo( params: { repo: RepoDesignation; @@ -58,8 +58,8 @@ export function pathsInfo( * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. */ fetch?: typeof fetch; - } & Partial -): Promise<(PathInfo)[]>; + } & Partial +): Promise; export async function pathsInfo( params: { @@ -72,14 +72,16 @@ export async function pathsInfo( * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. */ fetch?: typeof fetch; - } & Partial + } & Partial ): Promise { const accessToken = checkCredentials(params); const repoId = toRepoId(params.repo); const hubUrl = params.hubUrl ?? HUB_URL; - const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info/${encodeURIComponent(params.revision ?? "main")}`; + const url = `${hubUrl}/api/${repoId.type}s/${repoId.name}/paths-info/${encodeURIComponent( + params.revision ?? "main" + )}`; const resp = await (params.fetch ?? fetch)(url, { method: "POST", @@ -87,8 +89,8 @@ export async function pathsInfo( ...(params.credentials && { Authorization: `Bearer ${accessToken}`, }), - 'Accept': 'application/json', - 'Content-Type': 'application/json' + Accept: "application/json", + "Content-Type": "application/json", }, body: JSON.stringify({ paths: params.paths, @@ -101,7 +103,7 @@ export async function pathsInfo( } const json: unknown = await resp.json(); - if(!Array.isArray(json)) throw new Error('malformed response: expected array'); + if (!Array.isArray(json)) throw new Error("malformed response: expected array"); return json.map((item: PathInfo) => ({ path: item.path, @@ -111,10 +113,12 @@ export async function pathsInfo( size: item.size, // expand fields securityFileStatus: item.securityFileStatus, - lastCommit: item.lastCommit ? { - date: new Date(item.lastCommit.date), - title: item.lastCommit.title, - id: item.lastCommit.id, - }: undefined, + lastCommit: item.lastCommit + ? { + date: new Date(item.lastCommit.date), + title: item.lastCommit.title, + id: item.lastCommit.id, + } + : undefined, })); } diff --git a/packages/hub/src/lib/space-info.spec.ts b/packages/hub/src/lib/space-info.spec.ts index aafa9b88f4..ea966f98bc 100644 --- a/packages/hub/src/lib/space-info.spec.ts +++ b/packages/hub/src/lib/space-info.spec.ts @@ -19,9 +19,9 @@ describe("spaceInfo", () => { }); it("should return the space info with author", async () => { - const info: SpaceEntry & Pick = await spaceInfo({ + const info: SpaceEntry & Pick = await spaceInfo({ name: "huggingfacejs/client-side-oauth", - additionalFields: ['author'], + additionalFields: ["author"], }); expect(info).toEqual({ id: "659835e689010f9c7aed608d", @@ -30,15 +30,15 @@ describe("spaceInfo", () => { likes: expect.any(Number), private: false, sdk: "static", - author: 'huggingfacejs', + author: "huggingfacejs", }); }); it("should return the space info for a given revision", async () => { - const info: SpaceEntry & Pick = await spaceInfo({ + const info: SpaceEntry & Pick = await spaceInfo({ name: "huggingfacejs/client-side-oauth", - additionalFields: ['sha'], - revision: 'e410a9ff348e6bed393b847711e793282d7c672e' + additionalFields: ["sha"], + revision: "e410a9ff348e6bed393b847711e793282d7c672e", }); expect(info).toEqual({ id: "659835e689010f9c7aed608d", @@ -47,7 +47,7 @@ describe("spaceInfo", () => { likes: expect.any(Number), private: false, sdk: "static", - sha: 'e410a9ff348e6bed393b847711e793282d7c672e', + sha: "e410a9ff348e6bed393b847711e793282d7c672e", }); }); }); diff --git a/packages/hub/src/lib/space-info.ts b/packages/hub/src/lib/space-info.ts index fcbfee60dd..9422353825 100644 --- a/packages/hub/src/lib/space-info.ts +++ b/packages/hub/src/lib/space-info.ts @@ -32,7 +32,9 @@ export async function spaceInfo< ]).toString(); const response = await (params.fetch || fetch)( - `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent(params.revision ?? "HEAD")}?${search.toString()}`, + `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}/revision/${encodeURIComponent( + params.revision ?? "HEAD" + )}?${search.toString()}`, { headers: { ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), diff --git a/packages/tasks-gen/.prettierignore b/packages/tasks-gen/.prettierignore new file mode 100644 index 0000000000..cdbeac2bd0 --- /dev/null +++ b/packages/tasks-gen/.prettierignore @@ -0,0 +1,5 @@ +dist +snippets-fixtures +node_modules +README.md +pnpm-lock.yaml \ No newline at end of file diff --git a/packages/tasks-gen/README.md b/packages/tasks-gen/README.md index d78e62450c..6adbe32110 100644 --- a/packages/tasks-gen/README.md +++ b/packages/tasks-gen/README.md @@ -14,6 +14,7 @@ pnpm generate-snippets-fixtures ``` If some logic has been updated, you should see the result with a + ``` git diff # the diff has to be committed if correct @@ -64,4 +65,3 @@ To update the specs manually, run: ``` pnpm inference-tgi-import ``` - diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index bd21daac7a..0d2e7bba3e 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,6 @@ packages: + - "packages/blob" + - "packages/dduf" - "packages/hub" - "packages/inference" - "packages/doc-internal"