@huggingface/blob and @huggingface/dduf packages

.github/workflows/blob-publish.yml

@@ -0,0 +1,70 @@
+name: Blob - Version and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      newversion:
+        type: choice
+        description: "Semantic Version Bump Type"
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+concurrency:
+  group: "push-to-main"
+
+defaults:
+  run:
+    working-directory: packages/blob
+
+jobs:
+  version_and_release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Needed to push the tag and the commit on the main branch, otherwise we get:
+          # > Run git push --follow-tags
+          # remote: error: GH006: Protected branch update failed for refs/heads/main.
+          # remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - run: corepack enable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: |
+            packages/blob/pnpm-lock.yaml
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
+          registry-url: "https://registry.npmjs.org"
+      - run: pnpm install
+      - run: git config --global user.name machineuser
+      - run: git config --global user.email [email protected]
+      - run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
+          # Update package.json with the new version
+          node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
+          git commit . -m "🔖 @huggingface/blob $BUMPED_VERSION"
+          git tag "blob-v$BUMPED_VERSION"
+
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - run: git pull --rebase && git push --follow-tags
+      # hack - reuse actions/setup-node@v3 just to set a new registry
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          registry-url: "https://npm.pkg.github.com"
+      # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
+      # - run: pnpm publish --no-git-checks .
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Update Doc"
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          event-type: doc-build
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}

.github/workflows/dduf-publish.yml

@@ -0,0 +1,74 @@
+name: DDUF - Version and Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      newversion:
+        type: choice
+        description: "Semantic Version Bump Type"
+        default: patch
+        options:
+          - patch
+          - minor
+          - major
+
+concurrency:
+  group: "push-to-main"
+
+defaults:
+  run:
+    working-directory: packages/dduf
+
+jobs:
+  version_and_release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          # Needed to push the tag and the commit on the main branch, otherwise we get:
+          # > Run git push --follow-tags
+          # remote: error: GH006: Protected branch update failed for refs/heads/main.
+          # remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}
+      - run: corepack enable
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          cache: "pnpm"
+          cache-dependency-path: |
+            packages/dduf/pnpm-lock.yaml
+          # setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token.  REQUIRED
+          registry-url: "https://registry.npmjs.org"
+      - run: pnpm install
+      - run: git config --global user.name machineuser
+      - run: git config --global user.email [email protected]
+      - run: |
+          PACKAGE_VERSION=$(node -p "require('./package.json').version")
+          BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
+          # Update package.json with the new version
+          node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
+          git commit . -m "🔖 @huggingface/dduf $BUMPED_VERSION"
+          git tag "dduf-v$BUMPED_VERSION"
+
+
+      - name: "Check Deps are published before publishing this package"
+        run: pnpm -w check-deps blob
+
+      - run: pnpm publish --no-git-checks .
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
+      - run: git pull --rebase && git push --follow-tags
+      # hack - reuse actions/setup-node@v3 just to set a new registry
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "20"
+          registry-url: "https://npm.pkg.github.com"
+      # Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
+      # - run: pnpm publish --no-git-checks .
+      #   env:
+      #     NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: "Update Doc"
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          event-type: doc-build
+          token: ${{ secrets.BOT_ACCESS_TOKEN }}

packages/blob/.eslintignore

@@ -0,0 +1,2 @@
+dist
+sha256.js

packages/blob/.prettierignore

@@ -0,0 +1,5 @@
+pnpm-lock.yaml
+# In order to avoid code samples to have tabs, they don't display well on npm
+README.md
+dist
+sha256.js

packages/blob/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Hugging Face
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/blob/README.md

@@ -0,0 +1,90 @@
+# 🤗 Hugging Face Blobs
+
+Utilities to convert a string or URL to a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob) object, whether it represents a local file or remote URL.
+
+`fetch` already returns a `Blob` object for remote URLs, but it loads the entire file in memory. This utility makes ad-hoc http range requests when calling `.slice()` on the blob, for example.
+
+## Install
+
+```console
+pnpm add @huggingface/blob
+
+npm add @huggingface/blob
+
+yarn add @huggingface/blob
+```
+
+### Deno
+
+```ts
+// esm.sh
+import { FileBlob } from "https://esm.sh/@huggingface/blob/FileBlob";
+import { WebBlob } from "https://esm.sh/@huggingface/blob/WebBlob";
+import { createBlob } from "https://esm.sh/@huggingface/blob";
+// or npm:
+import { FileBlob } from "npm:@huggingface/blob/FileBlob";
+import { WebBlob } from "npm:@huggingface/blob/WebBlob";
+import { createBlob } from "npm:@huggingface/blob";
+```
+
+## Usage
+
+
+```ts
+import { FileBlob } from "@huggingface/blob/FileBlob";
+import { WebBlob } from "@huggingface/blob/WebBlob";
+import { createBlob } from "@huggingface/blob";
+
+const fileBlob = await FileBlob.create("path/to/file");
+const webBlob = await WebBlob.create("https://url/to/file");
+
+const blob = await createBlob("..."); // Automatically detects if it's a file or web URL
+```
+
+## API
+
+### createBlob
+
+Creates a Blob object from a string or URL. Automatically detects if it's a file or web URL.
+
+```ts
+await createBlob("...", {
+  /**
+   * Custom fetch function to use, in case it resolves to a Web Blob.
+   * 
+   * Useful for adding headers, etc.
+   */
+  fetch: ...,
+});
+
+### FileBlob
+
+```ts
+await FileBlob.create("path/to/file");
+await FileBlob.create(new URL("file:///path/to/file"));
+```
+
+### WebBlob
+
+Creates a Blob object from a URL. If the file is less than 1MB (as indicated by the Content-Length header), by default it will be cached in memory in entirety upon blob creation.
+
+This class is useful for large files that do not need to be loaded all at once in memory, as it makes range requests for the data.
+
+```ts
+await WebBlob.create("https://url/to/file");
+await WebBlob.create(new URL("https://url/to/file"));
+
+await WebBlob.create("https://url/to/file", {
+  /**
+   * Custom fetch function to use. Useful for adding headers, etc.
+   */
+  fetch: ...,
+  /**
+   * If the file is less than the specified size, it will be cached in memory in entirety upon blob creation,
+   * instead of doing range requests for the data.
+   * 
+   * @default 1_000_000
+   */
+  cacheBelow: ...
+})
+```

packages/blob/index.ts

@@ -0,0 +1 @@
+export * from "./src/index";

packages/blob/package.json

@@ -0,0 +1,64 @@
+{
+	"name": "@huggingface/blob",
+	"packageManager": "[email protected]",
+	"version": "0.0.1",
+	"description": "Utilities to convert URLs and files to Blobs, internally used by Hugging Face libs",
+	"repository": "https://github.com/huggingface/huggingface.js.git",
+	"publishConfig": {
+		"access": "public"
+	},
+	"main": "./dist/index.js",
+	"module": "./dist/index.mjs",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"require": "./dist/index.js",
+			"import": "./dist/index.mjs"
+		},
+		"./package.json": "./package.json",
+		"./WebBlob": {
+			"require": "./dist/src/utils/WebBlob.js",
+			"import": "./dist/src/utils/WebBlob.mjs"
+		},
+		"./FileBlob": {
+			"require": "./dist/src/utils/FileBlob.js",
+			"import": "./dist/src/utils/FileBlob.mjs"
+		}
+	},
+	"browser": {
+		"./src/utils/FileBlob.ts": false,
+		"./dist/index.js": "./dist/browser/index.js",
+		"./dist/index.mjs": "./dist/browser/index.mjs"
+	},
+	"source": "index.ts",
+	"scripts": {
+		"lint": "eslint --quiet --fix --ext .cjs,.ts .",
+		"lint:check": "eslint --ext .cjs,.ts .",
+		"format": "prettier --write .",
+		"format:check": "prettier --check .",
+		"prepublishOnly": "pnpm run build",
+		"build": "tsup && tsc --emitDeclarationOnly --declaration && cp dist/index.d.ts dist/index.m.d.ts && cp dist/src/utils/FileBlob.d.ts dist/src/utils/FileBlob.m.d.ts && cp dist/src/utils/WebBlob.d.ts dist/src/utils/WebBlob.m.d.ts",
+		"prepare": "pnpm run build",
+		"test": "vitest run",
+		"test:browser": "vitest run --browser.name=chrome --browser.headless --config vitest-browser.config.mts",
+		"check": "tsc"
+	},
+	"files": [
+		"src",
+		"dist",
+		"index.ts",
+		"tsconfig.json"
+	],
+	"keywords": [
+		"huggingface",
+		"hugging",
+		"face",
+		"blob",
+		"lazy"
+	],
+	"author": "Hugging Face",
+	"license": "MIT",
+	"devDependencies": {
+		"@types/node": "^20.11.28"
+	}
+}

packages/blob/src/index.ts

@@ -0,0 +1 @@
+export { createBlob } from "./utils/createBlob";