Skip to content

Commit 4294d19

Browse files
coyotte508coyotte508
committed
🚧 @huggingace/blob package
1 parent 2a99455 commit 4294d19

27 files changed

+325
-132
lines changed

‎.github/workflows/blob-publish.yml‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
name: Blob - Version and Release
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
newversion:
7+
type: choice
8+
description: "Semantic Version Bump Type"
9+
default: patch
10+
options:
11+
- patch
12+
- minor
13+
- major
14+
15+
concurrency:
16+
group: "push-to-main"
17+
18+
defaults:
19+
run:
20+
working-directory: packages/blob
21+
22+
jobs:
23+
version_and_release:
24+
runs-on: ubuntu-latest
25+
steps:
26+
- uses: actions/checkout@v3
27+
with:
28+
# Needed to push the tag and the commit on the main branch, otherwise we get:
29+
# > Run git push --follow-tags
30+
# remote: error: GH006: Protected branch update failed for refs/heads/main.
31+
# remote: error: Changes must be made through a pull request. Required status check "lint" is expected.
32+
token: ${{ secrets.BOT_ACCESS_TOKEN }}
33+
- run: corepack enable
34+
- uses: actions/setup-node@v3
35+
with:
36+
node-version: "20"
37+
cache: "pnpm"
38+
cache-dependency-path: |
39+
packages/blob/pnpm-lock.yaml
40+
# setting a registry enables the NODE_AUTH_TOKEN env variable where we can set an npm token. REQUIRED
41+
registry-url: "https://registry.npmjs.org"
42+
- run: pnpm install
43+
- run: git config --global user.name machineuser
44+
- run: git config --global user.email [email protected]
45+
- run: |
46+
PACKAGE_VERSION=$(node -p "require('./package.json').version")
47+
BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')")
48+
# Update package.json with the new version
49+
node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
50+
git commit . -m "🔖 @huggingface/blob $BUMPED_VERSION"
51+
git tag "blob-v$BUMPED_VERSION"
52+
53+
- run: pnpm publish --no-git-checks .
54+
env:
55+
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
56+
- run: git pull --rebase && git push --follow-tags
57+
# hack - reuse actions/setup-node@v3 just to set a new registry
58+
- uses: actions/setup-node@v3
59+
with:
60+
node-version: "20"
61+
registry-url: "https://npm.pkg.github.com"
62+
# Disable for now, until github supports PATs for writing github packages (https://github.com/github/roadmap/issues/558)
63+
# - run: pnpm publish --no-git-checks .
64+
# env:
65+
# NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
66+
- name: "Update Doc"
67+
uses: peter-evans/repository-dispatch@v2
68+
with:
69+
event-type: doc-build
70+
token: ${{ secrets.BOT_ACCESS_TOKEN }}

‎.github/workflows/hub-publish.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ jobs:
5454
git tag "hub-v$BUMPED_VERSION"
5555
5656
- name: "Check Deps are published before publishing this package"
57-
run: pnpm -w check-deps tasks
57+
run: pnpm -w check-deps tasks && pnpm -w check-deps blob
5858

5959
- run: pnpm publish --no-git-checks .
6060
env:

‎packages/blob/.eslintignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
dist

‎packages/blob/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
.python_generated

‎packages/blob/.prettierignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pnpm-lock.yaml
2+
# In order to avoid code samples to have tabs, they don't display well on npm
3+
README.md
4+
dist
5+
.tshy

‎packages/blob/LICENSE‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2023 Hugging Face
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎packages/blob/README.md‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# 🤗 Hugging Face Blobs
2+
3+
Utilitie to convert a string or URL to a [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob) object, whether it represents a local file or remote URL.
4+
5+
`fetch` already returns a `Blob` object for remote URLs, but it loads the entire file in memory. This utility makes ad-hoc http range requests when calling `.slice()` on the blob, for example.
6+
7+
## Install
8+
9+
```console
10+
pnpm add @huggingface/blob
11+
12+
npm add @huggingface/blob
13+
14+
yarn add @huggingface/blob
15+
```
16+
17+
### Deno
18+
19+
```ts
20+
// esm.sh
21+
import { FileBlob, WebBlob } from "https://esm.sh/@huggingface/blob";
22+
// or npm:
23+
import { FileBlob, WebBlob } from "npm:@huggingface/blob";
24+
```
25+
26+
## Usage
27+
28+
29+
```ts
30+
import { FileBlob } from "@huggingface/blob/FileBlob";
31+
import { WebBlob } from "@huggingface/blob/WebBlob";
32+
import { createBlob } from "@huggingface/blob";
33+
34+
const fileBlob = await FileBlob.create("path/to/file");
35+
const webBlob = await WebBlob.create("https://url/to/file");
36+
37+
const blob = await createBlob("..."); // Automatically detects if it's a file or web URL
38+
```
39+
40+
## API
41+
42+
### createBlob
43+
44+
Creates a Blob object from a string or URL. Automatically detects if it's a file or web URL.
45+
46+
```ts
47+
await createBlob("...", {
48+
/**
49+
* Custom fetch function to use, in case it resolves to a Web Blob.
50+
*
51+
* Useful for adding headers, etc.
52+
*/
53+
fetch: ...,
54+
});
55+
56+
### FileBlob
57+
58+
```ts
59+
await FileBlob.create("path/to/file");
60+
await FileBlob.create(new URL("file:///path/to/file"));
61+
```
62+
63+
### WebBlob
64+
65+
Creates a Blob object from a URL. If the file is less than 1MB (as indicated by the Content-Length header), by default it will be cached in memory in entirety upon blob creation.
66+
67+
This class is useful for large files that do not need to be loaded all at once in memory, as it makes range requests for the data.
68+
69+
```ts
70+
await WebBlob.create("https://url/to/file");
71+
await WebBlob.create(new URL("https://url/to/file"));
72+
73+
await WebBlob.create("https://url/to/file", {
74+
/**
75+
* Custom fetch function to use. Useful for adding headers, etc.
76+
*/
77+
fetch: ...,
78+
/**
79+
* If the file is less than the specified size, it will be cached in memory in entirety upon blob creation,
80+
* instead of doing range requests for the data.
81+
*
82+
* @default 1_000_000
83+
*/
84+
cacheBelow: ...
85+
})
86+
```

‎packages/blob/package.json‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
{
2+
"name": "@huggingface/blob",
3+
"packageManager": "[email protected]",
4+
"version": "0.0.1",
5+
"description": "Utilities to convert URLs and files to Blobs, internally used by Hugging Face libs",
6+
"repository": "https://github.com/huggingface/huggingface.js.git",
7+
"publishConfig": {
8+
"access": "public"
9+
},
10+
"main": "./dist/index.js",
11+
"module": "./dist/index.mjs",
12+
"types": "./dist/index.d.ts",
13+
"exports": {
14+
".": {
15+
"types": "./dist/index.d.ts",
16+
"require": "./dist/index.js",
17+
"import": "./dist/index.mjs"
18+
},
19+
"./WebBlob": {
20+
"types": "./dist/utils/WebBlob.d.ts",
21+
"require": "./dist/utils/WebBlob.js",
22+
"import": "./dist/utils/WebBlob.mjs"
23+
},
24+
"./FileBlob": {
25+
"types": "./dist/utils/FileBlob.d.ts",
26+
"require": "./dist/utils/FileBlob.js",
27+
"import": "./dist/utils/FileBlob.mjs"
28+
}
29+
},
30+
"source": "./src/index.ts",
31+
"scripts": {
32+
"lint": "eslint --quiet --fix --ext .cjs,.ts .",
33+
"lint:check": "eslint --ext .cjs,.ts .",
34+
"format": "prettier --write .",
35+
"format:check": "prettier --check .",
36+
"prepublishOnly": "pnpm run build",
37+
"build": "tsup && tsc --emitDeclarationOnly --declaration",
38+
"prepare": "pnpm run build",
39+
"check": "tsc",
40+
"test": "vitest run"
41+
},
42+
"type": "module",
43+
"files": [
44+
"dist",
45+
"src",
46+
"tsconfig.json"
47+
],
48+
"keywords": [
49+
"huggingface",
50+
"blob"
51+
],
52+
"author": "Hugging Face",
53+
"license": "MIT",
54+
"browser": {
55+
"./src/utils/FileBlob.ts": false,
56+
"./dist/index.js": "./dist/browser/index.js",
57+
"./dist/index.mjs": "./dist/browser/index.mjs"
58+
}
59+
}

‎packages/blob/pnpm-lock.yaml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/hub/src/utils/FileBlob.spec.ts‎ renamed to ‎packages/blob/src/FileBlob.spec.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { open, stat } from "node:fs/promises";
22
import { TextDecoder } from "node:util";
33
import { describe, expect, it } from "vitest";
4-
import { FileBlob } from "./FileBlob";
4+
import { FileBlob } from "./FileBlob.js";
55

66
describe("FileBlob", () => {
77
it("should create a FileBlob with a slice on the entire file", async () => {

0 commit comments

Comments
 (0)