Skip to content

Commit 401a03c

Browse files
coyotte508coyotte508
authored
✨ Add uploadFile(s) and deleteFile(s) helpers (#126)
1 parent b5d8fb0 commit 401a03c

File tree

11 files changed

+474
-23
lines changed

11 files changed

+474
-23
lines changed

‎README.md

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ You can run our packages with vanilla JS, without any bundler, by using a CDN or
5555
## Usage example
5656

5757
```ts
58-
import { createRepo, commit } from "@huggingface/hub";
58+
import { createRepo, uploadFile } from "@huggingface/hub";
5959
import { HfInference } from "@huggingface/inference";
6060

6161
// use an access token from your free account
@@ -66,15 +66,14 @@ await createRepo({
6666
credentials: {accessToken: HF_ACCESS_TOKEN}
6767
});
6868

69-
await commit({
69+
await uploadFile({
7070
repo: {type: "model", name: "my-user/nlp-test"},
7171
credentials: {accessToken: HF_ACCESS_TOKEN},
72-
title: "Add model file",
73-
operations: [{
74-
operation: "addOrUpdate",
72+
// Can work with native File in browsers
73+
file: {
7574
path: "pytorch_model.bin",
76-
content: new Blob(...) // Can work with native File in browsers
77-
}]
75+
content: new Blob(...)
76+
}
7877
});
7978

8079
const inference = new HfInference(HF_ACCESS_TOKEN);

‎packages/hub/README.md

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ Check out the [full documentation](https://huggingface.co/docs/huggingface.js/hu
1313
For some of the calls, you need to create an account and generate an [access token](https://huggingface.co/settings/tokens).
1414

1515
```ts
16-
import { createRepo, commit, deleteRepo, listFiles, whoAmI } from "@huggingface/hub";
16+
import { createRepo, uploadFiles, deleteFile, deleteRepo, listFiles, whoAmI } from "@huggingface/hub";
1717
import type { RepoId, Credentials } from "@huggingface/hub";
1818

1919
const repo: RepoId = { type: "model", name: "myname/some-model" };
@@ -27,30 +27,30 @@ for await (const model of listModels({search: {owner: username}, credentials}))
2727

2828
await createRepo({ repo, credentials, license: "mit" });
2929

30-
await commit({
30+
await uploadFiles({
3131
repo,
3232
credentials,
33-
operations: [
33+
files: [
34+
// path + blob content
3435
{
35-
operation: "addOrUpdate",
3636
path: "file.txt",
3737
content: new Blob(["Hello World"]),
3838
},
39+
// Local file URL
40+
pathToFileURL("./pytorch-model.bin"),
41+
// Web URL
42+
new URL("https://huggingface.co/xlm-roberta-base/resolve/main/tokenizer.json"),
43+
// Path + Web URL
3944
{
40-
operation: "addOrUpdate",
41-
path: "pytorch-model.bin",
42-
// Only supported from the backend
43-
content: pathToFileURL("./pytorch-model.bin"),
44-
},
45-
{
46-
operation: "addOrUpdate",
47-
path: "tokenizer.json",
48-
// Copy xml-roberta-base's tokenizer.json directly from HF
49-
content: new URL("https://huggingface.co/xlm-roberta-base/raw/main/tokenizer.json"),
50-
},
45+
path: "myfile.bin",
46+
content: new URL("https://huggingface.co/bert-base-uncased/resolve/main/pytorch_model.bin")
47+
}
48+
// Can also work with native File in browsers
5149
],
5250
});
5351

52+
await deleteFile({repo, credentials, path: "myfile.bin"});
53+
5454
await (await downloadFile({ repo, path: "README.md" })).text();
5555

5656
for await (const fileInfo of listFiles({repo})) {

‎packages/hub/src/lib/commit.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ export interface CommitDeletedEntry {
2929
path: string;
3030
}
3131

32-
type ContentSource = Blob | URL;
32+
export type ContentSource = Blob | URL;
3333

3434
export interface CommitFile {
3535
operation: "addOrUpdate";

‎packages/hub/src/lib/deleteFile.spec.ts

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
import { assert, it, describe } from "vitest";
2+
3+
import { HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../consts";
4+
import type { RepoId } from "../types/public";
5+
import { insecureRandomString } from "../utils/insecureRandomString";
6+
import { createRepo } from "./create-repo";
7+
import { deleteRepo } from "./delete-repo";
8+
import { deleteFile } from "./deleteFile";
9+
import { downloadFile } from "./download-file";
10+
11+
describe("deleteFile", () => {
12+
it("should delete a file", async () => {
13+
const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`;
14+
const repo = { type: "model", name: repoName } satisfies RepoId;
15+
const credentials = {
16+
accessToken: TEST_ACCESS_TOKEN,
17+
};
18+
19+
try {
20+
const result = await createRepo({
21+
credentials,
22+
repo,
23+
files: [
24+
{ path: "file1", content: new Blob(["file1"]) },
25+
{ path: "file2", content: new Blob(["file2"]) },
26+
],
27+
});
28+
29+
assert.deepStrictEqual(result, {
30+
repoUrl: `${HUB_URL}/${repoName}`,
31+
});
32+
33+
let content = await downloadFile({
34+
repo,
35+
path: "file1",
36+
});
37+
38+
assert.strictEqual(await content?.text(), "file1");
39+
40+
await deleteFile({ path: "file1", repo, credentials });
41+
42+
content = await downloadFile({
43+
repo,
44+
path: "file1",
45+
});
46+
47+
assert.strictEqual(content, null);
48+
49+
content = await downloadFile({
50+
repo,
51+
path: "file2",
52+
});
53+
54+
assert.strictEqual(await content?.text(), "file2");
55+
} finally {
56+
await deleteRepo({
57+
repo,
58+
credentials,
59+
});
60+
}
61+
});
62+
});

‎packages/hub/src/lib/deleteFile.ts

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import type { Credentials, RepoId } from "../types/public";
2+
import type { CommitOutput, CommitParams } from "./commit";
3+
import { commit } from "./commit";
4+
5+
export function deleteFile(params: {
6+
credentials: Credentials;
7+
repo: RepoId;
8+
path: string;
9+
commitTitle?: CommitParams["title"];
10+
commitDescription?: CommitParams["description"];
11+
hubUrl?: CommitParams["hubUrl"];
12+
branch?: CommitParams["branch"];
13+
isPullRequest?: CommitParams["isPullRequest"];
14+
parentCommit?: CommitParams["parentCommit"];
15+
}): Promise<CommitOutput> {
16+
return commit({
17+
credentials: params.credentials,
18+
repo: params.repo,
19+
operations: [
20+
{
21+
operation: "delete",
22+
path: params.path,
23+
},
24+
],
25+
title: params.commitTitle ?? `Delete ${params.path}`,
26+
description: params.commitDescription,
27+
hubUrl: params.hubUrl,
28+
branch: params.branch,
29+
isPullRequest: params.isPullRequest,
30+
parentCommit: params.parentCommit,
31+
});
32+
}

‎packages/hub/src/lib/deleteFiles.spec.ts

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import { assert, it, describe } from "vitest";
2+
3+
import { HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../consts";
4+
import type { RepoId } from "../types/public";
5+
import { insecureRandomString } from "../utils/insecureRandomString";
6+
import { createRepo } from "./create-repo";
7+
import { deleteRepo } from "./delete-repo";
8+
import { deleteFiles } from "./deleteFiles";
9+
import { downloadFile } from "./download-file";
10+
11+
describe("deleteFiles", () => {
12+
it("should delete multiple files", async () => {
13+
const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`;
14+
const repo = { type: "model", name: repoName } satisfies RepoId;
15+
const credentials = {
16+
accessToken: TEST_ACCESS_TOKEN,
17+
};
18+
19+
try {
20+
const result = await createRepo({
21+
credentials,
22+
repo,
23+
files: [
24+
{ path: "file1", content: new Blob(["file1"]) },
25+
{ path: "file2", content: new Blob(["file2"]) },
26+
{ path: "file3", content: new Blob(["file3"]) },
27+
],
28+
});
29+
30+
assert.deepStrictEqual(result, {
31+
repoUrl: `${HUB_URL}/${repoName}`,
32+
});
33+
34+
let content = await downloadFile({
35+
repo,
36+
path: "file1",
37+
});
38+
39+
assert.strictEqual(await content?.text(), "file1");
40+
41+
content = await downloadFile({
42+
repo,
43+
path: "file2",
44+
});
45+
46+
assert.strictEqual(await content?.text(), "file2");
47+
48+
await deleteFiles({ paths: ["file1", "file2"], repo, credentials });
49+
50+
content = await downloadFile({
51+
repo,
52+
path: "file1",
53+
});
54+
55+
assert.strictEqual(content, null);
56+
57+
content = await downloadFile({
58+
repo,
59+
path: "file2",
60+
});
61+
62+
assert.strictEqual(content, null);
63+
64+
content = await downloadFile({
65+
repo,
66+
path: "file3",
67+
});
68+
69+
assert.strictEqual(await content?.text(), "file3");
70+
} finally {
71+
await deleteRepo({
72+
repo,
73+
credentials,
74+
});
75+
}
76+
});
77+
});

‎packages/hub/src/lib/deleteFiles.ts

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import type { Credentials, RepoId } from "../types/public";
2+
import type { CommitOutput, CommitParams } from "./commit";
3+
import { commit } from "./commit";
4+
5+
export function deleteFiles(params: {
6+
credentials: Credentials;
7+
repo: RepoId;
8+
paths: string[];
9+
commitTitle?: CommitParams["title"];
10+
commitDescription?: CommitParams["description"];
11+
hubUrl?: CommitParams["hubUrl"];
12+
branch?: CommitParams["branch"];
13+
isPullRequest?: CommitParams["isPullRequest"];
14+
parentCommit?: CommitParams["parentCommit"];
15+
}): Promise<CommitOutput> {
16+
return commit({
17+
credentials: params.credentials,
18+
repo: params.repo,
19+
operations: params.paths.map((path) => ({
20+
operation: "delete",
21+
path,
22+
})),
23+
title: params.commitTitle ?? `Deletes ${params.paths.length} files`,
24+
description: params.commitDescription,
25+
hubUrl: params.hubUrl,
26+
branch: params.branch,
27+
isPullRequest: params.isPullRequest,
28+
parentCommit: params.parentCommit,
29+
});
30+
}

‎packages/hub/src/lib/uploadFile.spec.ts

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
import { assert, it, describe } from "vitest";
2+
3+
import { HUB_URL, TEST_ACCESS_TOKEN, TEST_USER } from "../consts";
4+
import type { RepoId } from "../types/public";
5+
import { insecureRandomString } from "../utils/insecureRandomString";
6+
import { createRepo } from "./create-repo";
7+
import { deleteRepo } from "./delete-repo";
8+
import { downloadFile } from "./download-file";
9+
import { uploadFile } from "./uploadFile";
10+
11+
describe("uploadFile", () => {
12+
it("should upload a file", async () => {
13+
const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`;
14+
const repo = { type: "model", name: repoName } satisfies RepoId;
15+
const credentials = {
16+
accessToken: TEST_ACCESS_TOKEN,
17+
};
18+
19+
try {
20+
const result = await createRepo({
21+
credentials,
22+
repo,
23+
});
24+
25+
assert.deepStrictEqual(result, {
26+
repoUrl: `${HUB_URL}/${repoName}`,
27+
});
28+
29+
await uploadFile({ credentials, repo, file: { content: new Blob(["file1"]), path: "file1" } });
30+
await uploadFile({ credentials, repo, file: new URL("https://huggingface.co/gpt2/raw/main/config.json") });
31+
32+
let content = await downloadFile({
33+
repo,
34+
path: "file1",
35+
});
36+
37+
assert.strictEqual(await content?.text(), "file1");
38+
39+
content = await downloadFile({
40+
repo,
41+
path: "config.json",
42+
});
43+
44+
assert.strictEqual(
45+
(await content?.text())?.trim(),
46+
`
47+
{
48+
"activation_function": "gelu_new",
49+
"architectures": [
50+
"GPT2LMHeadModel"
51+
],
52+
"attn_pdrop": 0.1,
53+
"bos_token_id": 50256,
54+
"embd_pdrop": 0.1,
55+
"eos_token_id": 50256,
56+
"initializer_range": 0.02,
57+
"layer_norm_epsilon": 1e-05,
58+
"model_type": "gpt2",
59+
"n_ctx": 1024,
60+
"n_embd": 768,
61+
"n_head": 12,
62+
"n_layer": 12,
63+
"n_positions": 1024,
64+
"resid_pdrop": 0.1,
65+
"summary_activation": null,
66+
"summary_first_dropout": 0.1,
67+
"summary_proj_to_labels": true,
68+
"summary_type": "cls_index",
69+
"summary_use_proj": true,
70+
"task_specific_params": {
71+
"text-generation": {
72+
"do_sample": true,
73+
"max_length": 50
74+
}
75+
},
76+
"vocab_size": 50257
77+
}
78+
`.trim()
79+
);
80+
} finally {
81+
await deleteRepo({
82+
repo,
83+
credentials,
84+
});
85+
}
86+
});
87+
});

0 commit comments

Comments
 (0)