From 86956a3720920eb0450f74c3b16e8a9b0b0657c6 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 30 Apr 2025 18:55:16 +0200 Subject: [PATCH 01/34] add create-branch --- packages/hub/src/lib/create-branch.spec.ts | 159 +++++++++++++++++++++ packages/hub/src/lib/create-branch.ts | 50 +++++++ 2 files changed, 209 insertions(+) create mode 100644 packages/hub/src/lib/create-branch.spec.ts create mode 100644 packages/hub/src/lib/create-branch.ts diff --git a/packages/hub/src/lib/create-branch.spec.ts b/packages/hub/src/lib/create-branch.spec.ts new file mode 100644 index 0000000000..b616fb4ceb --- /dev/null +++ b/packages/hub/src/lib/create-branch.spec.ts @@ -0,0 +1,159 @@ +import { assert, it, describe } from "vitest"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { createBranch } from "./create-branch"; +import { uploadFile } from "./upload-file"; +import { downloadFile } from "./download-file"; + +describe("createBranch", () => { + it("should create a new branch from the default branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "new-branch", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "new-branch", + }); + + assert.equal(await content?.text(), "file content"); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); + + it("should create an empty branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "empty-branch", + empty: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "empty-branch", + }); + + assert.equal(content, null); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); + + it("should overwrite an existing branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await uploadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + file: { + path: "file.txt", + content: new Blob(["file content"]), + }, + }); + + await createBranch({ + repo, + branch: "overwrite-branch", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await createBranch({ + repo, + branch: "overwrite-branch", + overwrite: true, + empty: true, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + const content = await downloadFile({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + path: "file.txt", + revision: "overwrite-branch", + }); + + assert.equal(content, null); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts new file mode 100644 index 0000000000..072b88bfbf --- /dev/null +++ b/packages/hub/src/lib/create-branch.ts @@ -0,0 +1,50 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { AccessToken, RepoDesignation } from "../types/public"; + +export async function createBranch(params: { + repo: RepoDesignation; + /** + * Revision to create the branch from. Defaults to the default branch. + * + * Use empty: true to create an empty branch. + */ + revision?: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; + /** + * The name of the branch to create + */ + branch: string; + /** + * Use this to create an empty branch, with no commits. + */ + empty?: boolean; + /** + * Use this to overwrite the branch if it already exists. + */ + overwrite?: boolean; +}): Promise { + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + body: JSON.stringify({ + startingPoint: params.revision, + ...(params.empty && { emptyBranch: true }), + overwrite: params.overwrite, + }), + } + ); + + if (!res.ok) { + throw await createApiError(res); + } +} From b175cf498f0b8f95a938d073b3c0a92482539f43 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 30 Apr 2025 18:55:27 +0200 Subject: [PATCH 02/34] add delete-branch --- packages/hub/src/lib/delete-branch.spec.ts | 43 ++++++++++++++++++++++ packages/hub/src/lib/delete-branch.ts | 30 +++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 packages/hub/src/lib/delete-branch.spec.ts create mode 100644 packages/hub/src/lib/delete-branch.ts diff --git a/packages/hub/src/lib/delete-branch.spec.ts b/packages/hub/src/lib/delete-branch.spec.ts new file mode 100644 index 0000000000..dcd253214b --- /dev/null +++ b/packages/hub/src/lib/delete-branch.spec.ts @@ -0,0 +1,43 @@ +import { it, describe } from "vitest"; +import { TEST_ACCESS_TOKEN, TEST_HUB_URL, TEST_USER } from "../test/consts"; +import type { RepoId } from "../types/public"; +import { insecureRandomString } from "../utils/insecureRandomString"; +import { createRepo } from "./create-repo"; +import { deleteRepo } from "./delete-repo"; +import { createBranch } from "./create-branch"; +import { deleteBranch } from "./delete-branch"; + +describe("deleteBranch", () => { + it("should delete an existing branch", async () => { + const repoName = `${TEST_USER}/TEST-${insecureRandomString()}`; + const repo = { type: "model", name: repoName } satisfies RepoId; + + try { + await createRepo({ + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + repo, + }); + + await createBranch({ + repo, + branch: "branch-to-delete", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + + await deleteBranch({ + repo, + branch: "branch-to-delete", + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } finally { + await deleteRepo({ + repo, + accessToken: TEST_ACCESS_TOKEN, + hubUrl: TEST_HUB_URL, + }); + } + }); +}); diff --git a/packages/hub/src/lib/delete-branch.ts b/packages/hub/src/lib/delete-branch.ts new file mode 100644 index 0000000000..30cb575aea --- /dev/null +++ b/packages/hub/src/lib/delete-branch.ts @@ -0,0 +1,30 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { AccessToken, RepoDesignation } from "../types/public"; + +export async function deleteBranch(params: { + repo: RepoDesignation; + /** + * The name of the branch to delete + */ + branch: string; + hubUrl?: string; + accessToken?: AccessToken; + fetch?: typeof fetch; +}): Promise { + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + { + method: "DELETE", + headers: { + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + } + ); + + if (!res.ok) { + throw await createApiError(res); + } +} From 80faefbddd5792fa0e8be5014c3afef933635bd7 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 30 Apr 2025 19:00:37 +0200 Subject: [PATCH 03/34] cli file --- packages/hub/cli.ts | 285 ++++++++++++++++++++++++++++++++++ packages/hub/package.json | 3 + packages/hub/src/lib/index.ts | 2 + packages/hub/tsconfig.json | 2 +- 4 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 packages/hub/cli.ts diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts new file mode 100644 index 0000000000..b7d9ff6978 --- /dev/null +++ b/packages/hub/cli.ts @@ -0,0 +1,285 @@ +#! /usr/bin/env node + +import { parseArgs } from "node:util"; +import { typedEntries } from "./src/utils/typedEntries"; +import { createBranch, uploadFilesWithProgress } from "./src"; +import { pathToFileURL } from "node:url"; + +const command = process.argv[2]; +const args = process.argv.slice(3); + +type Camelize = T extends `${infer A}-${infer B}` ? `${A}${Camelize>}` : T; + +const commands = { + upload: { + description: "Upload a folder to a repo on the Hub", + args: [ + { + name: "repo-name" as const, + description: "The name of the repo to create", + positional: true, + required: true, + }, + { + name: "local-folder" as const, + description: "The local folder to upload. Defaults to the current working directory", + positional: true, + default: () => process.cwd(), + }, + // { + // name: "path-in-repo" as const, + // description: "The path in the repo to upload the folder to. Defaults to the root of the repo", + // positional: true, + // default: "/", + // }, + { + name: "quiet" as const, + short: "q", + description: "Suppress all output", + boolean: true, + }, + { + name: "repo-type" as const, + short: "t", + enum: ["dataset", "model", "space"], + default: "model", + description: + "The type of repo to upload to. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + short: "r", + description: "The revision to upload to. Defaults to the main branch", + default: "main", + }, + { + name: "from-revision" as const, + short: "c", + description: + "The revision to upload from. Defaults to the latest commit on main or on the branch if it exists.", + }, + { + name: "from-empty" as const, + short: "e", + boolean: true, + description: + "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", + }, + { + name: "token" as const, + short: "k", + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ], + }, +} satisfies Record< + string, + { + description: string; + args?: Array<{ + name: string; + short?: string; + positional?: boolean; + description?: string; + required?: boolean; + boolean?: boolean; + enum?: Array; + default?: string | (() => string); + }>; + } +>; + +type Command = keyof typeof commands; + +async function run() { + switch (command) { + case "help": { + const positionals = parseArgs({ allowPositionals: true, args }).positionals; + + if (positionals.length > 0 && positionals[0] in commands) { + const commandName = positionals[0] as Command; + console.log(detailedUsage(commandName)); + break; + } + + console.log( + `Available commands\n\n` + + typedEntries(commands) + .map(([name, { description }]) => `- ${usage(name)}: ${description}`) + .join("\n") + ); + break; + } + + case "upload": { + if (args[1] === "--help" || args[1] === "-h") { + console.log(usage("upload")); + break; + } + const parsedArgs = advParseArgs(args, "upload"); + const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet } = parsedArgs; + + if (revision && (fromEmpty || fromRevision)) { + await createBranch({ + branch: revision, + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + accessToken: token, + revision: fromRevision, + empty: fromEmpty ? true : undefined, + overwrite: true, + }); + } + + for await (const event of uploadFilesWithProgress({ + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + files: [pathToFileURL(localFolder)], + branch: revision, + accessToken: token, + })) { + if (!quiet) { + console.log(event); + } + } + break; + } + default: + throw new Error("Command not found: " + command); + } +} +run(); + +function usage(commandName: Command) { + const command = commands[commandName]; + + return `${commandName} ${(command.args || []) + .map((arg) => { + if (arg.positional) { + if (arg.required) { + return `<${arg.name}>`; + } else { + return `[${arg.name}]`; + } + } + return `[--${arg.name} ${arg.enum ? `{${arg.enum.join(",")}}` : arg.name.toLocaleUpperCase()}]`; + }) + .join("")}`.trim(); +} + +function detailedUsage(commandName: Command) { + let ret = `usage: ${usage(commandName)}\n\n`; + const command = commands[commandName]; + + if (command.args.some((p) => p.positional)) { + ret += `Positional arguments:\n`; + + for (const arg of command.args) { + if (arg.positional) { + ret += ` ${arg.name}: ${arg.description}\n`; + } + } + + ret += `\n`; + } + + if (command.args.some((p) => !p.positional)) { + ret += `Options:\n`; + + for (const arg of command.args) { + if (!arg.positional) { + ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; + } + } + + ret += `\n`; + } + + return ret; +} + +function advParseArgs( + args: string[], + commandName: C +): { + // Todo : better typing + [key in Camelize<(typeof commands)[C]["args"][number]["name"]>]: string; +} { + const { tokens } = parseArgs({ + options: Object.fromEntries( + commands[commandName].args + .filter((arg) => !arg.positional) + .map((arg) => { + const option = { + name: arg.name, + short: arg.short, + type: arg.boolean ? "boolean" : "string", + } as const; + return [arg.name, option]; + }) + ), + args, + allowPositionals: true, + strict: false, + tokens: true, + }); + + const command = commands[commandName]; + const expectedPositionals = command.args.filter((arg) => arg.positional); + const requiredPositionals = expectedPositionals.filter((arg) => arg.required).length; + const providedPositionals = tokens.filter((token) => token.kind === "positional").length; + + if (providedPositionals < requiredPositionals) { + throw new Error( + `Missing required positional arguments. Expected: ${requiredPositionals}, Provided: ${providedPositionals}` + ); + } + + if (providedPositionals > expectedPositionals.length) { + throw new Error( + `Too many positional arguments. Expected: ${expectedPositionals.length}, Provided: ${providedPositionals}` + ); + } + + const positionals = Object.fromEntries( + tokens.filter((token) => token.kind === "positional").map((token, i) => [expectedPositionals[i].name, token.value]) + ); + + const options = Object.fromEntries( + tokens + .filter((token) => token.kind === "option") + .map((token) => { + const arg = command.args.find((arg) => arg.name === token.name || arg.short === token.name); + if (!arg) { + throw new Error(`Unknown option: ${token.name}`); + } + + if (!token.value) { + throw new Error(`Missing value for option: ${token.name}`); + } + + if (arg.enum && !arg.enum.includes(token.value)) { + throw new Error(`Invalid value for option ${token.name}. Expected one of: ${arg.enum.join(", ")}`); + } + + return [arg.name, arg.boolean ? true : token.value]; + }) + ); + const defaults = Object.fromEntries( + command.args + .filter((arg) => arg.default) + .map((arg) => { + const value = typeof arg.default === "function" ? arg.default() : arg.default; + return [arg.name, value]; + }) + ); + return Object.fromEntries( + Object.entries({ ...defaults, ...positionals, ...options }).map(([name, val]) => [kebabToCamelCase(name), val]) + ) as { + [key in Camelize<(typeof commands)[C]["args"][number]["name"]>]: string; + }; +} + +function kebabToCamelCase(str: string) { + return str.replace(/-./g, (match) => match[1].toUpperCase()); +} diff --git a/packages/hub/package.json b/packages/hub/package.json index 15d2e31c8d..cbe58ac214 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -57,6 +57,9 @@ "hugging", "face" ], + "bin": { + "@huggingface/hub": "./dist/cli.js" + }, "author": "Hugging Face", "license": "MIT", "devDependencies": { diff --git a/packages/hub/src/lib/index.ts b/packages/hub/src/lib/index.ts index 24e239bdcb..d4b771f2b2 100644 --- a/packages/hub/src/lib/index.ts +++ b/packages/hub/src/lib/index.ts @@ -3,7 +3,9 @@ export * from "./check-repo-access"; export * from "./commit"; export * from "./count-commits"; export * from "./create-repo"; +export * from "./create-branch"; export * from "./dataset-info"; +export * from "./delete-branch"; export * from "./delete-file"; export * from "./delete-files"; export * from "./delete-repo"; diff --git a/packages/hub/tsconfig.json b/packages/hub/tsconfig.json index 254606a30e..9dd335c6b7 100644 --- a/packages/hub/tsconfig.json +++ b/packages/hub/tsconfig.json @@ -15,6 +15,6 @@ "declaration": true, "declarationMap": true }, - "include": ["src", "index.ts"], + "include": ["src", "index.ts", "cli.ts"], "exclude": ["dist"] } From 28f1050001b033c43675380c55d7e27b2d2afa58 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 14:43:56 +0200 Subject: [PATCH 04/34] document cli & name it to 'hfx' --- packages/hub/README.md | 26 ++++++++++++++++++++++++++ packages/hub/package.json | 2 +- packages/hub/src/lib/create-branch.ts | 2 ++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index 0633f06271..eda66aad65 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -93,6 +93,32 @@ for await (const fileInfo of hub.listFiles({repo})) { await hub.deleteRepo({ repo, accessToken: "hf_..." }); ``` +## CLI usage + +You can use `@huggingface/hub` in CLI mode to upload files and folders to your repo. + +```console +npx @huggingface/hub upload coyotte508/test-model . +npx @huggingface/hub upload datasets/coyotte508/test-dataset . +# Same thing +npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . +# Upload new data with 0 history in a separate branch +npx @huggingface/hub upload coyotte508/test-model . --revision release --empty + +npx @huggingface/hub --help +npx @huggingface/hub upload --help +``` + +You can also instal globally with `npm install -g @huggingface/hub`. Then you can do: + +```console +hfx upload coyotte508/test-model . +hfx upload --repo-type dataset coyotte508/test-dataset . --revision release --empty + +hfx --help +hfx upload --help +``` + ## OAuth Login It's possible to login using OAuth (["Sign in with HF"](https://huggingface.co/docs/hub/oauth)). diff --git a/packages/hub/package.json b/packages/hub/package.json index 1e322145ad..a7b082bd7b 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "@huggingface/hub": "./dist/cli.js" + "hfx": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts index 072b88bfbf..e47a0e0c26 100644 --- a/packages/hub/src/lib/create-branch.ts +++ b/packages/hub/src/lib/create-branch.ts @@ -23,6 +23,8 @@ export async function createBranch(params: { empty?: boolean; /** * Use this to overwrite the branch if it already exists. + * + * If you only specify `overwrite` and no `revision`/`empty`, and the branch already exists, it will be a no-op. */ overwrite?: boolean; }): Promise { From e886d5448f3ac29bbe2d2e4da2f4551f7cd8db91 Mon Sep 17 00:00:00 2001 From: "Eliott C." Date: Tue, 6 May 2025 15:01:49 +0200 Subject: [PATCH 05/34] Update packages/hub/package.json Co-authored-by: Julien Chaumond --- packages/hub/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/package.json b/packages/hub/package.json index a7b082bd7b..67bc120092 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "hfx": "./dist/cli.js" + "hf": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", From dda33e70ee050dc8982f29d09c1c0c42ddb5910d Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:21:09 +0200 Subject: [PATCH 06/34] fix TS --- packages/hub/cli.ts | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index b7d9ff6978..fb090431e7 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -5,6 +5,18 @@ import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; +// Didn't find the import from "node:util", so duplicated it here +type OptionToken = + | { kind: "option"; index: number; name: string; rawName: string; value: string; inlineValue: boolean } + | { + kind: "option"; + index: number; + name: string; + rawName: string; + value: undefined; + inlineValue: undefined; + }; + const command = process.argv[2]; const args = process.argv.slice(3); @@ -242,12 +254,14 @@ function advParseArgs( } const positionals = Object.fromEntries( - tokens.filter((token) => token.kind === "positional").map((token, i) => [expectedPositionals[i].name, token.value]) + tokens + .filter((token): token is { kind: "positional"; index: number; value: string } => token.kind === "positional") + .map((token, i) => [expectedPositionals[i].name, token.value]) ); const options = Object.fromEntries( tokens - .filter((token) => token.kind === "option") + .filter((token): token is OptionToken => token.kind === "option") .map((token) => { const arg = command.args.find((arg) => arg.name === token.name || arg.short === token.name); if (!arg) { From 0de8bea4cac2292536782ea66166bf3cd550596a Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:21:36 +0200 Subject: [PATCH 07/34] switch back to hfx for now @julien-c, see slack --- packages/hub/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/package.json b/packages/hub/package.json index 67bc120092..a7b082bd7b 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "hf": "./dist/cli.js" + "hfx": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", From 13569ac5d71c1bfb0cf15a35f9874b37775e6002 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:26:10 +0200 Subject: [PATCH 08/34] fix repo param in url --- packages/hub/src/lib/create-branch.ts | 3 ++- packages/hub/src/lib/delete-branch.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts index e47a0e0c26..fb80103d80 100644 --- a/packages/hub/src/lib/create-branch.ts +++ b/packages/hub/src/lib/create-branch.ts @@ -1,6 +1,7 @@ import { HUB_URL } from "../consts"; import { createApiError } from "../error"; import type { AccessToken, RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; export async function createBranch(params: { repo: RepoDesignation; @@ -29,7 +30,7 @@ export async function createBranch(params: { overwrite?: boolean; }): Promise { const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, { method: "POST", headers: { diff --git a/packages/hub/src/lib/delete-branch.ts b/packages/hub/src/lib/delete-branch.ts index 30cb575aea..4c8b0bda5f 100644 --- a/packages/hub/src/lib/delete-branch.ts +++ b/packages/hub/src/lib/delete-branch.ts @@ -1,6 +1,7 @@ import { HUB_URL } from "../consts"; import { createApiError } from "../error"; import type { AccessToken, RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; export async function deleteBranch(params: { repo: RepoDesignation; @@ -13,7 +14,7 @@ export async function deleteBranch(params: { fetch?: typeof fetch; }): Promise { const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${params.repo}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, { method: "DELETE", headers: { From 618bbce325a7861b40ef4ca94838b2413850792a Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:30:07 +0200 Subject: [PATCH 09/34] fixup! fix repo param in url --- packages/hub/src/lib/create-branch.ts | 3 ++- packages/hub/src/lib/delete-branch.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/hub/src/lib/create-branch.ts b/packages/hub/src/lib/create-branch.ts index fb80103d80..100e4d1b94 100644 --- a/packages/hub/src/lib/create-branch.ts +++ b/packages/hub/src/lib/create-branch.ts @@ -29,8 +29,9 @@ export async function createBranch(params: { */ overwrite?: boolean; }): Promise { + const repoId = toRepoId(params.repo); const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, { method: "POST", headers: { diff --git a/packages/hub/src/lib/delete-branch.ts b/packages/hub/src/lib/delete-branch.ts index 4c8b0bda5f..70227b185e 100644 --- a/packages/hub/src/lib/delete-branch.ts +++ b/packages/hub/src/lib/delete-branch.ts @@ -13,8 +13,9 @@ export async function deleteBranch(params: { accessToken?: AccessToken; fetch?: typeof fetch; }): Promise { + const repoId = toRepoId(params.repo); const res = await (params.fetch ?? fetch)( - `${params.hubUrl ?? HUB_URL}/api/repos/${toRepoId(params.repo)}/branch/${encodeURIComponent(params.branch)}`, + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}/branch/${encodeURIComponent(params.branch)}`, { method: "DELETE", headers: { From d982af9a46dd422c8144c80bf74f9e5440b27ea6 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:39:53 +0200 Subject: [PATCH 10/34] hfjs --- packages/hub/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/package.json b/packages/hub/package.json index a7b082bd7b..022653e6f0 100644 --- a/packages/hub/package.json +++ b/packages/hub/package.json @@ -58,7 +58,7 @@ "face" ], "bin": { - "hfx": "./dist/cli.js" + "hfjs": "./dist/cli.js" }, "author": "Hugging Face", "license": "MIT", From ef81d23f7f48dee6a0f22b797e8099cb0a1ae592 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:41:30 +0200 Subject: [PATCH 11/34] fixup! hfjs --- packages/hub/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index eda66aad65..6d2668ba01 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -112,11 +112,11 @@ npx @huggingface/hub upload --help You can also instal globally with `npm install -g @huggingface/hub`. Then you can do: ```console -hfx upload coyotte508/test-model . -hfx upload --repo-type dataset coyotte508/test-dataset . --revision release --empty +hfjs upload coyotte508/test-model . +hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --empty -hfx --help -hfx upload --help +hfjs --help +hfjs upload --help ``` ## OAuth Login From f9a57471120958b3b9c943d110a2ffdf8e805a91 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:48:18 +0200 Subject: [PATCH 12/34] doc --- packages/hub/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index 6d2668ba01..80808f9877 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -103,7 +103,7 @@ npx @huggingface/hub upload datasets/coyotte508/test-dataset . # Same thing npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . # Upload new data with 0 history in a separate branch -npx @huggingface/hub upload coyotte508/test-model . --revision release --empty +npx @huggingface/hub upload coyotte508/test-model . --revision release --from-empty npx @huggingface/hub --help npx @huggingface/hub upload --help @@ -113,7 +113,7 @@ You can also instal globally with `npm install -g @huggingface/hub`. Then you ca ```console hfjs upload coyotte508/test-model . -hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --empty +hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --from-empty hfjs --help hfjs upload --help From fa312489b90ca40cc3bf1b288e76d6a16c559c68 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 15:59:57 +0200 Subject: [PATCH 13/34] remove all shorthands (except 'q') --- packages/hub/cli.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index fb090431e7..28bfd09fc3 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -52,7 +52,6 @@ const commands = { }, { name: "repo-type" as const, - short: "t", enum: ["dataset", "model", "space"], default: "model", description: @@ -60,26 +59,22 @@ const commands = { }, { name: "revision" as const, - short: "r", description: "The revision to upload to. Defaults to the main branch", default: "main", }, { name: "from-revision" as const, - short: "c", description: "The revision to upload from. Defaults to the latest commit on main or on the branch if it exists.", }, { name: "from-empty" as const, - short: "e", boolean: true, description: "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", }, { name: "token" as const, - short: "k", description: "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", default: process.env.HF_TOKEN, From 0fa9af71383a2a4e8b0cd6db1c8cc89d90b8d8f8 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 16:32:41 +0200 Subject: [PATCH 14/34] cli fixes --- package.json | 3 +++ packages/hub/cli.ts | 15 ++++++++++++--- packages/hub/tsup.config.ts | 5 +++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 0cbf7b9228..36fa9dab54 100644 --- a/package.json +++ b/package.json @@ -30,5 +30,8 @@ "vite": "^5.0.2", "vitest": "^0.34.6", "webdriverio": "^8.6.7" + }, + "dependencies": { + "@huggingface/hub": "link:packages/hub" } } diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 28bfd09fc3..84237a70ce 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -102,6 +102,8 @@ type Command = keyof typeof commands; async function run() { switch (command) { + case undefined: + case "--help": case "help": { const positionals = parseArgs({ allowPositionals: true, args }).positionals; @@ -117,11 +119,17 @@ async function run() { .map(([name, { description }]) => `- ${usage(name)}: ${description}`) .join("\n") ); + + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + + if (command === undefined) { + process.exitCode = 1; + } break; } case "upload": { - if (args[1] === "--help" || args[1] === "-h") { + if (args[0] === "--help" || args[0] === "-h") { console.log(usage("upload")); break; } @@ -171,7 +179,7 @@ function usage(commandName: Command) { } return `[--${arg.name} ${arg.enum ? `{${arg.enum.join(",")}}` : arg.name.toLocaleUpperCase()}]`; }) - .join("")}`.trim(); + .join(" ")}`.trim(); } function detailedUsage(commandName: Command) { @@ -219,8 +227,9 @@ function advParseArgs( .map((arg) => { const option = { name: arg.name, - short: arg.short, + ...(arg.short && { short: arg.short }), type: arg.boolean ? "boolean" : "string", + default: typeof arg.default === "function" ? arg.default() : arg.default, } as const; return [arg.name, option]; }) diff --git a/packages/hub/tsup.config.ts b/packages/hub/tsup.config.ts index 6be4e128a6..adbb9fdfbd 100644 --- a/packages/hub/tsup.config.ts +++ b/packages/hub/tsup.config.ts @@ -1,14 +1,15 @@ import type { Options } from "tsup"; -const baseConfig: Options = { +const baseConfig = { entry: ["./index.ts"], format: ["cjs", "esm"], outDir: "dist", clean: true, -}; +} satisfies Options; const nodeConfig: Options = { ...baseConfig, + entry: [...baseConfig.entry, "./cli.ts"], platform: "node", }; From 23498e0aa778fb8d1540bd0b76c1eba3e852ac48 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 16:47:12 +0200 Subject: [PATCH 15/34] cli fixes --- packages/hub/cli.ts | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 84237a70ce..dfb5f416b8 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -73,6 +73,10 @@ const commands = { description: "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", }, + { + name: "commit-message" as const, + description: "The commit message to use. Defaults to 'Add [x] files'", + }, { name: "token" as const, description: @@ -134,7 +138,8 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "upload"); - const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet } = parsedArgs; + const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet, commitMessage } = + parsedArgs; if (revision && (fromEmpty || fromRevision)) { await createBranch({ @@ -152,6 +157,8 @@ async function run() { files: [pathToFileURL(localFolder)], branch: revision, accessToken: token, + commitTitle: commitMessage?.trim().split("\n")[0], + commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), })) { if (!quiet) { console.log(event); @@ -272,12 +279,14 @@ function advParseArgs( throw new Error(`Unknown option: ${token.name}`); } - if (!token.value) { - throw new Error(`Missing value for option: ${token.name}`); - } + if (!arg.boolean) { + if (!token.value) { + throw new Error(`Missing value for option: ${token.name}: ${JSON.stringify(token)}`); + } - if (arg.enum && !arg.enum.includes(token.value)) { - throw new Error(`Invalid value for option ${token.name}. Expected one of: ${arg.enum.join(", ")}`); + if (arg.enum && !arg.enum.includes(token.value)) { + throw new Error(`Invalid value for option ${token.name}. Expected one of: ${arg.enum.join(", ")}`); + } } return [arg.name, arg.boolean ? true : token.value]; From f735cfa8514463ae9315f1c6b1300614151f739d Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 16:59:27 +0200 Subject: [PATCH 16/34] fixup! cli fixes --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index 36fa9dab54..0cbf7b9228 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,5 @@ "vite": "^5.0.2", "vitest": "^0.34.6", "webdriverio": "^8.6.7" - }, - "dependencies": { - "@huggingface/hub": "link:packages/hub" } } From f831b3cd6e4750f7bf9f74f5b207a658e4362be4 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 17:21:14 +0200 Subject: [PATCH 17/34] add hidden hub-url CLI param --- packages/hub/cli.ts | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index dfb5f416b8..0391a26350 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -4,6 +4,7 @@ import { parseArgs } from "node:util"; import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; +import { HUB_URL } from "./src/consts"; // Didn't find the import from "node:util", so duplicated it here type OptionToken = @@ -83,6 +84,13 @@ const commands = { "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", default: process.env.HF_TOKEN, }, + { + name: "hub-url" as const, + description: + "The URL of the Hub to upload to. Defaults to https://huggingface.co. Use this to upload to a private Hub.", + hidden: true, + default: HUB_URL, + }, ], }, } satisfies Record< @@ -97,6 +105,7 @@ const commands = { required?: boolean; boolean?: boolean; enum?: Array; + hidden?: boolean; default?: string | (() => string); }>; } @@ -138,8 +147,18 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "upload"); - const { repoName, localFolder, repoType, revision, fromEmpty, fromRevision, token, quiet, commitMessage } = - parsedArgs; + const { + repoName, + localFolder, + repoType, + revision, + fromEmpty, + fromRevision, + token, + quiet, + commitMessage, + hubUrl, + } = parsedArgs; if (revision && (fromEmpty || fromRevision)) { await createBranch({ @@ -149,6 +168,7 @@ async function run() { revision: fromRevision, empty: fromEmpty ? true : undefined, overwrite: true, + hubUrl, }); } @@ -159,6 +179,7 @@ async function run() { accessToken: token, commitTitle: commitMessage?.trim().split("\n")[0], commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), + hubUrl, })) { if (!quiet) { console.log(event); @@ -176,6 +197,7 @@ function usage(commandName: Command) { const command = commands[commandName]; return `${commandName} ${(command.args || []) + .filter((arg) => !arg.hidden) .map((arg) => { if (arg.positional) { if (arg.required) { @@ -205,11 +227,11 @@ function detailedUsage(commandName: Command) { ret += `\n`; } - if (command.args.some((p) => !p.positional)) { + if (command.args.some((p) => !p.positional && !p.hidden)) { ret += `Options:\n`; for (const arg of command.args) { - if (!arg.positional) { + if (!arg.positional && !arg.hidden) { ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; } } From 3aacda1bcf832ea20ebd7125935d9b5c2ac44705 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 6 May 2025 17:30:33 +0200 Subject: [PATCH 18/34] fix uploading to new rev not from empty --- packages/hub/cli.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 0391a26350..c880a9954c 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -160,7 +160,7 @@ async function run() { hubUrl, } = parsedArgs; - if (revision && (fromEmpty || fromRevision)) { + if (revision && revision !== "main") { await createBranch({ branch: revision, repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, From 4aefa05997f4a12b2fa0fa3fa36333cb194a1867 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 15:10:20 +0200 Subject: [PATCH 19/34] Also support pathInRepo positional arg --- packages/hub/cli.ts | 27 ++++++++++++++++++++------- packages/hub/src/utils/createBlobs.ts | 5 ++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index c880a9954c..3784115421 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -5,6 +5,8 @@ import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; import { HUB_URL } from "./src/consts"; +import { stat } from "node:fs/promises"; +import { basename, join } from "node:path"; // Didn't find the import from "node:util", so duplicated it here type OptionToken = @@ -39,12 +41,12 @@ const commands = { positional: true, default: () => process.cwd(), }, - // { - // name: "path-in-repo" as const, - // description: "The path in the repo to upload the folder to. Defaults to the root of the repo", - // positional: true, - // default: "/", - // }, + { + name: "path-in-repo" as const, + description: "The path in the repo to upload the folder to. Defaults to the root of the repo", + positional: true, + default: ".", + }, { name: "quiet" as const, short: "q", @@ -158,6 +160,7 @@ async function run() { quiet, commitMessage, hubUrl, + pathInRepo, } = parsedArgs; if (revision && revision !== "main") { @@ -172,9 +175,19 @@ async function run() { }); } + const isFile = (await stat(localFolder)).isFile(); + const files = isFile + ? [ + { + content: pathToFileURL(localFolder), + path: join(pathInRepo, `${basename(localFolder)}`).replace(/^[.]?\//, ""), + }, + ] + : [{ content: pathToFileURL(localFolder), path: pathInRepo.replace(/^[.]?\//, "") }]; + for await (const event of uploadFilesWithProgress({ repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, - files: [pathToFileURL(localFolder)], + files, branch: revision, accessToken: token, commitTitle: commitMessage?.trim().split("\n")[0], diff --git a/packages/hub/src/utils/createBlobs.ts b/packages/hub/src/utils/createBlobs.ts index 1a261c4c48..625bef5fde 100644 --- a/packages/hub/src/utils/createBlobs.ts +++ b/packages/hub/src/utils/createBlobs.ts @@ -38,7 +38,10 @@ export async function createBlobs( return Promise.all( paths.map(async (path) => ({ - path: `${destPath}/${path.relativePath}`.replace(/\/[.]$/, "").replaceAll("//", "/"), + path: `${destPath}/${path.relativePath}` + .replace(/\/[.]$/, "") + .replaceAll("//", "/") + .replace(/^[.]?\//, ""), blob: await FileBlob.create(new URL(path.path)), })) ); From 03e615dd74dc4a7fa962ce786b8449dda39ac4ba Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:30:08 +0200 Subject: [PATCH 20/34] Add create-branch to the cli --- packages/hub/README.md | 7 +- packages/hub/cli.ts | 141 ++++++++++++++++++++++------------------- 2 files changed, 81 insertions(+), 67 deletions(-) diff --git a/packages/hub/README.md b/packages/hub/README.md index 80808f9877..625cd5e4e2 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -103,7 +103,8 @@ npx @huggingface/hub upload datasets/coyotte508/test-dataset . # Same thing npx @huggingface/hub upload --repo-type dataset coyotte508/test-dataset . # Upload new data with 0 history in a separate branch -npx @huggingface/hub upload coyotte508/test-model . --revision release --from-empty +npx @huggingface/hub create-branch coyotte508/test-model release --empty +npx @huggingface/hub upload coyotte508/test-model . --revision release npx @huggingface/hub --help npx @huggingface/hub upload --help @@ -113,7 +114,9 @@ You can also instal globally with `npm install -g @huggingface/hub`. Then you ca ```console hfjs upload coyotte508/test-model . -hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release --from-empty + +hfjs create-branch --repo-type dataset coyotte508/test-dataset release --empty +hfjs upload --repo-type dataset coyotte508/test-dataset . --revision release hfjs --help hfjs upload --help diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 3784115421..ca33ebb6ba 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -4,9 +4,9 @@ import { parseArgs } from "node:util"; import { typedEntries } from "./src/utils/typedEntries"; import { createBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; -import { HUB_URL } from "./src/consts"; import { stat } from "node:fs/promises"; import { basename, join } from "node:path"; +import { HUB_URL } from "./src/consts"; // Didn't find the import from "node:util", so duplicated it here type OptionToken = @@ -25,6 +25,17 @@ const args = process.argv.slice(3); type Camelize = T extends `${infer A}-${infer B}` ? `${A}${Camelize>}` : T; +interface ArgDef { + name: string; + short?: string; + positional?: boolean; + description?: string; + required?: boolean; + boolean?: boolean; + enum?: Array; + default?: string | (() => string); +} + const commands = { upload: { description: "Upload a folder to a repo on the Hub", @@ -65,17 +76,6 @@ const commands = { description: "The revision to upload to. Defaults to the main branch", default: "main", }, - { - name: "from-revision" as const, - description: - "The revision to upload from. Defaults to the latest commit on main or on the branch if it exists.", - }, - { - name: "from-empty" as const, - boolean: true, - description: - "This will create an empty branch and upload the files to it. This will erase all previous commits on the branch if it exists.", - }, { name: "commit-message" as const, description: "The commit message to use. Defaults to 'Add [x] files'", @@ -86,30 +86,48 @@ const commands = { "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", default: process.env.HF_TOKEN, }, + ], + }, + "create-branch": { + description: "Create a new branch in a repo, or update an existing one", + args: [ + { + name: "repo-name" as const, + description: "The name of the repo to create", + positional: true, + required: true, + }, + { + name: "branch" as const, + description: "The name of the branch to create", + positional: true, + required: true, + }, { - name: "hub-url" as const, + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + default: "model", + description: + "The type of repo to create. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, description: - "The URL of the Hub to upload to. Defaults to https://huggingface.co. Use this to upload to a private Hub.", - hidden: true, - default: HUB_URL, + "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists.", + default: "main", + }, + { + name: "empty" as const, + boolean: true, + description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", }, ], - }, + } as const, } satisfies Record< string, { description: string; - args?: Array<{ - name: string; - short?: string; - positional?: boolean; - description?: string; - required?: boolean; - boolean?: boolean; - enum?: Array; - hidden?: boolean; - default?: string | (() => string); - }>; + args?: ArgDef[]; } >; @@ -149,31 +167,7 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "upload"); - const { - repoName, - localFolder, - repoType, - revision, - fromEmpty, - fromRevision, - token, - quiet, - commitMessage, - hubUrl, - pathInRepo, - } = parsedArgs; - - if (revision && revision !== "main") { - await createBranch({ - branch: revision, - repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, - accessToken: token, - revision: fromRevision, - empty: fromEmpty ? true : undefined, - overwrite: true, - hubUrl, - }); - } + const { repoName, localFolder, repoType, revision, token, quiet, commitMessage, pathInRepo } = parsedArgs; const isFile = (await stat(localFolder)).isFile(); const files = isFile @@ -192,7 +186,7 @@ async function run() { accessToken: token, commitTitle: commitMessage?.trim().split("\n")[0], commitDescription: commitMessage?.trim().split("\n").slice(1).join("\n").trim(), - hubUrl, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, })) { if (!quiet) { console.log(event); @@ -200,6 +194,24 @@ async function run() { } break; } + case "create-branch": { + if (args[0] === "--help" || args[0] === "-h") { + console.log(usage("create-branch")); + break; + } + const parsedArgs = advParseArgs(args, "create-branch"); + const { repoName, branch, revision, empty, repoType } = parsedArgs; + + await createBranch({ + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + branch, + accessToken: process.env.HF_TOKEN, + revision, + empty: empty ? true : undefined, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + }); + break; + } default: throw new Error("Command not found: " + command); } @@ -209,8 +221,7 @@ run(); function usage(commandName: Command) { const command = commands[commandName]; - return `${commandName} ${(command.args || []) - .filter((arg) => !arg.hidden) + return `${commandName} ${((command.args as ArgDef[]) || []) .map((arg) => { if (arg.positional) { if (arg.required) { @@ -228,10 +239,10 @@ function detailedUsage(commandName: Command) { let ret = `usage: ${usage(commandName)}\n\n`; const command = commands[commandName]; - if (command.args.some((p) => p.positional)) { + if ((command.args as ArgDef[]).some((p) => p.positional)) { ret += `Positional arguments:\n`; - for (const arg of command.args) { + for (const arg of command.args as ArgDef[]) { if (arg.positional) { ret += ` ${arg.name}: ${arg.description}\n`; } @@ -240,11 +251,11 @@ function detailedUsage(commandName: Command) { ret += `\n`; } - if (command.args.some((p) => !p.positional && !p.hidden)) { + if ((command.args as ArgDef[]).some((p) => !p.positional)) { ret += `Options:\n`; - for (const arg of command.args) { - if (!arg.positional && !arg.hidden) { + for (const arg of command.args as ArgDef[]) { + if (!arg.positional) { ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; } } @@ -264,7 +275,7 @@ function advParseArgs( } { const { tokens } = parseArgs({ options: Object.fromEntries( - commands[commandName].args + (commands[commandName].args as ArgDef[]) .filter((arg) => !arg.positional) .map((arg) => { const option = { @@ -283,7 +294,7 @@ function advParseArgs( }); const command = commands[commandName]; - const expectedPositionals = command.args.filter((arg) => arg.positional); + const expectedPositionals = (command.args as ArgDef[]).filter((arg) => arg.positional); const requiredPositionals = expectedPositionals.filter((arg) => arg.required).length; const providedPositionals = tokens.filter((token) => token.kind === "positional").length; @@ -309,7 +320,7 @@ function advParseArgs( tokens .filter((token): token is OptionToken => token.kind === "option") .map((token) => { - const arg = command.args.find((arg) => arg.name === token.name || arg.short === token.name); + const arg = (command.args as ArgDef[]).find((arg) => arg.name === token.name || arg.short === token.name); if (!arg) { throw new Error(`Unknown option: ${token.name}`); } @@ -328,7 +339,7 @@ function advParseArgs( }) ); const defaults = Object.fromEntries( - command.args + (commands[commandName].args as ArgDef[]) .filter((arg) => arg.default) .map((arg) => { const value = typeof arg.default === "function" ? arg.default() : arg.default; From 0dec5a0e622de917c4ea66644eef5c328b5817ca Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:35:24 +0200 Subject: [PATCH 21/34] token param too --- packages/hub/cli.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index ca33ebb6ba..6525cf1516 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -121,6 +121,12 @@ const commands = { boolean: true, description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, ], } as const, } satisfies Record< @@ -200,12 +206,12 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "create-branch"); - const { repoName, branch, revision, empty, repoType } = parsedArgs; + const { repoName, branch, revision, empty, repoType, token } = parsedArgs; await createBranch({ repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, branch, - accessToken: process.env.HF_TOKEN, + accessToken: token, revision, empty: empty ? true : undefined, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, From 3b84137c11f6d7aaff5c16532bfe449dd6a4e7a3 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:45:43 +0200 Subject: [PATCH 22/34] better usage text --- packages/hub/cli.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 6525cf1516..378c93396e 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -236,7 +236,9 @@ function usage(commandName: Command) { return `[${arg.name}]`; } } - return `[--${arg.name} ${arg.enum ? `{${arg.enum.join(",")}}` : arg.name.toLocaleUpperCase()}]`; + return `[--${arg.name}${ + arg.enum ? ` {${arg.enum.join(",")}}` : arg.boolean ? "" : " " + arg.name.toLocaleUpperCase() + }]`; }) .join(" ")}`.trim(); } @@ -262,7 +264,9 @@ function detailedUsage(commandName: Command) { for (const arg of command.args as ArgDef[]) { if (!arg.positional) { - ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}: ${arg.description}\n`; + ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}${ + arg.enum ? ` {${arg.enum.join(",")}}` : arg.boolean ? "" : " " + arg.name.toLocaleUpperCase() + }: ${arg.description}\n`; } } From 43aae0441935c90a663193a52717b0a98e58f77e Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:50:24 +0200 Subject: [PATCH 23/34] Add --force param to create-branch --- packages/hub/cli.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 378c93396e..49f99280d3 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -121,6 +121,13 @@ const commands = { boolean: true, description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", }, + { + name: "force" as const, + short: "f", + boolean: true, + description: + "Overwrite the branch if it already exists. Otherwise, throws an error if the branch already exists. No-ops if no revision is provided and the branch exists.", + }, { name: "token" as const, description: @@ -206,7 +213,7 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "create-branch"); - const { repoName, branch, revision, empty, repoType, token } = parsedArgs; + const { repoName, branch, revision, empty, repoType, token, force } = parsedArgs; await createBranch({ repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, @@ -214,6 +221,7 @@ async function run() { accessToken: token, revision, empty: empty ? true : undefined, + overwrite: force ? true : undefined, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, }); break; From e25930ede87e8197a70822d1aa3dcfc7079285ce Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:52:08 +0200 Subject: [PATCH 24/34] fixup! Add --force param to create-branch --- packages/hub/cli.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 49f99280d3..2809bff79f 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -114,7 +114,6 @@ const commands = { name: "revision" as const, description: "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists.", - default: "main", }, { name: "empty" as const, From be57754380a8da1c0e83b480a5b13a65575e7f86 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 17:54:55 +0200 Subject: [PATCH 25/34] more detailed usage --- packages/hub/cli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 2809bff79f..20d9047b7d 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -175,7 +175,7 @@ async function run() { case "upload": { if (args[0] === "--help" || args[0] === "-h") { - console.log(usage("upload")); + console.log(detailedUsage("upload")); break; } const parsedArgs = advParseArgs(args, "upload"); @@ -208,7 +208,7 @@ async function run() { } case "create-branch": { if (args[0] === "--help" || args[0] === "-h") { - console.log(usage("create-branch")); + console.log(detailedUsage("create-branch")); break; } const parsedArgs = advParseArgs(args, "create-branch"); From 8f96728c90299eaaa47b48aec8179dfeca5a00b7 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 7 May 2025 23:28:08 +0200 Subject: [PATCH 26/34] update repo name description --- packages/hub/cli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 20d9047b7d..00935645cb 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -42,7 +42,7 @@ const commands = { args: [ { name: "repo-name" as const, - description: "The name of the repo to create", + description: "The name of the repo to upload to", positional: true, required: true, }, @@ -93,7 +93,7 @@ const commands = { args: [ { name: "repo-name" as const, - description: "The name of the repo to create", + description: "The name of the repo to create the branch in", positional: true, required: true, }, From 0be6a40080876ee9d9a72f4363aa43aff3fa5436 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 13 May 2025 21:26:23 +0200 Subject: [PATCH 27/34] add delete-branch and version commands --- packages/hub/cli.ts | 66 +++++++++++++++++++++++++++++++++++++- packages/hub/tsconfig.json | 1 + 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 00935645cb..e9164e5fb2 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -2,11 +2,12 @@ import { parseArgs } from "node:util"; import { typedEntries } from "./src/utils/typedEntries"; -import { createBranch, uploadFilesWithProgress } from "./src"; +import { createBranch, deleteBranch, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; import { stat } from "node:fs/promises"; import { basename, join } from "node:path"; import { HUB_URL } from "./src/consts"; +import { version } from "./package.json"; // Didn't find the import from "node:util", so duplicated it here type OptionToken = @@ -80,6 +81,11 @@ const commands = { name: "commit-message" as const, description: "The commit message to use. Defaults to 'Add [x] files'", }, + { + name: "private" as const, + description: "If creating a new repo, make it private", + boolean: true, + }, { name: "token" as const, description: @@ -135,6 +141,40 @@ const commands = { }, ], } as const, + "delete-branch": { + description: "Delete a branch in a repo", + args: [ + { + name: "repo-name" as const, + description: "The name of the repo to delete the branch from", + positional: true, + required: true, + }, + { + name: "branch" as const, + description: "The name of the branch to delete", + positional: true, + required: true, + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + default: "model", + description: + "The type of repo to delete the branch from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ], + }, + version: { + description: "Print the version of the CLI", + args: [], + }, } satisfies Record< string, { @@ -225,6 +265,30 @@ async function run() { }); break; } + case "delete-branch": { + if (args[0] === "--help" || args[0] === "-h") { + console.log(detailedUsage("delete-branch")); + break; + } + const parsedArgs = advParseArgs(args, "delete-branch"); + const { repoName, branch, repoType, token } = parsedArgs; + + await deleteBranch({ + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + branch, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + }); + break; + } + case "version": { + if (args[0] === "--help" || args[0] === "-h") { + console.log(detailedUsage("version")); + break; + } + console.log(`hfjs version: ${version}`); + break; + } default: throw new Error("Command not found: " + command); } diff --git a/packages/hub/tsconfig.json b/packages/hub/tsconfig.json index 9dd335c6b7..c66aef0cff 100644 --- a/packages/hub/tsconfig.json +++ b/packages/hub/tsconfig.json @@ -6,6 +6,7 @@ "moduleResolution": "node", "target": "ES2022", "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, "strict": true, "noImplicitAny": true, "strictNullChecks": true, From ea0e2bcd0d58b6ce4ef2dddbdb20efbb76ac854e Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 13 May 2025 21:34:01 +0200 Subject: [PATCH 28/34] add repoExists function --- packages/hub/src/lib/index.ts | 1 + packages/hub/src/lib/repo-exists.spec.ts | 13 +++++++ packages/hub/src/lib/repo-exists.ts | 43 ++++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 packages/hub/src/lib/repo-exists.spec.ts create mode 100644 packages/hub/src/lib/repo-exists.ts diff --git a/packages/hub/src/lib/index.ts b/packages/hub/src/lib/index.ts index d4b771f2b2..cd777784f7 100644 --- a/packages/hub/src/lib/index.ts +++ b/packages/hub/src/lib/index.ts @@ -23,6 +23,7 @@ export * from "./oauth-handle-redirect"; export * from "./oauth-login-url"; export * from "./parse-safetensors-metadata"; export * from "./paths-info"; +export * from "./repo-exists"; export * from "./snapshot-download"; export * from "./space-info"; export * from "./upload-file"; diff --git a/packages/hub/src/lib/repo-exists.spec.ts b/packages/hub/src/lib/repo-exists.spec.ts new file mode 100644 index 0000000000..c4bbb192f2 --- /dev/null +++ b/packages/hub/src/lib/repo-exists.spec.ts @@ -0,0 +1,13 @@ +import { describe, expect, it } from "vitest"; +import { repoExists } from "./repo-exists"; + +describe("repoExists", () => { + it("should check if a repo exists", async () => { + const exists1 = await repoExists({ repo: { type: "model", name: "openai-community/gpt2" } }); + + expect(exists1).toBe(true); + + const exists2 = await repoExists({ repo: { type: "model", name: "openai-community/gpt9000" } }); + expect(exists2).toBe(false); + }); +}); diff --git a/packages/hub/src/lib/repo-exists.ts b/packages/hub/src/lib/repo-exists.ts new file mode 100644 index 0000000000..b53cae0e25 --- /dev/null +++ b/packages/hub/src/lib/repo-exists.ts @@ -0,0 +1,43 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { RepoDesignation } from "../types/public"; +import { toRepoId } from "../utils/toRepoId"; + +export async function repoExists(params: { + repo: RepoDesignation; + + hubUrl?: string; + /** + * An optional Git revision id which can be a branch name, a tag, or a commit hash. + */ + revision?: string; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + accessToken?: string; +}): Promise { + const repoId = toRepoId(params.repo); + + const res = await (params.fetch ?? fetch)( + `${params.hubUrl ?? HUB_URL}/api/${repoId.type}s/${repoId.name}?expand[]=likes`, + { + method: "GET", + headers: { + ...(params.accessToken && { + Authorization: `Bearer ${params.accessToken}`, + }), + }, + } + ); + + if (res.status === 404 || res.status === 401) { + return false; + } + + if (!res.ok) { + throw await createApiError(res); + } + + return true; +} From f6e98933c460a886e442eaa859f785a1312cc840 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 13 May 2025 21:38:15 +0200 Subject: [PATCH 29/34] create repo when it does not exist --- packages/hub/cli.ts | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index e9164e5fb2..79f76123c1 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -2,7 +2,7 @@ import { parseArgs } from "node:util"; import { typedEntries } from "./src/utils/typedEntries"; -import { createBranch, deleteBranch, uploadFilesWithProgress } from "./src"; +import { createBranch, createRepo, deleteBranch, repoExists, uploadFilesWithProgress } from "./src"; import { pathToFileURL } from "node:url"; import { stat } from "node:fs/promises"; import { basename, join } from "node:path"; @@ -219,7 +219,33 @@ async function run() { break; } const parsedArgs = advParseArgs(args, "upload"); - const { repoName, localFolder, repoType, revision, token, quiet, commitMessage, pathInRepo } = parsedArgs; + const { + repoName, + localFolder, + repoType, + revision, + token, + quiet, + commitMessage, + pathInRepo, + private: isPrivate, + } = parsedArgs; + + const repoId = repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName; + + if ( + !(await repoExists({ repo: repoId, revision, accessToken: token, hubUrl: process.env.HF_ENDPOINT ?? HUB_URL })) + ) { + if (!quiet) { + console.log(`Repo ${repoName} does not exist. Creating it...`); + } + await createRepo({ + repo: repoId, + accessToken: token, + private: !!isPrivate, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + }); + } const isFile = (await stat(localFolder)).isFile(); const files = isFile @@ -232,7 +258,7 @@ async function run() { : [{ content: pathToFileURL(localFolder), path: pathInRepo.replace(/^[.]?\//, "") }]; for await (const event of uploadFilesWithProgress({ - repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + repo: repoId, files, branch: revision, accessToken: token, From 4a8e319fde53a2f17436d47a600f68b6ccdb8832 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Tue, 13 May 2025 21:42:21 +0200 Subject: [PATCH 30/34] Update default commit message --- packages/hub/cli.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index 79f76123c1..cb3be6198f 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -79,7 +79,8 @@ const commands = { }, { name: "commit-message" as const, - description: "The commit message to use. Defaults to 'Add [x] files'", + description: "The commit message to use. Defaults to 'Upload files using @huggingface/hub'", + default: "Upload files using @huggingface/hub", }, { name: "private" as const, From 468dc2c3de750b2779429a36db7b73fea20d0bb3 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 14 May 2025 17:09:38 +0200 Subject: [PATCH 31/34] create-branch => branch create and delete-branch => branch delete --- package.json | 2 +- packages/hub/cli.ts | 641 ++++++++++++------- packages/inference/src/lib/getDefaultTask.ts | 3 +- pnpm-lock.yaml | 95 ++- 4 files changed, 454 insertions(+), 287 deletions(-) diff --git a/package.json b/package.json index 0cbf7b9228..80bcdac821 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "tshy": "^3.0.2", "tsup": "^6.7.0", "tsx": "^4.7.0", - "typescript": "^5.4.2", + "typescript": "^5.8.3", "vite": "^5.0.2", "vitest": "^0.34.6", "webdriverio": "^8.6.7" diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index cb3be6198f..cf6d6c5a3c 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -21,8 +21,8 @@ type OptionToken = inlineValue: undefined; }; -const command = process.argv[2]; -const args = process.argv.slice(3); +// const command = process.argv[2]; // Replaced by mainCommandName and subCommandName +// const args = process.argv.slice(3); // Replaced by cliArgs type Camelize = T extends `${infer A}-${infer B}` ? `${A}${Camelize>}` : T; @@ -34,7 +34,17 @@ interface ArgDef { required?: boolean; boolean?: boolean; enum?: Array; - default?: string | (() => string); + default?: string | boolean | (() => string | boolean); // Allow boolean defaults +} + +interface SingleCommand { + description: string; + args: readonly ArgDef[]; +} + +interface CommandGroup { + description: string; + subcommands: Record; } const commands = { @@ -93,133 +103,178 @@ const commands = { "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", default: process.env.HF_TOKEN, }, - ], - }, - "create-branch": { - description: "Create a new branch in a repo, or update an existing one", - args: [ - { - name: "repo-name" as const, - description: "The name of the repo to create the branch in", - positional: true, - required: true, - }, - { - name: "branch" as const, - description: "The name of the branch to create", - positional: true, - required: true, - }, - { - name: "repo-type" as const, - enum: ["dataset", "model", "space"], - default: "model", - description: - "The type of repo to create. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", - }, - { - name: "revision" as const, - description: - "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists.", + ] as const, + } satisfies SingleCommand, + branch: { + description: "Manage repository branches", + subcommands: { + create: { + description: "Create a new branch in a repo, or update an existing one", + args: [ + { + name: "repo-name" as const, + description: "The name of the repo to create the branch in", + positional: true, + required: true, + }, + { + name: "branch" as const, + description: "The name of the branch to create", + positional: true, + required: true, + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + default: "model", + description: + "The type of repo to create. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "revision" as const, + description: + "The revision to create the branch from. Defaults to the main branch, or existing branch if it exists.", + }, + { + name: "empty" as const, + boolean: true, + description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", + }, + { + name: "force" as const, + short: "f", + boolean: true, + description: + "Overwrite the branch if it already exists. Otherwise, throws an error if the branch already exists. No-ops if no revision is provided and the branch exists.", + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, }, - { - name: "empty" as const, - boolean: true, - description: "Create an empty branch. This will erase all previous commits on the branch if it exists.", + delete: { + description: "Delete a branch in a repo", + args: [ + { + name: "repo-name" as const, + description: "The name of the repo to delete the branch from", + positional: true, + required: true, + }, + { + name: "branch" as const, + description: "The name of the branch to delete", + positional: true, + required: true, + }, + { + name: "repo-type" as const, + enum: ["dataset", "model", "space"], + default: "model", + description: + "The type of repo to delete the branch from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", + }, + { + name: "token" as const, + description: + "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", + default: process.env.HF_TOKEN, + }, + ] as const, }, - { - name: "force" as const, - short: "f", - boolean: true, - description: - "Overwrite the branch if it already exists. Otherwise, throws an error if the branch already exists. No-ops if no revision is provided and the branch exists.", - }, - { - name: "token" as const, - description: - "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", - default: process.env.HF_TOKEN, - }, - ], - } as const, - "delete-branch": { - description: "Delete a branch in a repo", - args: [ - { - name: "repo-name" as const, - description: "The name of the repo to delete the branch from", - positional: true, - required: true, - }, - { - name: "branch" as const, - description: "The name of the branch to delete", - positional: true, - required: true, - }, - { - name: "repo-type" as const, - enum: ["dataset", "model", "space"], - default: "model", - description: - "The type of repo to delete the branch from. Defaults to model. You can also prefix the repo name with the type, e.g. datasets/username/repo-name", - }, - { - name: "token" as const, - description: - "The access token to use for authentication. If not provided, the HF_TOKEN environment variable will be used.", - default: process.env.HF_TOKEN, - }, - ], - }, + }, + } satisfies CommandGroup, version: { description: "Print the version of the CLI", - args: [], - }, -} satisfies Record< - string, - { - description: string; - args?: ArgDef[]; - } ->; - -type Command = keyof typeof commands; + args: [] as const, + } satisfies SingleCommand, +} satisfies Record; + +type TopLevelCommandName = keyof typeof commands; + +const mainCommandName = process.argv[2]; +let subCommandName: string | undefined; +let cliArgs: string[]; + +if ( + mainCommandName && + mainCommandName in commands && + commands[mainCommandName as keyof typeof commands] && + "subcommands" in commands[mainCommandName as keyof typeof commands] +) { + subCommandName = process.argv[3]; + cliArgs = process.argv.slice(4); +} else { + cliArgs = process.argv.slice(3); +} async function run() { - switch (command) { + switch (mainCommandName) { case undefined: case "--help": case "help": { - const positionals = parseArgs({ allowPositionals: true, args }).positionals; - - if (positionals.length > 0 && positionals[0] in commands) { - const commandName = positionals[0] as Command; - console.log(detailedUsage(commandName)); - break; - } - - console.log( - `Available commands\n\n` + - typedEntries(commands) - .map(([name, { description }]) => `- ${usage(name)}: ${description}`) - .join("\n") - ); - - console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); - - if (command === undefined) { - process.exitCode = 1; + const helpArgs = mainCommandName === "help" ? process.argv.slice(3) : []; + + if (helpArgs.length > 0) { + const cmdName = helpArgs[0] as TopLevelCommandName; + if (cmdName && commands[cmdName]) { + const cmdDef = commands[cmdName]; + if ("subcommands" in cmdDef) { + if (helpArgs.length > 1) { + const subCmdName = helpArgs[1]; + if ( + subCmdName in cmdDef.subcommands && + cmdDef.subcommands[subCmdName as keyof typeof cmdDef.subcommands] + ) { + console.log(detailedUsageForSubcommand(cmdName, subCmdName as keyof typeof cmdDef.subcommands)); + break; + } else { + console.error(`Error: Unknown subcommand '${subCmdName}' for command '${cmdName}'.`); + console.log(listSubcommands(cmdName, cmdDef)); + process.exitCode = 1; + break; + } + } else { + console.log(listSubcommands(cmdName, cmdDef)); + break; + } + } else { + console.log(detailedUsageForCommand(cmdName)); + break; + } + } else { + console.error(`Error: Unknown command '${cmdName}' for help.`); + process.exitCode = 1; + } + } else { + // General help + console.log( + `Hugging Face CLI Tools (hfjs)\n\nAvailable commands:\n\n` + + typedEntries(commands) + .map(([name, def]) => ` ${usage(name)}: ${def.description}`) + .join("\n") + ); + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + console.log( + "For commands with subcommands (like 'branch'), run `hfjs help ` or `hfjs --help`" + ); + if (mainCommandName === undefined) { + process.exitCode = 1; + } } break; } case "upload": { - if (args[0] === "--help" || args[0] === "-h") { - console.log(detailedUsage("upload")); + const cmdDef = commands.upload; + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + console.log(detailedUsageForCommand("upload")); break; } - const parsedArgs = advParseArgs(args, "upload"); + const parsedArgs = advParseArgs(cliArgs, cmdDef.args, "upload"); const { repoName, localFolder, @@ -273,192 +328,310 @@ async function run() { } break; } - case "create-branch": { - if (args[0] === "--help" || args[0] === "-h") { - console.log(detailedUsage("create-branch")); + case "branch": { + const branchCommandGroup = commands.branch; + const currentSubCommandName = subCommandName as keyof typeof branchCommandGroup.subcommands | undefined; + + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + if (currentSubCommandName && branchCommandGroup.subcommands[currentSubCommandName]) { + console.log(detailedUsageForSubcommand("branch", currentSubCommandName)); + } else { + console.log(listSubcommands("branch", branchCommandGroup)); + } break; } - const parsedArgs = advParseArgs(args, "create-branch"); - const { repoName, branch, revision, empty, repoType, token, force } = parsedArgs; - await createBranch({ - repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, - branch, - accessToken: token, - revision, - empty: empty ? true : undefined, - overwrite: force ? true : undefined, - hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, - }); - break; - } - case "delete-branch": { - if (args[0] === "--help" || args[0] === "-h") { - console.log(detailedUsage("delete-branch")); + if (!currentSubCommandName || !branchCommandGroup.subcommands[currentSubCommandName]) { + console.error(`Error: Missing or invalid subcommand for 'branch'.`); + console.log(listSubcommands("branch", branchCommandGroup)); + process.exitCode = 1; break; } - const parsedArgs = advParseArgs(args, "delete-branch"); - const { repoName, branch, repoType, token } = parsedArgs; - await deleteBranch({ - repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, - branch, - accessToken: token, - hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, - }); + const subCmdDef = branchCommandGroup.subcommands[currentSubCommandName]; + + switch (currentSubCommandName) { + case "create": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch create"); + const { repoName, branch, revision, empty, repoType, token, force } = parsedArgs; + + await createBranch({ + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + branch, + accessToken: token, + revision, + empty: empty ?? undefined, + overwrite: force ?? undefined, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + }); + console.log(`Branch '${branch}' created successfully in repo '${repoName}'.`); + break; + } + case "delete": { + const parsedArgs = advParseArgs(cliArgs, subCmdDef.args, "branch delete"); + const { repoName, branch, repoType, token } = parsedArgs; + + await deleteBranch({ + repo: repoType ? { type: repoType as "model" | "dataset" | "space", name: repoName } : repoName, + branch, + accessToken: token, + hubUrl: process.env.HF_ENDPOINT ?? HUB_URL, + }); + console.log(`Branch '${branch}' deleted successfully from repo '${repoName}'.`); + break; + } + default: + // Should be caught by the check above + console.error(`Error: Unknown subcommand '${currentSubCommandName}' for 'branch'.`); + console.log(listSubcommands("branch", branchCommandGroup)); + process.exitCode = 1; + break; + } break; } case "version": { - if (args[0] === "--help" || args[0] === "-h") { - console.log(detailedUsage("version")); + if (cliArgs[0] === "--help" || cliArgs[0] === "-h") { + console.log(detailedUsageForCommand("version")); break; } console.log(`hfjs version: ${version}`); break; } default: - throw new Error("Command not found: " + command); + console.error("Command not found: " + mainCommandName); + // Print general help + console.log( + `\nAvailable commands:\n\n` + + typedEntries(commands) + .map(([name, def]) => ` ${usage(name)}: ${def.description}`) + .join("\n") + ); + console.log("\nTo get help on a specific command, run `hfjs help ` or `hfjs --help`"); + process.exitCode = 1; + break; } } -run(); +run().catch((err) => { + console.error("\x1b[31mError:\x1b[0m", err.message); + if (process.env.DEBUG) { + console.error(err); + } + process.exitCode = 1; +}); + +function usage(commandName: TopLevelCommandName, subCommandName?: string): string { + const commandEntry = commands[commandName]; -function usage(commandName: Command) { - const command = commands[commandName]; + let cmdArgs: readonly ArgDef[]; + let fullCommandName = commandName as string; + + if ("subcommands" in commandEntry) { + if (subCommandName && subCommandName in commandEntry.subcommands) { + cmdArgs = commandEntry.subcommands[subCommandName as keyof typeof commandEntry.subcommands].args; + fullCommandName = `${commandName} ${subCommandName}`; + } else { + return `${commandName} `; + } + } else { + cmdArgs = commandEntry.args; + } - return `${commandName} ${((command.args as ArgDef[]) || []) + return `${fullCommandName} ${(cmdArgs || []) .map((arg) => { if (arg.positional) { - if (arg.required) { - return `<${arg.name}>`; - } else { - return `[${arg.name}]`; - } + return arg.required ? `<${arg.name}>` : `[${arg.name}]`; } - return `[--${arg.name}${ - arg.enum ? ` {${arg.enum.join(",")}}` : arg.boolean ? "" : " " + arg.name.toLocaleUpperCase() + return `[--${arg.name}${arg.short ? `|-${arg.short}` : ""}${ + arg.enum ? ` {${arg.enum.join("|")}}` : arg.boolean ? "" : ` <${arg.name.toUpperCase().replace(/-/g, "_")}>` }]`; }) .join(" ")}`.trim(); } -function detailedUsage(commandName: Command) { - let ret = `usage: ${usage(commandName)}\n\n`; - const command = commands[commandName]; +function _detailedUsage(args: readonly ArgDef[], usageLine: string, commandDescription?: string): string { + let ret = `usage: hfjs ${usageLine}\n`; + if (commandDescription) { + ret += `\n${commandDescription}\n`; + } - if ((command.args as ArgDef[]).some((p) => p.positional)) { - ret += `Positional arguments:\n`; + const positionals = args.filter((p) => p.positional); + const options = args.filter((p) => !p.positional); - for (const arg of command.args as ArgDef[]) { - if (arg.positional) { - ret += ` ${arg.name}: ${arg.description}\n`; - } + if (positionals.length > 0) { + ret += `\nPositional arguments:\n`; + for (const arg of positionals) { + ret += ` ${arg.name}\t${arg.description}${ + arg.default ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` : "" + }\n`; } - - ret += `\n`; } - if ((command.args as ArgDef[]).some((p) => !p.positional)) { - ret += `Options:\n`; - - for (const arg of command.args as ArgDef[]) { - if (!arg.positional) { - ret += ` --${arg.name}${arg.short ? `, -${arg.short}` : ""}${ - arg.enum ? ` {${arg.enum.join(",")}}` : arg.boolean ? "" : " " + arg.name.toLocaleUpperCase() - }: ${arg.description}\n`; - } + if (options.length > 0) { + ret += `\nOptions:\n`; + for (const arg of options) { + const nameAndAlias = `--${arg.name}${arg.short ? `, -${arg.short}` : ""}`; + const valueHint = arg.enum + ? `{${arg.enum.join("|")}}` + : arg.boolean + ? "" + : `<${arg.name.toUpperCase().replace(/-/g, "_")}>`; + ret += ` ${nameAndAlias}${valueHint ? " " + valueHint : ""}\t${arg.description}${ + arg.default !== undefined + ? ` (default: ${typeof arg.default === "function" ? arg.default() : arg.default})` + : "" + }\n`; } + } + ret += `\n`; + return ret; +} + +function detailedUsageForCommand(commandName: TopLevelCommandName): string { + const commandDef = commands[commandName]; + if ("subcommands" in commandDef) { + return listSubcommands(commandName, commandDef); + } + return _detailedUsage(commandDef.args, usage(commandName), commandDef.description); +} - ret += `\n`; +function detailedUsageForSubcommand( + commandName: TopLevelCommandName, + subCommandName: keyof CommandGroup["subcommands"] +): string { + const commandGroup = commands[commandName]; + if (!("subcommands" in commandGroup) || !(subCommandName in commandGroup.subcommands)) { + throw new Error(`Subcommand ${subCommandName as string} not found for ${commandName}`); } + const subCommandDef = commandGroup.subcommands[subCommandName as keyof typeof commandGroup.subcommands]; + return _detailedUsage(subCommandDef.args, usage(commandName, subCommandName as string), subCommandDef.description); +} +function listSubcommands(commandName: TopLevelCommandName, commandGroup: CommandGroup): string { + let ret = `usage: hfjs ${commandName} [options]\n\n`; + ret += `${commandGroup.description}\n\n`; + ret += `Available subcommands for '${commandName}':\n`; + ret += typedEntries(commandGroup.subcommands) + .map(([subName, subDef]) => ` ${subName}\t${subDef.description}`) + .join("\n"); + ret += `\n\nRun \`hfjs help ${commandName} \` for more information on a specific subcommand.`; return ret; } -function advParseArgs( +type ParsedArgsResult = { + [K in TArgsDef[number] as Camelize]: K["boolean"] extends true + ? boolean + : K["required"] extends true + ? string + : K["default"] extends undefined + ? string | undefined // Optional strings without default can be undefined + : string; // Strings with default or required are strings +}; + +function advParseArgs( args: string[], - commandName: C -): { - // Todo : better typing - [key in Camelize<(typeof commands)[C]["args"][number]["name"]>]: string; -} { + argDefs: TArgsDef, + commandNameForError: string +): ParsedArgsResult { const { tokens } = parseArgs({ options: Object.fromEntries( - (commands[commandName].args as ArgDef[]) + argDefs .filter((arg) => !arg.positional) .map((arg) => { - const option = { - name: arg.name, - ...(arg.short && { short: arg.short }), - type: arg.boolean ? "boolean" : "string", + const optionConfig = { + type: arg.boolean ? ("boolean" as const) : ("string" as const), + short: arg.short, default: typeof arg.default === "function" ? arg.default() : arg.default, - } as const; - return [arg.name, option]; + }; + if (arg.short) { + optionConfig.short = arg.short; + } + if (arg.default !== undefined) { + optionConfig.default = typeof arg.default === "function" ? arg.default() : arg.default; + } + return [arg.name, optionConfig]; }) ), args, allowPositionals: true, - strict: false, + strict: false, // We do custom validation based on tokens and argDefs tokens: true, }); - const command = commands[commandName]; - const expectedPositionals = (command.args as ArgDef[]).filter((arg) => arg.positional); - const requiredPositionals = expectedPositionals.filter((arg) => arg.required).length; - const providedPositionals = tokens.filter((token) => token.kind === "positional").length; + const expectedPositionals = argDefs.filter((arg) => arg.positional); + const providedPositionalTokens = tokens.filter((token) => token.kind === "positional"); - if (providedPositionals < requiredPositionals) { + if (providedPositionalTokens.length < expectedPositionals.filter((arg) => arg.required).length) { throw new Error( - `Missing required positional arguments. Expected: ${requiredPositionals}, Provided: ${providedPositionals}` + `Command '${commandNameForError}': Missing required positional arguments. Usage: hfjs ${usage( + commandNameForError.split(" ")[0] as TopLevelCommandName, + commandNameForError.split(" ")[1] + )}` ); } - if (providedPositionals > expectedPositionals.length) { + if (providedPositionalTokens.length > expectedPositionals.length) { throw new Error( - `Too many positional arguments. Expected: ${expectedPositionals.length}, Provided: ${providedPositionals}` + `Command '${commandNameForError}': Too many positional arguments. Usage: hfjs ${usage( + commandNameForError.split(" ")[0] as TopLevelCommandName, + commandNameForError.split(" ")[1] + )}` ); } - const positionals = Object.fromEntries( - tokens - .filter((token): token is { kind: "positional"; index: number; value: string } => token.kind === "positional") - .map((token, i) => [expectedPositionals[i].name, token.value]) - ); - - const options = Object.fromEntries( - tokens - .filter((token): token is OptionToken => token.kind === "option") - .map((token) => { - const arg = (command.args as ArgDef[]).find((arg) => arg.name === token.name || arg.short === token.name); - if (!arg) { - throw new Error(`Unknown option: ${token.name}`); - } + const result: Record = {}; - if (!arg.boolean) { - if (!token.value) { - throw new Error(`Missing value for option: ${token.name}: ${JSON.stringify(token)}`); - } + // Populate from defaults first + for (const argDef of argDefs) { + if (argDef.default !== undefined) { + result[argDef.name] = typeof argDef.default === "function" ? argDef.default() : argDef.default; + } else if (argDef.boolean) { + result[argDef.name] = false; // Booleans default to false if no other default + } + } - if (arg.enum && !arg.enum.includes(token.value)) { - throw new Error(`Invalid value for option ${token.name}. Expected one of: ${arg.enum.join(", ")}`); - } + // Populate positionals + providedPositionalTokens.forEach((token, i) => { + if (expectedPositionals[i]) { + result[expectedPositionals[i].name] = token.value; + } + }); + + // Populate options from tokens, overriding defaults + tokens + .filter((token): token is OptionToken => token.kind === "option") + .forEach((token) => { + const argDef = argDefs.find((def) => def.name === token.name || def.short === token.name); + if (!argDef) { + throw new Error(`Command '${commandNameForError}': Unknown option: ${token.rawName}`); + } + + if (argDef.boolean) { + result[argDef.name] = true; + } else { + if (token.value === undefined) { + throw new Error(`Command '${commandNameForError}': Missing value for option: ${token.rawName}`); + } + if (argDef.enum && !argDef.enum.includes(token.value)) { + throw new Error( + `Command '${commandNameForError}': Invalid value '${token.value}' for option ${ + token.rawName + }. Expected one of: ${argDef.enum.join(", ")}` + ); } + result[argDef.name] = token.value; + } + }); + + // Final check for required arguments + for (const argDef of argDefs) { + if (argDef.required && result[argDef.name] === undefined) { + throw new Error(`Command '${commandNameForError}': Missing required argument: ${argDef.name}`); + } + } - return [arg.name, arg.boolean ? true : token.value]; - }) - ); - const defaults = Object.fromEntries( - (commands[commandName].args as ArgDef[]) - .filter((arg) => arg.default) - .map((arg) => { - const value = typeof arg.default === "function" ? arg.default() : arg.default; - return [arg.name, value]; - }) - ); return Object.fromEntries( - Object.entries({ ...defaults, ...positionals, ...options }).map(([name, val]) => [kebabToCamelCase(name), val]) - ) as { - [key in Camelize<(typeof commands)[C]["args"][number]["name"]>]: string; - }; + Object.entries(result).map(([name, val]) => [kebabToCamelCase(name), val]) + ) as ParsedArgsResult; } function kebabToCamelCase(str: string) { diff --git a/packages/inference/src/lib/getDefaultTask.ts b/packages/inference/src/lib/getDefaultTask.ts index 38133d379b..51ddbde27b 100644 --- a/packages/inference/src/lib/getDefaultTask.ts +++ b/packages/inference/src/lib/getDefaultTask.ts @@ -53,7 +53,8 @@ export async function getDefaultTask( taskCache.set(key, { task: modelTask, date: new Date() }); if (taskCache.size > MAX_CACHE_ITEMS) { - taskCache.delete(taskCache.keys().next().value); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + taskCache.delete(taskCache.keys().next().value!); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bde5555164..ac4e071d92 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,10 +13,10 @@ importers: version: 22.14.1 '@typescript-eslint/eslint-plugin': specifier: ^7.2.0 - version: 7.2.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.4.2))(eslint@8.57.0)(typescript@5.4.2) + version: 7.2.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.8.3))(eslint@8.57.0)(typescript@5.8.3) '@typescript-eslint/parser': specifier: ^7.2.0 - version: 7.2.0(eslint@8.57.0)(typescript@5.4.2) + version: 7.2.0(eslint@8.57.0)(typescript@5.8.3) '@vitest/browser': specifier: ^0.34.6 version: 0.34.6(esbuild@0.17.16)(rollup@4.6.0)(vitest@0.34.6) @@ -49,22 +49,22 @@ importers: version: 3.0.2 tsup: specifier: ^6.7.0 - version: 6.7.0(postcss@8.4.31)(typescript@5.4.2) + version: 6.7.0(postcss@8.4.31)(typescript@5.8.3) tsx: specifier: ^4.7.0 version: 4.7.0 typescript: - specifier: ^5.4.2 - version: 5.4.2 + specifier: ^5.8.3 + version: 5.8.3 vite: specifier: ^5.0.2 version: 5.0.2(@types/node@22.14.1) vitest: specifier: ^0.34.6 - version: 0.34.6(@vitest/browser@0.34.6)(webdriverio@8.6.7(typescript@5.4.2)) + version: 0.34.6(@vitest/browser@0.34.6)(webdriverio@8.6.7(typescript@5.8.3)) webdriverio: specifier: ^8.6.7 - version: 8.6.7(typescript@5.4.2) + version: 8.6.7(typescript@5.8.3) packages: @@ -2631,13 +2631,8 @@ packages: resolution: {integrity: sha512-/OxDN6OtAk5KBpGb28T+HZc2M+ADtvRxXrKKbUwtsLgdoxgX13hyy7ek6bFRl5+aBs2yZzB0c4CnQfAtVypW/g==} engines: {node: '>= 0.4'} - typescript@5.4.2: - resolution: {integrity: sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ==} - engines: {node: '>=14.17'} - hasBin: true - - typescript@5.6.3: - resolution: {integrity: sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==} + typescript@5.8.3: + resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==} engines: {node: '>=14.17'} hasBin: true @@ -3249,13 +3244,13 @@ snapshots: '@types/node': 18.19.61 optional: true - '@typescript-eslint/eslint-plugin@7.2.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.4.2))(eslint@8.57.0)(typescript@5.4.2)': + '@typescript-eslint/eslint-plugin@7.2.0(@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.8.3))(eslint@8.57.0)(typescript@5.8.3)': dependencies: '@eslint-community/regexpp': 4.10.0 - '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.4.2) + '@typescript-eslint/parser': 7.2.0(eslint@8.57.0)(typescript@5.8.3) '@typescript-eslint/scope-manager': 7.2.0 - '@typescript-eslint/type-utils': 7.2.0(eslint@8.57.0)(typescript@5.4.2) - '@typescript-eslint/utils': 7.2.0(eslint@8.57.0)(typescript@5.4.2) + '@typescript-eslint/type-utils': 7.2.0(eslint@8.57.0)(typescript@5.8.3) + '@typescript-eslint/utils': 7.2.0(eslint@8.57.0)(typescript@5.8.3) '@typescript-eslint/visitor-keys': 7.2.0 debug: 4.3.4 eslint: 8.57.0 @@ -3263,22 +3258,22 @@ snapshots: ignore: 5.2.4 natural-compare: 1.4.0 semver: 7.6.0 - ts-api-utils: 1.3.0(typescript@5.4.2) + ts-api-utils: 1.3.0(typescript@5.8.3) optionalDependencies: - typescript: 5.4.2 + typescript: 5.8.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.4.2)': + '@typescript-eslint/parser@7.2.0(eslint@8.57.0)(typescript@5.8.3)': dependencies: '@typescript-eslint/scope-manager': 7.2.0 '@typescript-eslint/types': 7.2.0 - '@typescript-eslint/typescript-estree': 7.2.0(typescript@5.4.2) + '@typescript-eslint/typescript-estree': 7.2.0(typescript@5.8.3) '@typescript-eslint/visitor-keys': 7.2.0 debug: 4.3.4 eslint: 8.57.0 optionalDependencies: - typescript: 5.4.2 + typescript: 5.8.3 transitivePeerDependencies: - supports-color @@ -3287,21 +3282,21 @@ snapshots: '@typescript-eslint/types': 7.2.0 '@typescript-eslint/visitor-keys': 7.2.0 - '@typescript-eslint/type-utils@7.2.0(eslint@8.57.0)(typescript@5.4.2)': + '@typescript-eslint/type-utils@7.2.0(eslint@8.57.0)(typescript@5.8.3)': dependencies: - '@typescript-eslint/typescript-estree': 7.2.0(typescript@5.4.2) - '@typescript-eslint/utils': 7.2.0(eslint@8.57.0)(typescript@5.4.2) + '@typescript-eslint/typescript-estree': 7.2.0(typescript@5.8.3) + '@typescript-eslint/utils': 7.2.0(eslint@8.57.0)(typescript@5.8.3) debug: 4.3.4 eslint: 8.57.0 - ts-api-utils: 1.3.0(typescript@5.4.2) + ts-api-utils: 1.3.0(typescript@5.8.3) optionalDependencies: - typescript: 5.4.2 + typescript: 5.8.3 transitivePeerDependencies: - supports-color '@typescript-eslint/types@7.2.0': {} - '@typescript-eslint/typescript-estree@7.2.0(typescript@5.4.2)': + '@typescript-eslint/typescript-estree@7.2.0(typescript@5.8.3)': dependencies: '@typescript-eslint/types': 7.2.0 '@typescript-eslint/visitor-keys': 7.2.0 @@ -3310,20 +3305,20 @@ snapshots: is-glob: 4.0.3 minimatch: 9.0.3 semver: 7.6.0 - ts-api-utils: 1.3.0(typescript@5.4.2) + ts-api-utils: 1.3.0(typescript@5.8.3) optionalDependencies: - typescript: 5.4.2 + typescript: 5.8.3 transitivePeerDependencies: - supports-color - '@typescript-eslint/utils@7.2.0(eslint@8.57.0)(typescript@5.4.2)': + '@typescript-eslint/utils@7.2.0(eslint@8.57.0)(typescript@5.8.3)': dependencies: '@eslint-community/eslint-utils': 4.4.0(eslint@8.57.0) '@types/json-schema': 7.0.15 '@types/semver': 7.5.8 '@typescript-eslint/scope-manager': 7.2.0 '@typescript-eslint/types': 7.2.0 - '@typescript-eslint/typescript-estree': 7.2.0(typescript@5.4.2) + '@typescript-eslint/typescript-estree': 7.2.0(typescript@5.8.3) eslint: 8.57.0 semver: 7.6.0 transitivePeerDependencies: @@ -3343,7 +3338,7 @@ snapshots: magic-string: 0.30.5 modern-node-polyfills: 1.0.0(esbuild@0.17.16)(rollup@4.6.0) sirv: 2.0.3 - vitest: 0.34.6(@vitest/browser@0.34.6)(webdriverio@8.6.7(typescript@5.4.2)) + vitest: 0.34.6(@vitest/browser@0.34.6)(webdriverio@8.6.7(typescript@5.8.3)) transitivePeerDependencies: - esbuild - rollup @@ -3815,7 +3810,7 @@ snapshots: devtools-protocol@0.0.1119014: {} - devtools@8.6.6(typescript@5.4.2): + devtools@8.6.6(typescript@5.8.3): dependencies: '@types/node': 18.19.61 '@wdio/config': 8.6.6 @@ -3826,7 +3821,7 @@ snapshots: chrome-launcher: 0.15.1 edge-paths: 3.0.5 import-meta-resolve: 2.2.2 - puppeteer-core: 19.7.5(typescript@5.4.2) + puppeteer-core: 19.7.5(typescript@5.8.3) query-selector-shadow-dom: 1.0.1 ua-parser-js: 1.0.34 uuid: 9.0.0 @@ -5015,7 +5010,7 @@ snapshots: punycode@2.3.0: {} - puppeteer-core@19.7.5(typescript@5.4.2): + puppeteer-core@19.7.5(typescript@5.8.3): dependencies: chromium-bidi: 0.4.5(devtools-protocol@0.0.1094867) cross-fetch: 3.1.5 @@ -5029,7 +5024,7 @@ snapshots: unbzip2-stream: 1.4.3 ws: 8.12.1 optionalDependencies: - typescript: 5.4.2 + typescript: 5.8.3 transitivePeerDependencies: - bufferutil - encoding @@ -5440,9 +5435,9 @@ snapshots: tree-kill@1.2.2: {} - ts-api-utils@1.3.0(typescript@5.4.2): + ts-api-utils@1.3.0(typescript@5.8.3): dependencies: - typescript: 5.4.2 + typescript: 5.8.3 ts-interface-checker@0.1.13: {} @@ -5457,10 +5452,10 @@ snapshots: resolve-import: 2.0.0 rimraf: 6.0.1 sync-content: 2.0.1 - typescript: 5.6.3 + typescript: 5.8.3 walk-up-path: 4.0.0 - tsup@6.7.0(postcss@8.4.31)(typescript@5.4.2): + tsup@6.7.0(postcss@8.4.31)(typescript@5.8.3): dependencies: bundle-require: 4.0.1(esbuild@0.17.16) cac: 6.7.14 @@ -5478,7 +5473,7 @@ snapshots: tree-kill: 1.2.2 optionalDependencies: postcss: 8.4.31 - typescript: 5.4.2 + typescript: 5.8.3 transitivePeerDependencies: - supports-color - ts-node @@ -5532,9 +5527,7 @@ snapshots: is-typed-array: 1.1.13 possible-typed-array-names: 1.0.0 - typescript@5.4.2: {} - - typescript@5.6.3: {} + typescript@5.8.3: {} ua-parser-js@1.0.34: {} @@ -5605,7 +5598,7 @@ snapshots: '@types/node': 22.14.1 fsevents: 2.3.3 - vitest@0.34.6(@vitest/browser@0.34.6)(webdriverio@8.6.7(typescript@5.4.2)): + vitest@0.34.6(@vitest/browser@0.34.6)(webdriverio@8.6.7(typescript@5.8.3)): dependencies: '@types/chai': 4.3.11 '@types/chai-subset': 1.3.5 @@ -5633,7 +5626,7 @@ snapshots: why-is-node-running: 2.2.2 optionalDependencies: '@vitest/browser': 0.34.6(esbuild@0.17.16)(rollup@4.6.0)(vitest@0.34.6) - webdriverio: 8.6.7(typescript@5.4.2) + webdriverio: 8.6.7(typescript@5.8.3) transitivePeerDependencies: - less - lightningcss @@ -5662,7 +5655,7 @@ snapshots: - bufferutil - utf-8-validate - webdriverio@8.6.7(typescript@5.4.2): + webdriverio@8.6.7(typescript@5.8.3): dependencies: '@types/node': 18.19.61 '@wdio/config': 8.6.6 @@ -5675,7 +5668,7 @@ snapshots: aria-query: 5.1.3 css-shorthand-properties: 1.1.1 css-value: 0.0.1 - devtools: 8.6.6(typescript@5.4.2) + devtools: 8.6.6(typescript@5.8.3) devtools-protocol: 0.0.1119014 grapheme-splitter: 1.0.4 import-meta-resolve: 2.2.2 @@ -5683,7 +5676,7 @@ snapshots: lodash.clonedeep: 4.5.0 lodash.zip: 4.2.0 minimatch: 7.4.2 - puppeteer-core: 19.7.5(typescript@5.4.2) + puppeteer-core: 19.7.5(typescript@5.8.3) query-selector-shadow-dom: 1.0.1 resq: 1.11.0 rgb2hex: 0.2.5 From a597f08dc276ae456fb1d2fe59d5646bfdbeba64 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 14 May 2025 17:14:12 +0200 Subject: [PATCH 32/34] fixup! create-branch => branch create and delete-branch => branch delete --- packages/hub/cli.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index cf6d6c5a3c..aedd8fc32c 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -539,15 +539,11 @@ function advParseArgs( .map((arg) => { const optionConfig = { type: arg.boolean ? ("boolean" as const) : ("string" as const), - short: arg.short, - default: typeof arg.default === "function" ? arg.default() : arg.default, + ...(arg.short && { short: arg.short }), + ...(arg.default !== undefined && { + default: typeof arg.default === "function" ? arg.default() : arg.default, + }), }; - if (arg.short) { - optionConfig.short = arg.short; - } - if (arg.default !== undefined) { - optionConfig.default = typeof arg.default === "function" ? arg.default() : arg.default; - } return [arg.name, optionConfig]; }) ), From e0ad6f3ea37bd7691b48da31cb8d121d87c32151 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 14 May 2025 17:23:02 +0200 Subject: [PATCH 33/34] also upgrade typescriptt dep of doc-internal --- packages/doc-internal/pnpm-lock.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/doc-internal/pnpm-lock.yaml b/packages/doc-internal/pnpm-lock.yaml index 616ba1534e..b70901db22 100644 --- a/packages/doc-internal/pnpm-lock.yaml +++ b/packages/doc-internal/pnpm-lock.yaml @@ -20,10 +20,10 @@ importers: version: 18.14.5 typedoc: specifier: ^0.25.12 - version: 0.25.12(typescript@5.4.2) + version: 0.25.12(typescript@5.8.3) typedoc-plugin-markdown: specifier: ^3.17.1 - version: 3.17.1(typedoc@0.25.12(typescript@5.4.2)) + version: 3.17.1(typedoc@0.25.12(typescript@5.8.3)) packages: @@ -107,8 +107,8 @@ packages: peerDependencies: typescript: 4.6.x || 4.7.x || 4.8.x || 4.9.x || 5.0.x || 5.1.x || 5.2.x || 5.3.x || 5.4.x - typescript@5.4.2: - resolution: {integrity: sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ==} + typescript@5.8.3: + resolution: {integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==} engines: {node: '>=14.17'} hasBin: true @@ -196,20 +196,20 @@ snapshots: source-map@0.6.1: {} - typedoc-plugin-markdown@3.17.1(typedoc@0.25.12(typescript@5.4.2)): + typedoc-plugin-markdown@3.17.1(typedoc@0.25.12(typescript@5.8.3)): dependencies: handlebars: 4.7.7 - typedoc: 0.25.12(typescript@5.4.2) + typedoc: 0.25.12(typescript@5.8.3) - typedoc@0.25.12(typescript@5.4.2): + typedoc@0.25.12(typescript@5.8.3): dependencies: lunr: 2.3.9 marked: 4.3.0 minimatch: 9.0.3 shiki: 0.14.7 - typescript: 5.4.2 + typescript: 5.8.3 - typescript@5.4.2: {} + typescript@5.8.3: {} uglify-js@3.17.4: optional: true From a4a5a8db31ef0f83f93c2988aebb62acbcd5c170 Mon Sep 17 00:00:00 2001 From: coyotte508 Date: Wed, 14 May 2025 17:31:48 +0200 Subject: [PATCH 34/34] always display full stack trace --- packages/hub/cli.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/hub/cli.ts b/packages/hub/cli.ts index aedd8fc32c..be861ace5a 100644 --- a/packages/hub/cli.ts +++ b/packages/hub/cli.ts @@ -413,9 +413,9 @@ async function run() { } run().catch((err) => { console.error("\x1b[31mError:\x1b[0m", err.message); - if (process.env.DEBUG) { - console.error(err); - } + //if (process.env.DEBUG) { + console.error(err); + // } process.exitCode = 1; });