diff --git a/packages/hub/README.md b/packages/hub/README.md index 8fc8140f9e..6056dd6382 100644 --- a/packages/hub/README.md +++ b/packages/hub/README.md @@ -30,7 +30,7 @@ For some of the calls, you need to create an account and generate an [access tok Learn how to find free models using the hub package in this [interactive tutorial](https://scrimba.com/scrim/c7BbVPcd?pl=pkVnrP7uP). ```ts -import { createRepo, uploadFiles, uploadFilesWithProgress, deleteFile, deleteRepo, listFiles, whoAmI } from "@huggingface/hub"; +import { createRepo, uploadFiles, uploadFilesWithProgress, deleteFile, deleteRepo, listFiles, whoAmI, modelInfo, listModels } from "@huggingface/hub"; import type { RepoDesignation } from "@huggingface/hub"; const repo: RepoDesignation = { type: "model", name: "myname/some-model" }; @@ -41,6 +41,8 @@ for await (const model of listModels({search: {owner: username}, accessToken: "h console.log("My model:", model); } +const specificModel = await modelInfo({name: "openai-community/gpt2"}); + await createRepo({ repo, accessToken: "hf_...", license: "mit" }); await uploadFiles({ diff --git a/packages/hub/src/lib/dataset-info.spec.ts b/packages/hub/src/lib/dataset-info.spec.ts new file mode 100644 index 0000000000..8f944de6d2 --- /dev/null +++ b/packages/hub/src/lib/dataset-info.spec.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import { datasetInfo } from "./dataset-info"; + +describe("datasetInfo", () => { + it("should return the dataset info", async () => { + const info = await datasetInfo({ + name: "nyu-mll/glue", + }); + expect(info).toEqual({ + id: "621ffdd236468d709f181e3f", + downloads: expect.any(Number), + gated: false, + name: "nyu-mll/glue", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + }); + }); +}); diff --git a/packages/hub/src/lib/dataset-info.ts b/packages/hub/src/lib/dataset-info.ts new file mode 100644 index 0000000000..a18022b10f --- /dev/null +++ b/packages/hub/src/lib/dataset-info.ts @@ -0,0 +1,59 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiDatasetInfo } from "../types/api/api-dataset"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { pick } from "../utils/pick"; +import { type DATASET_EXPANDABLE_KEYS, DATASET_EXPAND_KEYS, type DatasetEntry } from "./list-datasets"; + +export async function datasetInfo< + const T extends Exclude<(typeof DATASET_EXPANDABLE_KEYS)[number], (typeof DATASET_EXPAND_KEYS)[number]> = never, +>( + params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * Set to limit the number of models returned. + */ + limit?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial +): Promise> { + const accessToken = params && checkCredentials(params); + + const search = new URLSearchParams([ + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), + ]).toString(); + + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/datasets/${params.name}?${search.toString()}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + Accepts: "application/json", + }, + } + ); + + if (!response.ok) { + createApiError(response); + } + + const data = await response.json(); + + return { + ...(params?.additionalFields && pick(data, params.additionalFields)), + id: data._id, + name: data.id, + private: data.private, + downloads: data.downloads, + likes: data.likes, + gated: data.gated, + updatedAt: new Date(data.lastModified), + } as DatasetEntry & Pick; +} diff --git a/packages/hub/src/lib/index.ts b/packages/hub/src/lib/index.ts index 4630d1764f..070667ef32 100644 --- a/packages/hub/src/lib/index.ts +++ b/packages/hub/src/lib/index.ts @@ -2,6 +2,7 @@ export * from "./cache-management"; export * from "./commit"; export * from "./count-commits"; export * from "./create-repo"; +export * from "./dataset-info"; export * from "./delete-file"; export * from "./delete-files"; export * from "./delete-repo"; @@ -13,9 +14,11 @@ export * from "./list-datasets"; export * from "./list-files"; export * from "./list-models"; export * from "./list-spaces"; +export * from "./model-info"; export * from "./oauth-handle-redirect"; export * from "./oauth-login-url"; export * from "./parse-safetensors-metadata"; +export * from "./space-info"; export * from "./upload-file"; export * from "./upload-files"; export * from "./upload-files-with-progress"; diff --git a/packages/hub/src/lib/list-datasets.ts b/packages/hub/src/lib/list-datasets.ts index 2b9524d10e..5bfbc4a154 100644 --- a/packages/hub/src/lib/list-datasets.ts +++ b/packages/hub/src/lib/list-datasets.ts @@ -6,7 +6,7 @@ import { checkCredentials } from "../utils/checkCredentials"; import { parseLinkHeader } from "../utils/parseLinkHeader"; import { pick } from "../utils/pick"; -const EXPAND_KEYS = [ +export const DATASET_EXPAND_KEYS = [ "private", "downloads", "gated", @@ -14,7 +14,7 @@ const EXPAND_KEYS = [ "lastModified", ] as const satisfies readonly (keyof ApiDatasetInfo)[]; -const EXPANDABLE_KEYS = [ +export const DATASET_EXPANDABLE_KEYS = [ "author", "cardData", "citation", @@ -45,7 +45,7 @@ export interface DatasetEntry { } export async function* listDatasets< - const T extends Exclude<(typeof EXPANDABLE_KEYS)[number], (typeof EXPAND_KEYS)[number]> = never, + const T extends Exclude<(typeof DATASET_EXPANDABLE_KEYS)[number], (typeof DATASET_EXPAND_KEYS)[number]> = never, >( params?: { search?: { @@ -77,7 +77,7 @@ export async function* listDatasets< ...(params?.search?.query ? { search: params.search.query } : undefined), }), ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), - ...EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...DATASET_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), ]).toString(); let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/datasets` + (search ? "?" + search : ""); diff --git a/packages/hub/src/lib/list-models.ts b/packages/hub/src/lib/list-models.ts index c4f0859c8a..0e62c8caf8 100644 --- a/packages/hub/src/lib/list-models.ts +++ b/packages/hub/src/lib/list-models.ts @@ -6,7 +6,7 @@ import { checkCredentials } from "../utils/checkCredentials"; import { parseLinkHeader } from "../utils/parseLinkHeader"; import { pick } from "../utils/pick"; -const EXPAND_KEYS = [ +export const MODEL_EXPAND_KEYS = [ "pipeline_tag", "private", "gated", @@ -15,7 +15,7 @@ const EXPAND_KEYS = [ "lastModified", ] as const satisfies readonly (keyof ApiModelInfo)[]; -const EXPANDABLE_KEYS = [ +export const MODEL_EXPANDABLE_KEYS = [ "author", "cardData", "config", @@ -51,7 +51,7 @@ export interface ModelEntry { } export async function* listModels< - const T extends Exclude<(typeof EXPANDABLE_KEYS)[number], (typeof EXPAND_KEYS)[number]> = never, + const T extends Exclude<(typeof MODEL_EXPANDABLE_KEYS)[number], (typeof MODEL_EXPAND_KEYS)[number]> = never, >( params?: { search?: { @@ -85,7 +85,7 @@ export async function* listModels< ...(params?.search?.query ? { search: params.search.query } : undefined), }), ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), - ...EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), ]).toString(); let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/models?${search}`; diff --git a/packages/hub/src/lib/list-spaces.ts b/packages/hub/src/lib/list-spaces.ts index 11b890ef1b..36a251afde 100644 --- a/packages/hub/src/lib/list-spaces.ts +++ b/packages/hub/src/lib/list-spaces.ts @@ -6,8 +6,13 @@ import { checkCredentials } from "../utils/checkCredentials"; import { parseLinkHeader } from "../utils/parseLinkHeader"; import { pick } from "../utils/pick"; -const EXPAND_KEYS = ["sdk", "likes", "private", "lastModified"] as const satisfies readonly (keyof ApiSpaceInfo)[]; -const EXPANDABLE_KEYS = [ +export const SPACE_EXPAND_KEYS = [ + "sdk", + "likes", + "private", + "lastModified", +] as const satisfies readonly (keyof ApiSpaceInfo)[]; +export const SPACE_EXPANDABLE_KEYS = [ "author", "cardData", "datasets", @@ -37,7 +42,7 @@ export interface SpaceEntry { } export async function* listSpaces< - const T extends Exclude<(typeof EXPANDABLE_KEYS)[number], (typeof EXPAND_KEYS)[number]> = never, + const T extends Exclude<(typeof SPACE_EXPANDABLE_KEYS)[number], (typeof SPACE_EXPAND_KEYS)[number]> = never, >( params?: { search?: { @@ -67,7 +72,9 @@ export async function* listSpaces< ...(params?.search?.query ? { search: params.search.query } : undefined), }), ...(params?.search?.tags?.map((tag) => ["filter", tag]) ?? []), - ...[...EXPAND_KEYS, ...(params?.additionalFields ?? [])].map((val) => ["expand", val] satisfies [string, string]), + ...[...SPACE_EXPAND_KEYS, ...(params?.additionalFields ?? [])].map( + (val) => ["expand", val] satisfies [string, string] + ), ]).toString(); let url: string | undefined = `${params?.hubUrl || HUB_URL}/api/spaces?${search}`; diff --git a/packages/hub/src/lib/model-info.spec.ts b/packages/hub/src/lib/model-info.spec.ts new file mode 100644 index 0000000000..cdbad71c16 --- /dev/null +++ b/packages/hub/src/lib/model-info.spec.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { modelInfo } from "./model-info"; + +describe("modelInfo", () => { + it("should return the model info", async () => { + const info = await modelInfo({ + name: "openai-community/gpt2", + }); + expect(info).toEqual({ + id: "621ffdc036468d709f17434d", + downloads: expect.any(Number), + gated: false, + name: "openai-community/gpt2", + updatedAt: expect.any(Date), + likes: expect.any(Number), + task: "text-generation", + private: false, + }); + }); +}); diff --git a/packages/hub/src/lib/model-info.ts b/packages/hub/src/lib/model-info.ts new file mode 100644 index 0000000000..e551d93896 --- /dev/null +++ b/packages/hub/src/lib/model-info.ts @@ -0,0 +1,60 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiModelInfo } from "../types/api/api-model"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { pick } from "../utils/pick"; +import { MODEL_EXPAND_KEYS, type MODEL_EXPANDABLE_KEYS, type ModelEntry } from "./list-models"; + +export async function modelInfo< + const T extends Exclude<(typeof MODEL_EXPANDABLE_KEYS)[number], (typeof MODEL_EXPANDABLE_KEYS)[number]> = never, +>( + params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * Set to limit the number of models returned. + */ + limit?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial +): Promise> { + const accessToken = params && checkCredentials(params); + + const search = new URLSearchParams([ + ...MODEL_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), + ]).toString(); + + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/models/${params.name}?${search.toString()}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + Accepts: "application/json", + }, + } + ); + + if (!response.ok) { + createApiError(response); + } + + const data = await response.json(); + + return { + ...(params?.additionalFields && pick(data, params.additionalFields)), + id: data._id, + name: data.id, + private: data.private, + task: data.pipeline_tag, + downloads: data.downloads, + gated: data.gated, + likes: data.likes, + updatedAt: new Date(data.lastModified), + } as ModelEntry & Pick; +} diff --git a/packages/hub/src/lib/space-info.spec.ts b/packages/hub/src/lib/space-info.spec.ts new file mode 100644 index 0000000000..1974a04c40 --- /dev/null +++ b/packages/hub/src/lib/space-info.spec.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from "vitest"; +import { spaceInfo } from "./space-info"; + +describe("spaceInfo", () => { + it("should return the space info", async () => { + const info = await spaceInfo({ + name: "huggingfacejs/client-side-oauth", + }); + expect(info).toEqual({ + id: "659835e689010f9c7aed608d", + name: "huggingfacejs/client-side-oauth", + updatedAt: expect.any(Date), + likes: expect.any(Number), + private: false, + sdk: "static", + }); + }); +}); diff --git a/packages/hub/src/lib/space-info.ts b/packages/hub/src/lib/space-info.ts new file mode 100644 index 0000000000..d6dcefc184 --- /dev/null +++ b/packages/hub/src/lib/space-info.ts @@ -0,0 +1,59 @@ +import { HUB_URL } from "../consts"; +import { createApiError } from "../error"; +import type { ApiSpaceInfo } from "../types/api/api-space"; +import type { CredentialsParams } from "../types/public"; +import { checkCredentials } from "../utils/checkCredentials"; +import { pick } from "../utils/pick"; +import type { SPACE_EXPANDABLE_KEYS, SpaceEntry } from "./list-spaces"; +import { SPACE_EXPAND_KEYS } from "./list-spaces"; + +export async function spaceInfo< + const T extends Exclude<(typeof SPACE_EXPANDABLE_KEYS)[number], (typeof SPACE_EXPAND_KEYS)[number]> = never, +>( + params: { + name: string; + hubUrl?: string; + additionalFields?: T[]; + /** + * Set to limit the number of models returned. + */ + limit?: number; + /** + * Custom fetch function to use instead of the default one, for example to use a proxy or edit headers. + */ + fetch?: typeof fetch; + } & Partial +): Promise> { + const accessToken = params && checkCredentials(params); + + const search = new URLSearchParams([ + ...SPACE_EXPAND_KEYS.map((val) => ["expand", val] satisfies [string, string]), + ...(params?.additionalFields?.map((val) => ["expand", val] satisfies [string, string]) ?? []), + ]).toString(); + + const response = await (params.fetch || fetch)( + `${params?.hubUrl || HUB_URL}/api/spaces/${params.name}?${search.toString()}`, + { + headers: { + ...(accessToken ? { Authorization: `Bearer ${accessToken}` } : {}), + Accepts: "application/json", + }, + } + ); + + if (!response.ok) { + createApiError(response); + } + + const data = await response.json(); + + return { + ...(params?.additionalFields && pick(data, params.additionalFields)), + id: data._id, + name: data.id, + sdk: data.sdk, + likes: data.likes, + private: data.private, + updatedAt: new Date(data.lastModified), + } as SpaceEntry & Pick; +}