Merge branch 'main' into add-workflow-update-specs

Author: Wauplin

.github/workflows/test.yml

@@ -104,7 +104,7 @@ jobs:
         working-directory: e2e
         run: |
           sleep 3
-          pnpm i --filter root --filter inference... --filter hub... --frozen-lockfile
+          pnpm i --filter root --filter inference... --filter hub... --filter tasks-gen --frozen-lockfile
           pnpm --filter inference --filter hub --filter tasks publish --force --no-git-checks --registry http://localhost:4874/
 
       - name: E2E test - test yarn install

package.json

@@ -23,6 +23,7 @@
 		"prettier": "^3.1.0",
 		"prettier-plugin-svelte": "^3.1.2",
 		"semver": "^7.5.0",
+		"tshy": "^3.0.2",
 		"tsup": "^6.7.0",
 		"tsx": "^4.7.0",
 		"typescript": "^5.4.2",

packages/gguf/src/gguf.spec.ts

@@ -33,7 +33,7 @@ describe("gguf", () => {
 			const arrayBuf = await res.arrayBuffer();
 			fs.writeFileSync(".cache/model.gguf", Buffer.from(arrayBuf));
 		}
-	});
+	}, 30_000);
 
 	it("should parse a llama2 7b", async () => {
 		const { metadata, tensorInfos } = await gguf(URL_LLAMA);

packages/tasks-gen/package.json

@@ -0,0 +1,26 @@
+{
+	"name": "@huggingface/tasks-gen",
+	"packageManager": "[email protected]",
+	"version": "0.13.1",
+	"description": "Utilities to generate files for @huggingface/tasks, and avoid unnecessary deps in the published package",
+	"repository": "https://github.com/huggingface/huggingface.js.git",
+	"private": true,
+	"scripts": {
+		"lint": "eslint --quiet --fix --ext .cjs,.ts .",
+		"lint:check": "eslint --ext .cjs,.ts .",
+		"format": "prettier --write .",
+		"format:check": "prettier --check .",
+		"check": "tsc",
+		"inference-codegen": "tsx scripts/inference-codegen.ts && prettier --write ../tasks/src/tasks/*/inference.ts",
+		"inference-tgi-import": "tsx scripts/inference-tgi-import.ts && prettier --write ../tasks/src/tasks/text-generation/spec/*.json && prettier --write ../tasks/src/tasks/chat-completion/spec/*.json",
+		"inference-tei-import": "tsx scripts/inference-tei-import.ts && prettier --write ../tasks/src/tasks/feature-extraction/spec/*.json"
+	},
+	"type": "module",
+	"author": "Hugging Face",
+	"license": "MIT",
+	"devDependencies": {
+		"@types/node": "^20.11.5",
+		"quicktype-core": "https://github.com/huggingface/quicktype/raw/pack-18.0.17/packages/quicktype-core/quicktype-core-18.0.17.tgz",
+		"type-fest": "^3.13.1"
+	}
+}

packages/tasks-gen/pnpm-lock.yaml

@@ -22,8 +22,6 @@ const PYTHON_HEADER_FILE = `
 #   - specs:  https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks.
 `;
 
-const PYTHON_DIR = "./.python_generated";
-
 const rootDirFinder = function (): string {
 	let currentPath = path.normalize(import.meta.url);
 
@@ -172,8 +170,10 @@ async function postProcessOutput(path2generated: string, outputSpec: Record<stri
 	return;
 }
 
-const rootDir = rootDirFinder();
+const rootDir = path.join(rootDirFinder(), "..", "tasks");
 const tasksDir = path.join(rootDir, "src", "tasks");
+const PYTHON_DIR = path.join(rootDir, ".python_generated");
+
 const allTasks = await Promise.all(
 	(await fs.readdir(tasksDir, { withFileTypes: true }))
 		.filter((entry) => entry.isDirectory())

packages/tasks/scripts/inference-codegen.ts renamed to packages/tasks-gen/scripts/inference-codegen.ts

@@ -24,7 +24,7 @@ const rootDirFinder = function (): string {
 	return "/";
 };
 
-const rootDir = rootDirFinder();
+const rootDir = path.join(rootDirFinder(), "..", "tasks");
 const tasksDir = path.join(rootDir, "src", "tasks");
 
 function toCamelCase(str: string, joiner = "") {

packages/tasks/scripts/inference-tei-import.ts renamed to packages/tasks-gen/scripts/inference-tei-import.ts

@@ -24,7 +24,7 @@ const rootDirFinder = function (): string {
 	return "/";
 };
 
-const rootDir = rootDirFinder();
+const rootDir = path.join(rootDirFinder(), "..", "tasks");
 const tasksDir = path.join(rootDir, "src", "tasks");
 
 function toCamelCase(str: string, joiner = "") {

packages/tasks/scripts/inference-tgi-import.ts renamed to packages/tasks-gen/scripts/inference-tgi-import.ts

@@ -0,0 +1,8 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "rootDir": "../src",
+    "module": "nodenext",
+    "moduleResolution": "nodenext"
+  }
+}

packages/tasks/.tshy/build.json

@@ -0,0 +1,16 @@
+{
+  "extends": "./build.json",
+  "include": [
+    "../src/**/*.ts",
+    "../src/**/*.cts",
+    "../src/**/*.tsx",
+    "../src/**/*.json"
+  ],
+  "exclude": [
+    "../src/**/*.mts",
+    "../src/package.json"
+  ],
+  "compilerOptions": {
+    "outDir": "../.tshy-build/commonjs"
+  }
+}