♻️ Move generation code to tasks-gen

Author: coyotte508

packages/tasks-gen/package.json

@@ -0,0 +1,26 @@
+{
+	"name": "@huggingface/tasks-gen",
+	"packageManager": "[email protected]",
+	"version": "0.13.1",
+	"description": "Utilities to generate files for @huggingface/tasks, and avoid unnecessary deps in the published package",
+	"repository": "https://github.com/huggingface/huggingface.js.git",
+	"private": true,
+	"scripts": {
+		"lint": "eslint --quiet --fix --ext .cjs,.ts .",
+		"lint:check": "eslint --ext .cjs,.ts .",
+		"format": "prettier --write .",
+		"format:check": "prettier --check .",
+		"check": "tsc",
+		"inference-codegen": "tsx scripts/inference-codegen.ts && prettier --write ../tasks/src/tasks/*/inference.ts",
+		"inference-tgi-import": "tsx scripts/inference-tgi-import.ts && prettier --write ../tasks/src/tasks/text-generation/spec/*.json && prettier --write ../tasks/src/tasks/chat-completion/spec/*.json",
+		"inference-tei-import": "tsx scripts/inference-tei-import.ts && prettier --write ../tasks/src/tasks/feature-extraction/spec/*.json"
+	},
+	"type": "module",
+	"author": "Hugging Face",
+	"license": "MIT",
+	"devDependencies": {
+		"@types/node": "^20.11.5",
+		"quicktype-core": "https://github.com/huggingface/quicktype/raw/pack-18.0.17/packages/quicktype-core/quicktype-core-18.0.17.tgz",
+		"type-fest": "^3.13.1"
+	}
+}

packages/tasks-gen/pnpm-lock.yaml

@@ -22,8 +22,6 @@ const PYTHON_HEADER_FILE = `
 #   - specs:  https://github.com/huggingface/huggingface.js/tree/main/packages/tasks/src/tasks.
 `;
 
-const PYTHON_DIR = "./.python_generated";
-
 const rootDirFinder = function (): string {
 	let currentPath = path.normalize(import.meta.url);
 
@@ -172,8 +170,10 @@ async function postProcessOutput(path2generated: string, outputSpec: Record<stri
 	return;
 }
 
-const rootDir = rootDirFinder();
+const rootDir = path.join(rootDirFinder(), "..", "tasks");
 const tasksDir = path.join(rootDir, "src", "tasks");
+const PYTHON_DIR = path.join(rootDir, ".python_generated");
+
 const allTasks = await Promise.all(
 	(await fs.readdir(tasksDir, { withFileTypes: true }))
 		.filter((entry) => entry.isDirectory())

packages/tasks/scripts/inference-codegen.ts renamed to packages/tasks-gen/scripts/inference-codegen.ts

@@ -24,7 +24,7 @@ const rootDirFinder = function (): string {
 	return "/";
 };
 
-const rootDir = rootDirFinder();
+const rootDir = path.join(rootDirFinder(), "..", "tasks");
 const tasksDir = path.join(rootDir, "src", "tasks");
 
 function toCamelCase(str: string, joiner = "") {

packages/tasks/scripts/inference-tei-import.ts renamed to packages/tasks-gen/scripts/inference-tei-import.ts

@@ -24,7 +24,7 @@ const rootDirFinder = function (): string {
 	return "/";
 };
 
-const rootDir = rootDirFinder();
+const rootDir = path.join(rootDirFinder(), "..", "tasks");
 const tasksDir = path.join(rootDir, "src", "tasks");
 
 function toCamelCase(str: string, joiner = "") {

packages/tasks/scripts/inference-tgi-import.ts renamed to packages/tasks-gen/scripts/inference-tgi-import.ts

@@ -35,10 +35,7 @@
 		"watch": "npm-run-all --parallel watch:esm watch:cjs",
 		"prepare": "pnpm run build",
 		"check": "tsc",
-		"test": "vitest run",
-		"inference-codegen": "tsx scripts/inference-codegen.ts && prettier --write src/tasks/*/inference.ts",
-		"inference-tgi-import": "tsx scripts/inference-tgi-import.ts && prettier --write src/tasks/text-generation/spec/*.json && prettier --write src/tasks/chat-completion/spec/*.json",
-		"inference-tei-import": "tsx scripts/inference-tei-import.ts && prettier --write src/tasks/feature-extraction/spec/*.json"
+		"test": "vitest run"
 	},
 	"type": "module",
 	"files": [
@@ -51,11 +48,6 @@
 	],
 	"author": "Hugging Face",
 	"license": "MIT",
-	"devDependencies": {
-		"@types/node": "^20.11.5",
-		"quicktype-core": "https://github.com/huggingface/quicktype/raw/pack-18.0.17/packages/quicktype-core/quicktype-core-18.0.17.tgz",
-		"type-fest": "^3.13.1"
-	},
 	"tshy": {
 		"exports": {
 			"./package.json": "./package.json",

packages/tasks/package.json

@@ -5,6 +5,7 @@ packages:
   - "packages/agents"
   - "packages/languages"
   - "packages/tasks"
+  - "packages/tasks-gen"
   - "packages/gguf"
   - "packages/jinja"
   - "packages/space-header"