Merge branch 'main' into fix-widget-model-for-image-text-to-text

Author: Wauplin

.github/workflows/tasks-publish.yml

@@ -49,6 +49,44 @@ jobs:
           node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');"
           git commit . -m "🔖 @huggingface/tasks $BUMPED_VERSION"
           git tag "tasks-v$BUMPED_VERSION"
+
+      - name: Make sure that the latest version of @huggingface/gguf is consistent with the local version
+        run: |
+          LOCAL_GGUF_VERSION=$(node -p "require('./package.json').version")
+          REMOTE_GGUF_VERSION=$(npm view @huggingface/gguf version)
+
+          # If the versions are different, error
+          if [ "$LOCAL_GGUF_VERSION" != "$REMOTE_GGUF_VERSION" ]; then
+            echo "Error: The local @huggingface/gguf package version ($LOCAL_GGUF_VERSION) differs from the remote version ($REMOTE_GGUF_VERSION). Release halted."
+            exit 1
+          fi
+
+          npm pack @huggingface/gguf
+          mv huggingface-gguf-$LOCAL_GGUF_VERSION.tgz gguf-local.tgz
+
+          npm pack @huggingface/gguf@$REMOTE_GGUF_VERSION
+          mv huggingface-gguf-$REMOTE_GGUF_VERSION.tgz gguf-remote.tgz
+
+          # Compute checksum of local tar. We need to extract both tar since the remote compression might be different
+          tar -xf gguf-local.tgz
+          LOCAL_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          echo "Local package checksum: $LOCAL_CHECKSUM"
+
+          rm -Rf package
+
+          tar -xf gguf-remote.tgz
+          REMOTE_CHECKSUM=$(cd package && tar --mtime='1970-01-01' --mode=755 -cf - . | sha256sum | cut -d' ' -f1)
+          echo "Remote package checksum: $REMOTE_CHECKSUM"
+
+          rm -Rf package
+
+          if [ "$LOCAL_CHECKSUM" != "$REMOTE_CHECKSUM" ]; then
+            echo "Checksum Verification Failed: The local @huggingface/gguf package differs from the remote version. Release halted. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM"
+            exit 1
+          fi
+          echo "Checksum Verification Successful: The local and remote @huggingface/gguf packages are consistent. Proceeding with the @huggingface/widgets package release. Local Checksum: $LOCAL_CHECKSUM, Remote Checksum: $REMOTE_CHECKSUM."
+        working-directory: packages/gguf
+
       - run: pnpm publish --no-git-checks .
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/test.yml

@@ -105,7 +105,7 @@ jobs:
         run: |
           sleep 3
           pnpm i --filter root --filter inference... --filter hub... --frozen-lockfile
-          pnpm --filter inference --filter hub --filter tasks publish --force --no-git-checks --registry http://localhost:4874/
+          pnpm --filter inference --filter hub --filter tasks --filter gguf publish --force --no-git-checks --registry http://localhost:4874/
 
       - name: E2E test - test yarn install
         working-directory: e2e/ts

e2e/deno/deno.json

@@ -0,0 +1 @@
+{}

e2e/mock-registry-config.yaml

@@ -4,6 +4,9 @@ auth:
       foo:
         name: test
         password: test
+web:
+  # login with foo / test
+  enable: true
 store:
   memory:
     limit: 1000

packages/tasks/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@huggingface/tasks",
 	"packageManager": "[email protected]",
-	"version": "0.12.22",
+	"version": "0.12.23",
 	"description": "List of ML tasks for huggingface.co/tasks",
 	"repository": "https://github.com/huggingface/huggingface.js.git",
 	"publishConfig": {

packages/tasks/src/model-libraries-snippets.ts

@@ -1,5 +1,5 @@
 import type { ModelData } from "./model-data";
-import type { WidgetExampleTextInput } from "./widget-example";
+import type { WidgetExampleTextInput, WidgetExampleSentenceSimilarityInput } from "./widget-example";
 import { LIBRARY_TASK_MAPPING } from "./library-to-tasks";
 
 const TAG_CUSTOM_CODE = "custom_code";
@@ -704,13 +704,32 @@ export const sampleFactory = (model: ModelData): string[] => [
 	`python -m sample_factory.huggingface.load_from_hub -r ${model.id} -d ./train_dir`,
 ];
 
+function get_widget_examples_from_st_model(model: ModelData): string[] | undefined {
+	const widgetExample = model.widgetData?.[0] as WidgetExampleSentenceSimilarityInput | undefined;
+	if (widgetExample) {
+		return [widgetExample.source_sentence, ...widgetExample.sentences];
+	}
+}
+
 export const sentenceTransformers = (model: ModelData): string[] => {
 	const remote_code_snippet = model.tags.includes(TAG_CUSTOM_CODE) ? ", trust_remote_code=True" : "";
+	const exampleSentences = get_widget_examples_from_st_model(model) ?? [
+		"The weather is lovely today.",
+		"It's so sunny outside!",
+		"He drove to the stadium.",
+	];
 
 	return [
 		`from sentence_transformers import SentenceTransformer
 
-model = SentenceTransformer("${model.id}"${remote_code_snippet})`,
+model = SentenceTransformer("${model.id}"${remote_code_snippet})
+
+sentences = ${JSON.stringify(exampleSentences, null, 4)}
+embeddings = model.encode(sentences)
+
+similarities = model.similarity(embeddings, embeddings)
+print(similarities.shape)
+# [${exampleSentences.length}, ${exampleSentences.length}]`,
 	];
 };