Skip to content

Commit f00b6aa

Browse files
mishig25mishig25
authored
[Conversational Snippet] Test non-streaming versions (huggingface#1027)
[Conversational Snippet] Test non-streaming versions
1 parent 27b995d commit f00b6aa

File tree

5 files changed

+137
-15
lines changed

5 files changed

+137
-15
lines changed

‎packages/tasks/src/snippets/curl.spec.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { ModelDataMinimal } from "./types";
22
import { describe, expect, it } from "vitest";
3-
import { snippetTextGeneration } from "./curl";
3+
import { getCurlInferenceSnippet } from "./curl";
44

55
describe("inference API snippets", () => {
66
it("conversational llm", async () => {
@@ -10,7 +10,7 @@ describe("inference API snippets", () => {
1010
tags: ["conversational"],
1111
inference: "",
1212
};
13-
const snippet = snippetTextGeneration(model, "api_token");
13+
const snippet = getCurlInferenceSnippet(model, "api_token");
1414

1515
expect(snippet.content)
1616
.toEqual(`curl 'https://api-inference.huggingface.co/models/meta-llama/Llama-3.1-8B-Instruct/v1/chat/completions' \\
@@ -29,14 +29,40 @@ describe("inference API snippets", () => {
2929
}'`);
3030
});
3131

32+
it("conversational llm non-streaming", async () => {
33+
const model: ModelDataMinimal = {
34+
id: "meta-llama/Llama-3.1-8B-Instruct",
35+
pipeline_tag: "text-generation",
36+
tags: ["conversational"],
37+
inference: "",
38+
};
39+
const snippet = getCurlInferenceSnippet(model, "api_token", { streaming: false });
40+
41+
expect(snippet.content)
42+
.toEqual(`curl 'https://api-inference.huggingface.co/models/meta-llama/Llama-3.1-8B-Instruct/v1/chat/completions' \\
43+
-H "Authorization: Bearer api_token" \\
44+
-H 'Content-Type: application/json' \\
45+
--data '{
46+
"model": "meta-llama/Llama-3.1-8B-Instruct",
47+
"messages": [
48+
{
49+
"role": "user",
50+
"content": "What is the capital of France?"
51+
}
52+
],
53+
"max_tokens": 500,
54+
"stream": false
55+
}'`);
56+
});
57+
3258
it("conversational vlm", async () => {
3359
const model: ModelDataMinimal = {
3460
id: "meta-llama/Llama-3.2-11B-Vision-Instruct",
3561
pipeline_tag: "image-text-to-text",
3662
tags: ["conversational"],
3763
inference: "",
3864
};
39-
const snippet = snippetTextGeneration(model, "api_token");
65+
const snippet = getCurlInferenceSnippet(model, "api_token");
4066

4167
expect(snippet.content)
4268
.toEqual(`curl 'https://api-inference.huggingface.co/models/meta-llama/Llama-3.2-11B-Vision-Instruct/v1/chat/completions' \\

‎packages/tasks/src/snippets/curl.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,13 @@ export const curlSnippets: Partial<
105105
"image-segmentation": snippetFile,
106106
};
107107

108-
export function getCurlInferenceSnippet(model: ModelDataMinimal, accessToken: string): InferenceSnippet {
108+
export function getCurlInferenceSnippet(
109+
model: ModelDataMinimal,
110+
accessToken: string,
111+
opts?: Record<string, unknown>
112+
): InferenceSnippet {
109113
return model.pipeline_tag && model.pipeline_tag in curlSnippets
110-
? curlSnippets[model.pipeline_tag]?.(model, accessToken) ?? { content: "" }
114+
? curlSnippets[model.pipeline_tag]?.(model, accessToken, opts) ?? { content: "" }
111115
: { content: "" };
112116
}
113117

‎packages/tasks/src/snippets/js.spec.ts‎

Lines changed: 65 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import type { InferenceSnippet, ModelDataMinimal } from "./types";
22
import { describe, expect, it } from "vitest";
3-
import { snippetTextGeneration } from "./js";
3+
import { getJsInferenceSnippet } from "./js";
44

55
describe("inference API snippets", () => {
66
it("conversational llm", async () => {
@@ -10,7 +10,7 @@ describe("inference API snippets", () => {
1010
tags: ["conversational"],
1111
inference: "",
1212
};
13-
const snippet = snippetTextGeneration(model, "api_token") as InferenceSnippet[];
13+
const snippet = getJsInferenceSnippet(model, "api_token") as InferenceSnippet[];
1414

1515
expect(snippet[0].content).toEqual(`import { HfInference } from "@huggingface/inference"
1616
@@ -38,14 +38,41 @@ for await (const chunk of stream) {
3838
}`);
3939
});
4040

41+
it("conversational llm non-streaming", async () => {
42+
const model: ModelDataMinimal = {
43+
id: "meta-llama/Llama-3.1-8B-Instruct",
44+
pipeline_tag: "text-generation",
45+
tags: ["conversational"],
46+
inference: "",
47+
};
48+
const snippet = getJsInferenceSnippet(model, "api_token", { streaming: false }) as InferenceSnippet[];
49+
50+
expect(snippet[0].content).toEqual(`import { HfInference } from "@huggingface/inference"
51+
52+
const client = new HfInference("api_token")
53+
54+
const chatCompletion = await client.chatCompletion({
55+
model: "meta-llama/Llama-3.1-8B-Instruct",
56+
messages: [
57+
{
58+
role: "user",
59+
content: "What is the capital of France?"
60+
}
61+
],
62+
max_tokens: 500
63+
});
64+
65+
console.log(chatCompletion.choices[0].message);`);
66+
});
67+
4168
it("conversational vlm", async () => {
4269
const model: ModelDataMinimal = {
4370
id: "meta-llama/Llama-3.2-11B-Vision-Instruct",
4471
pipeline_tag: "image-text-to-text",
4572
tags: ["conversational"],
4673
inference: "",
4774
};
48-
const snippet = snippetTextGeneration(model, "api_token") as InferenceSnippet[];
75+
const snippet = getJsInferenceSnippet(model, "api_token") as InferenceSnippet[];
4976

5077
expect(snippet[0].content).toEqual(`import { HfInference } from "@huggingface/inference"
5178
@@ -75,6 +102,41 @@ const stream = client.chatCompletionStream({
75102
max_tokens: 500
76103
});
77104
105+
for await (const chunk of stream) {
106+
if (chunk.choices && chunk.choices.length > 0) {
107+
const newContent = chunk.choices[0].delta.content;
108+
out += newContent;
109+
console.log(newContent);
110+
}
111+
}`);
112+
});
113+
114+
it("conversational llm", async () => {
115+
const model: ModelDataMinimal = {
116+
id: "meta-llama/Llama-3.1-8B-Instruct",
117+
pipeline_tag: "text-generation",
118+
tags: ["conversational"],
119+
inference: "",
120+
};
121+
const snippet = getJsInferenceSnippet(model, "api_token") as InferenceSnippet[];
122+
123+
expect(snippet[0].content).toEqual(`import { HfInference } from "@huggingface/inference"
124+
125+
const client = new HfInference("api_token")
126+
127+
let out = "";
128+
129+
const stream = client.chatCompletionStream({
130+
model: "meta-llama/Llama-3.1-8B-Instruct",
131+
messages: [
132+
{
133+
role: "user",
134+
content: "What is the capital of France?"
135+
}
136+
],
137+
max_tokens: 500
138+
});
139+
78140
for await (const chunk of stream) {
79141
if (chunk.choices && chunk.choices.length > 0) {
80142
const newContent = chunk.choices[0].delta.content;

‎packages/tasks/src/snippets/js.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ for await (const chunk of stream) {
109109
return [
110110
{
111111
client: "huggingface.js",
112-
content: `import { HfInference } from '@huggingface/inference'
112+
content: `import { HfInference } from "@huggingface/inference"
113113
114114
const client = new HfInference("${accessToken || `{API_TOKEN}`}")
115115
@@ -292,10 +292,11 @@ export const jsSnippets: Partial<
292292

293293
export function getJsInferenceSnippet(
294294
model: ModelDataMinimal,
295-
accessToken: string
295+
accessToken: string,
296+
opts?: Record<string, unknown>
296297
): InferenceSnippet | InferenceSnippet[] {
297298
return model.pipeline_tag && model.pipeline_tag in jsSnippets
298-
? jsSnippets[model.pipeline_tag]?.(model, accessToken) ?? { content: "" }
299+
? jsSnippets[model.pipeline_tag]?.(model, accessToken, opts) ?? { content: "" }
299300
: { content: "" };
300301
}
301302

‎packages/tasks/src/snippets/python.spec.ts‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import type { ModelDataMinimal } from "./types";
1+
import type { InferenceSnippet, ModelDataMinimal } from "./types";
22
import { describe, expect, it } from "vitest";
3-
import { snippetConversational } from "./python";
3+
import { getPythonInferenceSnippet } from "./python";
44

55
describe("inference API snippets", () => {
66
it("conversational llm", async () => {
@@ -10,7 +10,7 @@ describe("inference API snippets", () => {
1010
tags: ["conversational"],
1111
inference: "",
1212
};
13-
const snippet = snippetConversational(model, "api_token");
13+
const snippet = getPythonInferenceSnippet(model, "api_token") as InferenceSnippet[];
1414

1515
expect(snippet[0].content).toEqual(`from huggingface_hub import InferenceClient
1616
@@ -34,14 +34,43 @@ for chunk in stream:
3434
print(chunk.choices[0].delta.content, end="")`);
3535
});
3636

37+
it("conversational llm non-streaming", async () => {
38+
const model: ModelDataMinimal = {
39+
id: "meta-llama/Llama-3.1-8B-Instruct",
40+
pipeline_tag: "text-generation",
41+
tags: ["conversational"],
42+
inference: "",
43+
};
44+
const snippet = getPythonInferenceSnippet(model, "api_token", { streaming: false }) as InferenceSnippet[];
45+
46+
expect(snippet[0].content).toEqual(`from huggingface_hub import InferenceClient
47+
48+
client = InferenceClient(api_key="api_token")
49+
50+
messages = [
51+
{
52+
"role": "user",
53+
"content": "What is the capital of France?"
54+
}
55+
]
56+
57+
completion = client.chat.completions.create(
58+
model="meta-llama/Llama-3.1-8B-Instruct",
59+
messages=messages,
60+
max_tokens=500
61+
)
62+
63+
print(completion.choices[0].message)`);
64+
});
65+
3766
it("conversational vlm", async () => {
3867
const model: ModelDataMinimal = {
3968
id: "meta-llama/Llama-3.2-11B-Vision-Instruct",
4069
pipeline_tag: "image-text-to-text",
4170
tags: ["conversational"],
4271
inference: "",
4372
};
44-
const snippet = snippetConversational(model, "api_token");
73+
const snippet = getPythonInferenceSnippet(model, "api_token") as InferenceSnippet[];
4574

4675
expect(snippet[0].content).toEqual(`from huggingface_hub import InferenceClient
4776

0 commit comments

Comments
 (0)