diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..e98debb --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "launch", + "name": "Launch Program", + "skipFiles": [ + "/**" + ], + "program": "${workspaceFolder}/src/cli.ts", + "preLaunchTask": "tsc: build - tsconfig.json", + "outFiles": [ + "${workspaceFolder}/dist/**/*.js" + ], + } + ] +} \ No newline at end of file diff --git a/package.json b/package.json index ac02d63..7b12dee 100644 --- a/package.json +++ b/package.json @@ -23,5 +23,8 @@ "esbuild-register": "^3.5.0", "typescript": "^5.5.3", "uvu": "^0.5.6" + }, + "dependencies": { + "openai": "^5.11.0" } } diff --git a/src/cli.ts b/src/cli.ts index 4ef02b7..5bdc876 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,7 +26,7 @@ async function main() { // console.log(g.pp()); // console.log(JSON.stringify(g.serialize(), null, 1)); - const session = new Session(process.env["AZURE_GUIDANCE_URL"]); + const session = new Session("http://localhost:8000/v1", "Qwen/Qwen3-1.7B"); const seq = session.generation({ grammar: g, messages: [ diff --git a/src/client.ts b/src/client.ts index 71092ca..3db9690 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1,3 +1,4 @@ +import OpenAI from 'openai'; import { AssistantPrompt, InitialRunResponse, @@ -9,7 +10,6 @@ import { RunUsageResponse, } from "./api"; import { Gen, GrammarNode } from "./grammarnode"; -import { postAndRead } from "./nodefetch"; import { assert, uint8ArrayConcat, @@ -17,45 +17,9 @@ import { utf8decode, } from "./util"; -function mkUrl(path: string, connString: string) { - const match = /^(.*?)(#.*)?$/.exec(connString); - let url = match[1] || ""; - const fragment = match[2] || ""; - - let headers: Record = {}; - let info = "no auth header"; - - if (fragment) { - const params = new URLSearchParams(fragment.slice(1)); // remove the leading '#' - if (params.has("key")) { - const key = params.get("key"); - headers = { "api-key": key }; - info = `api-key: ${key.slice(0, 2)}...${key.slice(-2)}`; - } else if (params.has("auth")) { - const key = params.get("auth"); - headers = { authorization: "Bearer " + key }; - info = `authorization: Bearer ${key.slice(0, 2)}...${key.slice(-2)}`; - } - } - - if (url.endsWith("/")) { - url = url.slice(0, -1); - } - if (url.endsWith("/run")) { - url = url.slice(0, -4) + "/" + path; - } else if (url.endsWith("/guidance") && path === "run") { - // no change - } else { - url += "/" + path; - } - info = `${url} (${info})`; - - return { url, headers, info }; -} export interface RequestOptions { - url: string; info?: string; // included in headers?: Record; method?: string; @@ -71,28 +35,30 @@ export interface GenerationOptions { } export class Session { - constructor(private connectionString: string) { - const info = mkUrl("run", connectionString); - if (!(info.url.startsWith("http://") || info.url.startsWith("https://"))) - throw new Error("Invalid URL: " + connectionString); - if (Object.keys(info.headers).length == 0) - throw new Error("No key in connection string"); - } + private oai_client: OpenAI; + private model: string; - resolvePath(url: string) { - return mkUrl(url, this.connectionString); + constructor(baseUri: string, targetModel: string) { + this.oai_client = new OpenAI({ baseURL: baseUri, apiKey: "" }); + this.model = targetModel; } async request(options: RequestOptions) { - const info = this.resolvePath(options.url); - return await postAndRead({ - ...options, - url: info.url, - headers: { - ...info.headers, - ...(options.headers ?? {}), - }, - }); + console.log("Options.data:", options.data); + console.log("Messages:", options.data?.messages); + console.log("guided_grammar:", options.data?.grammar); + const response = await this.oai_client.chat.completions.create({ + model: this.model, + messages: options.data?.messages ?? [], + }, /*{ + extra_body: { + "guided_decoding_backend": "guidance", + "guided_grammar": options.data?.grammar?.serialize(), + } + }*/); + console.log("Response:", response); + console.log("Messages:", response.choices[0].message.content); + return response; } generation(options: GenerationOptions) { @@ -101,7 +67,7 @@ export class Session { } export abstract class Generation { - constructor(protected options: GenerationOptions) {} + constructor(protected options: GenerationOptions) { } lastUsage: RunUsageResponse; logLevel = 1; @@ -112,9 +78,9 @@ export abstract class Generation { started = false; warnings: string[] = []; - onText = (s: OutText) => {}; - onLog = (s: string) => {}; - onWarning = (warn: string) => {}; + onText = (s: OutText) => { }; + onLog = (s: string) => { }; + onWarning = (warn: string) => { }; onError = (err: string) => { throw new Error("Server error: " + err); }; @@ -139,7 +105,7 @@ export abstract class Generation { return this.listCaptures.get(name)?.map((v) => v.str); } - destroy() {} + destroy() { } abstract run(): Promise; @@ -181,12 +147,8 @@ class SessionGeneration extends Generation { }; assert(!this.started); this.started = true; - if (this.logLevel >= 4) { - console.log(`POST ${this.session.resolvePath("run").info}`); - console.log(JSON.stringify(arg)); - } + console.log("arg: RunRequest", JSON.stringify(arg)); await this.session.request({ - url: "run", data: arg, lineCb: (s) => this.handleLine(s), }); diff --git a/src/nodefetch.ts b/src/nodefetch.ts deleted file mode 100644 index 1c74ab8..0000000 --- a/src/nodefetch.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { RequestOptions } from "./client"; - -export async function postAndRead(options: RequestOptions): Promise<{}> { - let { url, headers = {}, method, data, lineCb, info } = options; - - if (info === undefined) { - info = url; - } - - if (data && !method) { - method = "POST"; - } - - let body = undefined; - if (typeof data === "object") { - body = JSON.stringify(data); - headers["Content-Type"] = "application/json"; - } - - const response = await fetch(url, { - method, - headers, - body, - }); - - if (!response.ok) { - const pref = - `Invalid HTTP response.\nRequest: ${method} ${info}\n` + - `Status: ${response.status} ${response.statusText}\n`; - let text = ""; - try { - text = "Body: " + (await response.text()); - } catch (e) { - text = e.toString(); - } - throw new Error(pref + text); - } - - if (!lineCb) return await response.json(); - - const reader = response.body.getReader(); - const decoder = new TextDecoder(); - let partialData = ""; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - partialData += decoder.decode(value, { stream: true }); - - let lines = partialData.split("\n"); - partialData = lines.pop() || ""; - - for (let line of lines) { - if (line.startsWith("data: ")) { - const message = line.substring(6).trim(); - lineCb(message); - } - } - } - - return null; -}