feat: adding text-to-audio example

- using `InferenceAPI` to perform `text-to-audio`.
- encoding `wave` audio tensors from the rust land

Author: kallebysantos

examples/ort-raw-session/index.ts

@@ -1,3 +1,76 @@
+/// /// <reference path="./types.d.ts" />
+
+/*
+const modelUrl = 'https://huggingface.co/kalleby/hp-to-miles/resolve/main/model.onnx?download=true';
+const modelConfigUrl =
+  'https://huggingface.co/kalleby/hp-to-miles/resolve/main/config.json?download=true';
+
+const model = await Supabase.ai.RawSession.fromUrl(modelUrl);
+const modelConfig = await fetch(modelConfigUrl).then((r) => r.json());
+
+Deno.serve(async (req: Request) => {
+  const params = new URL(req.url).searchParams;
+  const inputValue = parseInt(params.get('value'));
+
+  const input = new Supabase.ai.RawTensor('float32', [inputValue], [1, 1]);
+    .minMaxNormalize(modelConfig.input.min, modelConfig.input.max);
+
+  const output = await model.run({
+    'dense_dense1_input': input,
+  });
+
+  console.log('output', output);
+
+  const outputTensor = output['dense_Dense4']
+    .minMaxUnnormalize(modelConfig.label.min, modelConfig.label.max);
+
+  return Response.json({ result: outputTensor.data });
+});
+*/
+
+// transformers.js Compatible:
+// import { Tensor } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected]';
+//  const rawTensor = new Supabase.ai.RawTensor('string', urls, [urls.length]);
+//  console.log('raw tensor', rawTensor );
+//
+//  const tensor = new Tensor(rawTensor);
+//  console.log('hf tensor', tensor);
+//
+// 'hf tensor operations'
+//  tensor.min(); tensor.max(); tensor.norm() ....
+
+// const modelUrl =
+//   'https://huggingface.co/pirocheto/phishing-url-detection/resolve/main/model.onnx?download=true';
+
+/*
+const { Tensor, RawSession } = Supabase.ai;
+
+const model = await RawSession.fromHuggingFace('pirocheto/phishing-url-detection', {
+  path: {
+    template: `{REPO_ID}/resolve/{REVISION}/{MODEL_FILE}?donwload=true`,
+    modelFile: 'model.onnx',
+  },
+});
+
+console.log('session', model);
+
+Deno.serve(async (_req: Request) => {
+  const urls = [
+    'https://clubedemilhagem.com/home.php',
+    'http://www.medicalnewstoday.com/articles/188939.php',
+    'https://magalu-crediarioluiza.com/Produto_20203/produto.php?sku=1',
+  ];
+
+  const inputs = new Tensor('string', urls, [urls.length]);
+  console.log('tensor', inputs.data);
+
+  const output = await model.run({ inputs });
+  console.log(output);
+
+  return Response.json({ result: output.probabilities });
+});
+*/
+
 const { Tensor, RawSession } = Supabase.ai;
 
 const session = await RawSession.fromHuggingFace('kallebysantos/vehicle-emission', {
@@ -27,14 +100,15 @@ Deno.serve(async (_req: Request) => {
   }];
 
   // Parsing objects to tensor input
-  const inputTensors = {};
+  const inputTensors: Record<string, Supabase.Tensor<'float32'>> = {};
   session.inputs.forEach((inputKey) => {
     const values = carsBatchInput.map((item) => item[inputKey]);
 
     inputTensors[inputKey] = new Tensor('float32', values, [values.length, 1]);
   });
 
   const { emissions } = await session.run(inputTensors);
+  console.log(emissions);
   // [ 289.01, 199.53]
 
   return Response.json({ result: emissions });

examples/ort-raw-session/types.d.ts

@@ -91,6 +91,8 @@ declare namespace Supabase {
     size: number;
 
     constructor(type: T, data: TensorDataTypeMap[T], dims: number[]);
+
+    tryEncodeAudio(sampleRate: number): Promise<ArrayBuffer>;
   }
 
   export class RawSession {

examples/text-to-audio/index.ts

@@ -0,0 +1,103 @@
+// Setup type definitions for built-in Supabase Runtime APIs
+import 'jsr:@supabase/functions-js/edge-runtime.d.ts';
+import { PreTrainedTokenizer } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected]';
+
+// import 'phonemize' code from Kokoro.js repo
+import { phonemize } from './phonemizer.js';
+
+const { Tensor, RawSession } = Supabase.ai;
+
+const STYLE_DIM = 256;
+const SAMPLE_RATE = 24000;
+const MODEL_ID = 'onnx-community/Kokoro-82M-ONNX';
+
+// https://huggingface.co/onnx-community/Kokoro-82M-ONNX#samples
+const ALLOWED_VOICES = [
+  'af_bella',
+  'af_nicole',
+  'af_sarah',
+  'af_sky',
+  'am_adam',
+  'am_michael',
+  'bf_emma',
+  'bf_isabella',
+  'bm_george',
+  'bm_lewis',
+];
+
+const session = await RawSession.fromHuggingFace(MODEL_ID);
+
+Deno.serve(async (req) => {
+  const params = new URL(req.url).searchParams;
+  const text = params.get('text') ?? 'Hello from Supabase!';
+  const voice = params.get('voice') ?? 'af_bella';
+
+  if (!ALLOWED_VOICES.includes(voice)) {
+    return Response.json({
+      error: `invalid voice '${voice}'`,
+      must_be_one_of: ALLOWED_VOICES,
+    }, { status: 400 });
+  }
+
+  const tokenizer = await loadTokenizer();
+  const language = voice.at(0); // 'a'merican | 'b'ritish
+  const phonemes = await phonemize(text, language);
+  const { input_ids } = tokenizer(phonemes, {
+    truncation: true,
+  });
+
+  // Select voice style based on number of input tokens
+  const num_tokens = Math.max(
+    input_ids.dims.at(-1) - 2, // Without padding;
+    0,
+  );
+
+  const voiceStyle = await loadVoiceStyle(voice, num_tokens);
+
+  const { waveform } = await session.run({
+    input_ids,
+    style: voiceStyle,
+    speed: new Tensor('float32', [1], [1]),
+  });
+
+  // Do `wave` encoding from rust backend
+  const audio = await waveform.tryEncodeAudio(SAMPLE_RATE);
+
+  return new Response(audio, {
+    headers: {
+      'Content-Type': 'audio/wav',
+    },
+  });
+});
+
+async function loadVoiceStyle(voice: string, num_tokens: number) {
+  const voice_url =
+    `https://huggingface.co/onnx-community/Kokoro-82M-ONNX/resolve/main/voices/${voice}.bin?download=true`;
+
+  console.log('loading voice:', voice_url);
+
+  const voiceBuffer = await fetch(voice_url).then(async (res) => await res.arrayBuffer());
+
+  const offset = num_tokens * STYLE_DIM;
+  const voiceData = new Float32Array(voiceBuffer).slice(
+    offset,
+    offset + STYLE_DIM,
+  );
+
+  return new Tensor('float32', voiceData, [1, STYLE_DIM]);
+}
+
+async function loadTokenizer() {
+  // BUG: invalid 'h' not JSON. That's why we need to manually fetch the assets
+  // const tokenizer = await AutoTokenizer.from_pretrained(MODEL_ID);
+
+  const tokenizerData = await fetch(
+    'https://huggingface.co/onnx-community/Kokoro-82M-ONNX/resolve/main/tokenizer.json?download=true',
+  ).then(async (res) => await res.json());
+
+  const tokenizerConfig = await fetch(
+    'https://huggingface.co/onnx-community/Kokoro-82M-ONNX/resolve/main/tokenizer_config.json?download=true',
+  ).then(async (res) => await res.json());
+
+  return new PreTrainedTokenizer(tokenizerData, tokenizerConfig);
+}