helper

Author: kirklandsign

examples/models/voxtral/multimodal.cpp

@@ -104,7 +104,11 @@ MultimodalInput loadPreprocessedAudio(const std::string& audio_path) {
   ET_LOG(Info, "audio_data len = %zu", n_floats);
 
   std::vector<float> audio_data(n_floats);
+  ET_LOG(Info, "audio_data size = %zu", audio_data.size());
   f.read(reinterpret_cast<char*>(audio_data.data()), n_floats * sizeof(float));
+    ET_LOG(Info, "First 5 floats in audio_data is %f, %f, %f, %f, %f",
+         audio_data[10000], audio_data[10001], audio_data[10002], audio_data[10003], audio_data[10004]);
+
   f.close();
 
   auto audio = ::executorch::extension::llm::Audio(
@@ -331,6 +335,12 @@ int32_t main(int32_t argc, char** argv) {
 
   // Generate
   ET_LOG(Info, "Starting generation...");
+  for (const auto& input : inputs) {
+    ET_LOG(
+        Info,
+        "Input : %s",
+        input.to_string().c_str());
+  }
   auto error = runner->generate(inputs, config);
   if (error != ::executorch::runtime::Error::Ok) {
     ET_LOG(Error, "Failed to generate with multimodal runner");

extension/android/jni/jni_layer_llama.cpp

@@ -222,7 +222,15 @@ class ExecuTorchLlmJni : public facebook::jni::HybridClass<ExecuTorchLlmJni> {
           .echo = static_cast<bool>(echo),
           .seq_len = seq_len,
           .temperature = temperature_,
+          .max_new_tokens = 256,
       };
+      for (const auto& input : inputs) {
+        ET_LOG(
+            Error,
+            "Prefill input: %s",
+            input.to_string().c_str());
+        }
+
       multi_modal_runner_->generate(
           std::move(inputs),
           config,
@@ -342,6 +350,9 @@ class ExecuTorchLlmJni : public facebook::jni::HybridClass<ExecuTorchLlmJni> {
       for (int i = 0; i < data_size; i++) {
         data_f[i] = data_jfloat[i];
       }
+      ET_LOG(Error, "First 5 elements of data_f: %f, %f, %f, %f, %f",
+             data_f[10000], data_f[10001], data_f[10002], data_f[10003], data_f[
+      10004]);
       llm::Audio audio{std::move(data_f), batch_size, n_bins, n_frames};
       prefill_inputs_.emplace_back(llm::MultimodalInput{std::move(audio)});
     }
@@ -412,8 +423,7 @@ class ExecuTorchLlmJni : public facebook::jni::HybridClass<ExecuTorchLlmJni> {
         makeNativeMethod(
             "appendAudioInput", ExecuTorchLlmJni::append_audio_input),
         makeNativeMethod(
-            "appendAudioInputFloat",
-            ExecuTorchLlmJni::append_audio_input_float),
+            "appendAudioInputFloat", ExecuTorchLlmJni::append_audio_input_float),
         makeNativeMethod(
             "appendRawAudioInput", ExecuTorchLlmJni::append_raw_audio_input),
         makeNativeMethod(

extension/llm/runner/audio.h

@@ -29,6 +29,11 @@ struct ET_EXPERIMENTAL RawAudio {
   int32_t batch_size;
   int32_t n_channels; // For mono, use n_channels = 1.
   int32_t n_samples;
+
+  std::string to_string() const {
+    return "RawAudio: " + std::to_string(batch_size) + "x" +
+        std::to_string(n_channels) + "x" + std::to_string(n_samples);
+  }
 };
 
 /**
@@ -146,6 +151,21 @@ class ET_EXPERIMENTAL Audio final {
     return ::executorch::runtime::Error::NotSupported;
   }
 
+  std::string to_string() const {
+    std::string result = "Audio: ";
+    if (is_uint8()) {
+      result += "uint8_t";
+    } else if (is_float()) {
+      result += "float";
+    } else {
+      result += "unknown";
+    }
+    result += " data, batch_size: " + std::to_string(get_batch_size()) +
+        ", n_bins: " + std::to_string(get_n_bins()) +
+        ", n_frames: " + std::to_string(get_n_frames());
+    return result;
+  }
+
  private:
   // Members
   std::variant<std::vector<uint8_t>, std::vector<float>> data_;

extension/llm/runner/image.h

@@ -110,6 +110,21 @@ class ET_EXPERIMENTAL Image {
     return ::executorch::runtime::Error::NotSupported;
   }
 
+  std::string to_string() const {
+    std::string result = "Image: ";
+    if (is_uint8()) {
+      result += "uint8_t";
+    } else if (is_float()) {
+      result += "float";
+    } else {
+      result += "unknown";
+    }
+    result += "width: " + std::to_string(width_) + ", ";
+    result += "height: " + std::to_string(height_) + ", ";
+    result += "channels: " + std::to_string(channels_);
+    return result;
+  }
+
  private:
   // Assuming NCHW format
   std::variant<std::vector<uint8_t>, std::vector<float>> data_;

extension/llm/runner/multimodal_input.h

@@ -57,6 +57,20 @@ class ET_EXPERIMENTAL MultimodalInput {
   // Destructor
   ~MultimodalInput() = default;
 
+  std::string to_string() const noexcept {
+    if (is_text()) {
+      return "Text: \"" + get_text() + "\"";
+    } else if (is_image()) {
+      return get_image().to_string();
+    } else if (is_audio()) {
+      return get_audio().to_string();
+    } else if (is_raw_audio()) {
+      return get_raw_audio().to_string();
+    } else {
+      return "Unsupported input type";
+    }
+  }
+
   /**
    * Check if this input contains text data.
    * @return true if this input contains text, false otherwise.