+
+
#include "executorch/extension/llm/runner/multimodal_runner.h"
+
+// Initialize multimodal model (e.g., Voxtral, LLaVA)
+auto runner = MultimodalRunner::create(
+ "model.pte", // Text model
+ "vision.pte", // Vision encoder
+ "audio.pte", // Audio encoder
+ tokenizer_path,
+ temperature
+);
+
+// Run inference with audio + image + text
+auto result = runner->generate_multimodal(
+ "Describe what you hear and see",
+ audio_tensor, // Audio input
+ image_tensor, // Image input
+ max_tokens
+);
+
+// Stream response tokens
+for (const auto& token : result.tokens) {
+ std::cout << token << std::flush;
+}
+
+
+
+
import ExecuTorch
+import AVFoundation
+
+// Initialize multimodal runner with audio support
+let runner = try MultimodalRunner(
+ modelPath: "model.pte",
+ visionPath: "vision.pte",
+ audioPath: "audio.pte",
+ tokenizerPath: tokenizerPath,
+ temperature: 0.7
+)
+
+// Process audio and image inputs
+let audioTensor = AudioProcessor.preprocess(audioURL)
+let imageTensor = ImageProcessor.preprocess(uiImage)
+
+// Generate with audio + vision + text
+let result = try runner.generateMultimodal(
+ prompt: "Describe what you hear and see",
+ audio: audioTensor,
+ image: imageTensor,
+ maxTokens: 512
+)
+
+// Stream tokens to UI
+result.tokens.forEach { token in
+ DispatchQueue.main.async {
+ responseText += token
+ }
+}
+
+
+
+
import org.pytorch.executorch.MultimodalRunner
+import android.media.MediaRecorder
+
+// Initialize multimodal runner with audio
+val runner = MultimodalRunner.create(
+ modelPath = "model.pte",
+ visionPath = "vision.pte",
+ audioPath = "audio.pte",
+ tokenizerPath = tokenizerPath,
+ temperature = 0.7f
+)
+
+// Process audio and image inputs
+val audioTensor = AudioProcessor.preprocess(audioFile)
+val imageTensor = ImageProcessor.preprocess(bitmap)
+
+// Generate with audio + vision + text
+val result = runner.generateMultimodal(
+ prompt = "Describe what you hear and see",
+ audio = audioTensor,
+ image = imageTensor,
+ maxTokens = 512
+)
+
+// Display streaming response
+result.tokens.forEach { token ->
+ runOnUiThread {
+ responseView.append(token)
+ }
+}
+
+
+ ExecuTorch
+