C# binding into STFT pre-processing

src/csharp/NativeMethods.cs

@@ -147,7 +147,7 @@ internal class NativeLib
         [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)]
         public static extern IntPtr /* OgaResult* */ OgaGenerator_AppendTokenSequences(IntPtr /* OgaGenerator* */ generator,
                                                                                        IntPtr /* const OgaSequences* */ sequences);
-                                                                                       
+
 
         // This function is used to rewind the generator to the given newLength.
         [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)]
@@ -354,6 +354,21 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq
                                                                                                   IntPtr /* const Audios* */ audios,
                                                                                                   out IntPtr /* OgaNamedTensors** */ namedTensors);
 
+        [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)]
+        public static extern int /* extError_t */ OgaSplitSignalSegments(
+            IntPtr /* const OgaTensor* */ input,
+            IntPtr /* const OgaTensor* */ sr_tensor,
+            IntPtr /* const OgaTensor* */ frame_ms_tensor,
+            IntPtr /* const OgaTensor* */ hop_ms_tensor,
+            IntPtr /* const OgaTensor* */ energy_threshold_db_tensor,
+            IntPtr /* OgaTensor* */ output0);
+
+        [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)]
+        public static extern int /* extError_t */ OgaMergeSignalSegments(
+            IntPtr /* const OgaTensor* */ segments_tensor,
+            IntPtr /* const OgaTensor* */ merge_gap_ms_tensor,
+            IntPtr /* OgaTensor* */ output0);
+
         [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)]
         public static extern unsafe IntPtr /* OgaResult* */ OgaProcessorDecode(IntPtr /* const OgaMultiModalProcessor* */ processor,
                                                                                int* /* const int32_t* */ sequence,

src/csharp/SignalProcessor.cs

@@ -0,0 +1,259 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
+using System.Runtime.InteropServices;
+
+namespace Microsoft.ML.OnnxRuntimeGenAI
+{
+    public static class SignalProcessor
+    {
+        private const int ET_Float32 = 1;
+        private const int ET_Int64 = 7;
+
+        private static readonly ConcurrentDictionary<IntPtr, GCHandle> _tensorPins = new();
+
+        #region Native wrappers
+
+        public static void SplitSignalSegments(
+            IntPtr inputTensor,
+            IntPtr srTensor,
+            IntPtr frameMsTensor,
+            IntPtr hopMsTensor,
+            IntPtr energyThresholdDbTensor,
+            IntPtr outputTensor)
+        {
+            int err = NativeMethods.OgaSplitSignalSegments(
+                inputTensor,
+                srTensor,
+                frameMsTensor,
+                hopMsTensor,
+                energyThresholdDbTensor,
+                outputTensor);
+
+            if (err != 0)
+                throw new InvalidOperationException($"OgaSplitSignalSegments failed with error code {err}");
+        }
+
+        public static void MergeSignalSegments(
+            IntPtr segmentsTensor,
+            IntPtr mergeGapMsTensor,
+            IntPtr outputTensor)
+        {
+            int err = NativeMethods.OgaMergeSignalSegments(
+                segmentsTensor,
+                mergeGapMsTensor,
+                outputTensor);
+
+            if (err != 0)
+                throw new InvalidOperationException($"OgaMergeSignalSegments failed with error code {err}");
+        }
+
+        #endregion
+
+        #region Tensor creation helpers (fixed)
+
+        /// <summary>
+        /// Create a tensor view over a managed float[].
+        /// </summary>
+        public static IntPtr CreateFloatTensorFromArray(float[] data, long[] shape)
+        {
+            if (data == null) throw new ArgumentNullException(nameof(data));
+            if (shape == null) throw new ArgumentNullException(nameof(shape));
+
+            // The underlying buffer is pinned for the lifetime of the tensor.
+            var handle = GCHandle.Alloc(data, GCHandleType.Pinned);
+
+            IntPtr tensor;
+            var status = NativeMethods.OgaCreateTensorFromBuffer(
+                handle.AddrOfPinnedObject(),
+                shape,
+                (UIntPtr)shape.Length,
+                (ElementType)ET_Float32,
+                out tensor);
+
+            if (status.ToInt64() != 0)
+            {
+                handle.Free();
+                throw new InvalidOperationException($"OgaCreateTensorFromBuffer(float) failed with {status.ToInt64()}");
+            }
+
+            if (!_tensorPins.TryAdd(tensor, handle))
+            {
+                handle.Free();
+                throw new InvalidOperationException("Failed to track pinned buffer for float tensor.");
+            }
+
+            return tensor;
+        }
+
+        /// <summary>
+        /// Create a tensor view over a managed long[].
+        /// </summary>
+        public static IntPtr CreateInt64TensorFromArray(long[] data, long[] shape)
+        {
+            if (data == null) throw new ArgumentNullException(nameof(data));
+            if (shape == null) throw new ArgumentNullException(nameof(shape));
+
+            // The underlying buffer is pinned for the lifetime of the tensor.
+            var handle = GCHandle.Alloc(data, GCHandleType.Pinned);
+
+            IntPtr tensor;
+            var status = NativeMethods.OgaCreateTensorFromBuffer(
+                handle.AddrOfPinnedObject(),
+                shape,
+                (UIntPtr)shape.Length,
+                (ElementType)ET_Int64,
+                out tensor);
+
+            if (status.ToInt64() != 0)
+            {
+                handle.Free();
+                throw new InvalidOperationException($"OgaCreateTensorFromBuffer(int64) failed with {status.ToInt64()}");
+            }
+
+            if (!_tensorPins.TryAdd(tensor, handle))
+            {
+                handle.Free();
+                throw new InvalidOperationException("Failed to track pinned buffer for int64 tensor.");
+            }
+
+            return tensor;
+        }
+
+        /// <summary>
+        /// Create an output tensor that points at a caller-owned long[] buffer.
+        /// </summary>
+        public static IntPtr CreateOutputInt64Tensor(long[] backingBuffer, long rows, long cols)
+        {
+            if (backingBuffer == null) throw new ArgumentNullException(nameof(backingBuffer));
+            if (rows * cols > backingBuffer.LongLength)
+                throw new ArgumentException("backingBuffer too small for requested shape");
+
+            long[] shape = new long[] { rows, cols };
+
+            // The buffer is pinned for the lifetime of the tensor.
+            var handle = GCHandle.Alloc(backingBuffer, GCHandleType.Pinned);
+
+            IntPtr tensor;
+            var status = NativeMethods.OgaCreateTensorFromBuffer(
+                handle.AddrOfPinnedObject(),
+                shape,
+                (UIntPtr)shape.Length,
+                (ElementType)ET_Int64,
+                out tensor);
+
+            if (status.ToInt64() != 0)
+            {
+                handle.Free();
+                throw new InvalidOperationException($"OgaCreateTensorFromBuffer(output int64) failed with {status.ToInt64()}");
+            }
+
+            if (!_tensorPins.TryAdd(tensor, handle))
+            {
+                handle.Free();
+                throw new InvalidOperationException("Failed to track pinned buffer for output tensor.");
+            }
+
+            return tensor;
+        }
+
+        /// <summary>
+        /// Destroy a tensor and release its pinned managed buffer (if any).
+        /// </summary>
+        private static void SafeDestroyTensor(IntPtr tensor)
+        {
+            if (tensor == IntPtr.Zero)
+                return;
+
+            NativeMethods.OgaDestroyTensor(tensor);
+
+            if (_tensorPins.TryRemove(tensor, out var handle))
+            {
+                if (handle.IsAllocated)
+                    handle.Free();
+            }
+        }
+
+        #endregion
+
+        /// <summary>
+        /// Runs STFT over the input signal and finds the areas of high energy with start/end timestamps in ms.
+        /// </summary>
+        public static (double Start, double End)[] SplitAndMergeSegments(
+            float[] inputSignal,
+            int sampleRate,
+            int frameMs,
+            int hopMs,
+            float energyThresholdDb,
+            int mergeGapMs)
+        {
+            if (inputSignal == null || inputSignal.Length == 0)
+                throw new ArgumentException("Input array cannot be null or empty", nameof(inputSignal));
+
+            const int MaxSegs = 1024;
+
+            long[] splitBacking = new long[MaxSegs * 2];
+            long[] mergedBacking = new long[MaxSegs * 2];
+
+            IntPtr input = IntPtr.Zero, sr = IntPtr.Zero, frame = IntPtr.Zero, hop = IntPtr.Zero,
+                   thr = IntPtr.Zero, splitOut = IntPtr.Zero, mergeGap = IntPtr.Zero, mergedOut = IntPtr.Zero;
+
+            try
+            {
+                long[] inputShape = new long[] { 1, inputSignal.Length };
+
+                input = CreateFloatTensorFromArray(inputSignal, inputShape);
+                sr = CreateInt64TensorFromArray(new long[] { sampleRate }, new long[] { 1 });
+                frame = CreateInt64TensorFromArray(new long[] { frameMs }, new long[] { 1 });
+                hop = CreateInt64TensorFromArray(new long[] { hopMs }, new long[] { 1 });
+                thr = CreateFloatTensorFromArray(new float[] { energyThresholdDb }, new long[] { 1 });
+
+                splitOut = CreateOutputInt64Tensor(splitBacking, MaxSegs, 2);
+                mergedOut = CreateOutputInt64Tensor(mergedBacking, MaxSegs, 2);
+
+                SplitSignalSegments(input, sr, frame, hop, thr, splitOut);
+
+                long[] splitShapeBuf = new long[2];
+                Result.VerifySuccess(NativeMethods.OgaTensorGetShape(splitOut, splitShapeBuf, (UIntPtr)splitShapeBuf.Length));
+                long splitRows = splitShapeBuf[0];
+                long splitCols = splitShapeBuf[1];
+
+                mergeGap = CreateInt64TensorFromArray(new long[] { mergeGapMs }, new long[] { 1 });
+                MergeSignalSegments(splitOut, mergeGap, mergedOut);
+
+                long[] mergedShapeBuf = new long[2];
+                Result.VerifySuccess(NativeMethods.OgaTensorGetShape(mergedOut, mergedShapeBuf, (UIntPtr)mergedShapeBuf.Length));
+                long mergedRows = mergedShapeBuf[0];
+                long mergedCols = mergedShapeBuf[1];
+
+                if (mergedCols != 2)
+                    throw new InvalidOperationException($"Expected merged output with 2 columns, got {mergedCols}");
+
+                var result = new List<(double Start, double End)>();
+                for (int i = 0; i < mergedRows; ++i)
+                {
+                    long start = mergedBacking[i * 2 + 0];
+                    long end = mergedBacking[i * 2 + 1];
+                    if (start == 0 && end == 0) continue;
+                    result.Add((start, end));
+                }
+
+                return result.ToArray();
+            }
+            finally
+            {
+                SafeDestroyTensor(input);
+                SafeDestroyTensor(sr);
+                SafeDestroyTensor(frame);
+                SafeDestroyTensor(hop);
+                SafeDestroyTensor(thr);
+                SafeDestroyTensor(splitOut);
+                SafeDestroyTensor(mergeGap);
+                SafeDestroyTensor(mergedOut);
+            }
+        }
+    }
+}

src/models/processor.cpp

@@ -198,8 +198,93 @@ std::unique_ptr<OrtValue> ProcessTensor<float, int64_t>(OrtxTensor* tensor, Ort:
   return tensor_value;
 }
 
+template <typename T>
+OrtxTensor* MakeOrtxTensor(Generators::Tensor* src) {
+  if (!src) {
+    throw std::runtime_error("Null tensor passed to MakeOrtxTensor");
+  }
+
+  T* data = const_cast<T*>(src->GetData<T>());
+  std::vector<int64_t> shape = src->GetShape();
+
+  auto* ort = new Ort::Custom::Tensor<T>(shape, data);
+  return reinterpret_cast<OrtxTensor*>(ort);
+}
+
+template <typename T>
+const OrtxTensor* MakeOrtxTensorConst(const Generators::Tensor* src) {
+  if (!src) {
+    throw std::runtime_error("Null tensor passed to MakeOrtxTensorConst");
+  }
+
+  const T* data = src->GetData<T>();
+  std::vector<int64_t> shape = src->GetShape();
+
+  auto* ort = new Ort::Custom::Tensor<T>(shape, const_cast<T*>(data));
+  return reinterpret_cast<const OrtxTensor*>(ort);
+}
+
+void* SplitSignalSegments(
+    const Generators::Tensor* input,
+    const Generators::Tensor* sr_tensor,
+    const Generators::Tensor* frame_ms_tensor,
+    const Generators::Tensor* hop_ms_tensor,
+    const Generators::Tensor* energy_threshold_db_tensor,
+    Generators::Tensor* output0) {
+  if (!input || !sr_tensor || !frame_ms_tensor || !hop_ms_tensor ||
+      !energy_threshold_db_tensor || !output0) {
+    throw std::runtime_error("Null tensor argument passed to OgaSplitSignalSegments");
+  }
+
+  const OrtxTensor* in_tensor = Generators::MakeOrtxTensorConst<float>(input);
+  const OrtxTensor* sr_tensor_obj = Generators::MakeOrtxTensorConst<int64_t>(sr_tensor);
+  const OrtxTensor* frame_tensor = Generators::MakeOrtxTensorConst<int64_t>(frame_ms_tensor);
+  const OrtxTensor* hop_tensor = Generators::MakeOrtxTensorConst<int64_t>(hop_ms_tensor);
+  const OrtxTensor* thr_tensor = Generators::MakeOrtxTensorConst<float>(energy_threshold_db_tensor);
+  OrtxTensor* out_tensor = Generators::MakeOrtxTensor<int64_t>(output0);
+
+  extError_t err = OrtxSplitSignalSegments(
+      in_tensor,
+      sr_tensor_obj,
+      frame_tensor,
+      hop_tensor,
+      thr_tensor,
+      out_tensor);
+
+  if (err != kOrtxOK) {
+    throw std::runtime_error(OrtxGetLastErrorMessage());
+  }
+  return nullptr;
+}
+
+void* MergeSignalSegments(
+    const Generators::Tensor* segments_tensor,
+    const Generators::Tensor* merge_gap_ms_tensor,
+    Generators::Tensor* output0) {
+  if (!segments_tensor || !merge_gap_ms_tensor || !output0) {
+    throw std::runtime_error("Null tensor argument passed to OgaMergeSignalSegments");
+  }
+
+  const OrtxTensor* seg_tensor = Generators::MakeOrtxTensorConst<int64_t>(segments_tensor);
+  const OrtxTensor* gap_tensor = Generators::MakeOrtxTensorConst<int64_t>(merge_gap_ms_tensor);
+  OrtxTensor* out_tensor = Generators::MakeOrtxTensor<int64_t>(output0);
+
+  extError_t err = OrtxMergeSignalSegments(
+      seg_tensor,
+      gap_tensor,
+      out_tensor);
+
+  if (err != kOrtxOK) {
+    throw std::runtime_error(OrtxGetLastErrorMessage());
+  }
+  return nullptr;
+}
+
 template std::unique_ptr<OrtValue> ProcessTensor<float>(OrtxTensor* tensor, Ort::Allocator& allocator);
 template std::unique_ptr<OrtValue> ProcessTensor<int64_t>(OrtxTensor* tensor, Ort::Allocator& allocator);
 template std::unique_ptr<OrtValue> ProcessTensor<bool>(OrtxTensor* tensor, Ort::Allocator& allocator);
 
+template const OrtxTensor* MakeOrtxTensorConst<float>(const Generators::Tensor*);
+template const OrtxTensor* MakeOrtxTensorConst<int64_t>(const Generators::Tensor*);
+template OrtxTensor* MakeOrtxTensor<int64_t>(Generators::Tensor*);
 }  // namespace Generators