main.py

from beamforming import *
from PyQt5.QtWidgets import QApplication, QMainWindow, QWidget, QHBoxLayout, QVBoxLayout, QLabel
from PyQt5.QtCore import Qt, QTimer, QRectF, QThread, pyqtSignal, QObject, QPointF
from PyQt5.QtGui import QImage, QPixmap, QPainter, QPen, QColor, QPolygonF, QCursor, QFont
from collections import deque
import time
import matplotlib.cm as cm
import matplotlib.colors as mcolors
from stream_audio import *
from server import ArrayServer
from inference import *
from scipy import signal
import os
import numpy as np

chunk=2**11
model_audio_winsize = 2**12 // chunk
rate=44100
channels = 16

save_dir = "data/"

# Heatmap visualizer widget
class HeatmapVisualizerWidget(QWidget):
    def __init__(self, chunk, rate, save_duration=1, parent=None,
                 min_val=0.0, max_val=1.0, colormap='viridis'):
        super().__init__(parent)
        self.setFixedSize(400, 400)  # Fixed size prevents stretching
        self.frame = 0
        self.play = True
        self.saved_data = []  # Incoming heatmap data from the worker
        self.t = time.time()

        # Enable keyboard focus
        self.setFocusPolicy(Qt.StrongFocus)
        self.setFocus()

        self.chunk = chunk
        self.rate = rate

        # Parameters for color normalization and colormap selection.
        self.min_val = min_val
        self.max_val = max_val
        self.colormap_name = colormap
        self.cmap = cm.get_cmap(self.colormap_name)

        self.timer = QTimer(self)
        self.timer.timeout.connect(self.update_visualization)
        self.timer.start(20)

    def print_framerate(self):
        framerate = 1/(time.time()-self.t)
        framerate = round(framerate, 2)
        print(f"Heatmap Vis: {framerate} FPS")
        self.t = time.time()

    def setColorRange(self, min_val, max_val):
        """Set the min and max values used for normalization."""
        self.min_val = min_val
        self.max_val = max_val

    def setColormap(self, colormap):
        """Set the colormap to use (e.g., 'viridis', 'jet', etc.)."""
        self.colormap_name = colormap
        self.cmap = cm.get_cmap(self.colormap_name)

    def append_heatmap_data(self, data):
        """
        Slot: Called when new heatmap data is available.
        The data is appended to the saved_data list.
        """
        self.saved_data.append(data)
        while len(self.saved_data) > 10:
            self.saved_data.pop(0)

    def updateHeatmap(self, new_data):
        # Normalize and apply the colormap.
        norm = mcolors.Normalize(vmin=self.min_val, vmax=self.max_val)
        rgba_data = (self.cmap(norm(new_data)) * 255).astype(np.uint8)
        # Transpose to shape (height, width, 4)
        rgba_data = np.transpose(rgba_data, (1, 0, 2))
        height, width, _ = rgba_data.shape

        # Compute bytes per line: 4 bytes per pixel.
        bytes_per_line = width * 4

        # Keep a reference to rgba_data so that the memory isn't freed.
        self.cached_rgba = rgba_data

        # Convert the numpy array to bytes.
        img_bytes = rgba_data.tobytes()

        # Create the QImage with the proper arguments.
        self.cached_image = QImage(img_bytes, width, height, bytes_per_line, QImage.Format_RGBA8888)

        self.update()  # Trigger a repaint

    def paintEvent(self, event):
        painter = QPainter(self)
        painter.fillRect(self.rect(), Qt.white)
        if hasattr(self, 'cached_image'):
            painter.drawImage(self.rect(), self.cached_image)

    def keyPressEvent(self, event):
        if event.key() == Qt.Key_Space:
            self.play = not self.play
        event.accept()

    def update_visualization(self):
        self.frame += 1
        # Get data from saved_data if available; otherwise use zeros.
        if len(self.saved_data) > 0 and self.play:
            heatmap = self.saved_data.pop(0)
            self.updateHeatmap(heatmap)
        # self.print_framerate()

class SpectrogramVisualizerWidget(QWidget):
    def __init__(self, chunk, rate, parent=None):
        super().__init__(parent)
        self.setFixedSize(400, 150)
        self.chunk = chunk
        self.rate = rate
        self.play = True

        self.max_frames = int((0.5 * rate) / chunk)
        self.saved_data = deque(maxlen=self.max_frames)
        self.t = time.time()

        # Initialize the offscreen pixmap
        self.offscreen = QPixmap(self.size())
        self.offscreen.fill(Qt.white)

        # Enable keyboard focus
        self.setFocusPolicy(Qt.StrongFocus)
        self.setFocus()

        self.timer = QTimer(self)
        self.timer.timeout.connect(self.update_visualization)
        self.timer.start(20)

        # Parameters for spectrogram
        self.nperseg = 256  # Number of samples per segment
        self.noverlap = 128  # Number of overlapping samples

        self.prediction_results = deque(maxlen=100)  # Store up to 100 predictions

        # Connect the velocityReady signal to a new slot
        self.velocity_data = deque(maxlen=100)  # Store up to 100 velocity values

    def append_audio_data(self, audio_mono):
        """Append new audio data for spectrogram visualization."""
        if audio_mono is not None and len(audio_mono) > 0:
            # Compute the spectrogram using SciPy
            f, t, Sxx = signal.spectrogram(audio_mono, fs=self.rate, nperseg=self.nperseg, noverlap=self.noverlap)
            # Convert power spectrogram to dB scale
            Sxx_dB = 10 * np.log10(Sxx + 1e-10)  # Add a small value to avoid log(0)
            self.saved_data.append(Sxx_dB)

    def append_velocity_data(self, velocity):
        """Append new velocity data for visualization."""
        self.velocity_data.append(velocity)
        self.update()  # Trigger a repaint

    def update_visualization(self):
        if not self.saved_data or not self.play:
            return

        # Create a new pixmap for the updated drawing
        new_pix = QPixmap(self.size())
        new_pix.fill(Qt.white)
        painter = QPainter(new_pix)

        # Use the Viridis colormap
        viridis = cm.get_cmap('viridis')

        # Combine all spectrogram slices into a single 2D array and flip it vertically
        combined_spectrogram = np.flipud(np.hstack(self.saved_data))

        # Normalize the combined spectrogram data for visualization
        min_val = np.percentile(combined_spectrogram, 0)
        max_val = np.percentile(combined_spectrogram, 99)
        # min_val = -100
        # max_val = -99.998
        normalized = (combined_spectrogram - min_val) / (max_val - min_val)

        # Map the normalized values to colors using the Viridis colormap
        rgba_data = (viridis(normalized) * 255).astype(np.uint8)
        height, width, _ = rgba_data.shape

        # Create a QImage from the RGBA data
        img = QImage(rgba_data.data, width, height, QImage.Format_RGBA8888)

        # Scale the image to fit the widget's size
        scaled_img = img.scaled(self.size(), Qt.IgnoreAspectRatio, Qt.SmoothTransformation)

        # Draw the scaled image onto the widget
        painter.drawImage(0, 0, scaled_img)

        # Draw the rolling line of prediction results
        if self.prediction_results:
            painter.setPen(QPen(Qt.red, 2))
            width = self.width()
            height = self.height()
            num_results = len(self.prediction_results)
            x_step = width / max(num_results, 1)

            # Create points for the line
            points = [
                QPointF(i * x_step, height - (result[0] * height))
                for i, result in enumerate(self.prediction_results)
            ]

            # Draw the line
            painter.drawPolyline(QPolygonF(points))

        # Draw the rolling line of velocity data
        if self.velocity_data:
            painter.setPen(QPen(Qt.green, 2))
            width = self.width()
            height = self.height()
            num_velocities = len(self.velocity_data)
            x_step = width / max(num_velocities, 1)

            # Create points for the line
            points = [
                QPointF(i * x_step, height - (velocity[0] * height))
                for i, velocity in enumerate(self.velocity_data)
            ]

            # Draw the line
            painter.drawPolyline(QPolygonF(points))

        painter.end()
        self.offscreen = new_pix
        self.update()  # Trigger a repaint

    def paintEvent(self, event):
        painter = QPainter(self)
        painter.drawPixmap(0, 0, self.offscreen)

    def keyPressEvent(self, event):
        if event.key() == Qt.Key_Space:
            self.play = not self.play
        if event.key() == Qt.Key_Right:
            self.window().check_and_get_task()
        event.accept()

    def setPredictionResult(self, result):
        """Set the prediction result and update the visualization."""
        if isinstance(result, np.ndarray):
            self.prediction_results.append(result)
            self.update()  # Trigger a repaint


class TouchVisualizerWidget(QWidget):
    def __init__(self, parent=None, min_val=0.0, max_val=1.0, colormap='viridis'):
        super().__init__(parent)
        self.setFixedSize(100, 100)  # Set a fixed size for the patch
        self.probability = 0.0  # Initial probability
        self.min_val = min_val
        self.max_val = max_val
        self.colormap_name = colormap
        self.cmap = cm.get_cmap(self.colormap_name)
        self.play = True

        # Enable keyboard focus
        self.setFocusPolicy(Qt.StrongFocus)
        self.setFocus()

    def setProbability(self, probability):
        """Set the probability and update the visualization."""
        self.probability = probability
        self.update()  # Trigger a repaint

    def paintEvent(self, event):
        painter = QPainter(self)
        painter.fillRect(self.rect(), Qt.white)

        # Normalize the probability to the range [0, 1]
        norm_prob = (self.probability - self.min_val) / (self.max_val - self.min_val)
        norm_prob = np.clip(norm_prob, 0, 1)  # Ensure it's within [0, 1]

        # Get the color from the colormap
        color = self.cmap(norm_prob)
        qcolor = QColor.fromRgbF(color[0], color[1], color[2], color[3])

        # Fill the widget with the color
        painter.fillRect(self.rect(), qcolor)

        # Draw the probability as text
        painter.setPen(Qt.black)
        if self.probability > 1:
            idx = int(self.probability)
            labels = ['', 'draw', 'scrub', 'sandpaper', 'zipper', 'sccisor','powerdrill', 'e-toothbrush', 'spray']
            painter.drawText(self.rect(), Qt.AlignCenter, f"{labels[idx]}")
        else:
            painter.drawText(self.rect(), Qt.AlignCenter, f"{self.probability:.2f}")

    def keyPressEvent(self, event):
        if event.key() == Qt.Key_Space:
            self.play = not self.play
        if event.key() == Qt.Key_Right:
            self.window().check_and_get_task()
        event.accept()

class ProbabilityVisualizerWidget(QWidget):
    def __init__(self, parent=None):
        super().__init__(parent)
        self.setFixedSize(400, 150)
        self.play = True
        self.probability_data = deque(maxlen=100)  # Store up to 100 probability values
        self.t = time.time()

        # Initialize the offscreen pixmap
        self.offscreen = QPixmap(self.size())
        self.offscreen.fill(Qt.white)

        # Enable keyboard focus
        self.setFocusPolicy(Qt.StrongFocus)
        self.setFocus()

        self.timer = QTimer(self)
        self.timer.timeout.connect(self.update_visualization)
        self.timer.start(20)

    def append_probability_data(self, probability):
        """Append new probability data for visualization."""
        self.probability_data.append(probability)
        self.update()  # Trigger a repaint

    def update_visualization(self):
        if not self.probability_data or not self.play:
            return

        # Create a new pixmap for the updated drawing
        new_pix = QPixmap(self.size())
        new_pix.fill(Qt.white)
        painter = QPainter(new_pix)

        # Draw the rolling line of probability data
        if self.probability_data:
            painter.setPen(QPen(Qt.black, 2))
            width = self.width()
            height = self.height()
            num_probabilities = len(self.probability_data)
            x_step = width / max(num_probabilities, 1)

            # Create points for the line
            points = [
                QPointF(i * x_step, height - (probability[0] * height))
                for i, probability in enumerate(self.probability_data)
            ]

            # Draw the line
            painter.drawPolyline(QPolygonF(points))

        painter.end()
        self.offscreen = new_pix
        self.update()  # Trigger a repaint

    def paintEvent(self, event):
        painter = QPainter(self)
        painter.drawPixmap(0, 0, self.offscreen)

    def keyPressEvent(self, event):
        if event.key() == Qt.Key_Space:
            self.play = not self.play
        event.accept()

# Unified Main Window that shows both visualizers
class MainWindow(QMainWindow):
    def __init__(self, chunk, rate, main, buffer_duration=0.03, save_duration=1):
        super().__init__()
        self.setWindowTitle("SoundBubble Demo (CHI 2026)")
        self.play = True

        # Create a central widget with a main vertical layout.
        central_widget = QWidget(self)
        self.setCentralWidget(central_widget)
        main_layout = QVBoxLayout()
        central_widget.setLayout(main_layout)

        # Main title
        main_title = QLabel("SoundBubble Demo")
        main_title.setAlignment(Qt.AlignCenter)
        main_title.setFixedHeight(30)
        main_title.setFont(QFont("Arial", 30, QFont.Bold))
        main_layout.addWidget(main_title)

        # Horizontal layout for visualizers
        layout = QHBoxLayout()
        main_layout.addLayout(layout)

        # Create instances of both visualizers.
        self.heatmapVisualizer = HeatmapVisualizerWidget(chunk, rate, save_duration)
        self.spectrogramVisualizer = SpectrogramVisualizerWidget(chunk, rate)
        self.touchVisualizer = TouchVisualizerWidget()
        self.probabilityVisualizer = ProbabilityVisualizerWidget()  # New probability visualizer
        
        # Add titled visualizer widgets to the layout.
        for title, widget in [
            ("Sound Pressure Level Map", self.heatmapVisualizer),
            ("Audio Spectrogram", self.spectrogramVisualizer),
            ("Prediction\nProbability", self.touchVisualizer),
            ("Prediction History ", self.probabilityVisualizer),
        ]:
            col = QVBoxLayout()
            label = QLabel(title)
            label.setAlignment(Qt.AlignCenter)
            label.setFixedHeight(30)
            col.addWidget(label)
            col.addWidget(widget)
            col.addStretch()
            layout.addLayout(col)

        # Key bindings help text
        keys_label = QLabel(
            "[B] Toggle sound bubble: passthrough <-> isolated output audio (use earphone)  |  "
            "[M] Toggle the sound pressure map  |  "
            "[S] Start swipe demo with trained model (drawing demo)  |  "
            "[Q] Quit model inference"
        )
        keys_label.setAlignment(Qt.AlignCenter)
        keys_label.setFixedHeight(20)
        keys_label.setFont(QFont("Arial", 15))
        main_layout.addWidget(keys_label)

        self.setFocusPolicy(Qt.StrongFocus)
        self.setFocus()

        # Setup the worker thread for audio data.
        self.worker_thread = QThread()
        self.audio_worker = main
        self.audio_worker.moveToThread(self.worker_thread)
        # Connect worker signals to the visualizers' slots.
        self.audio_worker.heatmapDataReady.connect(self.heatmapVisualizer.append_heatmap_data)
        self.audio_worker.spectrogramDataReady.connect(self.spectrogramVisualizer.append_audio_data)
        self.audio_worker.predictionResultReady.connect(self.probabilityVisualizer.append_probability_data)  # Connect to new widget
        self.audio_worker.predictionReady.connect(self.touchVisualizer.setProbability)
        self.audio_worker.velocityReady.connect(self.spectrogramVisualizer.append_velocity_data)
        # When the thread starts, run the worker.
        self.worker_thread.started.connect(self.audio_worker.run)
        self.worker_thread.setStackSize(8 * 1024 * 1024)  # 8MB stack to prevent overflow
        self.worker_thread.start()

        # Timer to regularly send touch position to the worker.
        self.touch_timer = QTimer(self)
        self.touch_timer.timeout.connect(self.send_touch_position)
        self.touch_timer.start(50)  # 50ms update rate

        # Initialize touch-related attributes
        self.touch_point = QPointF(200, 200)
        self.is_touching = False

    def closeEvent(self, event):
        # Stop the worker thread cleanly on exit.
        self.audio_worker.stop()
        self.worker_thread.quit()
        self.worker_thread.wait()
        event.accept()

    def send_touch_position(self):
        touch_pos = self.get_touch_position()
        is_touching = self.is_touching
        if self.audio_worker:
            self.audio_worker.set_touch_position(touch_pos, is_touching)

    def get_touch_position(self):
        # Return normalized position (0-1) relative to the window
        if not self.is_touching:
            return None

        # Calculate position relative to window
        rel_x = self.touch_point.x() / self.width()
        rel_y = self.touch_point.y() / self.height()

        # Normalize to 0-1 range within the window
        return (rel_x, rel_y)

    def mousePressEvent(self, event):
        self.is_touching = True
        self.touch_point = event.pos()
        self.update()

    def mouseMoveEvent(self, event):
        if self.is_touching:
            self.touch_point = event.pos()
            self.update()

    def mouseReleaseEvent(self, event):
        self.is_touching = False
        self.update()

    def paintEvent(self, event):
        super().paintEvent(event)
        if self.is_touching:
            painter = QPainter(self)
            painter.setRenderHint(QPainter.Antialiasing)
            painter.setPen(QPen(QColor(255, 0, 0), 3))
            painter.drawEllipse(self.touch_point, 10, 10)
            painter.end()

    def keyPressEvent(self, event):
        if event.key() == Qt.Key_Space:
            self.play = not self.play
            self.heatmapVisualizer.play = self.play
            self.audio_worker.empty_audio_buffer()
        elif event.key() == Qt.Key_S:
            self.audio_worker.isdemoapp = True
            mode = "swipe"
            model_name = load_model_name(name=f"soundcam_{mode}", path="./best_models/")
            model = load_model(model_name, mode=mode)
            self.audio_worker.model = model
            self.audio_worker.mode = mode
            self.audio_worker.prediction_buffer = deque(maxlen=1)
            print("Swipe Inference start")
        elif event.key() == Qt.Key_Q:
            self.audio_worker.model = None
            self.audio_worker.mode = None
            print("Inference stop")

        elif event.key() == Qt.Key_B: # toggle sound bubble in unity scene
            if self.audio_worker.unity_vis_mode != 1:
                self.audio_worker.unity_vis_mode = 1  # show sound bubble
            else:
                self.audio_worker.unity_vis_mode = 0

        elif event.key() == Qt.Key_M: # demo app mode
            self.audio_worker.isdemoapp = not self.audio_worker.isdemoapp

        elif event.key() == Qt.Key_Right:
            self.check_and_get_task()
        elif event.key() == Qt.Key_R:
            self.audio_worker._reset_save_data()
        event.accept()

class main_thread(QObject):
    heatmapDataReady = pyqtSignal(np.ndarray)   # For heatmap data (e.g., shape (41, 41))
    spectrogramDataReady = pyqtSignal(np.ndarray)  # New signal for spectrogram data
    predictionReady = pyqtSignal(float)  # New signal for prediction
    predictionResultReady = pyqtSignal(np.ndarray)  # New signal for prediction result
    velocityReady = pyqtSignal(np.ndarray)  # New signal for velocity

    def __init__(self, parent=None):
        super().__init__(parent)
        self._running = True
        self.t = time.time()
        self.framerate = []
        self.model = None
        self.mode = None
        self.unity_vis_mode = 0
        self.server = None
        self.audio_buffer = []  # Buffer to accumulate audio_mono
        self.taskset = []
        self.handtracking_error = ["⚠️"]
        self.isdemoapp = True

        # for complete data
        self.audio_save = []
        self.heatmap_save = []
        self.probability_save = []
        self.finger_save = []
        self.label_save = []
        self.touch_save = []

        # for temp data
        self._audio_save = []
        self._heatmap_save = []
        self._probability_save = []
        self._finger_save = []
        self._label_save = []
        self._touch_save = []
        self.touch_position = None
        self.is_touching = False
        
        # demo app
        self.startpoint = None
        self.result_tail = 0

    def append_save_data(self):
        if len(self._audio_save) > 0:
            self.audio_save.append(self._audio_save)
            self.heatmap_save.append(self._heatmap_save)
            self.probability_save.append(self._probability_save)
            self.finger_save.append(self._finger_save)
            self.label_save.append(self._label_save)
            self.touch_save.append(self._touch_save)
            return True
        else:
            return False

    def reset_save_data(self):
        self.audio_save = []
        self.heatmap_save = []
        self.probability_save = []
        self.finger_save = []
        self.label_save = []
        self.touch_save = []

    def _reset_save_data(self):
        self._audio_save = []
        self._heatmap_save = []
        self._probability_save = []
        self._finger_save = []
        self._label_save = []
        self._touch_save = []

    def save_data(self):
        path = f'./{save_dir}/{self.mode}'
        if not os.path.exists(path):
            os.makedirs(path, exist_ok=True)
        filename = f'./{path}/data_{time.strftime("%Y%m%d_%H%M%S")}.npz'
        np.savez(filename,
                audio=np.concatenate(self.audio_save, axis=0),
                heatmap=np.concatenate(self.heatmap_save, axis=0),
                probability=np.concatenate(self.probability_save, axis=0),
                finger=np.concatenate(self.finger_save, axis=0),
                label=np.concatenate(self.label_save, axis=0),
                touch=np.concatenate(self.touch_save, axis=0))
        print(f"Data saved to {filename}")
        self.reset_save_data()


    def print_framerate(self):
        self.framerate.append(time.time()-self.t)
        if len(self.framerate) > 10:
            self.framerate.pop(0)
        framerate = round(1/np.mean(self.framerate), 2)
        print(f"          Worker: {framerate} FPS")
        # print(f"          Worker: {framerate} FPS", end='\r')
        self.t = time.time()

    def empty_audio_buffer(self):
        self.audiostream.audio_buffer_input = []

    def save_audio(self):
        """Save the accumulated audio data to a file."""
        import wave
        import numpy as np

        # Convert the buffer to a single numpy array
        audio_data = np.concatenate(self.audio_buffer)

        # Normalize the audio data to the range of int16
        max_val = np.max(np.abs(audio_data))
        if max_val > 0:
            audio_data = (audio_data / max_val) * 32767  # Scale to int16 range

        # Define the file name and parameters
        file_name = "saved_audio.wav"
        n_channels = 1
        sampwidth = 2  # Sample width in bytes
        n_frames = len(audio_data)
        comptype = "NONE"
        compname = "not compressed"

        # Open a wave file and write the audio data
        with wave.open(file_name, 'wb') as wf:
            wf.setnchannels(n_channels)
            wf.setsampwidth(sampwidth)
            wf.setframerate(rate)
            wf.setcomptype(comptype, compname)
            wf.writeframes(audio_data.astype(np.int16).tobytes())

        print("Audio saved.")
        self.audio_buffer.clear()  # Clear the buffer after saving
        
    def simple_direction_classifier(self, joints, prediction):
        if joints is None:
            return 0

        # thumb_joint = joints[3, 4, 5, 19]  # thumb
        thumbtip = joints[19]
        index_vector1 = joints[6] - joints[7]  # index
        index_vector2 = joints[8] - joints[7]  # index
        if prediction < 0.5 and self.startpoint is not None:
            endpoint = thumbtip
            thumb_vector = endpoint - self.startpoint
            thumb_vector /= np.linalg.norm(thumb_vector)
            index_vector1 /= np.linalg.norm(index_vector1)
            index_vector2 /= np.linalg.norm(index_vector2)

            # Calculate the angle between thumb_vector and index_vector
            dot_product1 = np.dot(thumb_vector, index_vector1)
            dot_product2 = np.dot(thumb_vector, index_vector2)

            if dot_product1 > 0.3 and dot_product2 < 0.8:
                direction = "right"
            elif dot_product1 < -0.3 and dot_product2 < 0.8:
                direction = "left"
            else:
                direction = "backward"
            # print(direction, dot_product1, dot_product2)

            code = {
                "left": 1,
                "right": 2,
                "backward": 3
            }
            self.startpoint = None
            return code[direction]
        
        if prediction > 0.5 and self.startpoint is None:
            self.startpoint = thumbtip
        
        # return "No movement"
        return 0

    def track_object_demo_stage(self, prediction):
        if prediction == 5 and self.stage == 0: # scissors
            if self.stage_clk[0] == False:
                self.stage_clk[0] = True
                self.stage_clk[1] = time.time()
        elif prediction == 1 and self.stage == 1: # pen
            if self.stage_clk[0] == False:
                self.stage_clk[0] = True
                self.stage_clk[1] = time.time()
        
        elapsed_time = time.time() - self.stage_clk[1]
        if elapsed_time > 3.0 and self.stage_clk[0] == True:
            self.stage_clk[0] = False
            self.stage += 1
            self.stage_clk[1] = time.time()
        return self.stage

    def thumb_left_or_right(self, prediction, heatmap_right, heatmap_left):
        if prediction == 1:
            prob_right = np.mean(heatmap_right[12:17,12:17])
            prob_left = np.mean(heatmap_left[12:17,12:17])
            if prob_right > prob_left:
                return 1
            else:
                return 2
        return 0

    def lowpass_filter(self, data, cutoff, fs, order=5):
        """
        Apply a low-pass Butterworth filter.
        
        data   : np.ndarray, audio samples
        cutoff : float, cutoff frequency in Hz
        fs     : float, sampling rate in Hz
        order  : int, filter order (higher = sharper cutoff)
        """
        nyquist = 0.5 * fs
        normal_cutoff = cutoff / nyquist
        b, a = butter(order, normal_cutoff, btype='low', analog=False)
        y = filtfilt(b, a, data)  # zero-phase filtering
        return y

    def run(self):
        server = ArrayServer(host='0.0.0.0', port=8000)
        server.start()
        self.server = server
        audiostream = AudioStream(chunk, rate)
        audiostream.start()
        beamformer = Beamforming(chunk, channels, rate)
        self.audiostream = audiostream

        prev_src_xyz = np.zeros(3)
        prev_world_xyz = np.zeros(3)
        vel_buffer = deque(maxlen=2)
        prev_thumb_to_index_distance = 0
        distance_buffer = deque(maxlen=2)
        self.prediction_buffer = deque(maxlen=3)
        model_audio_win = []
        skipvis = 0
        while self._running:
            if len(audiostream.audio_buffer_input) > 0:
                audio = audiostream.audio_buffer_input.pop(0)
                model_audio_win.append(audio)
                if len(model_audio_win) > model_audio_winsize:
                    model_audio_win.pop(0)
                # extract any digit in string beamformer.micgeofile
                audio = audio[:,:channels]
                # normalize the signal strength
                audio_hp = audio
                audio_hp = high_pass_filter_multichannel(
                    data=audio,
                    cutoff_freq=6000,  
                    sample_rate=rate
                )
                _model_audio_win = np.concatenate(model_audio_win, axis=0)
                _model_audio_win = high_pass_filter_multichannel(
                    data=_model_audio_win,
                    cutoff_freq=6000,  
                    sample_rate=rate
                )

                thumb_to_index_distance = server.get_thumb_to_index_distance()
                world_xyz = server.get_world_fingertip_position()
                finger_joints = server.get_finger_joints()

                src_xyz = server.get_fingertip_position(name="index")

                if np.all(prev_src_xyz==src_xyz):
                    print("⚠️ no hand tracking!")
                heatmap = beamformer.process_audio_block(audio_hp, src_xyz)
                audio_mono_model, audio_all_model, offset = beamformer.get_offseted_audio(_model_audio_win, src_xyz)
                audio_mono, audio_all, _ = beamformer.get_offseted_audio(audio_hp, src_xyz)
                if not self.isdemoapp or self.unity_vis_mode == 1:
                    audio_mono = self.lowpass_filter(audio_mono, 18000, rate, order=5)
                time_delay = np.around(offset*1/44100*1000, 2)
                audio_all[:,-1] = audio_all[:,-1] * 1.2

                if not self.isdemoapp:
                    # playback_audio = np.array([audio_mono*60, audio_mono*60]).T
                    playback_audio = np.array([audio_mono*60, audio_mono*60]).T
                elif self.unity_vis_mode == 1:
                    # playback_audio = np.array([audio_mono*60, audio_mono*60]).T
                    playback_audio = np.array([audio_mono*60, audio_mono*60]).T
                else:
                    playback_audio = np.array([audio[:,0]*4, audio[:,0]*4]).T
                playback_audio = np.array([audio_mono*10, audio[:,0]*10]).T
                audiostream.add_play_queue(playback_audio)
                self.audio_buffer.append(audio_mono)  # Accumulate audio_mono
                if len(self.audio_buffer) * chunk / rate >= 5.0:
                    self.audio_buffer.pop(0)  # Maintain 5 second of audio
                # Visualize
                self.print_framerate()
                if not self.isdemoapp:
                    server.send_array(heatmap)
                
                self.heatmapDataReady.emit(np.rot90(heatmap, k=-1))
                _audio_mono = high_pass_filter_multichannel(
                    data=np.expand_dims(audio_mono, axis=1),
                    cutoff_freq=6000,  
                    sample_rate=rate
                )[:,0]
                self.spectrogramDataReady.emit(_audio_mono)  # Emit audio_mono for spectrogram

                vel = np.linalg.norm(world_xyz - prev_world_xyz)
                vel_buffer.append(vel)
                vel = np.mean(vel_buffer)
                distance_buffer.append(thumb_to_index_distance)
                distance = np.mean(distance_buffer)
                prediction = 0
                if self.model is not None:
                    prediction, probability = infer(self.model, heatmap, src_xyz, audio=audio_mono_model, index_vel=vel, thumb2index_dist=distance)
                    self.prediction_buffer.append(probability)
                    _prediction = 1 if np.mean(self.prediction_buffer) > 0.5 else 0
                    self.predictionReady.emit(probability)
                    self.predictionResultReady.emit(np.array([_prediction*0.8+0.1]))
                    prediction = _prediction
                
                if self.isdemoapp:
                    direction = self.simple_direction_classifier(finger_joints, prediction)
                    packet = np.ones((26,26))*999
                    mode_dict = {"None":0, "swipe":1}
                    packet[0,0] = mode_dict[str(self.mode)]
                    packet[0,1] = prediction
                    packet[0,2] = self.unity_vis_mode
                    packet[0,3:6] = world_xyz
                    packet[0,6] = direction
                    server.send_array(packet)

                # self.velocityReady.emit(np.array([vel*50]))  # Emit velocity
                prev_src_xyz = src_xyz
                prev_world_xyz = world_xyz
                prev_thumb_to_index_distance = thumb_to_index_distance
            else:
                time.sleep(0.001)  # Prevent busy-wait when no audio data

    def stop(self):
        self._running = False

    def set_touch_position(self, touch_pos, is_touching):
        """Set the touch position and touching state."""
        self.touch_position = touch_pos
        self.is_touching = is_touching

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = MainWindow(chunk=chunk, rate=rate, main=main_thread(), buffer_duration=0.003)
    window.show()
    sys.exit(app.exec_())