Output delay issue after aiortc audio send/recv

[chaeheechan]

What I'm trying to do is implement a video call using aiortc.

Until now, video transmission and reception are working well, but audio output after voice transmission and reception is a problem
I'm not sure if this is a problem with aiortc or a window, linux environment difference

Currently, the voice data is being transmitted to 960, but the reception is set to 1920, so the received data is divided into 960 and outputted, but the voice data is delayed, so it is being heard because there is a delay

In order to find a solution to this part, I'm trying to use a queue to output it or send and receive voice data to 1920 to solve it, but it's not working well.

When I make the data as a wav file before sending and check it, it seems to be a problem between the output after receiving it when I see that the voice data comes out well.

How should I solve this problem?

Video transmission and reception are currently in progress, but audio output is an issue. But I'm not sure if this is a problem with aiortc or the difference between Windows and Linux environments. So I tested client connectivity once in a Windows environment, and there was a delay in Windows - Windows communication as well

If you test voice input/output by yourself without transmitting or receiving any sound device, it's not a device problem since there's no problem with both server and client

First of all, what I want to know is how to solve the problem that delays occur when outputting after transmitting and receiving voice

---

Environment
Server: Windows, Client: Raspberry Pi Linux

---

The uploaded image is a comparison between the transmitted data wav and the received data wav file.

class MicrophoneAudioTrack(AudioStreamTrack):
    def __init__(self, device_index=1):
        super().__init__()
        self.q = queue.Queue()
        self.sample_rate = 48000
        self.frame_samples = 960
        self.samples_sent = 0
        self.wav_writer = wave.open("server_sent.wav", "wb")
        self.wav_writer.setnchannels(1)
        self.wav_writer.setsampwidth(2)
        self.wav_writer.setframerate(self.sample_rate)

        def callback(indata, frames, time_info, status):
            if status:
                print("[⚠️ 마이크 상태]", status)
            if indata.shape[0] == self.frame_samples:
                self.q.put(indata.copy())
                self.wav_writer.writeframes(indata.tobytes())
            else:
                print(f"[경고] 프레임 길이 불일치: {indata.shape}")

        self.stream = sd.InputStream(
            samplerate=self.sample_rate,
            blocksize=self.frame_samples,
            channels=1,
            dtype='int16',
            callback=callback,
            device=device_index
        )
        self.stream.start()

    async def recv(self):
        while self.q.empty():
            await asyncio.sleep(0.001)

        data = self.q.get()
        data = data.T  # shape (1, 960)
        frame = AudioFrame.from_ndarray(data, layout="mono", format="s16")
        frame.sample_rate = self.sample_rate
        frame.time_base = fractions.Fraction(1, self.sample_rate)
        frame.pts = self.samples_sent
        frame.duration = self.frame_samples
        self.samples_sent += self.frame_samples
        return frame

    def __del__(self):
        try:
            self.wav_writer.close()
        except Exception as e:
            print("[WAV 종료 오류]", e)

class MultiAudioPlayer:
    def __init__(self, sample_rate=48000, frame_samples=960, output_device=3):
        self.sample_rate = sample_rate
        self.frame_samples = frame_samples
        self.buffer = asyncio.Queue(maxsize=100)
        self.running = True
        self.stream = sd.OutputStream(
            samplerate=sample_rate,
            channels=1,
            dtype='int16',
            device=output_device,
            blocksize=frame_samples,
            latency='low'
        )
        self.wav_writer = wave.open("server_received.wav", "wb")
        self.wav_writer.setnchannels(1)
        self.wav_writer.setsampwidth(2)
        self.wav_writer.setframerate(sample_rate)
        asyncio.create_task(self.audio_loop())

    def add_frame(self, frame):
        try:
            pcm = frame.to_ndarray()
            if pcm.ndim == 2 and pcm.shape[0] > 1:
                pcm = pcm[0, :]
            elif pcm.ndim == 2:
                pcm = pcm.flatten()

            total_samples = pcm.shape[0]
            # 960 샘플 배수가 아니면 자르기
            if total_samples % self.frame_samples != 0:
                total_samples = (total_samples // self.frame_samples) * self.frame_samples
                pcm = pcm[:total_samples]

            chunks = np.split(pcm, total_samples // self.frame_samples)
            for chunk in chunks:
                try:
                    self.buffer.put_nowait(chunk)
                except asyncio.QueueFull:
                    print("[경고] 버퍼가 가득 찼습니다")
        except Exception as e:
            print(f"[MultiAudioPlayer] 오류: {e}")

    async def audio_loop(self):
        self.stream.start()
        print("[🎧 출력 시작]")
        silence = np.zeros(self.frame_samples, dtype=np.int16)

        try:
            while self.running:
                pcm = await self.buffer.get()

                # 프레임 크기 맞춤
                if pcm.shape[0] != self.frame_samples:
                    pad = np.zeros(self.frame_samples, dtype=np.int16)
                    n = min(len(pcm), self.frame_samples)
                    pad[:n] = pcm[:n]
                    pcm = pad

                self.stream.write(pcm)
                self.wav_writer.writeframes(pcm.tobytes())
        except Exception as e:
            print("[AudioLoop 예외]", e)
        finally:
            self.stream.stop()
            self.stream.close()
            self.wav_writer.close()

    def stop(self):
        self.running = False
        

async def play_audio(track):
    player = MultiAudioPlayer()
    try:
        while True:
            try:
                frame = await track.recv()
            except MediaStreamError:
                print("[🛑 오디오 트랙 종료됨]")
                break
            player.add_frame(frame)
    finally:
        player.stop()

async def offer(request):
    client_ip = request.remote
    params = await request.json()
    offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])

    pc = RTCPeerConnection()
    pcs.add(pc)
    clients[pc] = client_ip

    opus_caps = get_capabilities("audio")
    for transceiver in pc.getTransceivers():
        if transceiver.kind == "audio":
            transceiver.setCodecPreferences([
                codec for codec in opus_caps.codecs if codec.name == "opus"
            ])

    pc.addTransceiver("video", direction="sendrecv")
    pc.addTransceiver("audio", direction="sendrecv")
    pc.addTrack(CameraVideoTrack())
    pc.addTrack(MicrophoneAudioTrack())

    @pc.on("track")
    def on_track(track):
        if track.kind == "video":
            asyncio.create_task(consume_video(track, client_ip))
        elif track.kind == "audio":
            asyncio.create_task(play_audio(track))

    await pc.setRemoteDescription(offer)
    answer = await pc.createAnswer()
    await pc.setLocalDescription(answer)

    return web.json_response({
        "sdp": pc.localDescription.sdp,
        "type": pc.localDescription.type
    })

async def app_runner():
    app = web.Application()
    app.router.add_post("/offer", offer)
    runner = web.AppRunner(app)
    await runner.setup()
    site = web.TCPSite(runner, "0.0.0.0", 8080)
    await site.start()
    print("[서버] 8080 포트 대기 중...")

---