feat: add audio encoder

Author: lqs

audio.go

@@ -0,0 +1,81 @@
+package kvm
+
+import (
+	"fmt"
+	"net"
+	"os/exec"
+	"sync"
+	"syscall"
+	"time"
+)
+
+func startFFmpeg() (cmd *exec.Cmd, err error) {
+	binaryPath := "/userdata/jetkvm/bin/ffmpeg"
+	// Run the binary in the background
+	cmd = exec.Command(binaryPath,
+		"-f", "alsa",
+		"-channels", "2",
+		"-sample_rate", "48000",
+		"-i", "hw:1,0",
+		"-c:a", "libopus",
+		"-b:a", "64k", // ought to be enough for anybody
+		"-vbr", "off",
+		"-frame_duration", "20",
+		"-compression_level", "2",
+		"-f", "rtp",
+		"rtp://127.0.0.1:3333")
+
+	nativeOutputLock := sync.Mutex{}
+	nativeStdout := &nativeOutput{
+		mu:     &nativeOutputLock,
+		logger: nativeLogger.Info().Str("pipe", "stdout"),
+	}
+	nativeStderr := &nativeOutput{
+		mu:     &nativeOutputLock,
+		logger: nativeLogger.Info().Str("pipe", "stderr"),
+	}
+
+	// Redirect stdout and stderr to the current process
+	cmd.Stdout = nativeStdout
+	cmd.Stderr = nativeStderr
+
+	// Set the process group ID so we can kill the process and its children when this process exits
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid:   true,
+		Pdeathsig: syscall.SIGKILL,
+	}
+
+	// Start the command
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start binary: %w", err)
+	}
+
+	return
+}
+
+func StartNtpAudioServer(handleClient func(net.Conn)) {
+	scopedLogger := nativeLogger.With().
+		Logger()
+
+	listener, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1"), Port: 3333})
+	if err != nil {
+		scopedLogger.Warn().Err(err).Msg("failed to start server")
+		return
+	}
+
+	scopedLogger.Info().Msg("server listening")
+
+	go func() {
+		for {
+			cmd, err := startFFmpeg()
+			if err != nil {
+				scopedLogger.Error().Err(err).Msg("failed to start ffmpeg")
+			}
+			err = cmd.Wait()
+			scopedLogger.Error().Err(err).Msg("ffmpeg exited, restarting")
+			time.Sleep(2 * time.Second)
+		}
+	}()
+
+	go handleClient(listener)
+}

main.go

@@ -76,6 +76,7 @@ func Main() {
 	}()
 
 	initUsbGadget()
+	StartNtpAudioServer(handleAudioClient)
 
 	if err := setInitialVirtualMediaState(); err != nil {
 		logger.Warn().Err(err).Msg("failed to set initial virtual media state")

native.go

@@ -215,7 +215,7 @@ func handleVideoClient(conn net.Conn) {
 
 	scopedLogger.Info().Msg("native video socket client connected")
 
-	inboundPacket := make([]byte, maxFrameSize)
+	inboundPacket := make([]byte, maxVideoFrameSize)
 	lastFrame := time.Now()
 	for {
 		n, err := conn.Read(inboundPacket)
@@ -235,6 +235,31 @@ func handleVideoClient(conn net.Conn) {
 	}
 }
 
+func handleAudioClient(conn net.Conn) {
+	defer conn.Close()
+	scopedLogger := nativeLogger.With().
+		Str("type", "audio").
+		Logger()
+
+	scopedLogger.Info().Msg("native audio socket client connected")
+	inboundPacket := make([]byte, maxAudioFrameSize)
+	for {
+		n, err := conn.Read(inboundPacket)
+		if err != nil {
+			scopedLogger.Warn().Err(err).Msg("error during read")
+			return
+		}
+
+		logger.Info().Msgf("audio socket msg: %d", n)
+		
+		if currentSession != nil {
+			if _, err := currentSession.AudioTrack.Write(inboundPacket[:n]); err != nil {
+				scopedLogger.Warn().Err(err).Msg("error writing sample")
+			}
+		}
+	}
+}
+
 func ExtractAndRunNativeBin() error {
 	binaryPath := "/userdata/jetkvm/bin/jetkvm_native"
 	if err := ensureBinaryUpdated(binaryPath); err != nil {

ui/src/components/WebRTCVideo.tsx

@@ -711,7 +711,7 @@ export default function WebRTCVideo() {
                           controls={false}
                           onPlaying={onVideoPlaying}
                           onPlay={onVideoPlaying}
-                          muted={true}
+                          muted={false}
                           playsInline
                           disablePictureInPicture
                           controlsList="nofullscreen"

ui/src/routes/devices.$id.tsx

@@ -480,6 +480,8 @@ export default function KvmIdRoute() {
     };
 
     setTransceiver(pc.addTransceiver("video", { direction: "recvonly" }));
+    // Add audio transceiver to receive audio from the server
+    pc.addTransceiver("audio", { direction: "recvonly" });
 
     const rpcDataChannel = pc.createDataChannel("rpc");
     rpcDataChannel.onopen = () => {

video.go

@@ -5,7 +5,8 @@ import (
 )
 
 // max frame size for 1080p video, specified in mpp venc setting
-const maxFrameSize = 1920 * 1080 / 2
+const maxVideoFrameSize = 1920 * 1080 / 2
+const maxAudioFrameSize = 1500
 
 func writeCtrlAction(action string) error {
 	actionMessage := map[string]string{

webrtc.go

@@ -18,6 +18,7 @@ import (
 type Session struct {
 	peerConnection           *webrtc.PeerConnection
 	VideoTrack               *webrtc.TrackLocalStaticSample
+	AudioTrack               *webrtc.TrackLocalStaticRTP
 	ControlChannel           *webrtc.DataChannel
 	RPCChannel               *webrtc.DataChannel
 	HidChannel               *webrtc.DataChannel
@@ -136,22 +137,27 @@ func newSession(config SessionConfig) (*Session, error) {
 		return nil, err
 	}
 
-	rtpSender, err := peerConnection.AddTrack(session.VideoTrack)
+	session.AudioTrack, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus}, "audio", "kvm")
+	if err != nil {
+		return nil, err
+	}
+
+	videoRtpSender, err := peerConnection.AddTrack(session.VideoTrack)
+	if err != nil {
+		return nil, err
+	}
+
+	audioRtpSender, err := peerConnection.AddTrack(session.AudioTrack)
 	if err != nil {
 		return nil, err
 	}
 
 	// Read incoming RTCP packets
 	// Before these packets are returned they are processed by interceptors. For things
 	// like NACK this needs to be called.
-	go func() {
-		rtcpBuf := make([]byte, 1500)
-		for {
-			if _, _, rtcpErr := rtpSender.Read(rtcpBuf); rtcpErr != nil {
-				return
-			}
-		}
-	}()
+	go drainRtpSender(videoRtpSender)
+	go drainRtpSender(audioRtpSender)
+
 	var isConnected bool
 
 	peerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) {
@@ -203,6 +209,15 @@ func newSession(config SessionConfig) (*Session, error) {
 	return session, nil
 }
 
+func drainRtpSender(rtpSender *webrtc.RTPSender) {
+	rtcpBuf := make([]byte, 1500)
+	for {
+		if _, _, err := rtpSender.Read(rtcpBuf); err != nil {
+			return
+		}
+	}
+}
+
 var actionSessions = 0
 
 func onActiveSessionsChanged() {