feat: release keyPress automatically (#796)

* feat: release keyPress automatically

* send keepalive when pressing the key

* remove logging

* clean up logging

* chore: use unreliable channel to send keepalive events

* chore: use ordered unreliable channel for pointer events

* chore: adjust auto release key interval

* chore: update logging for kbdAutoReleaseLock

* chore: update comment for KEEPALIVE_INTERVAL

* fix: should cancelAutorelease when pressed is true

* fix: handshake won't happen if webrtc reconnects

* chore: add trace log for writeWithTimeout

* chore: add timeout for KeypressReport

* chore: use the proper key to send release command

* refactor: simplify HID RPC keyboard input handling and improve key state management

- Updated `handleHidRPCKeyboardInput` to return errors directly instead of keys down state.
- Refactored `rpcKeyboardReport` and `rpcKeypressReport` to return errors instead of states.
- Introduced a queue for managing key down state updates in the `Session` struct to prevent input handling stalls.
- Adjusted the `UpdateKeysDown` method to handle state changes more efficiently.
- Removed unnecessary logging and commented-out code for clarity.

* refactor: enhance keyboard auto-release functionality and key state management

* fix: correct Windows default auto-repeat delay comment from 1ms to 1s

* refactor: send keypress as early as possible

* refactor: replace console.warn with console.info for HID RPC channel events

* refactor: remove unused NewKeypressKeepAliveMessage function from HID RPC

* fix: handle error in key release process and log warnings

* fix: log warning on keypress report failure

* fix: update auto-release keyboard interval to 225

* refactor: enhance keep-alive handling and jitter compensation in HID RPC

- Implemented staleness guard to ignore outdated keep-alive packets.
- Added jitter compensation logic to adjust timer extensions based on packet arrival times.
- Introduced new methods for managing keep-alive state and reset functionality in the Session struct.
- Updated auto-release delay mechanism to use dynamic durations based on keep-alive timing.
- Adjusted keep-alive interval in the UI to improve responsiveness.

* gofmt

* clean up code

* chore: use dynamic duration for scheduleAutoRelease

* Use harcoded timer reset value for now

* fix: prevent nil pointer dereference when stopping timers in Close method

* refactor: remove nil check for kbdAutoReleaseTimers in DelayAutoReleaseWithDuration

* refactor: optimize dependencies in useHidRpc hooks

* refactor: streamline keep-alive timer management in useKeyboard hook

* refactor: clarify comments in useKeyboard hook for resetKeyboardState function

* refactor: reduce keysDownStateQueueSize

* refactor: close and reset keysDownStateQueue in newSession function

* chore: resolve conflicts

* resolve conflicts

---------

Co-authored-by: Adam Shiervani <[email protected]>

Author: ym

go.mod

@@ -12,6 +12,7 @@ require (
 	github.com/gin-contrib/logger v1.2.6
 	github.com/gin-gonic/gin v1.10.1
 	github.com/go-co-op/gocron/v2 v2.16.5
+	github.com/google/flatbuffers v25.2.10+incompatible
 	github.com/google/uuid v1.6.0
 	github.com/guregu/null/v6 v6.0.0
 	github.com/gwatts/rootcerts v0.0.0-20250901182336-dc5ae18bd79f
@@ -23,6 +24,7 @@ require (
 	github.com/prometheus/common v0.66.0
 	github.com/prometheus/procfs v0.17.0
 	github.com/psanford/httpreadat v0.1.0
+	github.com/rs/xid v1.6.0
 	github.com/rs/zerolog v1.34.0
 	github.com/sourcegraph/tf-dag v0.2.2-0.20250131204052-3e8ff1477b4f
 	github.com/stretchr/testify v1.11.1

go.sum

@@ -53,6 +53,8 @@ github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAu
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
+github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -152,6 +154,7 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
 github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
+github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
 github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
 github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=

hidrpc.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/jetkvm/kvm/internal/hidrpc"
 	"github.com/jetkvm/kvm/internal/usbgadget"
+	"github.com/rs/zerolog"
 )
 
 func handleHidRPCMessage(message hidrpc.Message, session *Session) {
@@ -26,21 +27,19 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
 		}
 		session.hidRPCAvailable = true
 	case hidrpc.TypeKeypressReport, hidrpc.TypeKeyboardReport:
-		keysDownState, err := handleHidRPCKeyboardInput(message)
-		if keysDownState != nil {
-			session.reportHidRPCKeysDownState(*keysDownState)
-		}
-		rpcErr = err
+		rpcErr = handleHidRPCKeyboardInput(message)
 	case hidrpc.TypeKeyboardMacroReport:
 		keyboardMacroReport, err := message.KeyboardMacroReport()
 		if err != nil {
 			logger.Warn().Err(err).Msg("failed to get keyboard macro report")
 			return
 		}
-		_, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Steps)
+		rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Steps)
 	case hidrpc.TypeCancelKeyboardMacroReport:
 		rpcCancelKeyboardMacro()
 		return
+	case hidrpc.TypeKeypressKeepAliveReport:
+		rpcErr = handleHidRPCKeypressKeepAlive(session)
 	case hidrpc.TypePointerReport:
 		pointerReport, err := message.PointerReport()
 		if err != nil {
@@ -64,8 +63,13 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
 	}
 }
 
-func onHidMessage(data []byte, session *Session) {
-	scopedLogger := hidRPCLogger.With().Bytes("data", data).Logger()
+func onHidMessage(msg hidQueueMessage, session *Session) {
+	data := msg.Data
+
+	scopedLogger := hidRPCLogger.With().
+		Str("channel", msg.channel).
+		Bytes("data", data).
+		Logger()
 	scopedLogger.Debug().Msg("HID RPC message received")
 
 	if len(data) < 1 {
@@ -80,7 +84,9 @@ func onHidMessage(data []byte, session *Session) {
 		return
 	}
 
-	scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger()
+	if scopedLogger.GetLevel() <= zerolog.DebugLevel {
+		scopedLogger = scopedLogger.With().Str("descr", message.String()).Logger()
+	}
 
 	t := time.Now()
 
@@ -97,27 +103,88 @@ func onHidMessage(data []byte, session *Session) {
 	}
 }
 
-func handleHidRPCKeyboardInput(message hidrpc.Message) (*usbgadget.KeysDownState, error) {
+// Tunables
+// Keep in mind
+// macOS default: 15 * 15 = 225ms https://discussions.apple.com/thread/1316947?sortBy=rank
+// Linux default: 250ms https://man.archlinux.org/man/kbdrate.8.en
+// Windows default: 1s `HKEY_CURRENT_USER\Control Panel\Accessibility\Keyboard Response\AutoRepeatDelay`
+
+const expectedRate = 50 * time.Millisecond       // expected keepalive interval
+const maxLateness = 50 * time.Millisecond        // max jitter we'll tolerate OR jitter budget
+const baseExtension = expectedRate + maxLateness // 100ms extension on perfect tick
+
+const maxStaleness = 225 * time.Millisecond // discard ancient packets outright
+
+func handleHidRPCKeypressKeepAlive(session *Session) error {
+	session.keepAliveJitterLock.Lock()
+	defer session.keepAliveJitterLock.Unlock()
+
+	now := time.Now()
+
+	// 1) Staleness guard: ensures packets that arrive far beyond the life of a valid key hold
+	// (e.g. after a network stall, retransmit burst, or machine sleep) are ignored outright.
+	// This prevents “zombie” keepalives from reviving a key that should already be released.
+	if !session.lastTimerResetTime.IsZero() && now.Sub(session.lastTimerResetTime) > maxStaleness {
+		return nil
+	}
+
+	validTick := true
+	timerExtension := baseExtension
+
+	if !session.lastKeepAliveArrivalTime.IsZero() {
+		timeSinceLastTick := now.Sub(session.lastKeepAliveArrivalTime)
+		lateness := timeSinceLastTick - expectedRate
+
+		if lateness > 0 {
+			if lateness <= maxLateness {
+				// --- Small lateness (within jitterBudget) ---
+				// This is normal jitter (e.g., Wi-Fi contention).
+				// We still accept the tick, but *reduce the extension*
+				// so that the total hold time stays aligned with REAL client side intent.
+				timerExtension -= lateness
+			} else {
+				// --- Large lateness (beyond jitterBudget) ---
+				// This is likely a retransmit stall or ordering delay.
+				// We reject the tick entirely and DO NOT extend,
+				// so the auto-release still fires on time.
+				validTick = false
+			}
+		}
+	}
+
+	if !validTick {
+		return nil
+	}
+	// Only valid ticks update our state and extend the timer.
+	session.lastKeepAliveArrivalTime = now
+	session.lastTimerResetTime = now
+	if gadget != nil {
+		gadget.DelayAutoReleaseWithDuration(timerExtension)
+	}
+
+	// On a miss: do not advance any state — keeps baseline stable.
+	return nil
+}
+
+func handleHidRPCKeyboardInput(message hidrpc.Message) error {
 	switch message.Type() {
 	case hidrpc.TypeKeypressReport:
 		keypressReport, err := message.KeypressReport()
 		if err != nil {
 			logger.Warn().Err(err).Msg("failed to get keypress report")
-			return nil, err
+			return err
 		}
-		keysDownState, rpcError := rpcKeypressReport(keypressReport.Key, keypressReport.Press)
-		return &keysDownState, rpcError
+		return rpcKeypressReport(keypressReport.Key, keypressReport.Press)
 	case hidrpc.TypeKeyboardReport:
 		keyboardReport, err := message.KeyboardReport()
 		if err != nil {
 			logger.Warn().Err(err).Msg("failed to get keyboard report")
-			return nil, err
+			return err
 		}
-		keysDownState, rpcError := rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys)
-		return &keysDownState, rpcError
+		return rpcKeyboardReport(keyboardReport.Modifier, keyboardReport.Keys)
 	}
 
-	return nil, fmt.Errorf("unknown HID RPC message type: %d", message.Type())
+	return fmt.Errorf("unknown HID RPC message type: %d", message.Type())
 }
 
 func reportHidRPC(params any, session *Session) {
@@ -127,7 +194,10 @@ func reportHidRPC(params any, session *Session) {
 	}
 
 	if !session.hidRPCAvailable || session.HidChannel == nil {
-		logger.Warn().Msg("HID RPC is not available, skipping reportHidRPC")
+		logger.Warn().
+			Bool("hidRPCAvailable", session.hidRPCAvailable).
+			Bool("HidChannel", session.HidChannel != nil).
+			Msg("HID RPC is not available, skipping reportHidRPC")
 		return
 	}
 
@@ -174,8 +244,10 @@ func (s *Session) reportHidRPCKeyboardLedState(state usbgadget.KeyboardState) {
 
 func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) {
 	if !s.hidRPCAvailable {
+		usbLogger.Debug().Interface("state", state).Msg("reporting keys down state")
 		writeJSONRPCEvent("keysDownState", state, s)
 	}
+	usbLogger.Debug().Interface("state", state).Msg("reporting keys down state, calling reportHidRPC")
 	reportHidRPC(state, s)
 }
 

internal/hidrpc/hidrpc.go

@@ -15,6 +15,7 @@ const (
 	TypePointerReport             MessageType = 0x03
 	TypeWheelReport               MessageType = 0x04
 	TypeKeypressReport            MessageType = 0x05
+	TypeKeypressKeepAliveReport   MessageType = 0x09
 	TypeMouseReport               MessageType = 0x06
 	TypeKeyboardMacroReport       MessageType = 0x07
 	TypeCancelKeyboardMacroReport MessageType = 0x08

internal/hidrpc/message.go

@@ -44,6 +44,8 @@ func (m *Message) String() string {
 			return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
 		}
 		return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
+	case TypeKeypressKeepAliveReport:
+		return "KeypressKeepAliveReport"
 	case TypeKeyboardMacroReport:
 		if len(m.d) < 5 {
 			return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d)