feat: send all paste keystrokes to backend (#789)

* feat: send all paste keystrokes to backend

* feat: cancel paste mode

* wip: send macro using hidRPC channel

* add delay

* feat: allow paste progress to be cancelled

* allow user to override delay

* chore: clear keysDownState

* fix: use currentSession.reportHidRPCKeyboardMacroState

* fix: jsonrpc.go:1142:21: Error return value is not checked (errcheck)

* fix: performance issue of Uint8Array concat

* chore: hide delay option when debugMode isn't enabled

* feat: use clientSide macro if backend doesn't support macros

* fix: update keysDownState handling

* minor issues

* refactor

* fix: send duplicated keyDownState

* chore: add max length for paste text

---------

Co-authored-by: Adam Shiervani <[email protected]>

Author: ym

cloud.go

@@ -475,6 +475,10 @@ func handleSessionRequest(
 
 	cloudLogger.Info().Interface("session", session).Msg("new session accepted")
 	cloudLogger.Trace().Interface("session", session).Msg("new session accepted")
+
+	// Cancel any ongoing keyboard macro when session changes
+	cancelKeyboardMacro()
+
 	currentSession = session
 	_ = wsjson.Write(context.Background(), c, gin.H{"type": "answer", "data": sd})
 	return nil

hidrpc.go

@@ -1,7 +1,9 @@
 package kvm
 
 import (
+	"errors"
 	"fmt"
+	"io"
 	"time"
 
 	"github.com/jetkvm/kvm/internal/hidrpc"
@@ -29,6 +31,16 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
 			session.reportHidRPCKeysDownState(*keysDownState)
 		}
 		rpcErr = err
+	case hidrpc.TypeKeyboardMacroReport:
+		keyboardMacroReport, err := message.KeyboardMacroReport()
+		if err != nil {
+			logger.Warn().Err(err).Msg("failed to get keyboard macro report")
+			return
+		}
+		_, rpcErr = rpcExecuteKeyboardMacro(keyboardMacroReport.Steps)
+	case hidrpc.TypeCancelKeyboardMacroReport:
+		rpcCancelKeyboardMacro()
+		return
 	case hidrpc.TypePointerReport:
 		pointerReport, err := message.PointerReport()
 		if err != nil {
@@ -128,6 +140,8 @@ func reportHidRPC(params any, session *Session) {
 		message, err = hidrpc.NewKeyboardLedMessage(params).Marshal()
 	case usbgadget.KeysDownState:
 		message, err = hidrpc.NewKeydownStateMessage(params).Marshal()
+	case hidrpc.KeyboardMacroState:
+		message, err = hidrpc.NewKeyboardMacroStateMessage(params.State, params.IsPaste).Marshal()
 	default:
 		err = fmt.Errorf("unknown HID RPC message type: %T", params)
 	}
@@ -143,6 +157,10 @@ func reportHidRPC(params any, session *Session) {
 	}
 
 	if err := session.HidChannel.Send(message); err != nil {
+		if errors.Is(err, io.ErrClosedPipe) {
+			logger.Debug().Err(err).Msg("HID RPC channel closed, skipping reportHidRPC")
+			return
+		}
 		logger.Warn().Err(err).Msg("failed to send HID RPC message")
 	}
 }
@@ -160,3 +178,10 @@ func (s *Session) reportHidRPCKeysDownState(state usbgadget.KeysDownState) {
 	}
 	reportHidRPC(state, s)
 }
+
+func (s *Session) reportHidRPCKeyboardMacroState(state hidrpc.KeyboardMacroState) {
+	if !s.hidRPCAvailable {
+		writeJSONRPCEvent("keyboardMacroState", state, s)
+	}
+	reportHidRPC(state, s)
+}

internal/hidrpc/hidrpc.go

@@ -10,14 +10,17 @@ import (
 type MessageType byte
 
 const (
-	TypeHandshake        MessageType = 0x01
-	TypeKeyboardReport   MessageType = 0x02
-	TypePointerReport    MessageType = 0x03
-	TypeWheelReport      MessageType = 0x04
-	TypeKeypressReport   MessageType = 0x05
-	TypeMouseReport      MessageType = 0x06
-	TypeKeyboardLedState MessageType = 0x32
-	TypeKeydownState     MessageType = 0x33
+	TypeHandshake                 MessageType = 0x01
+	TypeKeyboardReport            MessageType = 0x02
+	TypePointerReport             MessageType = 0x03
+	TypeWheelReport               MessageType = 0x04
+	TypeKeypressReport            MessageType = 0x05
+	TypeMouseReport               MessageType = 0x06
+	TypeKeyboardMacroReport       MessageType = 0x07
+	TypeCancelKeyboardMacroReport MessageType = 0x08
+	TypeKeyboardLedState          MessageType = 0x32
+	TypeKeydownState              MessageType = 0x33
+	TypeKeyboardMacroState        MessageType = 0x34
 )
 
 const (
@@ -29,10 +32,13 @@ func GetQueueIndex(messageType MessageType) int {
 	switch messageType {
 	case TypeHandshake:
 		return 0
-	case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardLedState, TypeKeydownState:
+	case TypeKeyboardReport, TypeKeypressReport, TypeKeyboardMacroReport, TypeKeyboardLedState, TypeKeydownState, TypeKeyboardMacroState:
 		return 1
 	case TypePointerReport, TypeMouseReport, TypeWheelReport:
 		return 2
+	// we don't want to block the queue for this message
+	case TypeCancelKeyboardMacroReport:
+		return 3
 	default:
 		return 3
 	}
@@ -98,3 +104,19 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message {
 		d: data,
 	}
 }
+
+// NewKeyboardMacroStateMessage creates a new keyboard macro state message.
+func NewKeyboardMacroStateMessage(state bool, isPaste bool) *Message {
+	data := make([]byte, 2)
+	if state {
+		data[0] = 1
+	}
+	if isPaste {
+		data[1] = 1
+	}
+
+	return &Message{
+		t: TypeKeyboardMacroState,
+		d: data,
+	}
+}

internal/hidrpc/message.go

@@ -1,6 +1,7 @@
 package hidrpc
 
 import (
+	"encoding/binary"
 	"fmt"
 )
 
@@ -43,6 +44,11 @@ func (m *Message) String() string {
 			return fmt.Sprintf("MouseReport{Malformed: %v}", m.d)
 		}
 		return fmt.Sprintf("MouseReport{DX: %d, DY: %d, Button: %d}", m.d[0], m.d[1], m.d[2])
+	case TypeKeyboardMacroReport:
+		if len(m.d) < 5 {
+			return fmt.Sprintf("KeyboardMacroReport{Malformed: %v}", m.d)
+		}
+		return fmt.Sprintf("KeyboardMacroReport{IsPaste: %v, Length: %d}", m.d[0] == uint8(1), binary.BigEndian.Uint32(m.d[1:5]))
 	default:
 		return fmt.Sprintf("Unknown{Type: %d, Data: %v}", m.t, m.d)
 	}
@@ -84,6 +90,55 @@ func (m *Message) KeyboardReport() (KeyboardReport, error) {
 	}, nil
 }
 
+// Macro ..
+type KeyboardMacroStep struct {
+	Modifier byte   // 1 byte
+	Keys     []byte // 6 bytes: hidKeyBufferSize
+	Delay    uint16 // 2 bytes
+}
+type KeyboardMacroReport struct {
+	IsPaste   bool
+	StepCount uint32
+	Steps     []KeyboardMacroStep
+}
+
+// HidKeyBufferSize is the size of the keys buffer in the keyboard report.
+const HidKeyBufferSize = 6
+
+// KeyboardMacroReport returns the keyboard macro report from the message.
+func (m *Message) KeyboardMacroReport() (KeyboardMacroReport, error) {
+	if m.t != TypeKeyboardMacroReport {
+		return KeyboardMacroReport{}, fmt.Errorf("invalid message type: %d", m.t)
+	}
+
+	isPaste := m.d[0] == uint8(1)
+	stepCount := binary.BigEndian.Uint32(m.d[1:5])
+
+	// check total length
+	expectedLength := int(stepCount)*9 + 5
+	if len(m.d) != expectedLength {
+		return KeyboardMacroReport{}, fmt.Errorf("invalid length: %d, expected: %d", len(m.d), expectedLength)
+	}
+
+	steps := make([]KeyboardMacroStep, 0, int(stepCount))
+	offset := 5
+	for i := 0; i < int(stepCount); i++ {
+		steps = append(steps, KeyboardMacroStep{
+			Modifier: m.d[offset],
+			Keys:     m.d[offset+1 : offset+7],
+			Delay:    binary.BigEndian.Uint16(m.d[offset+7 : offset+9]),
+		})
+
+		offset += 1 + HidKeyBufferSize + 2
+	}
+
+	return KeyboardMacroReport{
+		IsPaste:   isPaste,
+		Steps:     steps,
+		StepCount: stepCount,
+	}, nil
+}
+
 // PointerReport ..
 type PointerReport struct {
 	X      int
@@ -131,3 +186,20 @@ func (m *Message) MouseReport() (MouseReport, error) {
 		Button: uint8(m.d[2]),
 	}, nil
 }
+
+type KeyboardMacroState struct {
+	State   bool
+	IsPaste bool
+}
+
+// KeyboardMacroState returns the keyboard macro state report from the message.
+func (m *Message) KeyboardMacroState() (KeyboardMacroState, error) {
+	if m.t != TypeKeyboardMacroState {
+		return KeyboardMacroState{}, fmt.Errorf("invalid message type: %d", m.t)
+	}
+
+	return KeyboardMacroState{
+		State:   m.d[0] == uint8(1),
+		IsPaste: m.d[1] == uint8(1),
+	}, nil
+}

jsonrpc.go

@@ -1,6 +1,7 @@
 package kvm
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -10,12 +11,14 @@ import (
 	"path/filepath"
 	"reflect"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/pion/webrtc/v4"
 	"github.com/rs/zerolog"
 	"go.bug.st/serial"
 
+	"github.com/jetkvm/kvm/internal/hidrpc"
 	"github.com/jetkvm/kvm/internal/usbgadget"
 	"github.com/jetkvm/kvm/internal/utils"
 )
@@ -1056,6 +1059,106 @@ func rpcSetLocalLoopbackOnly(enabled bool) error {
 	return nil
 }
 
+var (
+	keyboardMacroCancel context.CancelFunc
+	keyboardMacroLock   sync.Mutex
+)
+
+// cancelKeyboardMacro cancels any ongoing keyboard macro execution
+func cancelKeyboardMacro() {
+	keyboardMacroLock.Lock()
+	defer keyboardMacroLock.Unlock()
+
+	if keyboardMacroCancel != nil {
+		keyboardMacroCancel()
+		logger.Info().Msg("canceled keyboard macro")
+		keyboardMacroCancel = nil
+	}
+}
+
+func setKeyboardMacroCancel(cancel context.CancelFunc) {
+	keyboardMacroLock.Lock()
+	defer keyboardMacroLock.Unlock()
+
+	keyboardMacroCancel = cancel
+}
+
+func rpcExecuteKeyboardMacro(macro []hidrpc.KeyboardMacroStep) (usbgadget.KeysDownState, error) {
+	cancelKeyboardMacro()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	setKeyboardMacroCancel(cancel)
+
+	s := hidrpc.KeyboardMacroState{
+		State:   true,
+		IsPaste: true,
+	}
+
+	if currentSession != nil {
+		currentSession.reportHidRPCKeyboardMacroState(s)
+	}
+
+	result, err := rpcDoExecuteKeyboardMacro(ctx, macro)
+
+	setKeyboardMacroCancel(nil)
+
+	s.State = false
+	if currentSession != nil {
+		currentSession.reportHidRPCKeyboardMacroState(s)
+	}
+
+	return result, err
+}
+
+func rpcCancelKeyboardMacro() {
+	cancelKeyboardMacro()
+}
+
+var keyboardClearStateKeys = make([]byte, hidrpc.HidKeyBufferSize)
+
+func isClearKeyStep(step hidrpc.KeyboardMacroStep) bool {
+	return step.Modifier == 0 && bytes.Equal(step.Keys, keyboardClearStateKeys)
+}
+
+func rpcDoExecuteKeyboardMacro(ctx context.Context, macro []hidrpc.KeyboardMacroStep) (usbgadget.KeysDownState, error) {
+	var last usbgadget.KeysDownState
+	var err error
+
+	logger.Debug().Interface("macro", macro).Msg("Executing keyboard macro")
+
+	for i, step := range macro {
+		delay := time.Duration(step.Delay) * time.Millisecond
+
+		last, err = rpcKeyboardReport(step.Modifier, step.Keys)
+		if err != nil {
+			logger.Warn().Err(err).Msg("failed to execute keyboard macro")
+			return last, err
+		}
+
+		// notify the device that the keyboard state is being cleared
+		if isClearKeyStep(step) {
+			gadget.UpdateKeysDown(0, keyboardClearStateKeys)
+		}
+
+		// Use context-aware sleep that can be cancelled
+		select {
+		case <-time.After(delay):
+			// Sleep completed normally
+		case <-ctx.Done():
+			// make sure keyboard state is reset
+			_, err := rpcKeyboardReport(0, keyboardClearStateKeys)
+			if err != nil {
+				logger.Warn().Err(err).Msg("failed to reset keyboard state")
+			}
+
+			logger.Debug().Int("step", i).Msg("Keyboard macro cancelled during sleep")
+			return last, ctx.Err()
+		}
+	}
+
+	return last, nil
+}
+
 var rpcHandlers = map[string]RPCHandler{
 	"ping":                   {Func: rpcPing},
 	"reboot":                 {Func: rpcReboot, Params: []string{"force"}},

ui/src/components/InfoBar.tsx

@@ -27,6 +27,7 @@ export default function InfoBar() {
 
   const { rpcDataChannel } = useRTCStore();
   const { debugMode, mouseMode, showPressedKeys } = useSettingsStore();
+  const { isPasteInProgress } = useHidStore();
 
   useEffect(() => {
     if (!rpcDataChannel) return;
@@ -108,7 +109,12 @@ export default function InfoBar() {
                 <span className="text-xs">{rpcHidStatus}</span>
               </div>
             )}
-
+            {isPasteInProgress && (
+              <div className="flex w-[156px] items-center gap-x-1">
+                <span className="text-xs font-semibold">Paste Mode:</span>
+                <span className="text-xs">Enabled</span>
+              </div>
+            )}
             {showPressedKeys && (
               <div className="flex items-center gap-x-1">
                 <span className="text-xs font-semibold">Keys:</span>