Skip to content

os level screenshots #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rgarcia merged 8 commits into from
Oct 15, 2025
Merged

os level screenshots #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
7291068
native screenshots
rgarcia Oct 8, 2025
b51a533
Merge branch 'main' into raf/kernel-149-os-level-screenshots
rgarcia Oct 14, 2025
0db9012
validate upper bounds on x,y
rgarcia Oct 15, 2025
1f267f4
Merge branch 'main' into raf/kernel-149-os-level-screenshots
rgarcia Oct 15, 2025
5eb5adc
bot comments
rgarcia Oct 15, 2025
5b2d329
bot comments
rgarcia Oct 15, 2025
20aab2f
add type endpoint
rgarcia Oct 15, 2025
4094aca
one more bot comment
rgarcia Oct 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 172 additions & 2 deletions server/cmd/api/api/computer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,11 @@ package api

import (
"context"
"encoding/base64"
"fmt"
"io"
"os"
"os/exec"
"strconv"

"github.com/onkernel/kernel-images/server/lib/logger"
Expand All @@ -12,16 +16,29 @@ import (
func (s *ApiService) MoveMouse(ctx context.Context, request oapi.MoveMouseRequestObject) (oapi.MoveMouseResponseObject, error) {
log := logger.FromContext(ctx)

s.stz.Disable(ctx)
defer s.stz.Enable(ctx)

// Validate request body
if request.Body == nil {
return oapi.MoveMouse400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "request body is required"}}, nil
}
body := *request.Body

// Ensure non-negative coordinates
// Get current resolution for bounds validation
screenWidth, screenHeight, _, err := s.getCurrentResolution(ctx)
if err != nil {
log.Error("failed to get current resolution", "error", err)
return oapi.MoveMouse500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "failed to get current display resolution"}}, nil
}

// Ensure non-negative coordinates and within screen bounds
if body.X < 0 || body.Y < 0 {
return oapi.MoveMouse400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "coordinates must be non-negative"}}, nil
}
if body.X >= screenWidth || body.Y >= screenHeight {
return oapi.MoveMouse400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: fmt.Sprintf("coordinates exceed screen bounds (max: %dx%d)", screenWidth-1, screenHeight-1)}}, nil
}

// Build xdotool arguments
args := []string{}
Expand Down Expand Up @@ -57,16 +74,29 @@ func (s *ApiService) MoveMouse(ctx context.Context, request oapi.MoveMouseReques
func (s *ApiService) ClickMouse(ctx context.Context, request oapi.ClickMouseRequestObject) (oapi.ClickMouseResponseObject, error) {
log := logger.FromContext(ctx)

s.stz.Disable(ctx)
defer s.stz.Enable(ctx)

// Validate request body
if request.Body == nil {
return oapi.ClickMouse400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "request body is required"}}, nil
}
body := *request.Body

// Ensure non-negative coordinates
// Get current resolution for bounds validation
screenWidth, screenHeight, _, err := s.getCurrentResolution(ctx)
if err != nil {
log.Error("failed to get current resolution", "error", err)
return oapi.ClickMouse500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "failed to get current display resolution"}}, nil
}

// Ensure non-negative coordinates and within screen bounds
if body.X < 0 || body.Y < 0 {
return oapi.ClickMouse400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "coordinates must be non-negative"}}, nil
}
if body.X >= screenWidth || body.Y >= screenHeight {
return oapi.ClickMouse400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: fmt.Sprintf("coordinates exceed screen bounds (max: %dx%d)", screenWidth-1, screenHeight-1)}}, nil
}

// Map button enum to xdotool button code. Default to left button.
btn := "1"
Expand Down Expand Up @@ -143,3 +173,143 @@ func (s *ApiService) ClickMouse(ctx context.Context, request oapi.ClickMouseRequ

return oapi.ClickMouse200Response{}, nil
}

func (s *ApiService) TakeScreenshot(ctx context.Context, request oapi.TakeScreenshotRequestObject) (oapi.TakeScreenshotResponseObject, error) {
log := logger.FromContext(ctx)

s.stz.Disable(ctx)
defer s.stz.Enable(ctx)

var body oapi.ScreenshotRequest
if request.Body != nil {
body = *request.Body
}

// Get current resolution for bounds validation
screenWidth, screenHeight, _, err := s.getCurrentResolution(ctx)
if err != nil {
log.Error("failed to get current resolution", "error", err)
return oapi.TakeScreenshot500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "failed to get current display resolution"}}, nil
}

// Determine display to use (align with other functions)
display := s.resolveDisplayFromEnv()

// Validate region if provided
if body.Region != nil {
r := body.Region
if r.X < 0 || r.Y < 0 || r.Width <= 0 || r.Height <= 0 {
return oapi.TakeScreenshot400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "invalid region dimensions"}}, nil
}
if r.X+r.Width > screenWidth || r.Y+r.Height > screenHeight {
return oapi.TakeScreenshot400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "region exceeds screen bounds"}}, nil
}
}

// Build ffmpeg command
args := []string{
"-f", "x11grab",
"-video_size", fmt.Sprintf("%dx%d", screenWidth, screenHeight),
"-i", display,
"-vframes", "1",
}

// Add crop filter if region is specified
if body.Region != nil {
r := body.Region
cropFilter := fmt.Sprintf("crop=%d:%d:%d:%d", r.Width, r.Height, r.X, r.Y)
args = append(args, "-vf", cropFilter)
}

// Output as PNG to stdout
args = append(args, "-f", "image2pipe", "-vcodec", "png", "-")

cmd := exec.CommandContext(ctx, "ffmpeg", args...)
cmd.Env = append(os.Environ(), fmt.Sprintf("DISPLAY=%s", display))

log.Debug("executing ffmpeg command", "args", args, "display", display)

stdout, err := cmd.StdoutPipe()
if err != nil {
log.Error("failed to create stdout pipe", "err", err)
return oapi.TakeScreenshot500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "internal error"}}, nil
}

stderr, err := cmd.StderrPipe()
if err != nil {
log.Error("failed to create stderr pipe", "err", err)
return oapi.TakeScreenshot500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "internal error"}}, nil
}

if err := cmd.Start(); err != nil {
log.Error("failed to start ffmpeg", "err", err)
return oapi.TakeScreenshot500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "failed to start ffmpeg"}}, nil
}

// Start a goroutine to drain stderr for logging to avoid blocking
go func() {
data, _ := io.ReadAll(stderr)
if len(data) > 0 {
// ffmpeg writes progress/info to stderr; include in debug logs
enc := base64.StdEncoding.EncodeToString(data)
log.Debug("ffmpeg stderr (base64)", "data_b64", enc)
}
}()

pr, pw := io.Pipe()
go func() {
_, copyErr := io.Copy(pw, stdout)
waitErr := cmd.Wait()
var closeErr error
if copyErr != nil {
closeErr = fmt.Errorf("streaming ffmpeg output: %w", copyErr)
log.Error("failed streaming ffmpeg output", "err", copyErr)
} else if waitErr != nil {
closeErr = fmt.Errorf("ffmpeg exited with error: %w", waitErr)
log.Error("ffmpeg exited with error", "err", waitErr)
}
if closeErr != nil {
_ = pw.CloseWithError(closeErr)
return
}
_ = pw.Close()
}()

return oapi.TakeScreenshot200ImagepngResponse{Body: pr, ContentLength: 0}, nil
}

func (s *ApiService) TypeText(ctx context.Context, request oapi.TypeTextRequestObject) (oapi.TypeTextResponseObject, error) {
log := logger.FromContext(ctx)

s.stz.Disable(ctx)
defer s.stz.Enable(ctx)

// Validate request body
if request.Body == nil {
return oapi.TypeText400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "request body is required"}}, nil
}
body := *request.Body

// Validate delay if provided
if body.Delay != nil && *body.Delay < 0 {
return oapi.TypeText400JSONResponse{BadRequestErrorJSONResponse: oapi.BadRequestErrorJSONResponse{Message: "delay must be >= 0 milliseconds"}}, nil
}

// Build xdotool arguments
args := []string{"type"}
if body.Delay != nil {
args = append(args, "--delay", strconv.Itoa(*body.Delay))
}
// Use "--" to terminate options and pass raw text
args = append(args, "--", body.Text)

log.Info("executing xdotool", "args", args)

output, err := defaultXdoTool.Run(ctx, args...)
if err != nil {
log.Error("xdotool command failed", "err", err, "output", string(output))
return oapi.TypeText500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "failed to type text"}}, nil
}

return oapi.TypeText200Response{}, nil
}
20 changes: 11 additions & 9 deletions server/cmd/api/api/display.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,11 @@ func (s *ApiService) PatchDisplay(ctx context.Context, req oapi.PatchDisplayRequ
}

// Get current resolution with refresh rate
currentWidth, currentHeight, currentRefreshRate := s.getCurrentResolution(ctx)
currentWidth, currentHeight, currentRefreshRate, err := s.getCurrentResolution(ctx)
if err != nil {
log.Error("failed to get current resolution", "error", err)
return oapi.PatchDisplay500JSONResponse{InternalErrorJSONResponse: oapi.InternalErrorJSONResponse{Message: "failed to get current display resolution"}}, nil
}
width := currentWidth
height := currentHeight
refreshRate := currentRefreshRate
Expand Down Expand Up @@ -88,7 +92,6 @@ func (s *ApiService) PatchDisplay(ctx context.Context, req oapi.PatchDisplayRequ
}

// Route to appropriate resolution change handler
var err error
if displayMode == "xorg" {
if s.isNekoEnabled() {
log.Info("using Neko API for Xorg resolution change")
Expand Down Expand Up @@ -312,7 +315,7 @@ func (s *ApiService) resolveDisplayFromEnv() string {
}

// getCurrentResolution returns the current display resolution and refresh rate by querying xrandr
func (s *ApiService) getCurrentResolution(ctx context.Context) (int, int, int) {
func (s *ApiService) getCurrentResolution(ctx context.Context) (int, int, int, error) {
log := logger.FromContext(ctx)
display := s.resolveDisplayFromEnv()

Expand All @@ -324,21 +327,20 @@ func (s *ApiService) getCurrentResolution(ctx context.Context) (int, int, int) {
out, err := cmd.Output()
if err != nil {
log.Error("failed to get current resolution", "error", err)
// Return default resolution on error
return 1024, 768, 60
return 0, 0, 0, fmt.Errorf("failed to execute xrandr command: %w", err)
}

resStr := strings.TrimSpace(string(out))
parts := strings.Split(resStr, "x")
if len(parts) != 2 {
log.Error("unexpected xrandr output format", "output", resStr)
return 1024, 768, 60
return 0, 0, 0, fmt.Errorf("unexpected xrandr output format: %s", resStr)
}

width, err := strconv.Atoi(parts[0])
if err != nil {
log.Error("failed to parse width", "error", err, "value", parts[0])
return 1024, 768, 60
return 0, 0, 0, fmt.Errorf("failed to parse width '%s': %w", parts[0], err)
}

// Parse height and refresh rate (e.g., "1080_60.00" -> height=1080, rate=60)
Expand All @@ -356,10 +358,10 @@ func (s *ApiService) getCurrentResolution(ctx context.Context) (int, int, int) {
height, err := strconv.Atoi(heightStr)
if err != nil {
log.Error("failed to parse height", "error", err, "value", heightStr)
return 1024, 768, 60
return 0, 0, 0, fmt.Errorf("failed to parse height '%s': %w", heightStr, err)
}

return width, height, refreshRate
return width, height, refreshRate, nil
}

// isNekoEnabled checks if Neko service is enabled
Expand Down
Loading
Loading