diff --git a/Cargo.lock b/Cargo.lock index f57b54a..aab6819 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2310,6 +2310,7 @@ name = "rbx-studio-mcp" version = "0.1.0" dependencies = [ "axum", + "base64 0.22.1", "clap 4.5.37", "color-eyre", "core-foundation 0.10.0", @@ -2324,6 +2325,7 @@ dependencies = [ "tokio", "tracing", "tracing-subscriber", + "urlencoding", "uuid", ] @@ -3473,6 +3475,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" diff --git a/Cargo.toml b/Cargo.toml index 938a837..5e3ec02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,9 +15,11 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } uuid = { version = "1", features = ["v4", "serde"] } axum = { version = "0.8", features = ["macros"] } reqwest = { version = "0.12", features = ["json"] } +urlencoding = "2.1" color-eyre = "0.6" clap = { version = "4.5.37", features = ["derive"] } roblox_install = "1.0.0" +base64 = "0.22" [target.'cfg(target_os = "macos")'.dependencies] native-dialog = "0.8.8" diff --git a/README.md b/README.md index d0ccaf0..57f1a4f 100644 --- a/README.md +++ b/README.md @@ -88,14 +88,57 @@ To make sure everything is set up correctly, follow these steps: which you can also verify in the console output. 1. Verify that Claude Desktop is correctly configured by clicking on the hammer icon for MCP tools beneath the text field where you enter prompts. This should open a window with the list of - available Roblox Studio tools (`insert_model` and `run_code`). + available Roblox Studio tools (`insert_model`, `run_code`, and `capture_screenshot`). **Note**: You can fix common issues with setup by restarting Studio and Claude Desktop. Claude sometimes is hidden in the system tray, so ensure you've exited it completely. +## Available Tools + +The MCP server provides the following tools for Claude to interact with Roblox Studio: + +### `run_code` +Runs Luau code in Roblox Studio and returns the printed output. Can be used to make changes or retrieve information from the currently open place. + +**Example prompts:** +- "Add a red part to the workspace" +- "List all parts in the workspace" +- "Move the camera to position (0, 50, 50)" + +### `insert_model` +Inserts a model from the Roblox marketplace into the workspace. Returns the inserted model name. + +**Example prompts:** +- "Insert a tree model" +- "Add a car from the marketplace" + +### `capture_screenshot` +Captures a screenshot of the Roblox Studio window and returns it as base64-encoded PNG data. This allows Claude to visually analyze your workspace, debug UI issues, or verify changes. + +**Example prompts:** +- "Take a screenshot of my workspace" +- "Show me what the current scene looks like" +- "Screenshot the studio and analyze the lighting" + +**Requirements:** +- **macOS**: Screen Recording permission must be granted to Terminal (or your MCP client) + - Go to **System Settings** → **Privacy & Security** → **Screen Recording** + - Enable **Terminal** (or your MCP client application) + - Restart Terminal/client after granting permission +- **Windows**: No additional permissions required + +**Note:** The screenshot captures the entire Roblox Studio window, including all panels and UI elements. The Studio window does not need to be focused or in the foreground. + ## Send requests 1. Open a place in Studio. 1. Type a prompt in Claude Desktop and accept any permissions to communicate with Studio. 1. Verify that the intended action is performed in Studio by checking the console, inspecting the data model in Explorer, or visually confirming the desired changes occurred in your place. + +## Using `insert_model` + +The `insert_model` tool searches the Roblox catalog for free models and inserts them into your place. For this to work, your place must have HTTP requests enabled: + +1. **Publish your place** — The place must be published to Roblox (it can be private). Go to **File > Publish to Roblox** in Studio. +2. **Enable HTTP requests** — In Studio, go to **Home > Game Settings > Security** and enable **Allow HTTP Requests**. diff --git a/plugin/src/Tools/InsertModel.luau b/plugin/src/Tools/InsertModel.luau index 885362c..fd9ad7d 100644 --- a/plugin/src/Tools/InsertModel.luau +++ b/plugin/src/Tools/InsertModel.luau @@ -22,23 +22,6 @@ local function getInsertPosition() end end -local InsertService = game:GetService("InsertService") - -type GetFreeModelsResponse = { - [number]: { - CurrentStartIndex: number, - TotalCount: number, - Results: { - [number]: { - Name: string, - AssetId: number, - AssetVersionId: number, - CreatorName: string, - }, - }, - }, -} - local function toTitleCase(str: string): string local function titleCase(first: string, rest: string) return first:upper() .. rest:lower() @@ -81,28 +64,10 @@ local function loadAsset(assetId: number): Instance? return collapseObjectsIntoContainer(objects) end -local function getAssets(query: string): number? - local results: GetFreeModelsResponse = InsertService:GetFreeModels(query, 0) - local assets = {} - for i, result in results[1].Results do - if i > 6 then - break - end - table.insert(assets, result.AssetId) - end - - return table.remove(assets, 1) -end - -local function insertFromMarketplace(query: string): string - local primaryResult = getAssets(query) - if not primaryResult then - error("Failed to find asset") - end - - local instance = loadAsset(primaryResult) +local function insertFromAssetId(assetId: number, query: string): string + local instance = loadAsset(assetId) if not instance then - error("Failed to load asset") + error("Failed to load asset with ID: " .. tostring(assetId)) end local name = toTitleCase(query) @@ -132,7 +97,12 @@ local function handleInsertModel(args: Types.ToolArgs): string? error("Missing query in InsertModel") end - return insertFromMarketplace(insertModelArgs.query) + -- Use asset_id provided by the Rust server (from catalog API) + if insertModelArgs.asset_id then + return insertFromAssetId(insertModelArgs.asset_id, insertModelArgs.query) + else + error("No asset_id provided - catalog search must be done server-side") + end end return handleInsertModel :: Types.ToolFunction diff --git a/plugin/src/Types.luau b/plugin/src/Types.luau index b9f1623..104f110 100644 --- a/plugin/src/Types.luau +++ b/plugin/src/Types.luau @@ -1,5 +1,6 @@ export type InsertModelArgs = { query: string, + asset_id: number?, } export type RunCodeArgs = { diff --git a/src/rbx_studio_server.rs b/src/rbx_studio_server.rs index 6469f5f..5349e60 100644 --- a/src/rbx_studio_server.rs +++ b/src/rbx_studio_server.rs @@ -2,6 +2,7 @@ use crate::error::Result; use axum::http::StatusCode; use axum::response::IntoResponse; use axum::{extract::State, Json}; +use base64::Engine; use color_eyre::eyre::{Error, OptionExt}; use rmcp::{ handler::server::tool::Parameters, @@ -19,6 +20,50 @@ use tokio::sync::{mpsc, watch, Mutex}; use tokio::time::Duration; use uuid::Uuid; +// Roblox catalog API response structures +#[derive(Debug, Deserialize)] +struct CatalogSearchResponse { + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct CatalogItem { + id: u64, +} + +/// Search the Roblox catalog for free models and return the first asset ID +async fn search_roblox_catalog(query: &str) -> std::result::Result { + let client = reqwest::Client::new(); + + // Category 3 = Models, salesTypeFilter 1 = Free + let url = format!( + "https://catalog.roblox.com/v1/search/items?category=3&keyword={}&limit=10&salesTypeFilter=1", + urlencoding::encode(query) + ); + + let response = client + .get(&url) + .header("Accept", "application/json") + .send() + .await + .map_err(|e| format!("Failed to search catalog: {}", e))?; + + if !response.status().is_success() { + return Err(format!("Catalog API returned status: {}", response.status())); + } + + let catalog: CatalogSearchResponse = response + .json() + .await + .map_err(|e| format!("Failed to parse catalog response: {}", e))?; + + catalog + .data + .first() + .map(|item| item.id) + .ok_or_else(|| format!("No free models found matching '{}'. Try a different search term.", query)) +} + pub const STUDIO_PLUGIN_PORT: u16 = 44755; const LONG_POLL_DURATION: Duration = Duration::from_secs(15); @@ -99,6 +144,14 @@ struct RunCode { struct InsertModel { #[schemars(description = "Query to search for the model")] query: String, + #[serde(skip_deserializing)] + #[schemars(skip)] + asset_id: Option, +} + +#[derive(Debug, Deserialize, Serialize, schemars::JsonSchema, Clone)] +struct CaptureScreenshot { + // No parameters for v1 - just capture the Studio window } #[derive(Debug, Deserialize, Serialize, schemars::JsonSchema, Clone)] @@ -131,10 +184,33 @@ impl RBXStudioServer { )] async fn insert_model( &self, - Parameters(args): Parameters, + Parameters(mut args): Parameters, ) -> Result { - self.generic_tool_run(ToolArgumentValues::InsertModel(args)) - .await + // Search the Roblox catalog from the server side (bypasses Lua HttpService restrictions) + match search_roblox_catalog(&args.query).await { + Ok(asset_id) => { + args.asset_id = Some(asset_id); + self.generic_tool_run(ToolArgumentValues::InsertModel(args)).await + } + Err(e) => Ok(CallToolResult::error(vec![Content::text(e)])), + } + } + + #[tool( + description = "Captures a screenshot of the Roblox Studio window and returns it as base64-encoded PNG data" + )] + async fn capture_screenshot( + &self, + Parameters(_args): Parameters, + ) -> Result { + // Rust-only implementation - no plugin communication needed + match Self::take_studio_screenshot().await { + Ok(base64_data) => Ok(CallToolResult::success(vec![Content::text(base64_data)])), + Err(e) => Ok(CallToolResult::error(vec![Content::text(format!( + "Failed to capture screenshot: {}", + e + ))])), + } } async fn generic_tool_run( @@ -167,6 +243,138 @@ impl RBXStudioServer { Err(err) => Ok(CallToolResult::error(vec![Content::text(err.to_string())])), } } + + #[cfg(target_os = "macos")] + async fn take_studio_screenshot() -> Result { + use std::process::Command; + use std::fs; + use std::io::Write; + + // Create temp files + let temp_screenshot = std::env::temp_dir().join(format!("roblox_studio_{}.png", Uuid::new_v4())); + let temp_swift = std::env::temp_dir().join(format!("get_window_{}.swift", Uuid::new_v4())); + + // Swift script to get window ID without requiring accessibility permissions + let swift_script = r#" +import Cocoa +import CoreGraphics + +let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] ?? [] + +for window in windowList { + if let ownerName = window[kCGWindowOwnerName as String] as? String, + ownerName.contains("Roblox"), + let windowNumber = window[kCGWindowNumber as String] as? Int { + print(windowNumber) + exit(0) + } +} +exit(1) +"#; + + // Write Swift script to temp file + let mut swift_file = fs::File::create(&temp_swift)?; + swift_file.write_all(swift_script.as_bytes())?; + drop(swift_file); + + // Get window ID for Roblox Studio using Swift + let window_id_output = Command::new("swift") + .arg(&temp_swift) + .output()?; + + // Clean up Swift temp file + let _ = fs::remove_file(&temp_swift); + + if !window_id_output.status.success() { + return Err(Error::msg("No Roblox Studio window found. Please open Roblox Studio.")); + } + + let window_id_str = String::from_utf8_lossy(&window_id_output.stdout); + let window_id = window_id_str.trim().parse::() + .map_err(|_| Error::msg("Failed to parse window ID"))?; + + // Capture the window + let capture = Command::new("screencapture") + .arg("-l") + .arg(window_id.to_string()) + .arg("-o") // Disable window shadow + .arg("-x") // No sound + .arg(&temp_screenshot) + .status()?; + + if !capture.success() { + return Err(Error::msg("Failed to capture screenshot")); + } + + // Read and encode the image + let image_data = fs::read(&temp_screenshot)?; + + // Clean up screenshot temp file + let _ = fs::remove_file(&temp_screenshot); + + // Encode to base64 + Ok(base64::engine::general_purpose::STANDARD.encode(&image_data)) + } + + #[cfg(target_os = "windows")] + async fn take_studio_screenshot() -> Result { + use std::process::Command; + use std::fs; + + // Create temp file for screenshot + let temp_path = std::env::temp_dir().join(format!("roblox_studio_{}.png", Uuid::new_v4())); + + // PowerShell script to capture Roblox Studio window + let ps_script = format!( + r#" + Add-Type -AssemblyName System.Windows.Forms + Add-Type -AssemblyName System.Drawing + + $process = Get-Process | Where-Object {{ $_.MainWindowTitle -like "*Roblox Studio*" }} | Select-Object -First 1 + if ($null -eq $process) {{ + Write-Error "No Roblox Studio window found" + exit 1 + }} + + $handle = $process.MainWindowHandle + $rect = New-Object RECT + [Win32]::GetWindowRect($handle, [ref]$rect) + + $width = $rect.Right - $rect.Left + $height = $rect.Bottom - $rect.Top + + $bitmap = New-Object System.Drawing.Bitmap $width, $height + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $graphics.CopyFromScreen($rect.Left, $rect.Top, 0, 0, $bitmap.Size) + + $bitmap.Save('{}', [System.Drawing.Imaging.ImageFormat]::Png) + "#, + temp_path.display() + ); + + let capture = Command::new("powershell") + .arg("-Command") + .arg(&ps_script) + .status()?; + + if !capture.success() { + return Err(Error::msg("Failed to capture screenshot. Is Roblox Studio running?")); + } + + // Read and encode the image + let image_data = fs::read(&temp_path)?; + + // Clean up temp file + let _ = fs::remove_file(&temp_path); + + // Encode to base64 + Ok(base64::engine::general_purpose::STANDARD.encode(&image_data)) + } + + #[cfg(not(any(target_os = "macos", target_os = "windows")))] + async fn take_studio_screenshot() -> Result { + Err(Error::msg("Screenshot capture is only supported on macOS and Windows")) + } } pub async fn request_handler(State(state): State) -> Result {