1818 # 4. Run demo: python demos/22_vm_demo.py
1919 # 5. Watch: open http://localhost:8430 -> Live VM tab
2020"""
21+
2122from __future__ import annotations
2223
2324import base64
3435from PIL import Image
3536
3637# ── Config ──────────────────────────────────────
37- VM_IP = os .environ .get ("VM_IP" , "192.168.64.13 " )
38- VNC_USER = os .environ .get ("VNC_USER" , "admin " )
39- VNC_PASS = os .environ .get ("VNC_PASS" , "admin " )
38+ VM_IP = os .environ .get ("VM_IP" , "10.0.0.1 " )
39+ VNC_USER = os .environ .get ("VNC_USER" , "changeme " )
40+ VNC_PASS = os .environ .get ("VNC_PASS" , "changeme " )
4041CUA_URL = f"http://{ VM_IP } :8000/cmd" # For run_command only
4142MODEL = "anthropic/claude-sonnet-4.6"
4243LABWORK_URL = os .environ .get ("LABWORK_URL" , "http://localhost:8430" )
5960
6061# ── VNC helpers (via vncdo CLI) ─────────────────
6162
63+
6264def _vncdo (* args : str , timeout : int = 15 ) -> subprocess .CompletedProcess :
6365 """Run vncdo command with ARD auth credentials."""
6466 cmd = [
6567 sys .executable .replace ("python" , "vncdo" ).replace (
66- "bin/python" , "bin/vncdo" ,
68+ "bin/python" ,
69+ "bin/vncdo" ,
6770 ),
68- "-s" , VM_IP ,
69- "--username" , VNC_USER ,
70- "--password" , VNC_PASS ,
71+ "-s" ,
72+ VM_IP ,
73+ "--username" ,
74+ VNC_USER ,
75+ "--password" ,
76+ VNC_PASS ,
7177 * args ,
7278 ]
7379 # Fallback: find vncdo next to python
@@ -106,17 +112,29 @@ def vnc_type_command(cmd: str) -> None:
106112 """
107113 # Triple-click to select all text in command line, then type + Return
108114 _vncdo (
109- "move" , str (CMD_X ), str (CMD_Y ),
110- "pause" , "0.2" ,
111- "click" , "1" ,
112- "pause" , "0.05" ,
113- "click" , "1" ,
114- "pause" , "0.05" ,
115- "click" , "1" ,
116- "pause" , "0.3" ,
117- "type" , cmd ,
118- "pause" , "0.3" ,
119- "key" , "return" ,
115+ "move" ,
116+ str (CMD_X ),
117+ str (CMD_Y ),
118+ "pause" ,
119+ "0.2" ,
120+ "click" ,
121+ "1" ,
122+ "pause" ,
123+ "0.05" ,
124+ "click" ,
125+ "1" ,
126+ "pause" ,
127+ "0.05" ,
128+ "click" ,
129+ "1" ,
130+ "pause" ,
131+ "0.3" ,
132+ "type" ,
133+ cmd ,
134+ "pause" ,
135+ "0.3" ,
136+ "key" ,
137+ "return" ,
120138 )
121139
122140
@@ -125,6 +143,7 @@ def vnc_type_command(cmd: str) -> None:
125143# CUA keyboard/mouse don't reach Java/Swing apps in macOS VMs.
126144# We only use CUA for run_command (launching apps, killing processes).
127145
146+
128147def _cua_cmd (command : str , params : dict | None = None ) -> dict :
129148 """Send a command to CUA server, return parsed response."""
130149 body : dict = {"command" : command }
@@ -145,6 +164,7 @@ def cua_run(command: str) -> dict:
145164
146165# ── Stream helpers ──────────────────────────────
147166
167+
148168def push_frame (jpeg_bytes : bytes ) -> None :
149169 """Push a JPEG frame to labwork-web MJPEG stream."""
150170 try :
@@ -190,41 +210,47 @@ def screenshot_and_push() -> tuple[str, bytes]:
190210
191211# ── VLM ─────────────────────────────────────────
192212
213+
193214def ask_sonnet (client : OpenAI , screenshot_b64 : str , question : str ) -> str :
194215 """Send screenshot + question to Sonnet 4.6, get text response."""
195216 response = client .chat .completions .create (
196217 model = MODEL ,
197218 max_tokens = 512 ,
198- messages = [{
199- "role" : "user" ,
200- "content" : [
201- {
202- "type" : "image_url" ,
203- "image_url" : {
204- "url" : f"data:image/png;base64,{ screenshot_b64 } " ,
219+ messages = [
220+ {
221+ "role" : "user" ,
222+ "content" : [
223+ {
224+ "type" : "image_url" ,
225+ "image_url" : {
226+ "url" : f"data:image/png;base64,{ screenshot_b64 } " ,
227+ },
228+ },
229+ {
230+ "type" : "text" ,
231+ "text" : (
232+ "This is a 1280x960 screenshot of macOS with "
233+ "Bruker TopSpin 5.0 NMR software.\n "
234+ f"{ question } \n \n "
235+ "Return ONLY JSON, no markdown fences."
236+ ),
205237 },
206- },
207- {
208- "type" : "text" ,
209- "text" : (
210- "This is a 1280x960 screenshot of macOS with "
211- "Bruker TopSpin 5.0 NMR software.\n "
212- f"{ question } \n \n "
213- "Return ONLY JSON, no markdown fences."
214- ),
215- },
216- ],
217- }],
238+ ],
239+ }
240+ ],
218241 )
219242 return response .choices [0 ].message .content .strip ()
220243
221244
222245def ask_verify (
223- client : OpenAI , screenshot_b64 : str , check : str ,
246+ client : OpenAI ,
247+ screenshot_b64 : str ,
248+ check : str ,
224249) -> dict :
225250 """Ask Sonnet to verify a condition."""
226251 text = ask_sonnet (
227- client , screenshot_b64 ,
252+ client ,
253+ screenshot_b64 ,
228254 f'{ check } \n Return: {{"ok": true/false, "description": "what you see"}}' ,
229255 )
230256 m = re .search (r"\{[^}]+\}" , text )
@@ -238,6 +264,7 @@ def ask_verify(
238264
239265# ── Pipeline Steps ──────────────────────────────
240266
267+
241268def step_ensure_topspin (ai : OpenAI ) -> bool :
242269 """Ensure TopSpin is open and visible."""
243270 push_log (">>> Ensure TopSpin visible" , status = "operating" )
@@ -247,7 +274,8 @@ def step_ensure_topspin(ai: OpenAI) -> bool:
247274
248275 b64 , _ = screenshot_and_push ()
249276 result = ask_verify (
250- ai , b64 ,
277+ ai ,
278+ b64 ,
251279 "Is Bruker TopSpin 5.0 open with its main window visible? "
252280 "Look for the TopSpin toolbar, spectrum area, and command line." ,
253281 )
@@ -282,7 +310,8 @@ def step_load_dataset(ai: OpenAI) -> bool:
282310
283311 b64 , _ = screenshot_and_push ()
284312 result = ask_verify (
285- ai , b64 ,
313+ ai ,
314+ b64 ,
286315 "Has NMR data been loaded? Look for a spectrum plot "
287316 "(FID or frequency domain) in the main panel, OR dataset "
288317 "info in the title area." ,
@@ -296,7 +325,9 @@ def step_load_dataset(ai: OpenAI) -> bool:
296325 time .sleep (5 )
297326 b64 , _ = screenshot_and_push ()
298327 result = ask_verify (
299- ai , b64 , "Is there ANY spectrum or data displayed in TopSpin?" ,
328+ ai ,
329+ b64 ,
330+ "Is there ANY spectrum or data displayed in TopSpin?" ,
300331 )
301332 ok = result .get ("ok" , False )
302333 push_log (f" { 'OK' if ok else 'WARN' } { result .get ('description' , '' )} " )
@@ -321,7 +352,8 @@ def step_run_command(
321352 if handle_dialog :
322353 for _dlg in range (3 ):
323354 dlg = ask_verify (
324- ai , b64 ,
355+ ai ,
356+ b64 ,
325357 "Is there a dialog/popup window visible in the CENTER? "
326358 "NOT a notification. Set ok=true ONLY for centered "
327359 "dialogs with Close/OK/Cancel buttons." ,
@@ -336,8 +368,7 @@ def step_run_command(
336368 result = ask_verify (ai , b64 , verify_prompt )
337369 ok = result .get ("ok" , False )
338370 push_log (
339- f" { 'OK' if ok else 'WARN' } { step_name } : "
340- f"{ result .get ('description' , '' )} " ,
371+ f" { 'OK' if ok else 'WARN' } { step_name } : { result .get ('description' , '' )} " ,
341372 )
342373 return ok
343374
@@ -348,7 +379,8 @@ def step_verify_result(ai: OpenAI) -> bool:
348379
349380 b64 , _ = screenshot_and_push ()
350381 result = ask_verify (
351- ai , b64 ,
382+ ai ,
383+ b64 ,
352384 "Describe the NMR spectrum visible in TopSpin. Report ok=true "
353385 "if you can see: (1) NMR peaks in the spectrum display, "
354386 "(2) a chemical shift axis (ppm) at the bottom, "
@@ -362,6 +394,7 @@ def step_verify_result(ai: OpenAI) -> bool:
362394
363395# ── Main ────────────────────────────────────────
364396
397+
365398def main () -> int :
366399 print (
367400 f"\n { B } { '=' * 55 } { RST } \n "
@@ -375,7 +408,11 @@ def main() -> int:
375408 api_key = os .environ .get ("OPENROUTER_API_KEY" )
376409 if not api_key :
377410 env_path = os .path .join (
378- os .path .dirname (__file__ ), ".." , ".." , "labwork-web" , ".env" ,
411+ os .path .dirname (__file__ ),
412+ ".." ,
413+ ".." ,
414+ "labwork-web" ,
415+ ".env" ,
379416 )
380417 if os .path .exists (env_path ):
381418 with open (env_path ) as f :
@@ -431,8 +468,7 @@ def main() -> int:
431468 cmd = "efp" ,
432469 step_name = "Fourier Transform" ,
433470 verify_prompt = (
434- "Has the spectrum changed after Fourier transform? "
435- "Look for frequency-domain peaks."
471+ "Has the spectrum changed after Fourier transform? Look for frequency-domain peaks."
436472 ),
437473 )
438474 results .append (("Fourier Transform" , ok ))
0 commit comments