diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..c5f683742 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# Exclude directories that are modified by concurrent processes (agents, tests) +# during Modal's image upload. The Docker image only needs current.tar.gz; +# all other repo files are delivered via the tarball. +.git/ +**/__pycache__/ +**/*.pyc +**/.pytest_cache/ +**/.test_output/ +.venv/ +**/.ruff_cache/ +.mngr/dev/ +test-results/ +tmr_*/ +node_modules/ diff --git a/.gitignore b/.gitignore index 67b59a92b..95f7a453a 100644 --- a/.gitignore +++ b/.gitignore @@ -248,6 +248,8 @@ tmr_*/ # Offload caches and local files .offload/** +.offload-image-cache +.offload-cache-key test-results current.tar.gz diff --git a/.mngr/settings.toml b/.mngr/settings.toml index cd2421f63..7a3c148e4 100644 --- a/.mngr/settings.toml +++ b/.mngr/settings.toml @@ -2,6 +2,9 @@ is_allowed_in_pytest = false #default_destroyed_host_persisted_seconds = 36000 default_destroyed_host_persisted_seconds = 10 +[work_dir_extra_paths] +test-results = "SHARE" + [pre_command_scripts] create = ["""bash -c './scripts/make_tar_of_repo.sh `cat .mngr/image_commit_hash` .mngr/dev/build'"""] diff --git a/.offload-base-commit b/.offload-base-commit index f47f153fb..3757d06d7 100644 --- a/.offload-base-commit +++ b/.offload-base-commit @@ -1 +1 @@ -73ce8f3cc81924aba41ecf000524975b18979f6a +36f9e3b6d88585b335655d893890adeb123d4a97 diff --git a/.offload-image-cache b/.offload-image-cache index f8d65f903..12a6bfa74 100644 --- a/.offload-image-cache +++ b/.offload-image-cache @@ -1 +1 @@ -im-Hg9iOacASyJ2Im5Gt1qJ91 +im-xNeY6u8qRzETya9SVpJWdi diff --git a/apps/minds/imbue/minds/forwarding_server/agent_creator.py b/apps/minds/imbue/minds/forwarding_server/agent_creator.py index c62d3e2d0..a6d10b3e8 100644 --- a/apps/minds/imbue/minds/forwarding_server/agent_creator.py +++ b/apps/minds/imbue/minds/forwarding_server/agent_creator.py @@ -288,6 +288,7 @@ class AgentCreator(MutableModel): _redirect_urls: dict[str, str] = PrivateAttr(default_factory=dict) _errors: dict[str, str] = PrivateAttr(default_factory=dict) _log_queues: dict[str, queue.Queue[str]] = PrivateAttr(default_factory=dict) + _threads: dict[str, threading.Thread] = PrivateAttr(default_factory=dict) _lock: threading.Lock = PrivateAttr(default_factory=threading.Lock) def start_creation( @@ -317,9 +318,25 @@ def start_creation( daemon=True, name="agent-creator-{}".format(agent_id), ) + with self._lock: + self._threads[str(agent_id)] = thread thread.start() return agent_id + def wait_for_completion(self, agent_id: AgentId, timeout: float) -> None: + """Wait for the background creation thread to finish.""" + with self._lock: + thread = self._threads.get(str(agent_id)) + if thread is not None: + thread.join(timeout=timeout) + + def close(self, timeout: float = 10.0) -> None: + """Wait for all background creation threads to finish.""" + with self._lock: + threads = list(self._threads.values()) + for thread in threads: + thread.join(timeout=timeout) + def get_creation_info(self, agent_id: AgentId) -> AgentCreationInfo | None: """Get the current creation status for an agent, or None if not tracked.""" with self._lock: diff --git a/apps/minds/imbue/minds/forwarding_server/agent_creator_test.py b/apps/minds/imbue/minds/forwarding_server/agent_creator_test.py index 384194466..51e7ed714 100644 --- a/apps/minds/imbue/minds/forwarding_server/agent_creator_test.py +++ b/apps/minds/imbue/minds/forwarding_server/agent_creator_test.py @@ -179,6 +179,9 @@ def test_agent_creator_start_creation_returns_agent_id_and_tracks_status(tmp_pat assert info.agent_id == agent_id assert info.status == AgentCreationStatus.CLONING + # Wait for the background thread to finish so git clone doesn't leak. + creator.wait_for_completion(agent_id, timeout=10.0) + def test_agent_creator_start_creation_with_custom_name(tmp_path: Path) -> None: """Verify start_creation accepts a custom agent name.""" @@ -189,6 +192,8 @@ def test_agent_creator_start_creation_with_custom_name(tmp_path: Path) -> None: info = creator.get_creation_info(agent_id) assert info is not None + creator.wait_for_completion(agent_id, timeout=10.0) + def test_agent_creator_get_log_queue_returns_none_for_unknown() -> None: creator = AgentCreator( @@ -205,6 +210,8 @@ def test_agent_creator_get_log_queue_returns_queue_for_tracked() -> None: q = creator.get_log_queue(agent_id) assert q is not None + creator.close() + def test_make_log_callback_puts_lines_into_queue() -> None: log_queue: queue_mod.Queue[str] = queue_mod.Queue() diff --git a/apps/minds/imbue/minds/forwarding_server/backend_resolver_test.py b/apps/minds/imbue/minds/forwarding_server/backend_resolver_test.py index dd46e4fed..21121eb1a 100644 --- a/apps/minds/imbue/minds/forwarding_server/backend_resolver_test.py +++ b/apps/minds/imbue/minds/forwarding_server/backend_resolver_test.py @@ -571,6 +571,8 @@ def test_stream_manager_full_snapshot_updates_agent_ids() -> None: ) with manager._cg: manager._handle_discovery_line(line) + for process in manager._events_processes.values(): + process.terminate() ids = manager.resolver.list_known_agent_ids() assert _AGENT_A in ids @@ -601,6 +603,9 @@ def test_stream_manager_host_ssh_info_populates_resolver() -> None: ssh_line = _make_host_ssh_info_line(host_id, ssh_data) manager._handle_discovery_line(ssh_line) + for process in manager._events_processes.values(): + process.terminate() + ssh_info = manager.resolver.get_ssh_info(_AGENT_A) assert ssh_info is not None assert ssh_info.host == "remote.example.com" @@ -620,6 +625,9 @@ def test_stream_manager_no_ssh_for_local_hosts() -> None: ) manager._handle_discovery_line(line) + for process in manager._events_processes.values(): + process.terminate() + assert manager.resolver.list_known_agent_ids() == (_AGENT_A,) assert manager.resolver.get_ssh_info(_AGENT_A) is None @@ -647,6 +655,11 @@ def test_stream_manager_mixed_local_and_remote() -> None: ssh_line = _make_host_ssh_info_line(remote_host_id, ssh_data) manager._handle_discovery_line(ssh_line) + # Terminate background mngr events processes before the CG exits, + # otherwise they time out on slow systems (e.g. Modal containers). + for process in manager._events_processes.values(): + process.terminate() + assert manager.resolver.get_ssh_info(_AGENT_A) is None ssh_info = manager.resolver.get_ssh_info(_AGENT_B) assert ssh_info is not None @@ -677,6 +690,9 @@ def test_stream_manager_ssh_info_before_full_snapshot() -> None: ) manager._handle_discovery_line(full_line) + for process in manager._events_processes.values(): + process.terminate() + ssh_info = manager.resolver.get_ssh_info(_AGENT_A) assert ssh_info is not None assert ssh_info.host == "remote.example.com" diff --git a/apps/minds/imbue/minds/forwarding_server/ssh_tunnel.py b/apps/minds/imbue/minds/forwarding_server/ssh_tunnel.py index 75a426773..eca3f13ca 100644 --- a/apps/minds/imbue/minds/forwarding_server/ssh_tunnel.py +++ b/apps/minds/imbue/minds/forwarding_server/ssh_tunnel.py @@ -19,7 +19,7 @@ _SELECT_TIMEOUT_SECONDS: Final[float] = 1.0 -_ACCEPT_TIMEOUT_SECONDS: Final[float] = 1.0 +_SHUTDOWN_POLL_SECONDS: Final[float] = 0.2 _SOCKET_POLL_SECONDS: Final[float] = 0.01 @@ -234,7 +234,7 @@ def _tunnel_accept_loop( server.bind(str(sock_path)) os.chmod(str(sock_path), 0o600) server.listen(8) - server.settimeout(_ACCEPT_TIMEOUT_SECONDS) + server.settimeout(_SHUTDOWN_POLL_SECONDS) while not shutdown_event.is_set(): try: diff --git a/apps/minds/imbue/minds/forwarding_server/ssh_tunnel_test.py b/apps/minds/imbue/minds/forwarding_server/ssh_tunnel_test.py index ad48d461b..a039e1b40 100644 --- a/apps/minds/imbue/minds/forwarding_server/ssh_tunnel_test.py +++ b/apps/minds/imbue/minds/forwarding_server/ssh_tunnel_test.py @@ -329,5 +329,5 @@ def test_tunnel_accept_loop_shutdown_event_stops_loop(short_tmp_path: Path) -> N _wait_for_socket(sock_path, timeout=10.0) shutdown_event.set() - accept_thread.join(timeout=3.0) + accept_thread.join(timeout=10.0) assert not accept_thread.is_alive() diff --git a/apps/minds/imbue/minds/forwarding_server/test_forwarding_server.py b/apps/minds/imbue/minds/forwarding_server/test_forwarding_server.py index 0359f1c6c..30e1d3f71 100644 --- a/apps/minds/imbue/minds/forwarding_server/test_forwarding_server.py +++ b/apps/minds/imbue/minds/forwarding_server/test_forwarding_server.py @@ -907,9 +907,9 @@ def test_landing_page_prefills_git_url_from_query_param(tmp_path: Path) -> None: ) _authenticate_client(client=client, auth_store=auth_store) - response = client.get("/", params={"git_url": "https://github.com/test/repo"}) + response = client.get("/", params={"git_url": "file:///nonexistent-repo"}) assert response.status_code == 200 - assert "https://github.com/test/repo" in response.text + assert "file:///nonexistent-repo" in response.text def test_create_page_shows_form(tmp_path: Path) -> None: @@ -969,7 +969,7 @@ def test_create_form_submit_returns_501_without_agent_creator(tmp_path: Path) -> ) _authenticate_client(client=client, auth_store=auth_store) - response = client.post("/create", data={"git_url": "https://github.com/test/repo"}) + response = client.post("/create", data={"git_url": "file:///nonexistent-repo"}) assert response.status_code == 501 @@ -983,7 +983,7 @@ def test_create_agent_api_returns_501_without_agent_creator(tmp_path: Path) -> N ) _authenticate_client(client=client, auth_store=auth_store) - response = client.post("/api/create-agent", json={"git_url": "https://github.com/test/repo"}) + response = client.post("/api/create-agent", json={"git_url": "file:///nonexistent-repo"}) assert response.status_code == 501 @@ -1025,15 +1025,16 @@ def _create_test_server_with_agent_creator( def test_create_form_submit_redirects_to_creating_page(tmp_path: Path) -> None: """POST /create with valid git_url redirects to /creating/{agent_id}.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, _creator = _create_test_server_with_agent_creator(tmp_path) response = client.post( "/create", - data={"git_url": "https://github.com/test/repo"}, + data={"git_url": "file:///nonexistent-repo"}, follow_redirects=False, ) assert response.status_code == 303 assert response.headers["location"].startswith("/creating/") + _creator.close() def test_create_form_submit_rejects_empty_git_url(tmp_path: Path) -> None: @@ -1046,39 +1047,46 @@ def test_create_form_submit_rejects_empty_git_url(tmp_path: Path) -> None: def test_create_form_submit_passes_agent_name(tmp_path: Path) -> None: """POST /create passes agent_name to the creator.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, agent_creator = _create_test_server_with_agent_creator(tmp_path) response = client.post( "/create", - data={"git_url": "https://github.com/test/repo", "agent_name": "my-agent"}, + data={"git_url": "file:///nonexistent-repo", "agent_name": "my-agent"}, follow_redirects=False, ) assert response.status_code == 303 + for aid in agent_creator._statuses: + agent_creator.wait_for_completion(AgentId(aid), timeout=10.0) + def test_create_agent_api_passes_agent_name(tmp_path: Path) -> None: """POST /api/create-agent passes agent_name to the creator.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, agent_creator = _create_test_server_with_agent_creator(tmp_path) response = client.post( "/api/create-agent", - json={"git_url": "https://github.com/test/repo", "agent_name": "my-agent"}, + json={"git_url": "file:///nonexistent-repo", "agent_name": "my-agent"}, ) assert response.status_code == 200 data = response.json() assert "agent_id" in data + agent_creator.wait_for_completion(AgentId(data["agent_id"]), timeout=10.0) + def test_create_agent_api_returns_agent_id(tmp_path: Path) -> None: """POST /api/create-agent returns JSON with agent_id and status.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, agent_creator = _create_test_server_with_agent_creator(tmp_path) - response = client.post("/api/create-agent", json={"git_url": "https://github.com/test/repo"}) + response = client.post("/api/create-agent", json={"git_url": "file:///nonexistent-repo"}) assert response.status_code == 200 data = response.json() assert "agent_id" in data assert data["status"] == "CLONING" + agent_creator.wait_for_completion(AgentId(data["agent_id"]), timeout=10.0) + def test_create_agent_api_rejects_empty_git_url(tmp_path: Path) -> None: """POST /api/create-agent with empty git_url returns 400.""" @@ -1105,12 +1113,14 @@ def test_creating_page_shows_status(tmp_path: Path) -> None: """GET /creating/{agent_id} shows the creating progress page.""" client, _, agent_creator = _create_test_server_with_agent_creator(tmp_path) - agent_id = agent_creator.start_creation("https://github.com/test/repo") + agent_id = agent_creator.start_creation("file:///nonexistent-repo") response = client.get("/creating/{}".format(agent_id)) assert response.status_code == 200 assert "Creating your mind" in response.text + agent_creator.wait_for_completion(agent_id, timeout=10.0) + def test_creating_page_returns_404_for_unknown(tmp_path: Path) -> None: """GET /creating/{agent_id} returns 404 for unknown agent creation.""" @@ -1124,7 +1134,7 @@ def test_creation_status_api_returns_status_for_tracked_agent(tmp_path: Path) -> """GET /api/create-agent/{id}/status returns a valid status for a tracked creation.""" client, _, agent_creator = _create_test_server_with_agent_creator(tmp_path) - agent_id = agent_creator.start_creation("https://github.com/test/repo") + agent_id = agent_creator.start_creation("file:///nonexistent-repo") response = client.get("/api/create-agent/{}/status".format(agent_id)) assert response.status_code == 200 @@ -1132,14 +1142,17 @@ def test_creation_status_api_returns_status_for_tracked_agent(tmp_path: Path) -> assert data["agent_id"] == str(agent_id) assert data["status"] in ("CLONING", "CREATING", "DONE", "FAILED") + agent_creator.wait_for_completion(agent_id, timeout=10.0) + def test_create_page_prefills_git_url_from_query(tmp_path: Path) -> None: """GET /create?git_url=... pre-fills the form.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, _creator = _create_test_server_with_agent_creator(tmp_path) - response = client.get("/create", params={"git_url": "https://github.com/test/repo"}) + response = client.get("/create", params={"git_url": "file:///nonexistent-repo"}) assert response.status_code == 200 - assert "https://github.com/test/repo" in response.text + assert "file:///nonexistent-repo" in response.text + _creator.close() def test_landing_page_shows_create_link_when_multiple_agents_known(tmp_path: Path) -> None: @@ -1186,7 +1199,7 @@ def test_create_form_submit_rejects_unauthenticated(tmp_path: Path) -> None: http_client=None, ) - response = client.post("/create", data={"git_url": "https://github.com/test/repo"}) + response = client.post("/create", data={"git_url": "file:///nonexistent-repo"}) assert response.status_code == 403 @@ -1199,7 +1212,7 @@ def test_create_agent_api_rejects_unauthenticated(tmp_path: Path) -> None: http_client=None, ) - response = client.post("/api/create-agent", json={"git_url": "https://github.com/test/repo"}) + response = client.post("/api/create-agent", json={"git_url": "file:///nonexistent-repo"}) assert response.status_code == 403 @@ -1255,12 +1268,14 @@ def test_creation_logs_sse_streams_events(tmp_path: Path) -> None: """GET /api/create-agent/{id}/logs returns SSE stream for a tracked creation.""" client, _, agent_creator = _create_test_server_with_agent_creator(tmp_path) - agent_id = agent_creator.start_creation("https://github.com/test/repo") + agent_id = agent_creator.start_creation("file:///nonexistent-repo") with client.stream("GET", "/api/create-agent/{}/logs".format(agent_id)) as response: assert response.status_code == 200 assert "text/event-stream" in response.headers.get("content-type", "") + agent_creator.wait_for_completion(agent_id, timeout=10.0) + def test_creating_page_rejects_unauthenticated(tmp_path: Path) -> None: """GET /creating/{id} returns 403 without authentication.""" @@ -1277,28 +1292,29 @@ def test_creating_page_rejects_unauthenticated(tmp_path: Path) -> None: def test_create_form_submit_passes_launch_mode(tmp_path: Path) -> None: """POST /create passes launch_mode to the creator.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, _creator = _create_test_server_with_agent_creator(tmp_path) response = client.post( "/create", data={ - "git_url": "https://github.com/test/repo", + "git_url": "file:///nonexistent-repo", "agent_name": "my-agent", "launch_mode": "DEV", }, follow_redirects=False, ) assert response.status_code == 303 + _creator.close() def test_create_agent_api_passes_launch_mode(tmp_path: Path) -> None: """POST /api/create-agent passes launch_mode to the creator.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, _creator = _create_test_server_with_agent_creator(tmp_path) response = client.post( "/api/create-agent", json={ - "git_url": "https://github.com/test/repo", + "git_url": "file:///nonexistent-repo", "agent_name": "my-agent", "launch_mode": "DEV", }, @@ -1306,22 +1322,24 @@ def test_create_agent_api_passes_launch_mode(tmp_path: Path) -> None: assert response.status_code == 200 data = response.json() assert "agent_id" in data + _creator.close() def test_create_agent_api_rejects_invalid_launch_mode(tmp_path: Path) -> None: """POST /api/create-agent returns 400 for an invalid launch_mode.""" - client, _, _ = _create_test_server_with_agent_creator(tmp_path) + client, _, _creator = _create_test_server_with_agent_creator(tmp_path) response = client.post( "/api/create-agent", json={ - "git_url": "https://github.com/test/repo", + "git_url": "file:///nonexistent-repo", "agent_name": "my-agent", "launch_mode": "INVALID_MODE", }, ) assert response.status_code == 400 assert "Invalid launch_mode" in response.json()["error"] + _creator.close() def test_create_form_shows_launch_mode_dropdown(tmp_path: Path) -> None: diff --git a/justfile b/justfile index 434e98744..7d5dc217e 100644 --- a/justfile +++ b/justfile @@ -31,6 +31,18 @@ test-offload args="": tmpdir=$(mktemp -d) trap "rm -rf $tmpdir" EXIT + # Invalidate offload's image cache when build inputs change. + # Offload only caches by image ID and doesn't track Dockerfile or base commit changes. + CACHE_KEY=$(cat .offload-base-commit libs/mngr/imbue/mngr/resources/Dockerfile offload-modal.toml | shasum -a 256 | cut -d' ' -f1) + CACHE_KEY_FILE=".offload-cache-key" + if [ -f "$CACHE_KEY_FILE" ] && [ "$(cat "$CACHE_KEY_FILE")" = "$CACHE_KEY" ]; then + echo "[test-offload] Image cache key matches, reusing cached image." + else + echo "[test-offload] Image cache key changed, clearing cached image." + rm -f .offload-image-cache + echo "$CACHE_KEY" > "$CACHE_KEY_FILE" + fi + ./scripts/make_tar_of_repo.sh $BASE_COMMIT $tmpdir export OFFLOAD_PATCH_UUID=`uv run python -c"import uuid;print(uuid.uuid4())"` mkdir -p /tmp/$OFFLOAD_PATCH_UUID diff --git a/libs/mngr/imbue/mngr/config/completion_writer_test.py b/libs/mngr/imbue/mngr/config/completion_writer_test.py index 81157210e..4560982c3 100644 --- a/libs/mngr/imbue/mngr/config/completion_writer_test.py +++ b/libs/mngr/imbue/mngr/config/completion_writer_test.py @@ -47,23 +47,20 @@ def test_get_completion_cache_dir_falls_back_to_default_host_dir( def test_write_cli_completions_cache_handles_oserror(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """write_cli_completions_cache should silently handle OSError.""" - # Point to a read-only directory so the atomic_write fails - read_only_dir = tmp_path / "readonly" - read_only_dir.mkdir() - read_only_dir.chmod(0o444) - monkeypatch.setenv("MNGR_COMPLETION_CACHE_DIR", str(read_only_dir)) + # Monkeypatch atomic_write to simulate a write failure. We can't use chmod + # because Modal sandboxes run as root, which bypasses permission checks. + monkeypatch.setenv("MNGR_COMPLETION_CACHE_DIR", str(tmp_path)) + + def _raise_oserror(*_args: object, **_kwargs: object) -> None: + raise OSError("simulated write failure") + + monkeypatch.setattr("imbue.mngr.config.completion_writer.atomic_write", _raise_oserror) - # Create a minimal click.Group group = click.Group(name="test", commands={"hello": click.Command("hello")}) - try: - # Should not raise despite filesystem error - write_cli_completions_cache(cli_group=group) - finally: - read_only_dir.chmod(0o755) - # Verify the cache file was NOT created (write failed silently). - # Check after restoring permissions so Path.exists() doesn't raise PermissionError. - assert not (read_only_dir / COMPLETION_CACHE_FILENAME).exists() + # Should not raise despite the OSError from atomic_write + write_cli_completions_cache(cli_group=group) + assert not (tmp_path / COMPLETION_CACHE_FILENAME).exists() def test_write_cli_completions_cache_writes_valid_json(completion_cache_dir: Path) -> None: diff --git a/libs/mngr_claude_mind/imbue/mngr_claude_mind/conftest.py b/libs/mngr_claude_mind/imbue/mngr_claude_mind/conftest.py index d9d091320..eaae40d7f 100644 --- a/libs/mngr_claude_mind/imbue/mngr_claude_mind/conftest.py +++ b/libs/mngr_claude_mind/imbue/mngr_claude_mind/conftest.py @@ -90,6 +90,10 @@ def __init__(self, temp_host_dir: Path) -> None: self.chat_script = commands_dir / "chat.sh" self.chat_script.write_text(load_llm_resource("chat.sh")) + + conv_db_path = commands_dir / "conversation_db.py" + conv_db_path.write_text(load_llm_resource("conversation_db.py")) + os.chmod(conv_db_path, 0o755) os.chmod(self.chat_script, 0o755) mngr_log_path = commands_dir / "mngr_log.sh" diff --git a/libs/mngr_kanpan/imbue/mngr_kanpan/tui_test.py b/libs/mngr_kanpan/imbue/mngr_kanpan/tui_test.py index 0cabdeab4..b15684ce5 100644 --- a/libs/mngr_kanpan/imbue/mngr_kanpan/tui_test.py +++ b/libs/mngr_kanpan/imbue/mngr_kanpan/tui_test.py @@ -980,7 +980,7 @@ def test_dispatch_markable_custom_command_toggles_off() -> None: def test_dispatch_immediate_custom_command_does_not_mark() -> None: entry = _make_entry() - cmd = CustomCommand(name="connect", command="mngr connect $MNGR_AGENT_NAME") + cmd = CustomCommand(name="connect", command="true") commands = {"c": cmd} state = _make_state_with_focus(entries=(entry,), commands=commands) _dispatch_command(state, "c", cmd) diff --git a/libs/mngr_llm/imbue/mngr_llm/provisioning.py b/libs/mngr_llm/imbue/mngr_llm/provisioning.py index 353c7aac0..abda92f88 100644 --- a/libs/mngr_llm/imbue/mngr_llm/provisioning.py +++ b/libs/mngr_llm/imbue/mngr_llm/provisioning.py @@ -25,7 +25,7 @@ from imbue.mngr_llm.data_types import ProvisioningSettings # Supporting service shell scripts to provision to $MNGR_AGENT_STATE_DIR/commands/. -_SERVICE_SCRIPT_FILES: Final[tuple[str, ...]] = ("chat.sh",) +_SERVICE_SCRIPT_FILES: Final[tuple[str, ...]] = ("chat.sh", "conversation_db.py") # Scripts provisioned to $MNGR_AGENT_STATE_DIR/commands/ttyd/ for URL-arg dispatch. # Tuples of (resource filename, target filename under commands/ttyd/). diff --git a/libs/mngr_llm/imbue/mngr_llm/resources/chat.sh b/libs/mngr_llm/imbue/mngr_llm/resources/chat.sh index c9f3fafd4..7ca3ded34 100644 --- a/libs/mngr_llm/imbue/mngr_llm/resources/chat.sh +++ b/libs/mngr_llm/imbue/mngr_llm/resources/chat.sh @@ -24,6 +24,8 @@ set -euo pipefail AGENT_DATA_DIR="${MNGR_AGENT_STATE_DIR:?MNGR_AGENT_STATE_DIR must be set}" LLM_TOOLS_DIR="${MNGR_AGENT_STATE_DIR}/commands/llm_tools" +# Standalone conversation DB script -- avoids the heavy mngr CLI startup cost. +_CONV_DB="${MNGR_AGENT_STATE_DIR}/commands/conversation_db.py" TALKING_PROMPT="${MNGR_AGENT_WORK_DIR:-}/talking/PROMPT.md" # Path to the llm database (LLM_USER_PATH is always set during provisioning) @@ -69,7 +71,7 @@ insert_conversation_record() { local created_at created_at=$(mngr_timestamp) - mngr llmdb insert "$_LLM_DB" "$conversation_id" "$tags" "$created_at" + python3 "$_CONV_DB" insert "$_LLM_DB" "$conversation_id" "$tags" "$created_at" log "Inserted conversation record: conversation_id=$conversation_id tags=$tags" } @@ -78,7 +80,7 @@ insert_conversation_record() { get_last_response_id() { local conversation_id="$1" if [ -f "$_LLM_DB" ]; then - mngr llmdb last-response-id "$_LLM_DB" "$conversation_id" + python3 "$_CONV_DB" last-response-id "$_LLM_DB" "$conversation_id" fi } @@ -151,7 +153,7 @@ build_tags_json() { lookup_conversation_by_name() { local name="$1" if [ -f "$_LLM_DB" ]; then - mngr llmdb lookup-by-name "$_LLM_DB" "$name" + python3 "$_CONV_DB" lookup-by-name "$_LLM_DB" "$name" fi } @@ -286,7 +288,7 @@ new_conversation() { local _max_rowid _max_rowid=0 if [ -f "$_LLM_DB" ]; then - _max_rowid=$(mngr llmdb max-rowid "$_LLM_DB" 2>/dev/null) + _max_rowid=$(python3 "$_CONV_DB" max-rowid "$_LLM_DB" 2>/dev/null) fi ( @@ -294,7 +296,7 @@ new_conversation() { for _i in $(seq 1 60); do sleep 1 if [ -f "$_LLM_DB" ]; then - _new_conversation_id=$(mngr llmdb poll-new "$_LLM_DB" "$_max_rowid") + _new_conversation_id=$(python3 "$_CONV_DB" poll-new "$_LLM_DB" "$_max_rowid") if [ -n "$_new_conversation_id" ]; then insert_conversation_record "$_new_conversation_id" "$tags" log "Recorded conversation for new conversation_id=$_new_conversation_id (rowid > $_max_rowid)" @@ -317,7 +319,7 @@ resume_conversation() { # Get the model from the mind_conversations table local model - model=$(mngr llmdb lookup-model "$_LLM_DB" "$conversation_id") + model=$(python3 "$_CONV_DB" lookup-model "$_LLM_DB" "$conversation_id") if [ -z "$model" ]; then model=$(get_model) fi @@ -366,7 +368,7 @@ list_conversations() { # Check if the mind_conversations table exists and has rows local _row_count - _row_count=$(mngr llmdb count "$_LLM_DB") + _row_count=$(python3 "$_CONV_DB" count "$_LLM_DB") if [ "$_row_count" = "0" ]; then echo "No conversations yet." return 0 diff --git a/libs/mngr_pair/imbue/mngr_pair/api.py b/libs/mngr_pair/imbue/mngr_pair/api.py index aaf3c28a7..42434865d 100644 --- a/libs/mngr_pair/imbue/mngr_pair/api.py +++ b/libs/mngr_pair/imbue/mngr_pair/api.py @@ -179,6 +179,10 @@ def is_running(self) -> bool: return False return self._started_event.is_set() + def wait_for_started(self, timeout: float) -> None: + """Block until unison has produced its first output line.""" + self._started_event.wait(timeout=timeout) + _UNISON = SystemDependency( binary="unison", diff --git a/libs/mngr_pair/imbue/mngr_pair/test_api.py b/libs/mngr_pair/imbue/mngr_pair/test_api.py index 87eea0eaf..42c784fb1 100644 --- a/libs/mngr_pair/imbue/mngr_pair/test_api.py +++ b/libs/mngr_pair/imbue/mngr_pair/test_api.py @@ -237,7 +237,9 @@ def test_pair_files_syncs_git_state_before_starting(pair_ctx: SyncTestContext, c # The file should now exist in target assert (pair_ctx.local_dir / "agent_commit.txt").exists() - # Stop immediately - we just want to test git sync + # Wait for unison to actually start before stopping, otherwise the + # resource guard fires because the unison binary was never invoked. + syncer.wait_for_started(timeout=10.0) syncer.stop() diff --git a/offload-modal.toml b/offload-modal.toml index 61508080b..61fb22484 100644 --- a/offload-modal.toml +++ b/offload-modal.toml @@ -4,7 +4,7 @@ max_parallel = 50 test_timeout_secs = 60 stream_output = true sandbox_project_root = "/code/mngr" -sandbox_init_cmd = "git apply /offload-upload/patch --allow-empty && uv sync --all-packages" +sandbox_init_cmd = "git apply /offload-upload/patch --allow-empty && git add -A && uv sync --all-packages" [provider] type = "modal" diff --git a/scripts/josh/coordinator_test.py b/scripts/josh/coordinator_test.py index 86e2bb32d..951586fdf 100644 --- a/scripts/josh/coordinator_test.py +++ b/scripts/josh/coordinator_test.py @@ -464,6 +464,8 @@ def test_task_deletion(self, tmp_path: Path): assert (task_dir / "task-one.json").exists() assert not (task_dir / "task-two.json").exists() + process_manager.terminate_all() + def test_no_deletion_on_initial_sync(self, tmp_path: Path): """Test that existing JSON files are not deleted during initial sync.""" # Create task file with one task @@ -495,6 +497,8 @@ def test_no_deletion_on_initial_sync(self, tmp_path: Path): assert (task_dir / "task-one.json").exists() assert orphan_json.exists() # Should still exist + process_manager.terminate_all() + def test_handler_terminated_on_deletion(self, tmp_path: Path): """Test that handlers are terminated when tasks are deleted.""" # Create task file @@ -537,6 +541,8 @@ def test_handler_terminated_on_deletion(self, tmp_path: Path): # Handler should be terminated assert "long-running-task" not in process_manager.active_handlers + process_manager.terminate_all() + def test_markdown_files_moved_on_deletion(self, tmp_path: Path): """Test that markdown files are moved to md_done when tasks are deleted.""" # Create initial task file with two tasks