-
Notifications
You must be signed in to change notification settings - Fork 507
feat(skills): add shell command execution in frontmatter for dynamic context #2582
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
c77a8e1
feat(skills): Add shell command execution in frontmatter for context …
openhands-agent b3314bf
add command to skill
VascoSch92 191e967
feedbacks
VascoSch92 72a5951
adress comments
VascoSch92 f7485f5
refactor after discussion
VascoSch92 216e827
add docstring as promised
VascoSch92 File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Some comments aren't visible on the classic Files Changed page.
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| """Command execution for dynamic skill context injection.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import re | ||
| import subprocess | ||
| from pathlib import Path | ||
| from typing import Final | ||
|
|
||
| from openhands.sdk.context.skills.exceptions import SkillError | ||
| from openhands.sdk.context.skills.types import CommandSpec | ||
| from openhands.sdk.logger import get_logger | ||
|
|
||
|
|
||
| logger = get_logger(__name__) | ||
|
|
||
| # 50KB per command | ||
| MAX_OUTPUT_SIZE: Final[int] = 50 * 1024 | ||
|
|
||
|
|
||
| def _execute_command(spec: CommandSpec, working_dir: Path | None = None) -> str: | ||
| """Execute a single command and return its output.""" | ||
| cwd = str(working_dir) if working_dir else None | ||
| try: | ||
| result = subprocess.run( | ||
| spec.command, | ||
|
VascoSch92 marked this conversation as resolved.
Outdated
VascoSch92 marked this conversation as resolved.
Outdated
|
||
| shell=True, | ||
| cwd=cwd, | ||
| capture_output=True, | ||
| text=True, | ||
| timeout=spec.timeout, | ||
| ) | ||
| if result.returncode != 0: | ||
| return _handle_error( | ||
| spec, f"Command exited with code {result.returncode}: {result.stderr}" | ||
| ) | ||
| output = result.stdout.strip() | ||
| if len(output.encode()) > MAX_OUTPUT_SIZE: | ||
| output = output.encode()[:MAX_OUTPUT_SIZE].decode("utf-8", errors="ignore") | ||
| output += "\n... [output truncated]" | ||
| return output | ||
|
|
||
| except subprocess.TimeoutExpired: | ||
| return _handle_error(spec, f"Command timed out after {spec.timeout}s") | ||
| except Exception as e: | ||
| return _handle_error(spec, f"Failed to execute command: {e}") | ||
|
|
||
|
|
||
| def _handle_error(spec: CommandSpec, message: str) -> str: | ||
| """Handle command execution error based on on_error setting.""" | ||
| if spec.on_error == "fail": | ||
| raise SkillError(message) | ||
| logger.warning("Skill command '%s' failed: %s", spec.name, message) | ||
| if spec.on_error == "empty": | ||
| return "" | ||
| return f"[Error: {message}]" | ||
|
|
||
|
|
||
| def _execute_commands( | ||
| commands: list[CommandSpec], | ||
| working_dir: Path | None = None, | ||
|
VascoSch92 marked this conversation as resolved.
Outdated
|
||
| ) -> dict[str, str]: | ||
| """Execute all commands and return name->output mapping.""" | ||
| return {spec.name: _execute_command(spec, working_dir) for spec in commands} | ||
|
|
||
|
|
||
| def render_content_with_commands( | ||
| content: str, | ||
| commands: list[CommandSpec], | ||
| working_dir: Path | None = None, | ||
| extra_vars: dict[str, str] | None = None, | ||
| ) -> str: | ||
| """Execute commands and substitute {{var_name}} patterns in content.""" | ||
| if not commands and not extra_vars: | ||
| return content | ||
|
|
||
| # Execute commands | ||
| variables = _execute_commands(commands, working_dir) if commands else {} | ||
| if extra_vars: | ||
| collisions = set(variables) & set(extra_vars) | ||
| if collisions: | ||
| logger.warning("extra_vars overriding command outputs: %s", collisions) | ||
|
VascoSch92 marked this conversation as resolved.
Outdated
|
||
| variables.update(extra_vars) | ||
|
|
||
| if not variables: | ||
| return content | ||
|
|
||
| # Substitute {{var_name}} patterns | ||
| def replace_var(match: re.Match[str]) -> str: | ||
| var_name = match.group(1) | ||
| if var_name in variables: | ||
| return variables[var_name] | ||
| return match.group(0) | ||
|
|
||
| return re.sub(r"\{\{(\w+)\}\}", replace_var, content) | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| """Tests for shell command execution in skill frontmatter.""" | ||
|
|
||
| from collections.abc import Callable | ||
| from pathlib import Path | ||
|
|
||
| import pytest | ||
|
|
||
| from openhands.sdk.context.skills import CommandSpec, Skill, SkillValidationError | ||
| from openhands.sdk.context.skills.exceptions import SkillError | ||
| from openhands.sdk.context.skills.execute import ( | ||
| _execute_command, | ||
| render_content_with_commands, | ||
| ) | ||
|
|
||
|
|
||
| def test_command_spec_defaults(): | ||
| """CommandSpec should have sensible defaults.""" | ||
| spec = CommandSpec(name="test", command="echo hello") | ||
| assert spec.timeout == 10.0 and spec.on_error == "message" | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| ("command", "on_error", "timeout", "check"), | ||
| [ | ||
| ("echo hello", "message", 10.0, lambda r: r == "hello"), | ||
| ("exit 1", "message", 10.0, lambda r: "[Error:" in r), | ||
| ("exit 1", "empty", 10.0, lambda r: r == ""), | ||
| ("sleep 5", "message", 0.1, lambda r: "timed out" in r), | ||
| ], | ||
| ) | ||
| def test_execute_command( | ||
| command: str, on_error: str, timeout: float, check: Callable[[str], bool] | ||
| ): | ||
| """_execute_command handles success, failure, and timeout correctly.""" | ||
| spec = CommandSpec(name="t", command=command, on_error=on_error, timeout=timeout) # type: ignore[arg-type] | ||
| assert check(_execute_command(spec)) | ||
|
|
||
|
|
||
| def test_execute_command_failure_raises(): | ||
| """on_error='fail' should raise SkillError.""" | ||
| with pytest.raises(SkillError): | ||
| _execute_command(CommandSpec(name="fail", command="exit 1", on_error="fail")) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| ("content", "commands", "extra_vars", "expected"), | ||
| [ | ||
| ("Hi {{x}}", [], None, "Hi {{x}}"), | ||
| ("Hi {{x}}", [CommandSpec(name="x", command="echo world")], None, "Hi world"), | ||
| ("Hi {{x}}", [], {"x": "world"}, "Hi world"), | ||
| ("Hi {{x}}", [], {"y": "z"}, "Hi {{x}}"), | ||
| ], | ||
| ) | ||
| def test_render_content( | ||
| content: str, commands: list, extra_vars: dict | None, expected: str | ||
| ): | ||
| """render_content_with_commands substitutes variables correctly.""" | ||
| assert ( | ||
| render_content_with_commands(content, commands, extra_vars=extra_vars) | ||
| == expected | ||
| ) | ||
|
|
||
|
|
||
| def test_skill_load_with_commands(tmp_path: Path): | ||
| """Skill.load parses commands from frontmatter.""" | ||
| skill_md = tmp_path / "test-skill" / "SKILL.md" | ||
| skill_md.parent.mkdir() | ||
| skill_md.write_text( | ||
| "---\nname: test-skill\ncommands:\n" | ||
| " - name: a\n command: echo A\n" | ||
| " - name: b\n command: echo B\n timeout: 5.0\n---\n{{a}} {{b}}" | ||
| ) | ||
| skill = Skill.load(skill_md) | ||
| assert len(skill.commands) == 2 and skill.commands[1].timeout == 5.0 | ||
|
|
||
|
|
||
| def test_skill_load_commands_validation(tmp_path: Path): | ||
| """Skill.load rejects invalid commands field.""" | ||
| path = tmp_path / "bad.md" | ||
| path.write_text("---\nname: s\ncommands: not-a-list\n---\n#") | ||
| with pytest.raises(SkillValidationError, match="commands must be a list"): | ||
| Skill.load(path) | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| ("content", "commands", "expected"), | ||
| [ | ||
| ("Hello", [], "Hello"), | ||
| ("Out: {{x}}", [CommandSpec(name="x", command="echo hi")], "Out: hi"), | ||
| ], | ||
| ) | ||
| def test_skill_render_content(content: str, commands: list, expected: str): | ||
| """Skill.render_content executes commands and substitutes.""" | ||
| assert ( | ||
| Skill(name="t", content=content, commands=commands).render_content() == expected | ||
| ) | ||
|
VascoSch92 marked this conversation as resolved.
Outdated
|
||
|
|
||
|
|
||
| def test_execute_command_respects_working_dir(tmp_path: Path): | ||
| """Commands should run in the specified working directory.""" | ||
| spec = CommandSpec(name="cwd", command="pwd") | ||
| result = _execute_command(spec, working_dir=tmp_path) | ||
| assert result == str(tmp_path.resolve()) | ||
|
VascoSch92 marked this conversation as resolved.
Outdated
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.