Merge branch 'GitHubSecurityLab:main' into devcontainer-configuration

Author: Kwstubbs

.github/workflows/release.yml

@@ -0,0 +1,30 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_tag:
+        description: 'Release tag (e.g. v1.0.0)'
+        required: false
+        default: 'latest'
+
+permissions:
+    contents: read
+    packages: write
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Create release
+        run: |
+          echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u GitHubSecurityLab --password-stdin
+          python release_tools/publish_docker.py release.txt main.py ghcr.io/githubsecuritylab/seclab-taskflow-agent ${{ github.event.inputs.release_tag }}

README.md

@@ -115,23 +115,21 @@ MY_TASKFLOWS=~/my_taskflows MY_DATA=~/codeql_databases CODEQL_DBS_BASE_PATH=/app
 
 For more advanced scenarios like e.g. making custom MCP server code available, you can alter the run script to mount your custom code into the image and configure your toolboxes to use said code accordingly.
 
-Example: a custom MCP server deployment via Docker image:
-
 ```sh
 export MY_MCP_SERVERS="$PWD"/mcp_servers
 export MY_TOOLBOXES="$PWD"/toolboxes
 export MY_PERSONALITIES="$PWD"/personalities
 export MY_TASKFLOWS="$PWD"/taskflows
 export MY_PROMPTS="$PWD"/prompts
+export MY_DATA="$PWD"/data
 
 if [ ! -f ".env" ]; then
     touch ".env"
 fi
 
 docker run \
-       --volume /var/run/docker.sock:/var/run/docker.sock \
        --volume "$PWD"/logs:/app/logs \
-       --mount type=bind,src="$PWD"/env,dst=/app/.env,ro \
+       --mount type=bind,src="$PWD"/.env,dst=/app/.env,ro \
        ${MY_DATA:+--mount type=bind,src=$MY_DATA,dst=/app/my_data} \
        ${MY_MCP_SERVERS:+--mount type=bind,src=$MY_MCP_SERVERS,dst=/app/my_mcp_servers,ro} \
        ${MY_TASKFLOWS:+--mount type=bind,src=$MY_TASKFLOWS,dst=/app/taskflows/my_taskflows,ro} \
@@ -141,19 +139,6 @@ docker run \
        "ghcr.io/githubsecuritylab/seclab-taskflow-agent" "$@"
 ```
 
-Our default run script makes the Docker socket available to the image, which contains the Docker cli, so 3rd party Docker based stdio MCP servers also function as normal.
-
-Example: a toolbox configuration using the official GitHub MCP Server via Docker:
-
-```yaml
-server_params:
-  kind: stdio
-  command: docker
-  args: ["run", "-i", "--rm", "-e", "GITHUB_PERSONAL_ACCESS_TOKEN", "ghcr.io/github/github-mcp-server"]
-  env:
-    GITHUB_PERSONAL_ACCESS_TOKEN: "{{ env GITHUB_PERSONAL_ACCESS_TOKEN }}"
-```
-
 ## Personalities
 
 Core characteristics for a single Agent. Configured through YAML files in `personalities/`.

available_tools.py

@@ -1,9 +1,49 @@
+import logging
+
+class VersionException(Exception):
+    pass
+
+class FileTypeException(Exception):
+    pass
+
 class AvailableTools:
     """
     This class is used for storing dictionaries of all the available
     personalities, taskflows, and prompts.
     """
-    def __init__(self, personalities: dict, taskflows: dict, prompts: dict):
-        self.personalities = personalities
-        self.taskflows = taskflows
-        self.prompts = prompts
+    def __init__(self, yamls: dict):
+        self.personalities = {}
+        self.taskflows = {}
+        self.prompts = {}
+        self.toolboxes = {}
+
+        # Iterate through all the yaml files and divide them into categories.
+        # Each file should contain a header like this:
+        #
+        #   seclab-taskflow-agent:
+        #     type: taskflow
+        #     version: 1
+        #
+        for path, yaml in yamls.items():
+            try:
+                header = yaml['seclab-taskflow-agent']
+                version = header['version']
+                if version != 1:
+                    raise VersionException(str(version))
+                filetype = header['type']
+                if filetype == 'personality':
+                    self.personalities.update({path: yaml})
+                elif filetype == 'taskflow':
+                    self.taskflows.update({path: yaml})
+                elif filetype == 'prompt':
+                    self.prompts.update({path: yaml})
+                elif filetype == 'toolbox':
+                    self.toolboxes.update({path: yaml})
+                else:
+                    raise FileTypeException(str(filetype))
+            except KeyError as err:
+                logging.error(f'{path} does not contain the key {err.args[0]}')
+            except VersionException as err:
+                logging.error(f'{path}: seclab-taskflow-agent version {err.args[0]} is not supported')
+            except FileTypeException as err:
+                logging.error(f'{path}: seclab-taskflow-agent file type {err.args[0]} is not supported')

docker/run.sh

@@ -5,7 +5,6 @@ if [ ! -f ".env" ]; then
 fi
 docker run -i \
        --platform linux/amd64 \
-       --volume /var/run/docker.sock:/var/run/docker.sock \
        --volume "$PWD/"logs:/app/logs \
        --mount type=bind,src="$PWD/".env,dst=/app/.env,ro \
        ${MY_DATA:+--mount type=bind,src=$MY_DATA,dst=/app/my_data} \

main.py

@@ -79,7 +79,8 @@ def parse_prompt_args(available_tools: AvailableTools,
     l = args[0].l
     return p, t, l, ' '.join(args[0].prompt), help_msg
 
-async def deploy_task_agents(agents: dict,
+async def deploy_task_agents(available_tools: AvailableTools,
+                             agents: dict,
                              prompt: str,
                              async_task: bool = False,
                              toolboxes_override: list = [],
@@ -120,7 +121,7 @@ async def deploy_task_agents(agents: dict,
     tool_filter = create_static_tool_filter(blocked_tool_names=blocked_tools) if blocked_tools else None
 
     # fetch mcp params
-    mcp_params = mcp_client_params(YamlParser('toolboxes').get_yaml_dict(recurse=True), toolboxes)
+    mcp_params = mcp_client_params(available_tools.toolboxes, toolboxes)
     for tb, (params, confirms, server_prompt, client_session_timeout) in mcp_params.items():
         server_prompts.append(server_prompt)
         # https://openai.github.io/openai-agents-python/mcp/
@@ -401,6 +402,7 @@ async def on_handoff_hook(
             raise ValueError("No such personality!")
 
         await deploy_task_agents(
+            available_tools,
             { p:personality },
             prompt,
             run_hooks=TaskRunHooks(
@@ -575,6 +577,7 @@ async def run_prompts(async_task=False, max_concurrent_tasks=5):
                         async def _deploy_task_agents(resolved_agents, prompt):
                             async with semaphore:
                                 result = await deploy_task_agents(
+                                    available_tools,
                                     # pass agents and prompt by assignment, they change in-loop
                                     resolved_agents,
                                     prompt,
@@ -626,9 +629,10 @@ async def _deploy_task_agents(resolved_agents, prompt):
 
 if __name__ == '__main__':
     available_tools = AvailableTools(
-        personalities = YamlParser('personalities').get_yaml_dict(),
-        taskflows = YamlParser('taskflows').get_yaml_dict(),
-        prompts = YamlParser('prompts').get_yaml_dict(dir_namespace=True))
+        YamlParser('personalities').get_yaml_dict() |
+        YamlParser('taskflows').get_yaml_dict() |
+        YamlParser('prompts').get_yaml_dict(dir_namespace=True) |
+        YamlParser('toolboxes').get_yaml_dict(recurse=True))
 
     p, t, l, user_prompt, help_msg = parse_prompt_args(available_tools)
 

mcp_servers/memcache/memcache_backend/sql_models.py

@@ -13,40 +13,4 @@ class KeyValue(Base):
     value: Mapped[str] = mapped_column(Text)
 
     def __repr__(self):
-        return f"<KeyValue(key={self.key}, value={self.value})>"
-    
-class AlertResults(Base):
-    __tablename__ = 'alert_results'
-
-    canonical_id: Mapped[int] = mapped_column(primary_key=True)
-    alert_id: Mapped[str]
-    repo: Mapped[str]
-    rule: Mapped[str]
-    language: Mapped[str]
-    location: Mapped[str]
-    result: Mapped[str] = mapped_column(Text)
-    created: Mapped[Optional[str]]
-    valid: Mapped[bool] = mapped_column(nullable=False, default=True)
-    completed: Mapped[bool] = mapped_column(nullable=False, default=False)
-
-    relationship('AlertFlowGraph', cascade='all, delete')
-
-    def __repr__(self):
-        return (f"<AlertResults(alert_id={self.alert_id}, repo={self.repo}, "
-                f"rule={self.rule}, language={self.language}, location={self.location}, "
-                f"result={self.result}, created_at={self.created}, valid={self.valid}, completed={self.completed})>")
-
-class AlertFlowGraph(Base):
-    __tablename__ = 'alert_flow_graph'
-
-    id: Mapped[int] = mapped_column(primary_key=True)
-    alert_canonical_id = Column(Integer, ForeignKey('alert_results.canonical_id', ondelete='CASCADE'))
-    flow_data: Mapped[str] = mapped_column(Text)
-    repo: Mapped[str]
-    prev: Mapped[Optional[str]]
-    next: Mapped[Optional[str]]
-    started: Mapped[bool] = mapped_column(nullable=False, default=False)
-
-    def __repr__(self):
-        return (f"<AlertFlowGraph(alert_canonical_id={self.alert_canonical_id}, "
-                f"flow_data={self.flow_data}, repo={self.repo}, prev={self.prev}, next={self.next}, started={self.started})>")
+        return f"<KeyValue(key={self.key}, value={self.value})>"    

misc/codeql_fetch_db.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # dep: https://github.com/cli/cli
 # e.g. get_codeql_db.sh ntp-project/ntp cpp
@@ -9,6 +9,7 @@
 # - csharp: The C# programming language
 # - go: The Go programming language
 # - java: The Java programming language
+# - javascript: The JavaScript programming language (including TypeScript)
 # - python: The Python programming language
 # - ruby: The Ruby programming language
 # - swift: The Swift programming language

personalities/assistant.yaml

@@ -1,3 +1,7 @@
+seclab-taskflow-agent:
+  type: personality
+  version: 1
+
 personality: |
   You are a helpful assistant.
 task: |

personalities/c_auditer.yaml

@@ -1,3 +1,7 @@
+seclab-taskflow-agent:
+  type: personality
+  version: 1
+
 personality: |
   Your name is Ronald. You are a C programming language security expert.
   You have the ability to call tools to aid you in your security reviews.

personalities/examples/apple_expert.yaml

@@ -1,5 +1,9 @@
+seclab-taskflow-agent:
+  type: personality
+  version: 1
+
 personality: |
-  You are an an apples expert.
+  You are an apples expert.
 
 task: |
   Answer any questions about apples.