Merge pull request #1 from Froot-NetSys/modify_workflow

Modify workflow

Author: Kolleida

.github/workflows/run-k8s.yml

@@ -0,0 +1,21 @@
+name: Run K8s Evaluation
+
+on:
+  push:
+    paths:
+      - 'k8s_scenario.toml'
+
+jobs:
+  run-k8s:
+    runs-on: ubuntu-latest
+    if: github.event.repository.fork == true || github.ref != 'refs/heads/main'
+    permissions:
+      contents: write
+      packages: read
+
+    steps:
+      - name: Run assessment
+        uses: ./.github/workflows/run-scenario.yml
+        with:
+          scenario-file: 'k8s_scenario.toml'
+          benchmark-app: 'k8s'

.github/workflows/run-malt.yml

@@ -0,0 +1,21 @@
+name: Run MALT Evaluation
+
+on:
+  push:
+    paths:
+      - 'malt_scenario.toml'
+
+jobs:
+  run-malt:
+    runs-on: ubuntu-latest
+    if: github.event.repository.fork == true || github.ref != 'refs/heads/main'
+    permissions:
+      contents: write
+      packages: read
+
+    steps:
+      - name: Run assessment
+        uses: ./.github/workflows/run-scenario.yml
+        with:
+          scenario-file: 'malt_scenario.toml'
+          benchmark-app: 'malt'

.github/workflows/run-route.yml

@@ -0,0 +1,21 @@
+name: Run Route Evaluation
+
+on:
+  push:
+    paths:
+      - 'route_scenario.toml'
+
+jobs:
+  run-route:
+    runs-on: ubuntu-latest
+    if: github.event.repository.fork == true || github.ref != 'refs/heads/main'
+    permissions:
+      contents: write
+      packages: read
+
+    steps:
+      - name: Run assessment
+        uses: ./.github/workflows/run-scenario.yml
+        with:
+          scenario-file: 'route_scenario.toml'
+          benchmark-app: 'route'

.github/workflows/run-scenario.yml

@@ -1,9 +1,18 @@
 name: Run Scenario
 
 on:
-  push:
-    paths:
-      - 'scenario.toml'
+  workflow_call:
+    inputs:
+      scenario-file:
+        description: 'Path to the scenario TOML file'
+        required: false
+        default: 'scenario.toml'
+        type: string
+      benchmark-app:
+        description: 'Type of application: malt, route, k8s'
+        required: true
+        default: 'malt'
+        type: string
 
 jobs:
   run:
@@ -22,11 +31,26 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Start KinD cluster (for k8s app)
+        if: ${{ inputs.benchmark-app == 'k8s' }}
+        uses: helm/kind-action@v1
+        with:
+          config: ./kind_config.yaml
+
+      - name: Create kubeconfig for KinD (for k8s app).
+        if: ${{ inputs.benchmark-app == 'k8s' }}
+        run: |
+          mkdir -p ./kubeconfig
+          cp "$HOME/.kube/config" "./kubeconfig/config"
+          chmod +r ./kubeconfig/config
+          # Modify the kubeconfig to use host.docker.internal
+          sed -i 's/0.0.0.0/host.docker.internal/g' ./kubeconfig/config
+
       - name: Install dependencies
         run: pip install tomli tomli-w pyyaml requests
 
       - name: Generate docker-compose.yml
-        run: python generate_compose.py --scenario scenario.toml
+        run: python generate_compose.py --scenario ${{ inputs.scenario-file }} --app ${{ inputs.benchmark-app }}
 
       - name: Pull images
         run: |
@@ -69,7 +93,10 @@ jobs:
           password: ${{ secrets.GHCR_TOKEN }}
 
       - name: Run assessment
-        run: docker compose up --timestamps --no-color --exit-code-from agentbeats-client --abort-on-container-exit
+        run: |
+          mkdir -p output
+          chmod 777 output
+          docker compose up --timestamps --no-color --exit-code-from agentbeats-client --abort-on-container-exit
 
       - name: Record provenance
         run: python record_provenance.py --compose docker-compose.yml --output output/provenance.json

generate_compose.py

@@ -62,6 +62,7 @@ def fetch_agent_info(agentbeats_id: str) -> dict:
     image: {green_image}
     platform: linux/amd64
     container_name: green-agent
+    privileged: {privileged}  # Needed for route-agent (to run Mininet).
     command: ["--host", "0.0.0.0", "--port", "{green_port}", "--card-url", "http://green-agent:{green_port}"]
     environment:{green_env}
     healthcheck:
@@ -73,6 +74,7 @@ def fetch_agent_info(agentbeats_id: str) -> dict:
     depends_on:{green_depends}
     networks:
       - agent-network
+    {k8s_service_options}
 
 {participant_services}
   agentbeats-client:
@@ -175,12 +177,19 @@ def format_depends_on(services: list) -> str:
     return "\n" + "\n".join(lines)
 
 
-def generate_docker_compose(scenario: dict[str, Any]) -> str:
+def generate_docker_compose(scenario: dict[str, Any], app: str) -> str:
     green = scenario["green_agent"]
     participants = scenario.get("participants", [])
 
     participant_names = [p["name"] for p in participants]
 
+    # Expose kubeconfig and localhost for k8s app to allow communication with kind cluster
+    k8s_service_options = """extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - ./kubeconfig:/root/.kube/:ro
+"""
+
     participant_services = "\n".join([
         PARTICIPANT_TEMPLATE.format(
             name=p["name"],
@@ -192,14 +201,21 @@ def generate_docker_compose(scenario: dict[str, Any]) -> str:
     ])
 
     all_services = ["green-agent"] + participant_names
+    green_env = green.get("env", {})
+
+    if app == "k8s":
+        # For k8s app, set KUBECONFIG env var for green agent
+        green_env["KUBECONFIG"] = "/root/.kube/config"
 
     return COMPOSE_TEMPLATE.format(
         green_image=green["image"],
         green_port=DEFAULT_PORT,
-        green_env=format_env_vars(green.get("env", {})),
+        green_env=format_env_vars(green_env),
         green_depends=format_depends_on(participant_names),
         participant_services=participant_services,
-        client_depends=format_depends_on(all_services)
+        client_depends=format_depends_on(all_services),
+        privileged=str(app == "route").lower(),
+        k8s_service_options=k8s_service_options if app == "k8s" else ""
     )
 
 
@@ -259,6 +275,7 @@ def generate_env_file(scenario: dict[str, Any]) -> str:
 def main():
     parser = argparse.ArgumentParser(description="Generate Docker Compose from scenario.toml")
     parser.add_argument("--scenario", type=Path)
+    parser.add_argument("--app", type=str, choices=["malt", "route", "k8s"], default="malt", help="What app to generate for.")
     args = parser.parse_args()
 
     if not args.scenario.exists():
@@ -268,7 +285,7 @@ def main():
     scenario = parse_scenario(args.scenario)
 
     with open(COMPOSE_PATH, "w") as f:
-        f.write(generate_docker_compose(scenario))
+        f.write(generate_docker_compose(scenario, args.app))
 
     with open(A2A_SCENARIO_PATH, "w") as f:
         f.write(generate_a2a_scenario(scenario))

k8s_scenario.toml

@@ -0,0 +1,17 @@
+[green_agent]
+agentbeats_id = ""
+env = { LOG_LEVEL = "INFO" }
+
+[[participants]]
+agentbeats_id = ""
+name = "k8s_operator"
+env = {}
+
+[config]
+prompt_type = "zeroshot_base"
+num_queries = 1
+benchmark_path = "assessment_error_config.json"
+output_dir = "dump"
+microservice_dir = "/microservices-demo"
+max_iterations = 10
+regenerate_config = true

kind_config.yaml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # WARNING: It is _strongly_ recommended that you keep this the default
+  # (127.0.0.1) for security reasons. However it is possible to change this.
+  apiServerAddress: "0.0.0.0"
+  # By default the API server listens on a random open port.
+  # You may choose a specific port but probably don't need to in most cases.
+  # Using a random port makes it easier to spin up multiple clusters.
+  apiServerPort: 6443
+kubeadmConfigPatches:
+- |
+  kind: ClusterConfiguration
+  apiServer:
+    certSANs:
+      - "0.0.0.0"
+      - "localhost"
+      - "127.0.0.1"
+      - "host.docker.internal"

malt_scenario.toml

@@ -0,0 +1,17 @@
+[green_agent]
+agentbeats_id = ""
+env = { LOG_LEVEL = "INFO" }
+
+[[participants]]
+agentbeats_id = ""
+name = "malt_operator"
+env = {}
+
+[config]
+prompt_type = "zeroshot_base" 
+num_queries = 1 
+complexity_level = ["level1", "level2"]
+output_dir = "dump" 
+output_file = "query_output.jsonl" 
+benchmark_path = "assessment_queries.jsonl" 
+regenerate_query = true

route_scenario.toml

@@ -0,0 +1,18 @@
+[green_agent]
+agentbeats_id = ""
+env = { LOG_LEVEL = "INFO" }
+
+[[participants]]
+agentbeats_id = ""
+name = "routing_operator"
+env = {}
+
+[config]
+prompt_type = "zeroshot_base"
+num_queries = 1
+benchmark_path = "assessment_error_config.json"
+regenerate_benchmark = true
+output_dir = "dump"
+max_iterations = 10
+num_switches = 2
+num_hosts_per_subnet = 1