kagent/.github/workflows/run-agent-framework-test.yaml at b1aa03c7292dad9c0eda733450499d418972e54f · kagent-dev/kagent · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
name: Run PE Test

on:
  workflow_dispatch:
env:
  CLUSTER_CTX: kind-kagent
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}

jobs:
  solve-challenge:
    outputs:
      scn1_ch1: ${{ steps.gen_output.outputs.scn1_ch1 }}
      # scn1_ch2: ${{ steps.gen_output.outputs.scn1_ch2 }}
      # scn1_ch3: ${{ steps.gen_output.outputs.scn1_ch3 }}
      # scn1_ch4: ${{ steps.gen_output.outputs.scn1_ch4 }}
      # scn1_ch5: ${{ steps.gen_output.outputs.scn1_ch5 }}
      # scn1_ch6: ${{ steps.gen_output.outputs.scn1_ch6 }}
      # scn1_ch7: ${{ steps.gen_output.outputs.scn1_ch7 }}
      # scn1_ch8: ${{ steps.gen_output.outputs.scn1_ch8 }}
      # scn1_ch9: ${{ steps.gen_output.outputs.scn1_ch9 }}
      # scn1_ch10: ${{ steps.gen_output.outputs.scn1_ch10 }}
      # scn1_ch11: ${{ steps.gen_output.outputs.scn1_ch11 }}
      # scn1_ch12: ${{ steps.gen_output.outputs.scn1_ch12 }}
      # scn1_ch13: ${{ steps.gen_output.outputs.scn1_ch13 }}
      # scn1_ch14: ${{ steps.gen_output.outputs.scn1_ch14 }}
      # scn1_ch15: ${{ steps.gen_output.outputs.scn1_ch15 }}
      # scn1_ch16: ${{ steps.gen_output.outputs.scn1_ch16 }}
      # scn1_ch17: ${{ steps.gen_output.outputs.scn1_ch17 }}
      # scn1_ch18: ${{ steps.gen_output.outputs.scn1_ch18 }}
      # scn1_ch19: ${{ steps.gen_output.outputs.scn1_ch19 }}
      # scn1_ch20: ${{ steps.gen_output.outputs.scn1_ch20 }}
      # scn1_ch21: ${{ steps.gen_output.outputs.scn1_ch21 }}
      # scn1_ch22: ${{ steps.gen_output.outputs.scn1_ch22 }}
      # scn1_ch23: ${{ steps.gen_output.outputs.scn1_ch23 }}
      # scn1_ch24: ${{ steps.gen_output.outputs.scn1_ch24 }}
    name: ${{ matrix.challenges.scenario }}/${{ matrix.challenges.challenge }}
    strategy:
      fail-fast: false
      matrix:
        challenges:
        - { name: scn1_ch1, scenario: scenario1, challenge: configmap-missing-key.yaml}
        # - { name: scn1_ch2, scenario: scenario1, challenge: configmap-misspelled.yaml}
        # - { name: scn1_ch3, scenario: scenario1, challenge: configmap-readonly.yaml}
        # - { name: scn1_ch4, scenario: scenario1, challenge: deployment-env-mismatch.yaml}
        # - { name: scn1_ch5, scenario: scenario1, challenge: deployment-impossible-pod-affinity.yaml}
        # - { name: scn1_ch6, scenario: scenario1, challenge: deployment-low-resources.yaml}
        # - { name: scn1_ch7, scenario: scenario1, challenge: deployment-pod-affinity-wrong-key.yaml}
        # - { name: scn1_ch8, scenario: scenario1, challenge: deployment-probe-failures.yaml}
        # - { name: scn1_ch9, scenario: scenario1, challenge: deployment-scaled-down.yaml}
        # - { name: scn1_ch10, scenario: scenario1, challenge: network-policy.yaml}
        # - { name: scn1_ch11, scenario: scenario1, challenge: pvc-wrong-accessmode.yaml}
        # - { name: scn1_ch12, scenario: scenario1, challenge: pvc-wrong-storageclass.yaml}
        # - { name: scn1_ch13, scenario: scenario1, challenge: secret-missing.yaml}
        # - { name: scn1_ch14, scenario: scenario1, challenge: secret-not-mounted.yaml}
        # - { name: scn1_ch15, scenario: scenario1, challenge: service-no-endpoint.yaml}
        # - { name: scn1_ch16, scenario: scenario1, challenge: serviceaccount-misspelled.yaml}
        # - { name: scn1_ch17, scenario: scenario1, challenge: serviceaccount-permissions.yaml}
        # - { name: scn1_ch18, scenario: scenario1, challenge: pod-hostport-conflict.yaml}
        # - { name: scn1_ch19, scenario: scenario1, challenge: pod-limit-range-exceeded.yaml}
        # - { name: scn1_ch20, scenario: scenario1, challenge: pod-resource-quota-exceeded.yaml}
        # - { name: scn1_ch21, scenario: scenario1, challenge: service-dns-resolution-fail.yaml}
        # - { name: scn1_ch22, scenario: scenario1, challenge: service-incorrect-port-number.yaml}
        # - { name: scn1_ch23, scenario: scenario1, challenge: missing-service-selector.yaml}
        # - { name: scn1_ch24, scenario: scenario1, challenge: pod-security-context-issue.yaml}
    runs-on: ['ubuntu-latest']

    steps:
    - uses: actions/checkout@v4

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3

    - name: 'Build Images'
      env:
        DOCKER_BUILD_ARGS: "--load --platform linux/amd64"
        DOCKER_BUILDER: "docker buildx"
        BUILD_ARGS: "--platform linux/amd64"
      run: |
        bash .github/data/agent-framework/0.setup.sh

    - name: Gather challenge information
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      run: |
        cat ${{ matrix.challenges.challenge }} | yq .spec.prompt

    - name: Gather challenge information
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      run: |
        cat ${{ matrix.challenges.challenge }} | yq .spec.prompt

    - name: Setup scenario
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      timeout-minutes: 3
      env:
        CLUSTER_CTX: kind-kagent
      run: |
        bash "./run.sh"
        pnpm i || npm i

    - name: Run Test
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      timeout-minutes: 3
      env:
        CLUSTER_CTX: kind-kagent
      run: |
        ../run-challenge.sh "${{ matrix.challenges.scenario }}" "${{ matrix.challenges.challenge }}"


    - name: Test final state
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      timeout-minutes: 1
      env:
        CLUSTER_CTX: kind-kagent
      run: |
        kubectl --context ${CLUSTER_CTX} get pods -A
        timeout --signal=INT 3m mocha ./test.js --timeout 10000 --retries 5

    - name: Save result
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      id: gen_output
      if: always()
      run: |
        NAME=$(cat ${{ matrix.challenges.challenge }} | yq .metadata.name)
        DESCRIPTION=$(cat ${{ matrix.challenges.challenge }} | yq .spec.description)
        job_outcome=$(echo '{}' | jq '. + {status: "${{ job.status }}"}' | jq --arg name "$NAME" '. + {name: $name}' | jq --arg description "$DESCRIPTION" '. + {description: $description}' | jq tostring)
        echo "${{ matrix.challenges.name }}=${job_outcome}" >> "$GITHUB_OUTPUT"

  summarize-results:
    needs: [solve-challenge]
    if: always()
    runs-on: ['ubuntu-latest']
    steps:
    - name: Summarize results
      run: |
        cat << 'EOF' | jq 'with_entries(.value |= (fromjson | fromjson) | .value |= del(.completion_tokens_details) | .value |= del(.prompt_tokens_details))' | yq eval -P > out.yaml
        ${{ toJSON(needs.solve-challenge.outputs) }}
        EOF
        cat out.yaml
        echo "success: $(cat out.yaml|grep "status: success" |wc -l)"
        echo "failures: $(cat out.yaml|grep "status: failure" |wc -l)"