kagent/.github/workflows/run-agent-framework-test.yaml at be836382dfb6f9cc52d8ee8295650a7f22c33792 · kagent-dev/kagent · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
name: Run PE Test

on:
  workflow_dispatch:
env:
  CLUSTER_CTX: kind-kagent
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  QDRANT_API_KEY: ${{ secrets.QDRANT_API_KEY }}

jobs:
  solve-challenge:
    outputs:
      scn1_ch1: ${{ steps.gen_output.outputs.scn1_ch1 }}
      # scn1_ch2: ${{ steps.gen_output.outputs.scn1_ch2 }}
      # scn1_ch3: ${{ steps.gen_output.outputs.scn1_ch3 }}
      # scn1_ch4: ${{ steps.gen_output.outputs.scn1_ch4 }}
      # scn1_ch5: ${{ steps.gen_output.outputs.scn1_ch5 }}
      # scn1_ch6: ${{ steps.gen_output.outputs.scn1_ch6 }}
      # scn1_ch7: ${{ steps.gen_output.outputs.scn1_ch7 }}
      # scn1_ch8: ${{ steps.gen_output.outputs.scn1_ch8 }}
      # scn1_ch9: ${{ steps.gen_output.outputs.scn1_ch9 }}
      # scn1_ch10: ${{ steps.gen_output.outputs.scn1_ch10 }}
      # scn1_ch11: ${{ steps.gen_output.outputs.scn1_ch11 }}
      # scn1_ch12: ${{ steps.gen_output.outputs.scn1_ch12 }}
      # scn1_ch13: ${{ steps.gen_output.outputs.scn1_ch13 }}
      # scn1_ch14: ${{ steps.gen_output.outputs.scn1_ch14 }}
      # scn1_ch15: ${{ steps.gen_output.outputs.scn1_ch15 }}
      # scn1_ch16: ${{ steps.gen_output.outputs.scn1_ch16 }}
      # scn1_ch17: ${{ steps.gen_output.outputs.scn1_ch17 }}
      # scn1_ch18: ${{ steps.gen_output.outputs.scn1_ch18 }}
      # scn1_ch19: ${{ steps.gen_output.outputs.scn1_ch19 }}
      # scn1_ch20: ${{ steps.gen_output.outputs.scn1_ch20 }}
      # scn1_ch21: ${{ steps.gen_output.outputs.scn1_ch21 }}
      # scn1_ch22: ${{ steps.gen_output.outputs.scn1_ch22 }}
      # scn1_ch23: ${{ steps.gen_output.outputs.scn1_ch23 }}
      # scn1_ch24: ${{ steps.gen_output.outputs.scn1_ch24 }}
    name: ${{ matrix.challenges.scenario }}/${{ matrix.challenges.challenge }}
    strategy:
      fail-fast: false
      matrix:
        challenges:
        - { name: scn1_ch1, scenario: scenario1, challenge: configmap-missing-key.yaml}
        # - { name: scn1_ch2, scenario: scenario1, challenge: configmap-misspelled.yaml}
        # - { name: scn1_ch3, scenario: scenario1, challenge: configmap-readonly.yaml}
        # - { name: scn1_ch4, scenario: scenario1, challenge: deployment-env-mismatch.yaml}
        # - { name: scn1_ch5, scenario: scenario1, challenge: deployment-impossible-pod-affinity.yaml}
        # - { name: scn1_ch6, scenario: scenario1, challenge: deployment-low-resources.yaml}
        # - { name: scn1_ch7, scenario: scenario1, challenge: deployment-pod-affinity-wrong-key.yaml}
        # - { name: scn1_ch8, scenario: scenario1, challenge: deployment-probe-failures.yaml}
        # - { name: scn1_ch9, scenario: scenario1, challenge: deployment-scaled-down.yaml}
        # - { name: scn1_ch10, scenario: scenario1, challenge: network-policy.yaml}
        # - { name: scn1_ch11, scenario: scenario1, challenge: pvc-wrong-accessmode.yaml}
        # - { name: scn1_ch12, scenario: scenario1, challenge: pvc-wrong-storageclass.yaml}
        # - { name: scn1_ch13, scenario: scenario1, challenge: secret-missing.yaml}
        # - { name: scn1_ch14, scenario: scenario1, challenge: secret-not-mounted.yaml}
        # - { name: scn1_ch15, scenario: scenario1, challenge: service-no-endpoint.yaml}
        # - { name: scn1_ch16, scenario: scenario1, challenge: serviceaccount-misspelled.yaml}
        # - { name: scn1_ch17, scenario: scenario1, challenge: serviceaccount-permissions.yaml}
        # - { name: scn1_ch18, scenario: scenario1, challenge: pod-hostport-conflict.yaml}
        # - { name: scn1_ch19, scenario: scenario1, challenge: pod-limit-range-exceeded.yaml}
        # - { name: scn1_ch20, scenario: scenario1, challenge: pod-resource-quota-exceeded.yaml}
        # - { name: scn1_ch21, scenario: scenario1, challenge: service-dns-resolution-fail.yaml}
        # - { name: scn1_ch22, scenario: scenario1, challenge: service-incorrect-port-number.yaml}
        # - { name: scn1_ch23, scenario: scenario1, challenge: missing-service-selector.yaml}
        # - { name: scn1_ch24, scenario: scenario1, challenge: pod-security-context-issue.yaml}
    runs-on: ['ubuntu-latest']

    steps:
    - uses: actions/checkout@v4

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3

    - name: 'Build Images'
      env:
        DOCKER_BUILD_ARGS: "--load --platform linux/amd64"
        DOCKER_BUILDER: "docker buildx"
        BUILD_ARGS: "--platform linux/amd64"
      run: |
        bash .github/data/agent-framework/0.setup.sh
        kagent version

    - name: Gather challenge information
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      run: |
        cat ${{ matrix.challenges.challenge }} | yq .spec.prompt

    - name: Gather challenge information
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      run: |
        cat ${{ matrix.challenges.challenge }} | yq .spec.prompt

    - name: Setup scenario
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      timeout-minutes: 3
      env:
        CLUSTER_CTX: kind-kagent
      run: |
        bash "./run.sh"
        pnpm i || npm i
        npm install -g mocha

    - name: Run Test
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      timeout-minutes: 3
      env:
        CLUSTER_CTX: kind-kagent
      run: |
        ../run-challenge.sh "${{ matrix.challenges.scenario }}" "${{ matrix.challenges.challenge }}"

    - name: Setup upterm session
      if: failure()
      uses: lhotari/action-upterm@v1

    - name: Test final state
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      timeout-minutes: 1
      env:
        CLUSTER_CTX: kind-kagent
      run: |
        kubectl --context ${CLUSTER_CTX} get pods -A
        timeout --signal=INT 3m mocha ./test.js --timeout 10000 --retries 5

    - name: Save result
      working-directory: .github/data/agent-framework/${{ matrix.challenges.scenario }}
      id: gen_output
      if: always()
      run: |
        NAME=$(cat ${{ matrix.challenges.challenge }} | yq .metadata.name)
        DESCRIPTION=$(cat ${{ matrix.challenges.challenge }} | yq .spec.description)
        job_outcome=$(echo '{}' | jq '. + {status: "${{ job.status }}"}' | jq --arg name "$NAME" '. + {name: $name}' | jq --arg description "$DESCRIPTION" '. + {description: $description}' | jq tostring)
        echo "${{ matrix.challenges.name }}=${job_outcome}" >> "$GITHUB_OUTPUT"

  summarize-results:
    needs: [solve-challenge]
    if: always()
    runs-on: ['ubuntu-latest']
    steps:
    - name: Summarize results
      run: |
        cat << 'EOF' | jq 'with_entries(.value |= (fromjson | fromjson) | .value |= del(.completion_tokens_details) | .value |= del(.prompt_tokens_details))' | yq eval -P > out.yaml
        ${{ toJSON(needs.solve-challenge.outputs) }}
        EOF
        cat out.yaml
        echo "success: $(cat out.yaml|grep "status: success" |wc -l)"
        echo "failures: $(cat out.yaml|grep "status: failure" |wc -l)"