|
18 | 18 | - '.github/workflows/vscode-automation.yml' |
19 | 19 | - '.github/scripts/build-vscode-automation.sh' |
20 | 20 | workflow_dispatch: |
21 | | - inputs: |
22 | | - automation_mode: |
23 | | - description: 'Automation test mode' |
24 | | - required: false |
25 | | - default: 'automated' |
26 | | - type: choice |
27 | | - options: |
28 | | - - automated |
29 | | - - interactive |
30 | | - test_scenarios: |
31 | | - description: 'Test scenarios to run (comma-separated: python,javascript,typescript or leave empty for all)' |
32 | | - required: false |
33 | | - type: string |
34 | | - log_level: |
35 | | - description: 'Log level for automation tests' |
36 | | - required: false |
37 | | - default: 'info' |
38 | | - type: choice |
39 | | - options: |
40 | | - - debug |
41 | | - - info |
42 | | - - warn |
43 | | - - error |
44 | 21 |
|
45 | 22 | env: |
46 | 23 | REGISTRY: ghcr.io |
@@ -181,172 +158,7 @@ jobs: |
181 | 158 | BUILD_DATE=${{ fromJSON(steps.meta.outputs.json).labels['org.opencontainers.image.created'] }} |
182 | 159 | VCS_REF=${{ github.sha }} |
183 | 160 |
|
184 | | - - name: Basic image test |
185 | | - if: github.event_name != 'pull_request' |
186 | | - run: | |
187 | | - # Get the first tag for testing |
188 | | - IMAGE_TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1) |
189 | | - |
190 | | - echo "🧪 Testing VSCode automation container: $IMAGE_TAG" |
191 | | - |
192 | | - # Basic container startup test |
193 | | - docker run --rm \ |
194 | | - -e AUTOMATION_MODE=test \ |
195 | | - -e LOG_LEVEL=debug \ |
196 | | - "$IMAGE_TAG" shell -c "echo 'Container startup test passed'" |
197 | | -
|
198 | | - # Phase 3: Run Automation Tests |
199 | | - run-automation-tests: |
200 | | - name: VSCode Automation Tests |
201 | | - runs-on: ubuntu-latest |
202 | | - needs: build-vscode-automation |
203 | | - if: github.event_name != 'pull_request' |
204 | | - permissions: |
205 | | - contents: read |
206 | | - packages: read |
207 | | - |
208 | | - strategy: |
209 | | - matrix: |
210 | | - scenario: [python-algorithms, javascript-api, typescript-utils] |
211 | | - fail-fast: false |
212 | | - |
213 | | - steps: |
214 | | - - name: Checkout code |
215 | | - uses: actions/checkout@v4 |
216 | | - |
217 | | - - name: Log in to Container Registry |
218 | | - uses: docker/login-action@v3 |
219 | | - with: |
220 | | - registry: ${{ env.REGISTRY }} |
221 | | - username: ${{ github.actor }} |
222 | | - password: ${{ secrets.GITHUB_TOKEN }} |
223 | | - |
224 | | - - name: Setup test environment |
225 | | - run: | |
226 | | - # Create results directory |
227 | | - mkdir -p automation-results/${{ matrix.scenario }} |
228 | | - |
229 | | - # Set test parameters |
230 | | - AUTOMATION_MODE="${{ github.event.inputs.automation_mode || 'automated' }}" |
231 | | - LOG_LEVEL="${{ github.event.inputs.log_level || 'info' }}" |
232 | | - TEST_SCENARIOS="${{ github.event.inputs.test_scenarios || matrix.scenario }}" |
233 | | - |
234 | | - echo "AUTOMATION_MODE=$AUTOMATION_MODE" >> $GITHUB_ENV |
235 | | - echo "LOG_LEVEL=$LOG_LEVEL" >> $GITHUB_ENV |
236 | | - echo "TEST_SCENARIOS=$TEST_SCENARIOS" >> $GITHUB_ENV |
237 | | -
|
238 | | - - name: Run automation test for ${{ matrix.scenario }} |
239 | | - env: |
240 | | - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |
241 | | - run: | |
242 | | - # Pull the VSCode automation image |
243 | | - IMAGE_NAME="${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}" |
244 | | - docker pull "$IMAGE_NAME" |
245 | | - |
246 | | - echo "🤖 Running automation test for scenario: ${{ matrix.scenario }}" |
247 | | - |
248 | | - # Run automation tests with timeout |
249 | | - timeout 1800 docker run --rm \ |
250 | | - -v "$(pwd)/automation-results/${{ matrix.scenario }}:/logs" \ |
251 | | - -e GITHUB_TOKEN="$GITHUB_TOKEN" \ |
252 | | - -e AUTOMATION_MODE="$AUTOMATION_MODE" \ |
253 | | - -e LOG_LEVEL="$LOG_LEVEL" \ |
254 | | - -e TEST_SCENARIO="${{ matrix.scenario }}" \ |
255 | | - -e TIMEOUT=1500 \ |
256 | | - --name "automation-${{ matrix.scenario }}-${{ github.run_id }}" \ |
257 | | - "$IMAGE_NAME" test || echo "Test completed with timeout or error" |
258 | | - |
259 | | - echo "✅ Automation test for ${{ matrix.scenario }} completed" |
260 | | -
|
261 | | - - name: Process test results for ${{ matrix.scenario }} |
262 | | - if: always() |
263 | | - run: | |
264 | | - RESULTS_DIR="automation-results/${{ matrix.scenario }}" |
265 | | - |
266 | | - # Check if results exist |
267 | | - if [ -f "$RESULTS_DIR/automation-results.json" ]; then |
268 | | - echo "📊 Processing results for ${{ matrix.scenario }}" |
269 | | - |
270 | | - # Extract key metrics |
271 | | - SUCCESS_RATE=$(jq -r '.summary.overallSuccessRate // 0' "$RESULTS_DIR/automation-results.json") |
272 | | - TOTAL_INTERACTIONS=$(jq -r '.summary.totalInteractions // 0' "$RESULTS_DIR/automation-results.json") |
273 | | - |
274 | | - echo "SUCCESS_RATE_${{ matrix.scenario }}=$SUCCESS_RATE" >> $GITHUB_ENV |
275 | | - echo "TOTAL_INTERACTIONS_${{ matrix.scenario }}=$TOTAL_INTERACTIONS" >> $GITHUB_ENV |
276 | | - |
277 | | - # Create summary for this scenario |
278 | | - echo "## 🤖 Automation Results: ${{ matrix.scenario }}" >> automation-results/${{ matrix.scenario }}/summary.md |
279 | | - echo "- **Success Rate**: ${SUCCESS_RATE}%" >> automation-results/${{ matrix.scenario }}/summary.md |
280 | | - echo "- **Total Interactions**: $TOTAL_INTERACTIONS" >> automation-results/${{ matrix.scenario }}/summary.md |
281 | | - echo "- **Timestamp**: $(date -u)" >> automation-results/${{ matrix.scenario }}/summary.md |
282 | | - else |
283 | | - echo "⚠️ No results file found for ${{ matrix.scenario }}" |
284 | | - echo "SUCCESS_RATE_${{ matrix.scenario }}=0" >> $GITHUB_ENV |
285 | | - echo "TOTAL_INTERACTIONS_${{ matrix.scenario }}=0" >> $GITHUB_ENV |
286 | | - fi |
287 | | -
|
288 | | - - name: Upload automation results for ${{ matrix.scenario }} |
289 | | - uses: actions/upload-artifact@v4 |
290 | | - if: always() |
291 | | - with: |
292 | | - name: vscode-automation-results-${{ matrix.scenario }} |
293 | | - path: automation-results/${{ matrix.scenario }}/ |
294 | | - retention-days: 30 |
295 | | - |
296 | | - # Phase 4: Aggregate Results and Summary |
297 | | - automation-summary: |
298 | | - name: Automation Test Summary |
299 | | - runs-on: ubuntu-latest |
300 | | - needs: run-automation-tests |
301 | | - if: always() && github.event_name != 'pull_request' |
302 | | - |
303 | | - steps: |
304 | | - - name: Download all automation results |
305 | | - uses: actions/download-artifact@v4 |
306 | | - with: |
307 | | - pattern: vscode-automation-results-* |
308 | | - path: all-results/ |
309 | | - |
310 | | - - name: Generate comprehensive summary |
311 | | - run: | |
312 | | - echo "## 🤖 VSCode Automation Pipeline Results" >> $GITHUB_STEP_SUMMARY |
313 | | - echo "" >> $GITHUB_STEP_SUMMARY |
314 | | - echo "**Pipeline Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY |
315 | | - echo "**Branch:** ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY |
316 | | - echo "**Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY |
317 | | - echo "**Timestamp:** $(date -u)" >> $GITHUB_STEP_SUMMARY |
318 | | - echo "" >> $GITHUB_STEP_SUMMARY |
319 | | - |
320 | | - # Check if we have results |
321 | | - if [ -d "all-results" ]; then |
322 | | - echo "### 📊 Test Scenarios Results:" >> $GITHUB_STEP_SUMMARY |
323 | | - |
324 | | - for scenario_dir in all-results/vscode-automation-results-*/; do |
325 | | - if [ -d "$scenario_dir" ]; then |
326 | | - scenario_name=$(basename "$scenario_dir" | sed 's/vscode-automation-results-//') |
327 | | - echo "#### $scenario_name" >> $GITHUB_STEP_SUMMARY |
328 | | - |
329 | | - if [ -f "$scenario_dir/summary.md" ]; then |
330 | | - cat "$scenario_dir/summary.md" >> $GITHUB_STEP_SUMMARY |
331 | | - else |
332 | | - echo "- ⚠️ No summary available" >> $GITHUB_STEP_SUMMARY |
333 | | - fi |
334 | | - echo "" >> $GITHUB_STEP_SUMMARY |
335 | | - fi |
336 | | - done |
337 | | - |
338 | | - echo "### 📋 Detailed Results" >> $GITHUB_STEP_SUMMARY |
339 | | - echo "Detailed results are available in the uploaded artifacts." >> $GITHUB_STEP_SUMMARY |
340 | | - else |
341 | | - echo "### ⚠️ No Results Available" >> $GITHUB_STEP_SUMMARY |
342 | | - echo "No automation test results were generated." >> $GITHUB_STEP_SUMMARY |
343 | | - fi |
344 | | - |
345 | | - echo "" >> $GITHUB_STEP_SUMMARY |
346 | | - echo "---" >> $GITHUB_STEP_SUMMARY |
347 | | - echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}\`" >> $GITHUB_STEP_SUMMARY |
348 | | -
|
349 | | - # Phase 5: Security Scan |
| 161 | + # Phase 3: Security Scan |
350 | 162 | security-scan: |
351 | 163 | name: Security Scan VSCode Automation |
352 | 164 | runs-on: ubuntu-latest |
|
0 commit comments