|
| 1 | +name: Autograding |
| 2 | +on: |
| 3 | + push: |
| 4 | + branches: [main] |
| 5 | + workflow_dispatch: |
| 6 | + |
| 7 | +env: |
| 8 | + AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }} |
| 9 | + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} |
| 10 | + AZURE_OPENAI_DEPLOYMENT: ${{ secrets.AZURE_OPENAI_DEPLOYMENT }} |
| 11 | + STUDENT_CORPUS_SEED: ${{ github.actor }} |
| 12 | + |
| 13 | +jobs: |
| 14 | + grade: |
| 15 | + runs-on: ubuntu-latest |
| 16 | + timeout-minutes: 15 |
| 17 | + steps: |
| 18 | + - uses: actions/checkout@v4 |
| 19 | + |
| 20 | + - uses: actions/setup-python@v5 |
| 21 | + with: |
| 22 | + python-version: "3.11" |
| 23 | + |
| 24 | + - name: Install dependencies |
| 25 | + run: pip install -r requirements.txt |
| 26 | + |
| 27 | + - name: Run student code |
| 28 | + run: python app/main.py |
| 29 | + continue-on-error: true |
| 30 | + |
| 31 | + - name: Clone autograder |
| 32 | + uses: actions/checkout@v4 |
| 33 | + with: |
| 34 | + repository: Tech901/ai102-autograder-tests |
| 35 | + token: ${{ secrets.AUTOGRADER_PAT }} |
| 36 | + path: .autograder |
| 37 | + |
| 38 | + # --- 27 tests, 100 points --- |
| 39 | + - name: "test_student_modified_select_service" |
| 40 | + id: test1 |
| 41 | + uses: classroom-resources/autograding-command-grader@v1 |
| 42 | + with: |
| 43 | + test-name: "test_student_modified_select_service" |
| 44 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_student_modified_select_service -v --tb=short" |
| 45 | + timeout: "5" |
| 46 | + max-score: "4" |
| 47 | + |
| 48 | + - name: "test_student_modified_estimate_cost" |
| 49 | + id: test2 |
| 50 | + uses: classroom-resources/autograding-command-grader@v1 |
| 51 | + with: |
| 52 | + test-name: "test_student_modified_estimate_cost" |
| 53 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_student_modified_estimate_cost -v --tb=short" |
| 54 | + timeout: "5" |
| 55 | + max-score: "4" |
| 56 | + |
| 57 | + - name: "test_design_has_architecture" |
| 58 | + id: test3 |
| 59 | + uses: classroom-resources/autograding-command-grader@v1 |
| 60 | + with: |
| 61 | + test-name: "test_design_has_architecture" |
| 62 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_design_has_architecture -v --tb=short" |
| 63 | + timeout: "5" |
| 64 | + max-score: "4" |
| 65 | + |
| 66 | + - name: "test_design_is_valid_json" |
| 67 | + id: test4 |
| 68 | + uses: classroom-resources/autograding-command-grader@v1 |
| 69 | + with: |
| 70 | + test-name: "test_design_is_valid_json" |
| 71 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_design_is_valid_json -v --tb=short" |
| 72 | + timeout: "5" |
| 73 | + max-score: "4" |
| 74 | + |
| 75 | + - name: "test_correct_services_per_department" |
| 76 | + id: test5 |
| 77 | + uses: classroom-resources/autograding-command-grader@v1 |
| 78 | + with: |
| 79 | + test-name: "test_correct_services_per_department" |
| 80 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_correct_services_per_department -v --tb=short" |
| 81 | + timeout: "5" |
| 82 | + max-score: "4" |
| 83 | + |
| 84 | + - name: "test_unique_justifications" |
| 85 | + id: test6 |
| 86 | + uses: classroom-resources/autograding-command-grader@v1 |
| 87 | + with: |
| 88 | + test-name: "test_unique_justifications" |
| 89 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_unique_justifications -v --tb=short" |
| 90 | + timeout: "5" |
| 91 | + max-score: "4" |
| 92 | + |
| 93 | + - name: "test_rai_not_empty" |
| 94 | + id: test7 |
| 95 | + uses: classroom-resources/autograding-command-grader@v1 |
| 96 | + with: |
| 97 | + test-name: "test_rai_not_empty" |
| 98 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_rai_not_empty -v --tb=short" |
| 99 | + timeout: "5" |
| 100 | + max-score: "4" |
| 101 | + |
| 102 | + - name: "test_min_distinct_services" |
| 103 | + id: test8 |
| 104 | + uses: classroom-resources/autograding-command-grader@v1 |
| 105 | + with: |
| 106 | + test-name: "test_min_distinct_services" |
| 107 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_min_distinct_services -v --tb=short" |
| 108 | + timeout: "5" |
| 109 | + max-score: "4" |
| 110 | + |
| 111 | + - name: "test_valid_sdk_names" |
| 112 | + id: test9 |
| 113 | + uses: classroom-resources/autograding-command-grader@v1 |
| 114 | + with: |
| 115 | + test-name: "test_valid_sdk_names" |
| 116 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_valid_sdk_names -v --tb=short" |
| 117 | + timeout: "5" |
| 118 | + max-score: "4" |
| 119 | + |
| 120 | + - name: "test_auth_model_mentioned" |
| 121 | + id: test10 |
| 122 | + uses: classroom-resources/autograding-command-grader@v1 |
| 123 | + with: |
| 124 | + test-name: "test_auth_model_mentioned" |
| 125 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_auth_model_mentioned -v --tb=short" |
| 126 | + timeout: "5" |
| 127 | + max-score: "4" |
| 128 | + |
| 129 | + - name: "test_comprehensive_audit_fields" |
| 130 | + id: test11 |
| 131 | + uses: classroom-resources/autograding-command-grader@v1 |
| 132 | + with: |
| 133 | + test-name: "test_comprehensive_audit_fields" |
| 134 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_comprehensive_audit_fields -v --tb=short" |
| 135 | + timeout: "5" |
| 136 | + max-score: "4" |
| 137 | + |
| 138 | + - name: "test_auth_comparison_mentions_three_models" |
| 139 | + id: test12 |
| 140 | + uses: classroom-resources/autograding-command-grader@v1 |
| 141 | + with: |
| 142 | + test-name: "test_auth_comparison_mentions_three_models" |
| 143 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_auth_comparison_mentions_three_models -v --tb=short" |
| 144 | + timeout: "5" |
| 145 | + max-score: "4" |
| 146 | + |
| 147 | + - name: "test_four_services_validated" |
| 148 | + id: test13 |
| 149 | + uses: classroom-resources/autograding-command-grader@v1 |
| 150 | + with: |
| 151 | + test-name: "test_four_services_validated" |
| 152 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_four_services_validated -v --tb=short" |
| 153 | + timeout: "5" |
| 154 | + max-score: "4" |
| 155 | + |
| 156 | + - name: "test_exact_math_311" |
| 157 | + id: test14 |
| 158 | + uses: classroom-resources/autograding-command-grader@v1 |
| 159 | + with: |
| 160 | + test-name: "test_exact_math_311" |
| 161 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_exact_math_311 -v --tb=short" |
| 162 | + timeout: "5" |
| 163 | + max-score: "4" |
| 164 | + |
| 165 | + - name: "test_exact_math_public_works" |
| 166 | + id: test15 |
| 167 | + uses: classroom-resources/autograding-command-grader@v1 |
| 168 | + with: |
| 169 | + test-name: "test_exact_math_public_works" |
| 170 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_exact_math_public_works -v --tb=short" |
| 171 | + timeout: "5" |
| 172 | + max-score: "4" |
| 173 | + |
| 174 | + - name: "test_budget_flag_logic" |
| 175 | + id: test16 |
| 176 | + uses: classroom-resources/autograding-command-grader@v1 |
| 177 | + with: |
| 178 | + test-name: "test_budget_flag_logic" |
| 179 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_budget_flag_logic -v --tb=short" |
| 180 | + timeout: "5" |
| 181 | + max-score: "4" |
| 182 | + |
| 183 | + - name: "test_total_equals_sum" |
| 184 | + id: test17 |
| 185 | + uses: classroom-resources/autograding-command-grader@v1 |
| 186 | + with: |
| 187 | + test-name: "test_total_equals_sum" |
| 188 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_total_equals_sum -v --tb=short" |
| 189 | + timeout: "5" |
| 190 | + max-score: "4" |
| 191 | + |
| 192 | + - name: "test_metric_names_valid" |
| 193 | + id: test18 |
| 194 | + uses: classroom-resources/autograding-command-grader@v1 |
| 195 | + with: |
| 196 | + test-name: "test_metric_names_valid" |
| 197 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_metric_names_valid -v --tb=short" |
| 198 | + timeout: "5" |
| 199 | + max-score: "4" |
| 200 | + |
| 201 | + - name: "test_alert_rules_have_condition_and_action" |
| 202 | + id: test19 |
| 203 | + uses: classroom-resources/autograding-command-grader@v1 |
| 204 | + with: |
| 205 | + test-name: "test_alert_rules_have_condition_and_action" |
| 206 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_alert_rules_have_condition_and_action -v --tb=short" |
| 207 | + timeout: "5" |
| 208 | + max-score: "4" |
| 209 | + |
| 210 | + - name: "test_result_json_is_valid_json" |
| 211 | + id: test20 |
| 212 | + uses: classroom-resources/autograding-command-grader@v1 |
| 213 | + with: |
| 214 | + test-name: "test_result_json_is_valid_json" |
| 215 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_result_json_is_valid_json -v --tb=short" |
| 216 | + timeout: "5" |
| 217 | + max-score: "3" |
| 218 | + |
| 219 | + - name: "test_no_leaked_endpoints" |
| 220 | + id: test21 |
| 221 | + uses: classroom-resources/autograding-command-grader@v1 |
| 222 | + with: |
| 223 | + test-name: "test_no_leaked_endpoints" |
| 224 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_no_leaked_endpoints -v --tb=short" |
| 225 | + timeout: "5" |
| 226 | + max-score: "3" |
| 227 | + |
| 228 | + - name: "test_no_hardcoded_secrets_ast" |
| 229 | + id: test22 |
| 230 | + uses: classroom-resources/autograding-command-grader@v1 |
| 231 | + with: |
| 232 | + test-name: "test_no_hardcoded_secrets_ast" |
| 233 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_no_hardcoded_secrets_ast -v --tb=short" |
| 234 | + timeout: "5" |
| 235 | + max-score: "3" |
| 236 | + |
| 237 | + - name: "test_pricing_version_present" |
| 238 | + id: test23 |
| 239 | + uses: classroom-resources/autograding-command-grader@v1 |
| 240 | + with: |
| 241 | + test-name: "test_pricing_version_present" |
| 242 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_pricing_version_present -v --tb=short" |
| 243 | + timeout: "5" |
| 244 | + max-score: "3" |
| 245 | + |
| 246 | + - name: "test_justification_minimum_length" |
| 247 | + id: test24 |
| 248 | + uses: classroom-resources/autograding-command-grader@v1 |
| 249 | + with: |
| 250 | + test-name: "test_justification_minimum_length" |
| 251 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_justification_minimum_length -v --tb=short" |
| 252 | + timeout: "5" |
| 253 | + max-score: "3" |
| 254 | + |
| 255 | + - name: "test_validation_has_required_fields" |
| 256 | + id: test25 |
| 257 | + uses: classroom-resources/autograding-command-grader@v1 |
| 258 | + with: |
| 259 | + test-name: "test_validation_has_required_fields" |
| 260 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_validation_has_required_fields -v --tb=short" |
| 261 | + timeout: "5" |
| 262 | + max-score: "3" |
| 263 | + |
| 264 | + - name: "test_shared_resources_complete" |
| 265 | + id: test26 |
| 266 | + uses: classroom-resources/autograding-command-grader@v1 |
| 267 | + with: |
| 268 | + test-name: "test_shared_resources_complete" |
| 269 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_shared_resources_complete -v --tb=short" |
| 270 | + timeout: "5" |
| 271 | + max-score: "3" |
| 272 | + |
| 273 | + - name: "test_alternative_differs_from_primary" |
| 274 | + id: test27 |
| 275 | + uses: classroom-resources/autograding-command-grader@v1 |
| 276 | + with: |
| 277 | + test-name: "test_alternative_differs_from_primary" |
| 278 | + command: "python -m pytest .autograder/activity02-architect-platform/test_hidden.py::test_alternative_differs_from_primary -v --tb=short" |
| 279 | + timeout: "5" |
| 280 | + max-score: "3" |
| 281 | + |
| 282 | + - name: Autograding Reporter |
| 283 | + uses: classroom-resources/autograding-grading-reporter@v1 |
| 284 | + env: |
| 285 | + TEST1_RESULTS: "${{ steps.test1.outputs.result }}" |
| 286 | + TEST2_RESULTS: "${{ steps.test2.outputs.result }}" |
| 287 | + TEST3_RESULTS: "${{ steps.test3.outputs.result }}" |
| 288 | + TEST4_RESULTS: "${{ steps.test4.outputs.result }}" |
| 289 | + TEST5_RESULTS: "${{ steps.test5.outputs.result }}" |
| 290 | + TEST6_RESULTS: "${{ steps.test6.outputs.result }}" |
| 291 | + TEST7_RESULTS: "${{ steps.test7.outputs.result }}" |
| 292 | + TEST8_RESULTS: "${{ steps.test8.outputs.result }}" |
| 293 | + TEST9_RESULTS: "${{ steps.test9.outputs.result }}" |
| 294 | + TEST10_RESULTS: "${{ steps.test10.outputs.result }}" |
| 295 | + TEST11_RESULTS: "${{ steps.test11.outputs.result }}" |
| 296 | + TEST12_RESULTS: "${{ steps.test12.outputs.result }}" |
| 297 | + TEST13_RESULTS: "${{ steps.test13.outputs.result }}" |
| 298 | + TEST14_RESULTS: "${{ steps.test14.outputs.result }}" |
| 299 | + TEST15_RESULTS: "${{ steps.test15.outputs.result }}" |
| 300 | + TEST16_RESULTS: "${{ steps.test16.outputs.result }}" |
| 301 | + TEST17_RESULTS: "${{ steps.test17.outputs.result }}" |
| 302 | + TEST18_RESULTS: "${{ steps.test18.outputs.result }}" |
| 303 | + TEST19_RESULTS: "${{ steps.test19.outputs.result }}" |
| 304 | + TEST20_RESULTS: "${{ steps.test20.outputs.result }}" |
| 305 | + TEST21_RESULTS: "${{ steps.test21.outputs.result }}" |
| 306 | + TEST22_RESULTS: "${{ steps.test22.outputs.result }}" |
| 307 | + TEST23_RESULTS: "${{ steps.test23.outputs.result }}" |
| 308 | + TEST24_RESULTS: "${{ steps.test24.outputs.result }}" |
| 309 | + TEST25_RESULTS: "${{ steps.test25.outputs.result }}" |
| 310 | + TEST26_RESULTS: "${{ steps.test26.outputs.result }}" |
| 311 | + TEST27_RESULTS: "${{ steps.test27.outputs.result }}" |
| 312 | + with: |
| 313 | + runners: test1,test2,test3,test4,test5,test6,test7,test8,test9,test10,test11,test12,test13,test14,test15,test16,test17,test18,test19,test20,test21,test22,test23,test24,test25,test26,test27 |
0 commit comments