Skip to content

Commit 165a061

Browse files
committed
feat: implement advanced CI/CD and automation features
- Add evalops cost command with LLM provider pricing calculations - Implement evalops budget command with budget.yaml support - Create budget validation system with quality gates and cost limits - Add multi-environment support (dev/staging/production) - Integrate budget checking into upload command with --check-budget flag - Create comprehensive GitHub Actions workflow with cost gates - Update README with detailed CI/CD integration documentation - Add support for automated PR comments and budget enforcement This completes the advanced CI/CD roadmap including: - Quality Gates and Performance Budgeting - Cost Estimation and Budget Configuration - Official GitHub Action integration - CI build failure on budget violations - Automated PR comment integration
1 parent ecc40a1 commit 165a061

File tree

8 files changed

+1226
-3
lines changed

8 files changed

+1226
-3
lines changed

.github/workflows/evalops-ci.yml

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: 'EvalOps CI/CD Pipeline'
2+
3+
on:
4+
pull_request:
5+
branches: [main]
6+
paths:
7+
- '**/*.eval.ts'
8+
- '**/*.eval.js'
9+
- 'evalops.yaml'
10+
- 'budget.yaml'
11+
push:
12+
branches: [main]
13+
14+
jobs:
15+
cost-estimation:
16+
name: 'Cost Estimation'
17+
runs-on: ubuntu-latest
18+
outputs:
19+
estimated-cost: ${{ steps.cost.outputs.total-cost }}
20+
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@v4
24+
25+
- name: Setup Node.js
26+
uses: actions/setup-node@v4
27+
with:
28+
node-version: '18'
29+
30+
- name: Install EvalOps CLI
31+
run: npm install -g evalops-cli
32+
33+
- name: Calculate Cost Estimate
34+
id: cost
35+
run: |
36+
COST=$(evalops cost --format json | jq -r '.totalCost')
37+
echo "total-cost=$COST" >> $GITHUB_OUTPUT
38+
echo "### 💰 Cost Estimate: \$${COST}" >> $GITHUB_STEP_SUMMARY
39+
40+
- name: Cost Gate Check
41+
run: |
42+
COST="${{ steps.cost.outputs.total-cost }}"
43+
MAX_COST="10.00"
44+
if (( $(echo "$COST > $MAX_COST" | bc -l) )); then
45+
echo "❌ Cost estimate \$${COST} exceeds maximum \$${MAX_COST}"
46+
exit 1
47+
fi
48+
echo "✅ Cost estimate \$${COST} is within budget"
49+
50+
quality-evaluation:
51+
name: 'Quality Evaluation'
52+
runs-on: ubuntu-latest
53+
needs: cost-estimation
54+
if: github.event_name == 'pull_request'
55+
56+
steps:
57+
- name: Checkout
58+
uses: actions/checkout@v4
59+
with:
60+
fetch-depth: 0 # Need full history for comparison
61+
62+
- name: Setup Node.js
63+
uses: actions/setup-node@v4
64+
with:
65+
node-version: '18'
66+
67+
- name: Run EvalOps Evaluation
68+
id: evaluation
69+
uses: evalops/evalops-action@v1
70+
with:
71+
api-key: ${{ secrets.EVALOPS_API_KEY }}
72+
config-file: ./evalops.yaml
73+
budget-file: ./budget.yaml
74+
environment: staging
75+
check-budget: true
76+
comment-pr: true
77+
fail-on-violation: true
78+
quality-threshold: '0.6'
79+
cost-threshold: '10.00'
80+
81+
deploy-evaluation:
82+
name: 'Deploy to Production'
83+
runs-on: ubuntu-latest
84+
needs: [cost-estimation, quality-evaluation]
85+
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
86+
environment: production
87+
88+
steps:
89+
- name: Checkout
90+
uses: actions/checkout@v4
91+
92+
- name: Setup Node.js
93+
uses: actions/setup-node@v4
94+
with:
95+
node-version: '18'
96+
97+
- name: Install EvalOps CLI
98+
run: npm install -g evalops-cli
99+
100+
- name: Deploy Evaluation
101+
env:
102+
EVALOPS_API_KEY: ${{ secrets.EVALOPS_API_KEY }}
103+
run: |
104+
# Deploy using official EvalOps Action with production settings
105+
evalops upload --check-budget --budget-file budget.yaml --environment production
106+
107+
- name: Create Release
108+
if: success()
109+
uses: actions/github-script@v7
110+
with:
111+
script: |
112+
const tagName = `evaluation-${Date.now()}`;
113+
114+
github.rest.repos.createRelease({
115+
owner: context.repo.owner,
116+
repo: context.repo.repo,
117+
tag_name: tagName,
118+
name: `Evaluation Release ${tagName}`,
119+
body: `
120+
## EvalOps Evaluation Deployed
121+
122+
- **Quality Score**: ${{ needs.quality-evaluation.outputs.quality-score || 'N/A' }}
123+
- **Cost**: $${{ needs.cost-estimation.outputs.estimated-cost }}
124+
- **Deploy Time**: ${new Date().toISOString()}
125+
126+
This release contains the latest code evaluation configuration and has passed all quality gates.
127+
`,
128+
draft: false,
129+
prerelease: false
130+
});

0 commit comments

Comments
 (0)