Try to add promptfoo into the ci flow

Author: fangpenlin

.github/workflows/ci.yml

@@ -18,6 +18,15 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+        cache: 'npm'
+    
+    - name: Install promptfoo
+      run: npm install -g promptfoo
+    
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4
       with:
@@ -36,6 +45,10 @@ jobs:
       run: |
         uv run pytest --cov=src/isnt_that_odd --cov-report=xml
     
+    - name: Run promptfoo tests
+      run: |
+        promptfoo eval --config promptfoo.yaml
+    
     - name: Upload coverage to Codecov
       uses: codecov/codecov-action@v3
       with:
@@ -48,6 +61,15 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
+    - name: Set up Node.js
+      uses: actions/setup-node@v4
+      with:
+        node-version: '18'
+        cache: 'npm'
+    
+    - name: Install promptfoo
+      run: npm install -g promptfoo
+    
     - name: Set up Python
       uses: actions/setup-python@v4
       with:
@@ -67,6 +89,10 @@ jobs:
         uv run ruff check src/ tests/
         uv run mypy src/
     
+    - name: Run promptfoo lint
+      run: |
+        promptfoo lint --config promptfoo.yaml
+    
   build:
     runs-on: ubuntu-latest
     needs: [test, lint]

Makefile

@@ -2,131 +2,114 @@
 description: Test prompts for determining if numbers are even or odd
 
 prompts:
-  - even_odd_prompt: |
-      You are a mathematical assistant. Your task is to determine if the given number is even or odd.
-
-      Number: {{number}}
-
-      Instructions:
-      1. A number is even if it is divisible by 2 with no remainder
-      2. A number is odd if it is not divisible by 2
-      3. For decimal numbers, only the integer part matters
-      4. For negative numbers, the same rules apply
-      5. Zero (0) is considered even
-
-      Please respond with ONLY a JSON object containing a boolean field "is_even":
-      - Set "is_even" to true if the number is even
-      - Set "is_even" to false if the number is odd
-
-      Example response format:
-      {"is_even": true}
-
-      Your response:
+  - id: even_odd_prompt
+    raw: file://src/isnt_that_odd/prompts/even_odd.txt
+    label: Even/Odd Determination Prompt
+    vars:
+      number: "{{number}}"
 
 providers:
-  - id: openai:gpt-3.5-turbo
+  - id: xai:grok-3
     config:
       temperature: 0.0
       max_tokens: 50
-      response_format:
-        type: json_object
 
 tests:
   - vars:
       number: 2
     assert:
-      - type: contains-json
-        value: '{"is_even": true}'
+      - type: contains
+        value: '"is_even": true'
       - type: contains
         value: "true"
     description: "Even number (2) should return true"
 
   - vars:
       number: 3
     assert:
-      - type: contains-json
-        value: '{"is_even": false}'
+      - type: contains
+        value: '"is_even": false'
       - type: contains
         value: "false"
     description: "Odd number (3) should return false"
 
   - vars:
       number: 0
     assert:
-      - type: contains-json
-        value: '{"is_even": true}'
+      - type: contains
+        value: '"is_even": true'
       - type: contains
         value: "true"
     description: "Zero (0) should return true (even)"
 
   - vars:
       number: -4
     assert:
-      - type: contains-json
-        value: '{"is_even": true}'
+      - type: contains
+        value: '"is_even": true'
       - type: contains
         value: "true"
     description: "Negative even number (-4) should return true"
 
   - vars:
       number: -7
     assert:
-      - type: contains-json
-        value: '{"is_even": false}'
+      - type: contains
+        value: '"is_even": false'
       - type: contains
         value: "false"
     description: "Negative odd number (-7) should return false"
 
   - vars:
       number: 10.5
     assert:
-      - type: contains-json
-        value: '{"is_even": true}'
+      - type: contains
+        value: '"is_even": true'
       - type: contains
         value: "true"
     description: "Decimal number (10.5) should return true (integer part 10 is even)"
 
   - vars:
       number: 15.8
     assert:
-      - type: contains-json
-        value: '{"is_even": false}'
+      - type: contains
+        value: '"is_even": false'
       - type: contains
         value: "false"
     description: "Decimal number (15.8) should return false (integer part 15 is odd)"
 
   - vars:
       number: 100
     assert:
-      - type: contains-json
-        value: '{"is_even": true}'
+      - type: contains
+        value: '"is_even": true'
       - type: contains
         value: "true"
     description: "Large even number (100) should return true"
 
   - vars:
       number: 999
     assert:
-      - type: contains-json
-        value: '{"is_even": false}'
+      - type: contains
+        value: '"is_even": false'
       - type: contains
         value: "false"
     description: "Large odd number (999) should return false"
 
   - vars:
       number: "42"
     assert:
-      - type: contains-json
-        value: '{"is_even": true}'
+      - type: contains
+        value: '"is_even": true'
       - type: contains
         value: "true"
     description: "String number ('42') should return true"
 
   - vars:
       number: "17"
     assert:
-      - type: contains-json
-        value: '{"is_even": false}'
+      - type: contains
+        value: '"is_even": false'
       - type: contains
         value: "false"
     description: "String number ('17') should return false"

promptfoo.yaml

@@ -1,4 +1,5 @@
 """Core functionality for determining if numbers are even using LLM APIs."""
+
 from typing import Optional
 from typing import Union
 
@@ -42,7 +43,20 @@ def _create_prompt(self, number: Union[int, float, str]) -> str:
         Returns:
             A formatted prompt string
         """
-        return f"""You are a mathematical assistant. Your task is to determine if the given number is even or odd.
+        import os
+        from pathlib import Path
+
+        # Get the directory where this module is located
+        current_dir = Path(__file__).parent
+        prompt_file = current_dir / "prompts" / "even_odd.txt"
+
+        try:
+            with open(prompt_file, "r", encoding="utf-8") as f:
+                prompt_template = f.read()
+            return prompt_template.format(number=number)
+        except FileNotFoundError:
+            # Fallback to hardcoded prompt if file not found
+            return f"""You are a mathematical assistant. Your task is to determine if the given number is even or odd.
 
 Number: {number}
 

src/isnt_that_odd/core.py

@@ -0,0 +1 @@
+"""Prompts package for the isn't that odd library."""

src/isnt_that_odd/prompts/__init__.py

@@ -0,0 +1,19 @@
+You are a mathematical assistant. Your task is to determine if the given number is even or odd.
+
+Number: {{number}}
+
+Instructions:
+1. A number is even if it is divisible by 2 with no remainder
+2. A number is odd if it is not divisible by 2
+3. For decimal numbers, only the integer part matters
+4. For negative numbers, the same rules apply
+5. Zero (0) is considered even
+
+Please respond with ONLY a JSON object containing a boolean field "is_even":
+- Set "is_even" to true if the number is even
+- Set "is_even" to false if the number is odd
+
+Example response format:
+{"is_even": true}
+
+Your response: