# .github/workflows/prompt-eval.yml
name: Prompt Eval
 
on:
  pull_request:
    paths:
      - 'prompts/**'
 
jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: '3.11' }
      - run: pip install -r requirements.txt
 
      - name: Run eval
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          python -m evals.run_all --output report.json
 
      - name: Comment PR
        uses: actions/github-script@v7
        with:
          script: |
            const report = require('./report.json');
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: `## Eval Report\n\nAccuracy: ${report.accuracy}%\n...`
            });
 
      - name: Fail on regression
        run: |
          python -m evals.regression_check --baseline 0.85