diff --git a/.github/workflows/skill-eval.yml b/.github/workflows/skill-eval.yml
deleted file mode 100644
index 5510232..0000000
--- a/.github/workflows/skill-eval.yml
+++ /dev/null
@@ -1,235 +0,0 @@
----
-name: Skill Quality Eval (promptfoo)
-
-'on':
- pull_request:
- types: [opened, synchronize, reopened]
- paths:
- - '**/SKILL.md'
- workflow_dispatch:
- inputs:
- skill:
- description: 'Specific skill eval config to run (e.g. copywriting)'
- required: false
-
-concurrency:
- group: skill-eval-${{ github.event.pull_request.number || github.run_id }}
- cancel-in-progress: true
-
-jobs:
- detect-changes:
- name: Detect changed skills
- runs-on: ubuntu-latest
- outputs:
- skills: ${{ steps.find-evals.outputs.skills }}
- has_evals: ${{ steps.find-evals.outputs.has_evals }}
- steps:
- - name: Checkout
- uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Find eval configs for changed skills
- id: find-evals
- run: |
- if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.skill }}" ]]; then
- SKILL="${{ github.event.inputs.skill }}"
- if [[ -f "eval/skills/${SKILL}.yaml" ]]; then
- echo "skills=[\"${SKILL}\"]" >> "$GITHUB_OUTPUT"
- echo "has_evals=true" >> "$GITHUB_OUTPUT"
- else
- echo "No eval config found for: ${SKILL}"
- echo "has_evals=false" >> "$GITHUB_OUTPUT"
- fi
- exit 0
- fi
-
- # Get changed SKILL.md files in this PR
- CHANGED=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- '**/SKILL.md' | grep -v '.gemini/' | grep -v '.codex/' | grep -v 'sample')
-
- if [[ -z "$CHANGED" ]]; then
- echo "No SKILL.md files changed."
- echo "has_evals=false" >> "$GITHUB_OUTPUT"
- exit 0
- fi
-
- echo "Changed SKILL.md files:"
- echo "$CHANGED"
-
- # Map changed skills to eval configs
- EVALS="[]"
- for skill_path in $CHANGED; do
- # Extract skill name from path (e.g. marketing-skill/copywriting/SKILL.md -> copywriting)
- skill_name=$(basename $(dirname "$skill_path"))
- eval_config="eval/skills/${skill_name}.yaml"
-
- if [[ -f "$eval_config" ]]; then
- EVALS=$(echo "$EVALS" | python3 -c "
- import json, sys
- arr = json.load(sys.stdin)
- name = '$skill_name'
- if name not in arr:
- arr.append(name)
- print(json.dumps(arr))
- ")
- echo " ✅ $skill_name → $eval_config"
- else
- echo " ⏭️ $skill_name → no eval config (skipping)"
- fi
- done
-
- echo "skills=$EVALS" >> "$GITHUB_OUTPUT"
- if [[ "$EVALS" == "[]" ]]; then
- echo "has_evals=false" >> "$GITHUB_OUTPUT"
- else
- echo "has_evals=true" >> "$GITHUB_OUTPUT"
- fi
-
- eval:
- name: "Eval: ${{ matrix.skill }}"
- needs: detect-changes
- if: needs.detect-changes.outputs.has_evals == 'true'
- runs-on: ubuntu-latest
- permissions:
- contents: read
- pull-requests: write
- timeout-minutes: 15
- strategy:
- fail-fast: false
- matrix:
- skill: ${{ fromJson(needs.detect-changes.outputs.skills) }}
- steps:
- - name: Checkout
- uses: actions/checkout@v4
-
- - name: Set up Node.js
- uses: actions/setup-node@v4
- with:
- node-version: 20
-
- - name: Run promptfoo eval
- id: eval
- continue-on-error: true
- env:
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- run: |
- npx promptfoo@latest eval \
- -c "eval/skills/${{ matrix.skill }}.yaml" \
- --no-cache \
- --output "/tmp/${{ matrix.skill }}-results.json" \
- --output-format json \
- 2>&1 | tee /tmp/eval-output.log
-
- echo "exit_code=$?" >> "$GITHUB_OUTPUT"
-
- - name: Parse results
- id: parse
- if: always()
- run: |
- RESULTS_FILE="/tmp/${{ matrix.skill }}-results.json"
- if [[ ! -f "$RESULTS_FILE" ]]; then
- echo "summary=⚠️ No results file generated" >> "$GITHUB_OUTPUT"
- exit 0
- fi
-
- python3 << 'PYEOF'
- import json, os
-
- with open(os.environ.get("RESULTS_FILE", f"/tmp/${{ matrix.skill }}-results.json")) as f:
- data = json.load(f)
-
- results = data.get("results", data.get("evalResults", []))
- total = len(results)
- passed = 0
- failed = 0
- details = []
-
- for r in results:
- test_pass = r.get("success", False)
- if test_pass:
- passed += 1
- else:
- failed += 1
-
- prompt_vars = r.get("vars", {})
- task = prompt_vars.get("task", "unknown")[:80]
-
- assertions = r.get("gradingResult", {}).get("componentResults", [])
- for a in assertions:
- status = "✅" if a.get("pass", False) else "❌"
- reason = a.get("reason", a.get("assertion", {}).get("value", ""))[:100]
- details.append(f" {status} {reason}")
-
- rate = (passed / total * 100) if total > 0 else 0
- icon = "✅" if rate >= 80 else "⚠️" if rate >= 50 else "❌"
-
- summary = f"{icon} **${{ matrix.skill }}**: {passed}/{total} tests passed ({rate:.0f}%)"
-
- # Write to file for comment step
- with open("/tmp/eval-summary.md", "w") as f:
- f.write(f"### {summary}\n\n")
- if details:
- f.write("Assertion details
\n\n")
- f.write("\n".join(details))
- f.write("\n\n