feat(quality): add skill completeness checks (#207)

Add _check_skill_completeness() method to quality checker that validates:
- Prerequisites/verification sections (helps Claude check conditions first)
- Error handling/troubleshooting guidance (common issues and solutions)
- Workflow steps (sequential instructions using first/then/next/finally)

This addresses G2.3 and G2.4 from the roadmap:
- G2.3: Add readability scoring (via workflow step detection)
- G2.4: Add completeness checker

New checks use info-level messages (not warnings) to avoid affecting
quality scores for existing skills while still providing helpful guidance.

Includes 4 new unit tests for completeness checks.

Contributed by the AI Writing Guide project.
This commit is contained in:
Joseph Magly
2026-01-01 11:54:48 -05:00
committed by GitHub
parent 9949cdcdca
commit 8a111eb526
2 changed files with 221 additions and 0 deletions

View File

@@ -258,6 +258,136 @@ See [this file](nonexistent.md) for more info.
self.assertFalse(report2.is_excellent)
class TestCompletenessChecks(unittest.TestCase):
"""Test completeness check functionality"""
def create_test_skill(self, tmpdir, skill_md_content):
"""Helper to create a test skill directory"""
skill_dir = Path(tmpdir) / "test-skill"
skill_dir.mkdir()
# Create SKILL.md
skill_md = skill_dir / "SKILL.md"
skill_md.write_text(skill_md_content, encoding='utf-8')
# Create references directory
refs_dir = skill_dir / "references"
refs_dir.mkdir()
(refs_dir / "index.md").write_text("# Index\n", encoding='utf-8')
return skill_dir
def test_checker_detects_prerequisites_section(self):
"""Test that checker detects prerequisites section"""
with tempfile.TemporaryDirectory() as tmpdir:
skill_md = """---
name: test
---
# Test Skill
## Prerequisites
Make sure you have:
- Python 3.10+
- pip installed
## Usage
Run the command.
"""
skill_dir = self.create_test_skill(tmpdir, skill_md)
checker = SkillQualityChecker(skill_dir)
report = checker.check_all()
# Should have info about found prerequisites
completeness_infos = [i for i in report.info if i.category == 'completeness']
self.assertTrue(any('prerequisites' in i.message.lower() or 'verification' in i.message.lower()
for i in completeness_infos))
def test_checker_detects_troubleshooting_section(self):
"""Test that checker detects troubleshooting section"""
with tempfile.TemporaryDirectory() as tmpdir:
skill_md = """---
name: test
---
# Test Skill
## Usage
Run the command.
## Troubleshooting
### Common Issues
If the command fails, check your permissions.
"""
skill_dir = self.create_test_skill(tmpdir, skill_md)
checker = SkillQualityChecker(skill_dir)
report = checker.check_all()
# Should have info about found troubleshooting
completeness_infos = [i for i in report.info if i.category == 'completeness']
self.assertTrue(any('troubleshoot' in i.message.lower() or 'error handling' in i.message.lower()
for i in completeness_infos))
def test_checker_detects_workflow_steps(self):
"""Test that checker detects workflow steps"""
with tempfile.TemporaryDirectory() as tmpdir:
skill_md = """---
name: test
---
# Test Skill
## Getting Started
First, install the dependencies.
Then, configure your environment.
Next, run the setup script.
Finally, verify the installation.
"""
skill_dir = self.create_test_skill(tmpdir, skill_md)
checker = SkillQualityChecker(skill_dir)
report = checker.check_all()
# Should have info about found workflow steps
completeness_infos = [i for i in report.info if i.category == 'completeness']
self.assertTrue(any('workflow' in i.message.lower() or 'step' in i.message.lower()
for i in completeness_infos))
def test_checker_suggests_adding_prerequisites(self):
"""Test that checker suggests adding prerequisites when missing"""
with tempfile.TemporaryDirectory() as tmpdir:
skill_md = """---
name: test
---
# Test Skill
## Usage
Just run the command.
"""
skill_dir = self.create_test_skill(tmpdir, skill_md)
checker = SkillQualityChecker(skill_dir)
report = checker.check_all()
# Should have info suggesting prerequisites
completeness_infos = [i for i in report.info if i.category == 'completeness']
self.assertTrue(any('consider' in i.message.lower() and 'prerequisites' in i.message.lower()
for i in completeness_infos))
class TestQualityCheckerCLI(unittest.TestCase):
"""Test quality checker CLI"""