feat(quality): add skill completeness checks (#207)
Add _check_skill_completeness() method to quality checker that validates: - Prerequisites/verification sections (helps Claude check conditions first) - Error handling/troubleshooting guidance (common issues and solutions) - Workflow steps (sequential instructions using first/then/next/finally) This addresses G2.3 and G2.4 from the roadmap: - G2.3: Add readability scoring (via workflow step detection) - G2.4: Add completeness checker New checks use info-level messages (not warnings) to avoid affecting quality scores for existing skills while still providing helpful guidance. Includes 4 new unit tests for completeness checks. Contributed by the AI Writing Guide project.
This commit is contained in:
Joseph Magly
GitHub
@@ -126,6 +126,9 @@ class SkillQualityChecker:
|
||||
# Link validation
|
||||
self._check_links()
|
||||
|
||||
# Completeness checks
|
||||
self._check_skill_completeness()
|
||||
|
||||
return self.report
|
||||
|
||||
def _check_skill_structure(self):
|
||||
@@ -363,6 +366,94 @@ class SkillQualityChecker:
|
||||
'SKILL.md'
|
||||
)
|
||||
|
||||
def _check_skill_completeness(self):
|
||||
"""Check skill completeness based on best practices.
|
||||
|
||||
Validates that skills include verification/prerequisites sections,
|
||||
error handling guidance, and clear workflow steps.
|
||||
"""
|
||||
if not self.skill_md_path.exists():
|
||||
return
|
||||
|
||||
content = self.skill_md_path.read_text(encoding='utf-8')
|
||||
|
||||
# Check for grounding/verification section (prerequisites)
|
||||
grounding_patterns = [
|
||||
r'before\s+(executing|running|proceeding|you\s+start)',
|
||||
r'verify\s+that',
|
||||
r'prerequisites?',
|
||||
r'requirements?:',
|
||||
r'make\s+sure\s+you\s+have',
|
||||
]
|
||||
has_grounding = any(
|
||||
re.search(pattern, content, re.IGNORECASE)
|
||||
for pattern in grounding_patterns
|
||||
)
|
||||
if has_grounding:
|
||||
self.report.add_info(
|
||||
'completeness',
|
||||
'✓ Found verification/prerequisites section',
|
||||
'SKILL.md'
|
||||
)
|
||||
else:
|
||||
self.report.add_info(
|
||||
'completeness',
|
||||
'Consider adding prerequisites section - helps Claude verify conditions first',
|
||||
'SKILL.md'
|
||||
)
|
||||
|
||||
# Check for error handling/troubleshooting guidance
|
||||
error_patterns = [
|
||||
r'if\s+.*\s+(fails?|errors?)',
|
||||
r'troubleshoot',
|
||||
r'common\s+(issues?|problems?)',
|
||||
r'error\s+handling',
|
||||
r'when\s+things\s+go\s+wrong',
|
||||
]
|
||||
has_error_handling = any(
|
||||
re.search(pattern, content, re.IGNORECASE)
|
||||
for pattern in error_patterns
|
||||
)
|
||||
if has_error_handling:
|
||||
self.report.add_info(
|
||||
'completeness',
|
||||
'✓ Found error handling/troubleshooting guidance',
|
||||
'SKILL.md'
|
||||
)
|
||||
else:
|
||||
self.report.add_info(
|
||||
'completeness',
|
||||
'Consider adding troubleshooting section for common issues',
|
||||
'SKILL.md'
|
||||
)
|
||||
|
||||
# Check for workflow steps (numbered or sequential indicators)
|
||||
step_patterns = [
|
||||
r'step\s+\d',
|
||||
r'##\s+\d\.',
|
||||
r'first,?\s+',
|
||||
r'then,?\s+',
|
||||
r'finally,?\s+',
|
||||
r'next,?\s+',
|
||||
]
|
||||
steps_found = sum(
|
||||
1 for pattern in step_patterns
|
||||
if re.search(pattern, content, re.IGNORECASE)
|
||||
)
|
||||
if steps_found >= 3:
|
||||
self.report.add_info(
|
||||
'completeness',
|
||||
f'✓ Found clear workflow indicators ({steps_found} step markers)',
|
||||
'SKILL.md'
|
||||
)
|
||||
elif steps_found > 0:
|
||||
self.report.add_info(
|
||||
'completeness',
|
||||
f'Some workflow guidance found ({steps_found} markers) - '
|
||||
'consider adding numbered steps for clarity',
|
||||
'SKILL.md'
|
||||
)
|
||||
|
||||
|
||||
def print_report(report: QualityReport, verbose: bool = False):
|
||||
"""Print quality report to console.
|
||||
|
||||
@@ -258,6 +258,136 @@ See [this file](nonexistent.md) for more info.
|
||||
self.assertFalse(report2.is_excellent)
|
||||
|
||||
|
||||
class TestCompletenessChecks(unittest.TestCase):
|
||||
"""Test completeness check functionality"""
|
||||
|
||||
def create_test_skill(self, tmpdir, skill_md_content):
|
||||
"""Helper to create a test skill directory"""
|
||||
skill_dir = Path(tmpdir) / "test-skill"
|
||||
skill_dir.mkdir()
|
||||
|
||||
# Create SKILL.md
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
skill_md.write_text(skill_md_content, encoding='utf-8')
|
||||
|
||||
# Create references directory
|
||||
refs_dir = skill_dir / "references"
|
||||
refs_dir.mkdir()
|
||||
(refs_dir / "index.md").write_text("# Index\n", encoding='utf-8')
|
||||
|
||||
return skill_dir
|
||||
|
||||
def test_checker_detects_prerequisites_section(self):
|
||||
"""Test that checker detects prerequisites section"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_md = """---
|
||||
name: test
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Make sure you have:
|
||||
- Python 3.10+
|
||||
- pip installed
|
||||
|
||||
## Usage
|
||||
|
||||
Run the command.
|
||||
"""
|
||||
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||
|
||||
checker = SkillQualityChecker(skill_dir)
|
||||
report = checker.check_all()
|
||||
|
||||
# Should have info about found prerequisites
|
||||
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||
self.assertTrue(any('prerequisites' in i.message.lower() or 'verification' in i.message.lower()
|
||||
for i in completeness_infos))
|
||||
|
||||
def test_checker_detects_troubleshooting_section(self):
|
||||
"""Test that checker detects troubleshooting section"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_md = """---
|
||||
name: test
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
## Usage
|
||||
|
||||
Run the command.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
If the command fails, check your permissions.
|
||||
"""
|
||||
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||
|
||||
checker = SkillQualityChecker(skill_dir)
|
||||
report = checker.check_all()
|
||||
|
||||
# Should have info about found troubleshooting
|
||||
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||
self.assertTrue(any('troubleshoot' in i.message.lower() or 'error handling' in i.message.lower()
|
||||
for i in completeness_infos))
|
||||
|
||||
def test_checker_detects_workflow_steps(self):
|
||||
"""Test that checker detects workflow steps"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_md = """---
|
||||
name: test
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
## Getting Started
|
||||
|
||||
First, install the dependencies.
|
||||
|
||||
Then, configure your environment.
|
||||
|
||||
Next, run the setup script.
|
||||
|
||||
Finally, verify the installation.
|
||||
"""
|
||||
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||
|
||||
checker = SkillQualityChecker(skill_dir)
|
||||
report = checker.check_all()
|
||||
|
||||
# Should have info about found workflow steps
|
||||
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||
self.assertTrue(any('workflow' in i.message.lower() or 'step' in i.message.lower()
|
||||
for i in completeness_infos))
|
||||
|
||||
def test_checker_suggests_adding_prerequisites(self):
|
||||
"""Test that checker suggests adding prerequisites when missing"""
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
skill_md = """---
|
||||
name: test
|
||||
---
|
||||
|
||||
# Test Skill
|
||||
|
||||
## Usage
|
||||
|
||||
Just run the command.
|
||||
"""
|
||||
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||
|
||||
checker = SkillQualityChecker(skill_dir)
|
||||
report = checker.check_all()
|
||||
|
||||
# Should have info suggesting prerequisites
|
||||
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||
self.assertTrue(any('consider' in i.message.lower() and 'prerequisites' in i.message.lower()
|
||||
for i in completeness_infos))
|
||||
|
||||
|
||||
class TestQualityCheckerCLI(unittest.TestCase):
|
||||
"""Test quality checker CLI"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user