feat(quality): add skill completeness checks (#207)
Add _check_skill_completeness() method to quality checker that validates: - Prerequisites/verification sections (helps Claude check conditions first) - Error handling/troubleshooting guidance (common issues and solutions) - Workflow steps (sequential instructions using first/then/next/finally) This addresses G2.3 and G2.4 from the roadmap: - G2.3: Add readability scoring (via workflow step detection) - G2.4: Add completeness checker New checks use info-level messages (not warnings) to avoid affecting quality scores for existing skills while still providing helpful guidance. Includes 4 new unit tests for completeness checks. Contributed by the AI Writing Guide project.
This commit is contained in:
@@ -126,6 +126,9 @@ class SkillQualityChecker:
|
|||||||
# Link validation
|
# Link validation
|
||||||
self._check_links()
|
self._check_links()
|
||||||
|
|
||||||
|
# Completeness checks
|
||||||
|
self._check_skill_completeness()
|
||||||
|
|
||||||
return self.report
|
return self.report
|
||||||
|
|
||||||
def _check_skill_structure(self):
|
def _check_skill_structure(self):
|
||||||
@@ -363,6 +366,94 @@ class SkillQualityChecker:
|
|||||||
'SKILL.md'
|
'SKILL.md'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _check_skill_completeness(self):
|
||||||
|
"""Check skill completeness based on best practices.
|
||||||
|
|
||||||
|
Validates that skills include verification/prerequisites sections,
|
||||||
|
error handling guidance, and clear workflow steps.
|
||||||
|
"""
|
||||||
|
if not self.skill_md_path.exists():
|
||||||
|
return
|
||||||
|
|
||||||
|
content = self.skill_md_path.read_text(encoding='utf-8')
|
||||||
|
|
||||||
|
# Check for grounding/verification section (prerequisites)
|
||||||
|
grounding_patterns = [
|
||||||
|
r'before\s+(executing|running|proceeding|you\s+start)',
|
||||||
|
r'verify\s+that',
|
||||||
|
r'prerequisites?',
|
||||||
|
r'requirements?:',
|
||||||
|
r'make\s+sure\s+you\s+have',
|
||||||
|
]
|
||||||
|
has_grounding = any(
|
||||||
|
re.search(pattern, content, re.IGNORECASE)
|
||||||
|
for pattern in grounding_patterns
|
||||||
|
)
|
||||||
|
if has_grounding:
|
||||||
|
self.report.add_info(
|
||||||
|
'completeness',
|
||||||
|
'✓ Found verification/prerequisites section',
|
||||||
|
'SKILL.md'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.report.add_info(
|
||||||
|
'completeness',
|
||||||
|
'Consider adding prerequisites section - helps Claude verify conditions first',
|
||||||
|
'SKILL.md'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check for error handling/troubleshooting guidance
|
||||||
|
error_patterns = [
|
||||||
|
r'if\s+.*\s+(fails?|errors?)',
|
||||||
|
r'troubleshoot',
|
||||||
|
r'common\s+(issues?|problems?)',
|
||||||
|
r'error\s+handling',
|
||||||
|
r'when\s+things\s+go\s+wrong',
|
||||||
|
]
|
||||||
|
has_error_handling = any(
|
||||||
|
re.search(pattern, content, re.IGNORECASE)
|
||||||
|
for pattern in error_patterns
|
||||||
|
)
|
||||||
|
if has_error_handling:
|
||||||
|
self.report.add_info(
|
||||||
|
'completeness',
|
||||||
|
'✓ Found error handling/troubleshooting guidance',
|
||||||
|
'SKILL.md'
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.report.add_info(
|
||||||
|
'completeness',
|
||||||
|
'Consider adding troubleshooting section for common issues',
|
||||||
|
'SKILL.md'
|
||||||
|
)
|
||||||
|
|
||||||
|
# Check for workflow steps (numbered or sequential indicators)
|
||||||
|
step_patterns = [
|
||||||
|
r'step\s+\d',
|
||||||
|
r'##\s+\d\.',
|
||||||
|
r'first,?\s+',
|
||||||
|
r'then,?\s+',
|
||||||
|
r'finally,?\s+',
|
||||||
|
r'next,?\s+',
|
||||||
|
]
|
||||||
|
steps_found = sum(
|
||||||
|
1 for pattern in step_patterns
|
||||||
|
if re.search(pattern, content, re.IGNORECASE)
|
||||||
|
)
|
||||||
|
if steps_found >= 3:
|
||||||
|
self.report.add_info(
|
||||||
|
'completeness',
|
||||||
|
f'✓ Found clear workflow indicators ({steps_found} step markers)',
|
||||||
|
'SKILL.md'
|
||||||
|
)
|
||||||
|
elif steps_found > 0:
|
||||||
|
self.report.add_info(
|
||||||
|
'completeness',
|
||||||
|
f'Some workflow guidance found ({steps_found} markers) - '
|
||||||
|
'consider adding numbered steps for clarity',
|
||||||
|
'SKILL.md'
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def print_report(report: QualityReport, verbose: bool = False):
|
def print_report(report: QualityReport, verbose: bool = False):
|
||||||
"""Print quality report to console.
|
"""Print quality report to console.
|
||||||
|
|||||||
@@ -258,6 +258,136 @@ See [this file](nonexistent.md) for more info.
|
|||||||
self.assertFalse(report2.is_excellent)
|
self.assertFalse(report2.is_excellent)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCompletenessChecks(unittest.TestCase):
|
||||||
|
"""Test completeness check functionality"""
|
||||||
|
|
||||||
|
def create_test_skill(self, tmpdir, skill_md_content):
|
||||||
|
"""Helper to create a test skill directory"""
|
||||||
|
skill_dir = Path(tmpdir) / "test-skill"
|
||||||
|
skill_dir.mkdir()
|
||||||
|
|
||||||
|
# Create SKILL.md
|
||||||
|
skill_md = skill_dir / "SKILL.md"
|
||||||
|
skill_md.write_text(skill_md_content, encoding='utf-8')
|
||||||
|
|
||||||
|
# Create references directory
|
||||||
|
refs_dir = skill_dir / "references"
|
||||||
|
refs_dir.mkdir()
|
||||||
|
(refs_dir / "index.md").write_text("# Index\n", encoding='utf-8')
|
||||||
|
|
||||||
|
return skill_dir
|
||||||
|
|
||||||
|
def test_checker_detects_prerequisites_section(self):
|
||||||
|
"""Test that checker detects prerequisites section"""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
skill_md = """---
|
||||||
|
name: test
|
||||||
|
---
|
||||||
|
|
||||||
|
# Test Skill
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
Make sure you have:
|
||||||
|
- Python 3.10+
|
||||||
|
- pip installed
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
Run the command.
|
||||||
|
"""
|
||||||
|
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||||
|
|
||||||
|
checker = SkillQualityChecker(skill_dir)
|
||||||
|
report = checker.check_all()
|
||||||
|
|
||||||
|
# Should have info about found prerequisites
|
||||||
|
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||||
|
self.assertTrue(any('prerequisites' in i.message.lower() or 'verification' in i.message.lower()
|
||||||
|
for i in completeness_infos))
|
||||||
|
|
||||||
|
def test_checker_detects_troubleshooting_section(self):
|
||||||
|
"""Test that checker detects troubleshooting section"""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
skill_md = """---
|
||||||
|
name: test
|
||||||
|
---
|
||||||
|
|
||||||
|
# Test Skill
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
Run the command.
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Common Issues
|
||||||
|
|
||||||
|
If the command fails, check your permissions.
|
||||||
|
"""
|
||||||
|
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||||
|
|
||||||
|
checker = SkillQualityChecker(skill_dir)
|
||||||
|
report = checker.check_all()
|
||||||
|
|
||||||
|
# Should have info about found troubleshooting
|
||||||
|
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||||
|
self.assertTrue(any('troubleshoot' in i.message.lower() or 'error handling' in i.message.lower()
|
||||||
|
for i in completeness_infos))
|
||||||
|
|
||||||
|
def test_checker_detects_workflow_steps(self):
|
||||||
|
"""Test that checker detects workflow steps"""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
skill_md = """---
|
||||||
|
name: test
|
||||||
|
---
|
||||||
|
|
||||||
|
# Test Skill
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
First, install the dependencies.
|
||||||
|
|
||||||
|
Then, configure your environment.
|
||||||
|
|
||||||
|
Next, run the setup script.
|
||||||
|
|
||||||
|
Finally, verify the installation.
|
||||||
|
"""
|
||||||
|
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||||
|
|
||||||
|
checker = SkillQualityChecker(skill_dir)
|
||||||
|
report = checker.check_all()
|
||||||
|
|
||||||
|
# Should have info about found workflow steps
|
||||||
|
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||||
|
self.assertTrue(any('workflow' in i.message.lower() or 'step' in i.message.lower()
|
||||||
|
for i in completeness_infos))
|
||||||
|
|
||||||
|
def test_checker_suggests_adding_prerequisites(self):
|
||||||
|
"""Test that checker suggests adding prerequisites when missing"""
|
||||||
|
with tempfile.TemporaryDirectory() as tmpdir:
|
||||||
|
skill_md = """---
|
||||||
|
name: test
|
||||||
|
---
|
||||||
|
|
||||||
|
# Test Skill
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
Just run the command.
|
||||||
|
"""
|
||||||
|
skill_dir = self.create_test_skill(tmpdir, skill_md)
|
||||||
|
|
||||||
|
checker = SkillQualityChecker(skill_dir)
|
||||||
|
report = checker.check_all()
|
||||||
|
|
||||||
|
# Should have info suggesting prerequisites
|
||||||
|
completeness_infos = [i for i in report.info if i.category == 'completeness']
|
||||||
|
self.assertTrue(any('consider' in i.message.lower() and 'prerequisites' in i.message.lower()
|
||||||
|
for i in completeness_infos))
|
||||||
|
|
||||||
|
|
||||||
class TestQualityCheckerCLI(unittest.TestCase):
|
class TestQualityCheckerCLI(unittest.TestCase):
|
||||||
"""Test quality checker CLI"""
|
"""Test quality checker CLI"""
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user