#!/usr/bin/env python3 """ Quality Checker for Claude Skills Validates skill quality, checks links, and generates quality reports. Usage: python3 quality_checker.py output/react/ python3 quality_checker.py output/godot/ --verbose """ import re import sys from dataclasses import dataclass, field from pathlib import Path @dataclass class QualityIssue: """Represents a quality issue found during validation.""" level: str # 'error', 'warning', 'info' category: str # 'enhancement', 'content', 'links', 'structure' message: str file: str | None = None line: int | None = None @dataclass class QualityReport: """Complete quality report for a skill.""" skill_name: str skill_path: Path errors: list[QualityIssue] = field(default_factory=list) warnings: list[QualityIssue] = field(default_factory=list) info: list[QualityIssue] = field(default_factory=list) def add_error(self, category: str, message: str, file: str = None, line: int = None): """Add an error to the report.""" self.errors.append(QualityIssue("error", category, message, file, line)) def add_warning(self, category: str, message: str, file: str = None, line: int = None): """Add a warning to the report.""" self.warnings.append(QualityIssue("warning", category, message, file, line)) def add_info(self, category: str, message: str, file: str = None, line: int = None): """Add info to the report.""" self.info.append(QualityIssue("info", category, message, file, line)) @property def has_errors(self) -> bool: """Check if there are any errors.""" return len(self.errors) > 0 @property def has_warnings(self) -> bool: """Check if there are any warnings.""" return len(self.warnings) > 0 @property def is_excellent(self) -> bool: """Check if quality is excellent (no errors, no warnings).""" return not self.has_errors and not self.has_warnings @property def quality_score(self) -> float: """Calculate quality score (0-100).""" # Start with perfect score score = 100.0 # Deduct points for issues score -= len(self.errors) * 15 # -15 per error score -= len(self.warnings) * 5 # -5 per warning # Never go below 0 return max(0.0, score) @property def quality_grade(self) -> str: """Get quality grade (A-F).""" score = self.quality_score if score >= 90: return "A" elif score >= 80: return "B" elif score >= 70: return "C" elif score >= 60: return "D" else: return "F" class SkillQualityChecker: """Validates skill quality and generates reports.""" def __init__(self, skill_dir: Path): """Initialize quality checker. Args: skill_dir: Path to skill directory """ self.skill_dir = Path(skill_dir) self.skill_md_path = self.skill_dir / "SKILL.md" self.references_dir = self.skill_dir / "references" self.report = QualityReport(skill_name=self.skill_dir.name, skill_path=self.skill_dir) def check_all(self) -> QualityReport: """Run all quality checks and return report. Returns: QualityReport: Complete quality report """ # Basic structure checks self._check_skill_structure() # Enhancement verification self._check_enhancement_quality() # Content quality checks self._check_content_quality() # Link validation self._check_links() # Completeness checks self._check_skill_completeness() return self.report def _check_skill_structure(self): """Check basic skill structure.""" # Check SKILL.md exists if not self.skill_md_path.exists(): self.report.add_error("structure", "SKILL.md file not found", str(self.skill_md_path)) return # Check references directory exists if not self.references_dir.exists(): self.report.add_warning( "structure", "references/ directory not found - skill may be incomplete", str(self.references_dir), ) elif not list(self.references_dir.rglob("*.md")): self.report.add_warning( "structure", "references/ directory is empty - no reference documentation found", str(self.references_dir), ) def _check_enhancement_quality(self): """Check if SKILL.md was properly enhanced.""" if not self.skill_md_path.exists(): return content = self.skill_md_path.read_text(encoding="utf-8") # Check for template indicators (signs it wasn't enhanced) template_indicators = [ "TODO:", "[Add description]", "[Framework specific tips]", "coming soon", ] for indicator in template_indicators: if indicator.lower() in content.lower(): self.report.add_warning( "enhancement", f'Found template placeholder: "{indicator}" - SKILL.md may not be enhanced', "SKILL.md", ) # Check for good signs of enhancement enhancement_indicators = { "code_examples": re.compile(r"```[\w-]+\n", re.MULTILINE), "real_examples": re.compile(r"Example:", re.IGNORECASE), "sections": re.compile(r"^## .+", re.MULTILINE), } code_blocks = len(enhancement_indicators["code_examples"].findall(content)) _real_examples = len(enhancement_indicators["real_examples"].findall(content)) sections = len(enhancement_indicators["sections"].findall(content)) # Quality thresholds if code_blocks == 0: self.report.add_warning( "enhancement", "No code examples found in SKILL.md - consider enhancing", "SKILL.md" ) elif code_blocks < 3: self.report.add_info( "enhancement", f"Only {code_blocks} code examples found - more examples would improve quality", "SKILL.md", ) else: self.report.add_info("enhancement", f"✓ Found {code_blocks} code examples", "SKILL.md") if sections < 4: self.report.add_warning( "enhancement", f"Only {sections} sections found - SKILL.md may be too basic", "SKILL.md", ) else: self.report.add_info("enhancement", f"✓ Found {sections} sections", "SKILL.md") def _check_content_quality(self): """Check content quality.""" if not self.skill_md_path.exists(): return content = self.skill_md_path.read_text(encoding="utf-8") # Check YAML frontmatter if not content.startswith("---"): self.report.add_error( "content", "Missing YAML frontmatter - SKILL.md must start with ---", "SKILL.md", 1 ) else: # Extract frontmatter try: frontmatter_match = re.match(r"^---\n(.*?)\n---", content, re.DOTALL) if frontmatter_match: frontmatter = frontmatter_match.group(1) # Check for required fields if "name:" not in frontmatter: self.report.add_error( "content", 'Missing "name:" field in YAML frontmatter', "SKILL.md", 2 ) # Check for description if "description:" in frontmatter: self.report.add_info( "content", "✓ YAML frontmatter includes description", "SKILL.md" ) else: self.report.add_error( "content", "Invalid YAML frontmatter format", "SKILL.md", 1 ) except Exception as e: self.report.add_error( "content", f"Error parsing YAML frontmatter: {e}", "SKILL.md", 1 ) # Check code block language tags code_blocks_without_lang = re.findall(r"```\n[^`]", content) if code_blocks_without_lang: self.report.add_warning( "content", f"Found {len(code_blocks_without_lang)} code blocks without language tags", "SKILL.md", ) # Check for "When to Use" section if "when to use" not in content.lower(): self.report.add_warning( "content", 'Missing "When to Use This Skill" section', "SKILL.md" ) else: self.report.add_info("content", '✓ Found "When to Use" section', "SKILL.md") # Check reference files if self.references_dir.exists(): ref_files = list(self.references_dir.rglob("*.md")) if ref_files: self.report.add_info( "content", f"✓ Found {len(ref_files)} reference files", "references/" ) # Check if references are mentioned in SKILL.md mentioned_refs = 0 for ref_file in ref_files: if ref_file.name in content: mentioned_refs += 1 if mentioned_refs == 0: self.report.add_warning( "content", "Reference files exist but none are mentioned in SKILL.md", "SKILL.md", ) def _check_links(self): """Check internal markdown links.""" if not self.skill_md_path.exists(): return content = self.skill_md_path.read_text(encoding="utf-8") # Find all markdown links [text](path) link_pattern = re.compile(r"\[([^\]]+)\]\(([^)]+)\)") links = link_pattern.findall(content) broken_links = [] for text, link in links: # Skip external links (http/https) if link.startswith("http://") or link.startswith("https://"): continue # Skip anchor links if link.startswith("#"): continue # Check if file exists (relative to SKILL.md) link_path = self.skill_dir / link if not link_path.exists(): broken_links.append((text, link)) if broken_links: for text, link in broken_links: self.report.add_warning("links", f"Broken link: [{text}]({link})", "SKILL.md") else: if links: internal_links = [link for t, link in links if not link.startswith("http")] if internal_links: self.report.add_info( "links", f"✓ All {len(internal_links)} internal links are valid", "SKILL.md" ) def _check_skill_completeness(self): """Check skill completeness based on best practices. Validates that skills include verification/prerequisites sections, error handling guidance, and clear workflow steps. """ if not self.skill_md_path.exists(): return content = self.skill_md_path.read_text(encoding="utf-8") # Check for grounding/verification section (prerequisites) grounding_patterns = [ r"before\s+(executing|running|proceeding|you\s+start)", r"verify\s+that", r"prerequisites?", r"requirements?:", r"make\s+sure\s+you\s+have", ] has_grounding = any( re.search(pattern, content, re.IGNORECASE) for pattern in grounding_patterns ) if has_grounding: self.report.add_info( "completeness", "✓ Found verification/prerequisites section", "SKILL.md" ) else: self.report.add_info( "completeness", "Consider adding prerequisites section - helps Claude verify conditions first", "SKILL.md", ) # Check for error handling/troubleshooting guidance error_patterns = [ r"if\s+.*\s+(fails?|errors?)", r"troubleshoot", r"common\s+(issues?|problems?)", r"error\s+handling", r"when\s+things\s+go\s+wrong", ] has_error_handling = any( re.search(pattern, content, re.IGNORECASE) for pattern in error_patterns ) if has_error_handling: self.report.add_info( "completeness", "✓ Found error handling/troubleshooting guidance", "SKILL.md" ) else: self.report.add_info( "completeness", "Consider adding troubleshooting section for common issues", "SKILL.md", ) # Check for workflow steps (numbered or sequential indicators) step_patterns = [ r"step\s+\d", r"##\s+\d\.", r"first,?\s+", r"then,?\s+", r"finally,?\s+", r"next,?\s+", ] steps_found = sum( 1 for pattern in step_patterns if re.search(pattern, content, re.IGNORECASE) ) if steps_found >= 3: self.report.add_info( "completeness", f"✓ Found clear workflow indicators ({steps_found} step markers)", "SKILL.md", ) elif steps_found > 0: self.report.add_info( "completeness", f"Some workflow guidance found ({steps_found} markers) - consider adding numbered steps for clarity", "SKILL.md", ) def print_report(report: QualityReport, verbose: bool = False): """Print quality report to console. Args: report: Quality report to print verbose: Show all info messages """ print("\n" + "=" * 60) print(f"QUALITY REPORT: {report.skill_name}") print("=" * 60) print() # Quality score print(f"Quality Score: {report.quality_score:.1f}/100 (Grade: {report.quality_grade})") print() # Errors if report.errors: print(f"❌ ERRORS ({len(report.errors)}):") for issue in report.errors: location = ( f" ({issue.file}:{issue.line})" if issue.file and issue.line else f" ({issue.file})" if issue.file else "" ) print(f" [{issue.category}] {issue.message}{location}") print() # Warnings if report.warnings: print(f"⚠️ WARNINGS ({len(report.warnings)}):") for issue in report.warnings: location = ( f" ({issue.file}:{issue.line})" if issue.file and issue.line else f" ({issue.file})" if issue.file else "" ) print(f" [{issue.category}] {issue.message}{location}") print() # Info (only in verbose mode) if verbose and report.info: print(f"ℹ️ INFO ({len(report.info)}):") for issue in report.info: location = f" ({issue.file})" if issue.file else "" print(f" [{issue.category}] {issue.message}{location}") print() # Summary if report.is_excellent: print("✅ EXCELLENT! No issues found.") elif not report.has_errors: print("✓ GOOD! No errors, but some warnings to review.") else: print("❌ NEEDS IMPROVEMENT! Please fix errors before packaging.") print() def main(): """Main entry point.""" import argparse parser = argparse.ArgumentParser( description="Check skill quality and generate report", formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: # Basic quality check python3 quality_checker.py output/react/ # Verbose mode (show all info) python3 quality_checker.py output/godot/ --verbose # Exit with error code if issues found python3 quality_checker.py output/django/ --strict """, ) parser.add_argument("skill_directory", help="Path to skill directory (e.g., output/react/)") parser.add_argument("--verbose", "-v", action="store_true", help="Show all info messages") parser.add_argument( "--strict", action="store_true", help="Exit with error code if any warnings or errors found" ) args = parser.parse_args() # Check if directory exists skill_dir = Path(args.skill_directory) if not skill_dir.exists(): print(f"❌ Directory not found: {skill_dir}") sys.exit(1) # Run quality checks checker = SkillQualityChecker(skill_dir) report = checker.check_all() # Print report print_report(report, verbose=args.verbose) # Exit code if args.strict and (report.has_errors or report.has_warnings) or report.has_errors: sys.exit(1) else: sys.exit(0) if __name__ == "__main__": main()