From 3272f9c59dbd4b30fe38aab7b12bf19d4322723a Mon Sep 17 00:00:00 2001 From: yusyus Date: Wed, 12 Nov 2025 23:01:28 +0300 Subject: [PATCH] feat: Add comprehensive quality checker for skills MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 & 3: Quality assurance before packaging New module: quality_checker.py - Enhancement verification (checks for template text, code examples, sections) - Structure validation (SKILL.md, references/ directory) - Content quality checks (YAML frontmatter, language tags, "When to Use" section) - Link validation (internal markdown links) - Quality scoring system (0-100 score + A-F grade) - Detailed reporting with errors, warnings, and info messages - CLI with --verbose and --strict modes Integration in package_skill.py: - Automatic quality checks before packaging - Display quality report with score and grade - Ask user to confirm if warnings/errors found - Add --skip-quality-check flag to bypass checks - Updated help examples Benefits: - Catch quality issues before packaging - Ensure SKILL.md is properly enhanced - Validate all links work - Give users confidence in skill quality - Comprehensive quality reports Addresses user request: "check some sort of quality check at the end like is links working, skill is good etc and give report the user" šŸ¤– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- src/skill_seekers/cli/package_skill.py | 49 ++- src/skill_seekers/cli/quality_checker.py | 480 +++++++++++++++++++++++ 2 files changed, 526 insertions(+), 3 deletions(-) create mode 100644 src/skill_seekers/cli/quality_checker.py diff --git a/src/skill_seekers/cli/package_skill.py b/src/skill_seekers/cli/package_skill.py index d45f844..cf251d0 100644 --- a/src/skill_seekers/cli/package_skill.py +++ b/src/skill_seekers/cli/package_skill.py @@ -23,6 +23,7 @@ try: format_file_size, validate_skill_directory ) + from quality_checker import SkillQualityChecker, print_report except ImportError: # If running from different directory, add cli to path sys.path.insert(0, str(Path(__file__).parent)) @@ -32,15 +33,17 @@ except ImportError: format_file_size, validate_skill_directory ) + from quality_checker import SkillQualityChecker, print_report -def package_skill(skill_dir, open_folder_after=True): +def package_skill(skill_dir, open_folder_after=True, skip_quality_check=False): """ Package a skill directory into a .zip file Args: skill_dir: Path to skill directory open_folder_after: Whether to open the output folder after packaging + skip_quality_check: Skip quality checks before packaging Returns: tuple: (success, zip_path) where success is bool and zip_path is Path or None @@ -53,6 +56,30 @@ def package_skill(skill_dir, open_folder_after=True): print(f"āŒ Error: {error_msg}") return False, None + # Run quality checks (unless skipped) + if not skip_quality_check: + print("\n" + "=" * 60) + print("QUALITY CHECK") + print("=" * 60) + + checker = SkillQualityChecker(skill_path) + report = checker.check_all() + + # Print report + print_report(report, verbose=False) + + # If there are errors or warnings, ask user to confirm + if report.has_errors or report.has_warnings: + print("=" * 60) + response = input("\nContinue with packaging? (y/n): ").strip().lower() + if response != 'y': + print("\nāŒ Packaging cancelled by user") + return False, None + print() + else: + print("=" * 60) + print() + # Create zip filename skill_name = skill_path.name zip_path = skill_path.parent / f"{skill_name}.zip" @@ -95,12 +122,18 @@ def main(): formatter_class=argparse.RawDescriptionHelpFormatter, epilog=""" Examples: - # Package skill and open folder + # Package skill with quality checks (recommended) skill-seekers package output/react/ # Package skill without opening folder skill-seekers package output/react/ --no-open + # Skip quality checks (faster, but not recommended) + skill-seekers package output/react/ --skip-quality-check + + # Package and auto-upload to Claude + skill-seekers package output/react/ --upload + # Get help skill-seekers package --help """ @@ -117,6 +150,12 @@ Examples: help='Do not open the output folder after packaging' ) + parser.add_argument( + '--skip-quality-check', + action='store_true', + help='Skip quality checks before packaging' + ) + parser.add_argument( '--upload', action='store_true', @@ -125,7 +164,11 @@ Examples: args = parser.parse_args() - success, zip_path = package_skill(args.skill_dir, open_folder_after=not args.no_open) + success, zip_path = package_skill( + args.skill_dir, + open_folder_after=not args.no_open, + skip_quality_check=args.skip_quality_check + ) if not success: sys.exit(1) diff --git a/src/skill_seekers/cli/quality_checker.py b/src/skill_seekers/cli/quality_checker.py new file mode 100644 index 0000000..8ff66c5 --- /dev/null +++ b/src/skill_seekers/cli/quality_checker.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +""" +Quality Checker for Claude Skills +Validates skill quality, checks links, and generates quality reports. + +Usage: + python3 quality_checker.py output/react/ + python3 quality_checker.py output/godot/ --verbose +""" + +import os +import re +import sys +from pathlib import Path +from typing import Dict, List, Tuple, Optional +from dataclasses import dataclass, field + + +@dataclass +class QualityIssue: + """Represents a quality issue found during validation.""" + level: str # 'error', 'warning', 'info' + category: str # 'enhancement', 'content', 'links', 'structure' + message: str + file: Optional[str] = None + line: Optional[int] = None + + +@dataclass +class QualityReport: + """Complete quality report for a skill.""" + skill_name: str + skill_path: Path + errors: List[QualityIssue] = field(default_factory=list) + warnings: List[QualityIssue] = field(default_factory=list) + info: List[QualityIssue] = field(default_factory=list) + + def add_error(self, category: str, message: str, file: str = None, line: int = None): + """Add an error to the report.""" + self.errors.append(QualityIssue('error', category, message, file, line)) + + def add_warning(self, category: str, message: str, file: str = None, line: int = None): + """Add a warning to the report.""" + self.warnings.append(QualityIssue('warning', category, message, file, line)) + + def add_info(self, category: str, message: str, file: str = None, line: int = None): + """Add info to the report.""" + self.info.append(QualityIssue('info', category, message, file, line)) + + @property + def has_errors(self) -> bool: + """Check if there are any errors.""" + return len(self.errors) > 0 + + @property + def has_warnings(self) -> bool: + """Check if there are any warnings.""" + return len(self.warnings) > 0 + + @property + def is_excellent(self) -> bool: + """Check if quality is excellent (no errors, no warnings).""" + return not self.has_errors and not self.has_warnings + + @property + def quality_score(self) -> float: + """Calculate quality score (0-100).""" + # Start with perfect score + score = 100.0 + + # Deduct points for issues + score -= len(self.errors) * 15 # -15 per error + score -= len(self.warnings) * 5 # -5 per warning + + # Never go below 0 + return max(0.0, score) + + @property + def quality_grade(self) -> str: + """Get quality grade (A-F).""" + score = self.quality_score + if score >= 90: + return 'A' + elif score >= 80: + return 'B' + elif score >= 70: + return 'C' + elif score >= 60: + return 'D' + else: + return 'F' + + +class SkillQualityChecker: + """Validates skill quality and generates reports.""" + + def __init__(self, skill_dir: Path): + """Initialize quality checker. + + Args: + skill_dir: Path to skill directory + """ + self.skill_dir = Path(skill_dir) + self.skill_md_path = self.skill_dir / "SKILL.md" + self.references_dir = self.skill_dir / "references" + self.report = QualityReport( + skill_name=self.skill_dir.name, + skill_path=self.skill_dir + ) + + def check_all(self) -> QualityReport: + """Run all quality checks and return report. + + Returns: + QualityReport: Complete quality report + """ + # Basic structure checks + self._check_skill_structure() + + # Enhancement verification + self._check_enhancement_quality() + + # Content quality checks + self._check_content_quality() + + # Link validation + self._check_links() + + return self.report + + def _check_skill_structure(self): + """Check basic skill structure.""" + # Check SKILL.md exists + if not self.skill_md_path.exists(): + self.report.add_error( + 'structure', + 'SKILL.md file not found', + str(self.skill_md_path) + ) + return + + # Check references directory exists + if not self.references_dir.exists(): + self.report.add_warning( + 'structure', + 'references/ directory not found - skill may be incomplete', + str(self.references_dir) + ) + elif not list(self.references_dir.glob('*.md')): + self.report.add_warning( + 'structure', + 'references/ directory is empty - no reference documentation found', + str(self.references_dir) + ) + + def _check_enhancement_quality(self): + """Check if SKILL.md was properly enhanced.""" + if not self.skill_md_path.exists(): + return + + content = self.skill_md_path.read_text(encoding='utf-8') + + # Check for template indicators (signs it wasn't enhanced) + template_indicators = [ + "TODO:", + "[Add description]", + "[Framework specific tips]", + "coming soon", + ] + + for indicator in template_indicators: + if indicator.lower() in content.lower(): + self.report.add_warning( + 'enhancement', + f'Found template placeholder: "{indicator}" - SKILL.md may not be enhanced', + 'SKILL.md' + ) + + # Check for good signs of enhancement + enhancement_indicators = { + 'code_examples': re.compile(r'```[\w-]+\n', re.MULTILINE), + 'real_examples': re.compile(r'Example:', re.IGNORECASE), + 'sections': re.compile(r'^## .+', re.MULTILINE), + } + + code_blocks = len(enhancement_indicators['code_examples'].findall(content)) + real_examples = len(enhancement_indicators['real_examples'].findall(content)) + sections = len(enhancement_indicators['sections'].findall(content)) + + # Quality thresholds + if code_blocks == 0: + self.report.add_warning( + 'enhancement', + 'No code examples found in SKILL.md - consider enhancing', + 'SKILL.md' + ) + elif code_blocks < 3: + self.report.add_info( + 'enhancement', + f'Only {code_blocks} code examples found - more examples would improve quality', + 'SKILL.md' + ) + else: + self.report.add_info( + 'enhancement', + f'āœ“ Found {code_blocks} code examples', + 'SKILL.md' + ) + + if sections < 4: + self.report.add_warning( + 'enhancement', + f'Only {sections} sections found - SKILL.md may be too basic', + 'SKILL.md' + ) + else: + self.report.add_info( + 'enhancement', + f'āœ“ Found {sections} sections', + 'SKILL.md' + ) + + def _check_content_quality(self): + """Check content quality.""" + if not self.skill_md_path.exists(): + return + + content = self.skill_md_path.read_text(encoding='utf-8') + + # Check YAML frontmatter + if not content.startswith('---'): + self.report.add_error( + 'content', + 'Missing YAML frontmatter - SKILL.md must start with ---', + 'SKILL.md', + 1 + ) + else: + # Extract frontmatter + try: + frontmatter_match = re.match(r'^---\n(.*?)\n---', content, re.DOTALL) + if frontmatter_match: + frontmatter = frontmatter_match.group(1) + + # Check for required fields + if 'name:' not in frontmatter: + self.report.add_error( + 'content', + 'Missing "name:" field in YAML frontmatter', + 'SKILL.md', + 2 + ) + + # Check for description + if 'description:' in frontmatter: + self.report.add_info( + 'content', + 'āœ“ YAML frontmatter includes description', + 'SKILL.md' + ) + else: + self.report.add_error( + 'content', + 'Invalid YAML frontmatter format', + 'SKILL.md', + 1 + ) + except Exception as e: + self.report.add_error( + 'content', + f'Error parsing YAML frontmatter: {e}', + 'SKILL.md', + 1 + ) + + # Check code block language tags + code_blocks_without_lang = re.findall(r'```\n[^`]', content) + if code_blocks_without_lang: + self.report.add_warning( + 'content', + f'Found {len(code_blocks_without_lang)} code blocks without language tags', + 'SKILL.md' + ) + + # Check for "When to Use" section + if 'when to use' not in content.lower(): + self.report.add_warning( + 'content', + 'Missing "When to Use This Skill" section', + 'SKILL.md' + ) + else: + self.report.add_info( + 'content', + 'āœ“ Found "When to Use" section', + 'SKILL.md' + ) + + # Check reference files + if self.references_dir.exists(): + ref_files = list(self.references_dir.glob('*.md')) + if ref_files: + self.report.add_info( + 'content', + f'āœ“ Found {len(ref_files)} reference files', + 'references/' + ) + + # Check if references are mentioned in SKILL.md + mentioned_refs = 0 + for ref_file in ref_files: + if ref_file.name in content: + mentioned_refs += 1 + + if mentioned_refs == 0: + self.report.add_warning( + 'content', + 'Reference files exist but none are mentioned in SKILL.md', + 'SKILL.md' + ) + + def _check_links(self): + """Check internal markdown links.""" + if not self.skill_md_path.exists(): + return + + content = self.skill_md_path.read_text(encoding='utf-8') + + # Find all markdown links [text](path) + link_pattern = re.compile(r'\[([^\]]+)\]\(([^)]+)\)') + links = link_pattern.findall(content) + + broken_links = [] + + for text, link in links: + # Skip external links (http/https) + if link.startswith('http://') or link.startswith('https://'): + continue + + # Skip anchor links + if link.startswith('#'): + continue + + # Check if file exists (relative to SKILL.md) + link_path = self.skill_dir / link + if not link_path.exists(): + broken_links.append((text, link)) + + if broken_links: + for text, link in broken_links: + self.report.add_warning( + 'links', + f'Broken link: [{text}]({link})', + 'SKILL.md' + ) + else: + if links: + internal_links = [l for t, l in links if not l.startswith('http')] + if internal_links: + self.report.add_info( + 'links', + f'āœ“ All {len(internal_links)} internal links are valid', + 'SKILL.md' + ) + + +def print_report(report: QualityReport, verbose: bool = False): + """Print quality report to console. + + Args: + report: Quality report to print + verbose: Show all info messages + """ + print("\n" + "=" * 60) + print(f"QUALITY REPORT: {report.skill_name}") + print("=" * 60) + print() + + # Quality score + print(f"Quality Score: {report.quality_score:.1f}/100 (Grade: {report.quality_grade})") + print() + + # Errors + if report.errors: + print(f"āŒ ERRORS ({len(report.errors)}):") + for issue in report.errors: + location = f" ({issue.file}:{issue.line})" if issue.file and issue.line else f" ({issue.file})" if issue.file else "" + print(f" [{issue.category}] {issue.message}{location}") + print() + + # Warnings + if report.warnings: + print(f"āš ļø WARNINGS ({len(report.warnings)}):") + for issue in report.warnings: + location = f" ({issue.file}:{issue.line})" if issue.file and issue.line else f" ({issue.file})" if issue.file else "" + print(f" [{issue.category}] {issue.message}{location}") + print() + + # Info (only in verbose mode) + if verbose and report.info: + print(f"ā„¹ļø INFO ({len(report.info)}):") + for issue in report.info: + location = f" ({issue.file})" if issue.file else "" + print(f" [{issue.category}] {issue.message}{location}") + print() + + # Summary + if report.is_excellent: + print("āœ… EXCELLENT! No issues found.") + elif not report.has_errors: + print("āœ“ GOOD! No errors, but some warnings to review.") + else: + print("āŒ NEEDS IMPROVEMENT! Please fix errors before packaging.") + + print() + + +def main(): + """Main entry point.""" + import argparse + + parser = argparse.ArgumentParser( + description="Check skill quality and generate report", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic quality check + python3 quality_checker.py output/react/ + + # Verbose mode (show all info) + python3 quality_checker.py output/godot/ --verbose + + # Exit with error code if issues found + python3 quality_checker.py output/django/ --strict +""" + ) + + parser.add_argument( + 'skill_directory', + help='Path to skill directory (e.g., output/react/)' + ) + + parser.add_argument( + '--verbose', '-v', + action='store_true', + help='Show all info messages' + ) + + parser.add_argument( + '--strict', + action='store_true', + help='Exit with error code if any warnings or errors found' + ) + + args = parser.parse_args() + + # Check if directory exists + skill_dir = Path(args.skill_directory) + if not skill_dir.exists(): + print(f"āŒ Directory not found: {skill_dir}") + sys.exit(1) + + # Run quality checks + checker = SkillQualityChecker(skill_dir) + report = checker.check_all() + + # Print report + print_report(report, verbose=args.verbose) + + # Exit code + if args.strict and (report.has_errors or report.has_warnings): + sys.exit(1) + elif report.has_errors: + sys.exit(1) + else: + sys.exit(0) + + +if __name__ == "__main__": + main()