94462a3657
Fixes 3 critical issues found during FastMCP real-world testing: 1. **C3.4 Config Extraction Parameter Mismatch** - Fixed: ConfigExtractor() called with invalid max_files parameter - Error: "ConfigExtractor.__init__() got an unexpected keyword argument 'max_files'" - Solution: Removed max_files and include_optional_deps parameters - Impact: Configuration section now works in ARCHITECTURE.md 2. **C3.3 How-To Guide Building NoneType Guard** - Fixed: Missing null check for guide_collection - Error: "'NoneType' object has no attribute 'get'" - Solution: Added guard: if guide_collection and guide_collection.total_guides > 0 - Impact: No more crashes when guide building fails 3. **Technology Stack Section Population** - Fixed: Empty Section 3 in ARCHITECTURE.md - Enhancement: Now pulls languages from GitHub data as fallback - Solution: Added dual-source language detection (C3.7 → GitHub) - Impact: Technology stack always shows something useful **Test Results After Fixes:** - ✅ All 3 sections now populate correctly - ✅ Graceful degradation still works - ✅ No errors in ARCHITECTURE.md generation **Files Modified:** - codebase_scraper.py: Fixed C3.4 call, added C3.3 null guard - unified_skill_builder.py: Enhanced Technology Stack section 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
785 lines
28 KiB
Python
785 lines
28 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Codebase Scraper CLI Tool
|
|
|
|
Standalone tool for analyzing local codebases without GitHub API.
|
|
Extracts code signatures, comments, and optionally generates API documentation.
|
|
|
|
Usage:
|
|
codebase-scraper --directory /path/to/repo --output output/codebase/
|
|
codebase-scraper --directory . --depth deep --languages Python,JavaScript
|
|
codebase-scraper --directory /path/to/repo --build-api-reference
|
|
|
|
Features:
|
|
- File tree walking with .gitignore support
|
|
- Multi-language code analysis (9 languages: Python, JavaScript/TypeScript, C/C++, C#, Go, Rust, Java, Ruby, PHP)
|
|
- API reference generation
|
|
- Comment extraction
|
|
- Dependency graph analysis
|
|
- Configurable depth levels
|
|
|
|
Credits:
|
|
- Language parsing patterns inspired by official language specifications
|
|
- NetworkX for dependency graph analysis: https://networkx.org/
|
|
- pathspec for .gitignore support: https://pypi.org/project/pathspec/
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import json
|
|
import argparse
|
|
import logging
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Any
|
|
|
|
# Add parent directory to path for imports
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from skill_seekers.cli.code_analyzer import CodeAnalyzer
|
|
from skill_seekers.cli.api_reference_builder import APIReferenceBuilder
|
|
from skill_seekers.cli.dependency_analyzer import DependencyAnalyzer
|
|
from skill_seekers.cli.config_extractor import ConfigExtractor
|
|
|
|
# Try to import pathspec for .gitignore support
|
|
try:
|
|
import pathspec
|
|
PATHSPEC_AVAILABLE = True
|
|
except ImportError:
|
|
PATHSPEC_AVAILABLE = False
|
|
|
|
# Configure logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s'
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
# Language extension mapping
|
|
LANGUAGE_EXTENSIONS = {
|
|
'.py': 'Python',
|
|
'.js': 'JavaScript',
|
|
'.jsx': 'JavaScript',
|
|
'.ts': 'TypeScript',
|
|
'.tsx': 'TypeScript',
|
|
'.cpp': 'C++',
|
|
'.cc': 'C++',
|
|
'.cxx': 'C++',
|
|
'.h': 'C++',
|
|
'.hpp': 'C++',
|
|
'.hxx': 'C++',
|
|
'.c': 'C',
|
|
'.cs': 'C#',
|
|
'.go': 'Go',
|
|
'.rs': 'Rust',
|
|
'.java': 'Java',
|
|
'.rb': 'Ruby',
|
|
'.php': 'PHP',
|
|
}
|
|
|
|
# Default directories to exclude
|
|
DEFAULT_EXCLUDED_DIRS = {
|
|
'node_modules', 'venv', '__pycache__', '.git', '.svn', '.hg',
|
|
'build', 'dist', 'target', '.pytest_cache', '.tox', '.mypy_cache',
|
|
'htmlcov', 'coverage', '.coverage', '.eggs', '*.egg-info',
|
|
'.idea', '.vscode', '.vs', '__pypackages__'
|
|
}
|
|
|
|
|
|
def detect_language(file_path: Path) -> str:
|
|
"""
|
|
Detect programming language from file extension.
|
|
|
|
Args:
|
|
file_path: Path to source file
|
|
|
|
Returns:
|
|
Language name or 'Unknown'
|
|
"""
|
|
extension = file_path.suffix.lower()
|
|
return LANGUAGE_EXTENSIONS.get(extension, 'Unknown')
|
|
|
|
|
|
def load_gitignore(directory: Path) -> Optional[pathspec.PathSpec]:
|
|
"""
|
|
Load .gitignore file and create pathspec matcher.
|
|
|
|
Args:
|
|
directory: Root directory to search for .gitignore
|
|
|
|
Returns:
|
|
PathSpec object if .gitignore found, None otherwise
|
|
"""
|
|
if not PATHSPEC_AVAILABLE:
|
|
logger.warning("pathspec not installed - .gitignore support disabled")
|
|
logger.warning("Install with: pip install pathspec")
|
|
return None
|
|
|
|
gitignore_path = directory / '.gitignore'
|
|
if not gitignore_path.exists():
|
|
logger.debug(f"No .gitignore found in {directory}")
|
|
return None
|
|
|
|
try:
|
|
with open(gitignore_path, 'r', encoding='utf-8') as f:
|
|
spec = pathspec.PathSpec.from_lines('gitwildmatch', f)
|
|
logger.info(f"Loaded .gitignore from {gitignore_path}")
|
|
return spec
|
|
except Exception as e:
|
|
logger.warning(f"Failed to load .gitignore: {e}")
|
|
return None
|
|
|
|
|
|
def should_exclude_dir(dir_name: str, excluded_dirs: set) -> bool:
|
|
"""
|
|
Check if directory should be excluded from analysis.
|
|
|
|
Args:
|
|
dir_name: Directory name
|
|
excluded_dirs: Set of directory names to exclude
|
|
|
|
Returns:
|
|
True if directory should be excluded
|
|
"""
|
|
return dir_name in excluded_dirs
|
|
|
|
|
|
def walk_directory(
|
|
root: Path,
|
|
patterns: Optional[List[str]] = None,
|
|
gitignore_spec: Optional[pathspec.PathSpec] = None,
|
|
excluded_dirs: Optional[set] = None
|
|
) -> List[Path]:
|
|
"""
|
|
Walk directory tree and collect source files.
|
|
|
|
Args:
|
|
root: Root directory to walk
|
|
patterns: Optional file patterns to include (e.g., ['*.py', '*.js'])
|
|
gitignore_spec: Optional PathSpec object for .gitignore rules
|
|
excluded_dirs: Set of directory names to exclude
|
|
|
|
Returns:
|
|
List of source file paths
|
|
"""
|
|
if excluded_dirs is None:
|
|
excluded_dirs = DEFAULT_EXCLUDED_DIRS
|
|
|
|
files = []
|
|
root = Path(root).resolve()
|
|
|
|
for dirpath, dirnames, filenames in os.walk(root):
|
|
current_dir = Path(dirpath)
|
|
|
|
# Filter out excluded directories (in-place modification)
|
|
dirnames[:] = [d for d in dirnames if not should_exclude_dir(d, excluded_dirs)]
|
|
|
|
for filename in filenames:
|
|
file_path = current_dir / filename
|
|
|
|
# Check .gitignore rules
|
|
if gitignore_spec:
|
|
try:
|
|
rel_path = file_path.relative_to(root)
|
|
if gitignore_spec.match_file(str(rel_path)):
|
|
logger.debug(f"Skipping (gitignore): {rel_path}")
|
|
continue
|
|
except ValueError:
|
|
# File is outside root, skip it
|
|
continue
|
|
|
|
# Check file extension
|
|
if file_path.suffix.lower() not in LANGUAGE_EXTENSIONS:
|
|
continue
|
|
|
|
# Check file patterns if provided
|
|
if patterns:
|
|
if not any(file_path.match(pattern) for pattern in patterns):
|
|
continue
|
|
|
|
files.append(file_path)
|
|
|
|
return sorted(files)
|
|
|
|
|
|
def analyze_codebase(
|
|
directory: Path,
|
|
output_dir: Path,
|
|
depth: str = 'deep',
|
|
languages: Optional[List[str]] = None,
|
|
file_patterns: Optional[List[str]] = None,
|
|
build_api_reference: bool = True,
|
|
extract_comments: bool = True,
|
|
build_dependency_graph: bool = True,
|
|
detect_patterns: bool = True,
|
|
extract_test_examples: bool = True,
|
|
build_how_to_guides: bool = True,
|
|
extract_config_patterns: bool = True,
|
|
enhance_with_ai: bool = True,
|
|
ai_mode: str = "auto"
|
|
) -> Dict[str, Any]:
|
|
"""
|
|
Analyze local codebase and extract code knowledge.
|
|
|
|
Args:
|
|
directory: Directory to analyze
|
|
output_dir: Output directory for results
|
|
depth: Analysis depth (surface, deep, full)
|
|
languages: Optional list of languages to analyze
|
|
file_patterns: Optional file patterns to include
|
|
build_api_reference: Generate API reference markdown
|
|
extract_comments: Extract inline comments
|
|
build_dependency_graph: Generate dependency graph and detect circular dependencies
|
|
detect_patterns: Detect design patterns (Singleton, Factory, Observer, etc.)
|
|
extract_test_examples: Extract usage examples from test files
|
|
build_how_to_guides: Build how-to guides from workflow examples (C3.3)
|
|
extract_config_patterns: Extract configuration patterns from config files (C3.4)
|
|
enhance_with_ai: Enhance patterns and examples with AI analysis (C3.6)
|
|
ai_mode: AI enhancement mode for how-to guides (auto, api, local, none)
|
|
|
|
Returns:
|
|
Analysis results dictionary
|
|
"""
|
|
logger.info(f"Analyzing codebase: {directory}")
|
|
logger.info(f"Depth: {depth}")
|
|
|
|
# Create output directory
|
|
output_dir = Path(output_dir)
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Load .gitignore
|
|
gitignore_spec = load_gitignore(directory)
|
|
|
|
# Walk directory tree
|
|
logger.info("Scanning directory tree...")
|
|
files = walk_directory(
|
|
directory,
|
|
patterns=file_patterns,
|
|
gitignore_spec=gitignore_spec
|
|
)
|
|
|
|
logger.info(f"Found {len(files)} source files")
|
|
|
|
# Filter by language if specified
|
|
if languages:
|
|
language_set = set(languages)
|
|
files = [f for f in files if detect_language(f) in language_set]
|
|
logger.info(f"Filtered to {len(files)} files for languages: {', '.join(languages)}")
|
|
|
|
# Initialize code analyzer
|
|
analyzer = CodeAnalyzer(depth=depth)
|
|
|
|
# Analyze each file
|
|
results = {'files': []}
|
|
analyzed_count = 0
|
|
|
|
for file_path in files:
|
|
try:
|
|
content = file_path.read_text(encoding='utf-8', errors='ignore')
|
|
language = detect_language(file_path)
|
|
|
|
if language == 'Unknown':
|
|
continue
|
|
|
|
# Analyze file
|
|
analysis = analyzer.analyze_file(str(file_path), content, language)
|
|
|
|
# Only include files with actual analysis results
|
|
if analysis and (analysis.get('classes') or analysis.get('functions')):
|
|
results['files'].append({
|
|
'file': str(file_path.relative_to(directory)),
|
|
'language': language,
|
|
**analysis
|
|
})
|
|
analyzed_count += 1
|
|
|
|
if analyzed_count % 10 == 0:
|
|
logger.info(f"Analyzed {analyzed_count}/{len(files)} files...")
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Error analyzing {file_path}: {e}")
|
|
continue
|
|
|
|
logger.info(f"✅ Successfully analyzed {analyzed_count} files")
|
|
|
|
# Save results
|
|
output_json = output_dir / 'code_analysis.json'
|
|
with open(output_json, 'w', encoding='utf-8') as f:
|
|
json.dump(results, f, indent=2)
|
|
|
|
logger.info(f"📁 Saved analysis to: {output_json}")
|
|
|
|
# Build API reference if requested
|
|
if build_api_reference and results['files']:
|
|
logger.info("Building API reference documentation...")
|
|
builder = APIReferenceBuilder(results)
|
|
api_output_dir = output_dir / 'api_reference'
|
|
generated_files = builder.build_reference(api_output_dir)
|
|
logger.info(f"✅ Generated {len(generated_files)} API reference files")
|
|
logger.info(f"📁 API reference: {api_output_dir}")
|
|
|
|
# Build dependency graph if requested (C2.6)
|
|
if build_dependency_graph:
|
|
logger.info("Building dependency graph...")
|
|
dep_analyzer = DependencyAnalyzer()
|
|
|
|
# Analyze dependencies for all files
|
|
for file_path in files:
|
|
try:
|
|
content = file_path.read_text(encoding='utf-8', errors='ignore')
|
|
language = detect_language(file_path)
|
|
|
|
if language != 'Unknown':
|
|
# Use relative path from directory for better graph readability
|
|
rel_path = str(file_path.relative_to(directory))
|
|
dep_analyzer.analyze_file(rel_path, content, language)
|
|
except Exception as e:
|
|
logger.warning(f"Error analyzing dependencies for {file_path}: {e}")
|
|
continue
|
|
|
|
# Build the graph
|
|
graph = dep_analyzer.build_graph()
|
|
|
|
# Detect circular dependencies
|
|
cycles = dep_analyzer.detect_cycles()
|
|
if cycles:
|
|
logger.warning(f"⚠️ Found {len(cycles)} circular dependencies:")
|
|
for i, cycle in enumerate(cycles[:5], 1): # Show first 5
|
|
cycle_str = ' → '.join(cycle) + f" → {cycle[0]}"
|
|
logger.warning(f" {i}. {cycle_str}")
|
|
if len(cycles) > 5:
|
|
logger.warning(f" ... and {len(cycles) - 5} more")
|
|
else:
|
|
logger.info("✅ No circular dependencies found")
|
|
|
|
# Save dependency graph data
|
|
dep_output_dir = output_dir / 'dependencies'
|
|
dep_output_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Export as JSON
|
|
dep_json = dep_output_dir / 'dependency_graph.json'
|
|
with open(dep_json, 'w', encoding='utf-8') as f:
|
|
json.dump(dep_analyzer.export_json(), f, indent=2)
|
|
logger.info(f"📁 Saved dependency graph: {dep_json}")
|
|
|
|
# Export as Mermaid diagram
|
|
mermaid_file = dep_output_dir / 'dependency_graph.mmd'
|
|
mermaid_file.write_text(dep_analyzer.export_mermaid())
|
|
logger.info(f"📁 Saved Mermaid diagram: {mermaid_file}")
|
|
|
|
# Save statistics
|
|
stats = dep_analyzer.get_statistics()
|
|
stats_file = dep_output_dir / 'statistics.json'
|
|
with open(stats_file, 'w', encoding='utf-8') as f:
|
|
json.dump(stats, f, indent=2)
|
|
logger.info(f"📊 Statistics: {stats['total_files']} files, "
|
|
f"{stats['total_dependencies']} dependencies, "
|
|
f"{stats['circular_dependencies']} cycles")
|
|
|
|
# Try to export as DOT (requires pydot)
|
|
try:
|
|
dot_file = dep_output_dir / 'dependency_graph.dot'
|
|
dep_analyzer.export_dot(str(dot_file))
|
|
except:
|
|
pass # pydot not installed, skip DOT export
|
|
|
|
# Detect design patterns if requested (C3.1)
|
|
if detect_patterns:
|
|
logger.info("Detecting design patterns...")
|
|
from skill_seekers.cli.pattern_recognizer import PatternRecognizer
|
|
|
|
pattern_recognizer = PatternRecognizer(depth=depth, enhance_with_ai=enhance_with_ai)
|
|
pattern_results = []
|
|
|
|
for file_path in files:
|
|
try:
|
|
content = file_path.read_text(encoding='utf-8', errors='ignore')
|
|
language = detect_language(file_path)
|
|
|
|
if language != 'Unknown':
|
|
report = pattern_recognizer.analyze_file(
|
|
str(file_path), content, language
|
|
)
|
|
|
|
if report.patterns:
|
|
pattern_results.append(report.to_dict())
|
|
except Exception as e:
|
|
logger.warning(f"Pattern detection failed for {file_path}: {e}")
|
|
continue
|
|
|
|
# Save pattern results
|
|
if pattern_results:
|
|
pattern_output = output_dir / 'patterns'
|
|
pattern_output.mkdir(parents=True, exist_ok=True)
|
|
|
|
pattern_json = pattern_output / 'detected_patterns.json'
|
|
with open(pattern_json, 'w', encoding='utf-8') as f:
|
|
json.dump(pattern_results, f, indent=2)
|
|
|
|
total_patterns = sum(len(r['patterns']) for r in pattern_results)
|
|
logger.info(f"✅ Detected {total_patterns} patterns in {len(pattern_results)} files")
|
|
logger.info(f"📁 Saved to: {pattern_json}")
|
|
else:
|
|
logger.info("No design patterns detected")
|
|
|
|
# Extract test examples if requested (C3.2)
|
|
if extract_test_examples:
|
|
logger.info("Extracting usage examples from test files...")
|
|
from skill_seekers.cli.test_example_extractor import TestExampleExtractor
|
|
|
|
# Create extractor
|
|
test_extractor = TestExampleExtractor(
|
|
min_confidence=0.5,
|
|
max_per_file=10,
|
|
languages=languages,
|
|
enhance_with_ai=enhance_with_ai
|
|
)
|
|
|
|
# Extract examples from directory
|
|
try:
|
|
example_report = test_extractor.extract_from_directory(
|
|
directory,
|
|
recursive=True
|
|
)
|
|
|
|
if example_report.total_examples > 0:
|
|
# Save results
|
|
examples_output = output_dir / 'test_examples'
|
|
examples_output.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Save as JSON
|
|
examples_json = examples_output / 'test_examples.json'
|
|
with open(examples_json, 'w', encoding='utf-8') as f:
|
|
json.dump(example_report.to_dict(), f, indent=2)
|
|
|
|
# Save as Markdown
|
|
examples_md = examples_output / 'test_examples.md'
|
|
examples_md.write_text(example_report.to_markdown(), encoding='utf-8')
|
|
|
|
logger.info(f"✅ Extracted {example_report.total_examples} test examples "
|
|
f"({example_report.high_value_count} high-value)")
|
|
logger.info(f"📁 Saved to: {examples_output}")
|
|
else:
|
|
logger.info("No test examples extracted")
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Test example extraction failed: {e}")
|
|
example_report = None
|
|
|
|
# Build how-to guides from workflow examples (C3.3)
|
|
if build_how_to_guides and extract_test_examples:
|
|
logger.info("Building how-to guides from workflow examples...")
|
|
try:
|
|
from skill_seekers.cli.how_to_guide_builder import HowToGuideBuilder
|
|
|
|
# Create guide builder
|
|
guide_builder = HowToGuideBuilder(enhance_with_ai=enhance_with_ai)
|
|
|
|
# Build guides from workflow examples
|
|
tutorials_dir = output_dir / 'tutorials'
|
|
|
|
# Get workflow examples from the example_report if available
|
|
if 'example_report' in locals() and example_report and example_report.total_examples > 0:
|
|
# Convert example_report to list of dicts for processing
|
|
examples_list = example_report.to_dict().get('examples', [])
|
|
|
|
guide_collection = guide_builder.build_guides_from_examples(
|
|
examples_list,
|
|
grouping_strategy='ai-tutorial-group',
|
|
output_dir=tutorials_dir,
|
|
enhance_with_ai=enhance_with_ai,
|
|
ai_mode=ai_mode
|
|
)
|
|
|
|
if guide_collection and guide_collection.total_guides > 0:
|
|
# Save collection summary
|
|
collection_json = tutorials_dir / 'guide_collection.json'
|
|
with open(collection_json, 'w', encoding='utf-8') as f:
|
|
json.dump(guide_collection.to_dict(), f, indent=2)
|
|
|
|
logger.info(f"✅ Built {guide_collection.total_guides} how-to guides")
|
|
logger.info(f"📁 Saved to: {tutorials_dir}")
|
|
else:
|
|
logger.info("No how-to guides generated (insufficient workflow examples)")
|
|
else:
|
|
logger.info("No workflow examples available for guide generation")
|
|
|
|
except Exception as e:
|
|
logger.warning(f"How-to guide building failed: {e}")
|
|
|
|
# Extract configuration patterns (C3.4)
|
|
if extract_config_patterns:
|
|
logger.info("Extracting configuration patterns...")
|
|
try:
|
|
config_extractor = ConfigExtractor()
|
|
|
|
# Extract config patterns from directory
|
|
extraction_result = config_extractor.extract_from_directory(directory)
|
|
|
|
if extraction_result.config_files:
|
|
# Convert to dict for enhancement
|
|
result_dict = config_extractor.to_dict(extraction_result)
|
|
|
|
# AI Enhancement (if enabled)
|
|
if enhance_with_ai and ai_mode != 'none':
|
|
try:
|
|
from skill_seekers.cli.config_enhancer import ConfigEnhancer
|
|
logger.info(f"🤖 Enhancing config analysis with AI (mode: {ai_mode})...")
|
|
enhancer = ConfigEnhancer(mode=ai_mode)
|
|
result_dict = enhancer.enhance_config_result(result_dict)
|
|
logger.info("✅ AI enhancement complete")
|
|
except Exception as e:
|
|
logger.warning(f"⚠️ Config AI enhancement failed: {e}")
|
|
|
|
# Save results
|
|
config_output = output_dir / 'config_patterns'
|
|
config_output.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Save as JSON
|
|
config_json = config_output / 'config_patterns.json'
|
|
with open(config_json, 'w', encoding='utf-8') as f:
|
|
json.dump(result_dict, f, indent=2)
|
|
|
|
# Save as Markdown (basic - AI enhancements in JSON only for now)
|
|
config_md = config_output / 'config_patterns.md'
|
|
config_md.write_text(extraction_result.to_markdown(), encoding='utf-8')
|
|
|
|
# Count total settings across all files
|
|
total_settings = sum(len(cf.settings) for cf in extraction_result.config_files)
|
|
total_patterns = sum(len(cf.patterns) for cf in extraction_result.config_files)
|
|
|
|
logger.info(f"✅ Extracted {len(extraction_result.config_files)} config files "
|
|
f"with {total_settings} settings and {total_patterns} detected patterns")
|
|
|
|
if 'ai_enhancements' in result_dict:
|
|
insights = result_dict['ai_enhancements'].get('overall_insights', {})
|
|
if insights.get('security_issues_found'):
|
|
logger.info(f"🔐 Security issues found: {insights['security_issues_found']}")
|
|
|
|
logger.info(f"📁 Saved to: {config_output}")
|
|
else:
|
|
logger.info("No configuration files found")
|
|
|
|
except Exception as e:
|
|
logger.warning(f"Config pattern extraction failed: {e}")
|
|
|
|
# Detect architectural patterns (C3.7)
|
|
# Always run this - it provides high-level overview
|
|
logger.info("Analyzing architectural patterns...")
|
|
from skill_seekers.cli.architectural_pattern_detector import ArchitecturalPatternDetector
|
|
|
|
arch_detector = ArchitecturalPatternDetector(enhance_with_ai=enhance_with_ai)
|
|
arch_report = arch_detector.analyze(directory, results['files'])
|
|
|
|
if arch_report.patterns:
|
|
arch_output = output_dir / 'architecture'
|
|
arch_output.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Save as JSON
|
|
arch_json = arch_output / 'architectural_patterns.json'
|
|
with open(arch_json, 'w', encoding='utf-8') as f:
|
|
json.dump(arch_report.to_dict(), f, indent=2)
|
|
|
|
logger.info(f"🏗️ Detected {len(arch_report.patterns)} architectural patterns")
|
|
for pattern in arch_report.patterns:
|
|
logger.info(f" - {pattern.pattern_name} (confidence: {pattern.confidence:.2f})")
|
|
logger.info(f"📁 Saved to: {arch_json}")
|
|
else:
|
|
logger.info("No clear architectural patterns detected")
|
|
|
|
return results
|
|
|
|
|
|
def main():
|
|
"""Command-line interface for codebase analysis."""
|
|
parser = argparse.ArgumentParser(
|
|
description='Analyze local codebases and extract code knowledge',
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Analyze current directory
|
|
codebase-scraper --directory . --output output/codebase/
|
|
|
|
# Deep analysis with API reference and dependency graph
|
|
codebase-scraper --directory /path/to/repo --depth deep --build-api-reference --build-dependency-graph
|
|
|
|
# Analyze only Python and JavaScript
|
|
codebase-scraper --directory . --languages Python,JavaScript
|
|
|
|
# Use file patterns
|
|
codebase-scraper --directory . --file-patterns "*.py,src/**/*.js"
|
|
|
|
# Full analysis with all features (default)
|
|
codebase-scraper --directory . --depth deep
|
|
|
|
# Surface analysis (fast, skip all analysis features)
|
|
codebase-scraper --directory . --depth surface --skip-api-reference --skip-dependency-graph --skip-patterns --skip-test-examples
|
|
|
|
# Skip specific features
|
|
codebase-scraper --directory . --skip-patterns --skip-test-examples
|
|
"""
|
|
)
|
|
|
|
parser.add_argument(
|
|
'--directory',
|
|
required=True,
|
|
help='Directory to analyze'
|
|
)
|
|
parser.add_argument(
|
|
'--output',
|
|
default='output/codebase/',
|
|
help='Output directory (default: output/codebase/)'
|
|
)
|
|
parser.add_argument(
|
|
'--depth',
|
|
choices=['surface', 'deep', 'full'],
|
|
default='deep',
|
|
help='Analysis depth (default: deep)'
|
|
)
|
|
parser.add_argument(
|
|
'--languages',
|
|
help='Comma-separated languages to analyze (e.g., Python,JavaScript,C++)'
|
|
)
|
|
parser.add_argument(
|
|
'--file-patterns',
|
|
help='Comma-separated file patterns (e.g., *.py,src/**/*.js)'
|
|
)
|
|
parser.add_argument(
|
|
'--skip-api-reference',
|
|
action='store_true',
|
|
default=False,
|
|
help='Skip API reference markdown documentation generation (default: enabled)'
|
|
)
|
|
parser.add_argument(
|
|
'--skip-dependency-graph',
|
|
action='store_true',
|
|
default=False,
|
|
help='Skip dependency graph and circular dependency detection (default: enabled)'
|
|
)
|
|
parser.add_argument(
|
|
'--skip-patterns',
|
|
action='store_true',
|
|
default=False,
|
|
help='Skip design pattern detection (Singleton, Factory, Observer, etc.) (default: enabled)'
|
|
)
|
|
parser.add_argument(
|
|
'--skip-test-examples',
|
|
action='store_true',
|
|
default=False,
|
|
help='Skip test example extraction (instantiation, method calls, configs, etc.) (default: enabled)'
|
|
)
|
|
parser.add_argument(
|
|
'--skip-how-to-guides',
|
|
action='store_true',
|
|
default=False,
|
|
help='Skip how-to guide generation from workflow examples (default: enabled)'
|
|
)
|
|
parser.add_argument(
|
|
'--skip-config-patterns',
|
|
action='store_true',
|
|
default=False,
|
|
help='Skip configuration pattern extraction from config files (JSON, YAML, TOML, ENV, etc.) (default: enabled)'
|
|
)
|
|
parser.add_argument(
|
|
'--ai-mode',
|
|
choices=['auto', 'api', 'local', 'none'],
|
|
default='auto',
|
|
help='AI enhancement mode for how-to guides: auto (detect best), api (Claude API), local (Claude Code CLI), none (disable) (default: auto)'
|
|
)
|
|
parser.add_argument(
|
|
'--no-comments',
|
|
action='store_true',
|
|
help='Skip comment extraction'
|
|
)
|
|
parser.add_argument(
|
|
'--verbose',
|
|
action='store_true',
|
|
help='Enable verbose logging'
|
|
)
|
|
|
|
# Check for deprecated flags
|
|
deprecated_flags = {
|
|
'--build-api-reference': '--skip-api-reference',
|
|
'--build-dependency-graph': '--skip-dependency-graph',
|
|
'--detect-patterns': '--skip-patterns',
|
|
'--extract-test-examples': '--skip-test-examples',
|
|
'--build-how-to-guides': '--skip-how-to-guides',
|
|
'--extract-config-patterns': '--skip-config-patterns'
|
|
}
|
|
|
|
for old_flag, new_flag in deprecated_flags.items():
|
|
if old_flag in sys.argv:
|
|
logger.warning(f"⚠️ DEPRECATED: {old_flag} is deprecated. "
|
|
f"All features are now enabled by default. "
|
|
f"Use {new_flag} to disable this feature.")
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Set logging level
|
|
if args.verbose:
|
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
|
|
# Validate directory
|
|
directory = Path(args.directory)
|
|
if not directory.exists():
|
|
logger.error(f"Directory not found: {directory}")
|
|
return 1
|
|
|
|
if not directory.is_dir():
|
|
logger.error(f"Not a directory: {directory}")
|
|
return 1
|
|
|
|
# Parse languages
|
|
languages = None
|
|
if args.languages:
|
|
languages = [lang.strip() for lang in args.languages.split(',')]
|
|
|
|
# Parse file patterns
|
|
file_patterns = None
|
|
if args.file_patterns:
|
|
file_patterns = [p.strip() for p in args.file_patterns.split(',')]
|
|
|
|
# Analyze codebase
|
|
try:
|
|
results = analyze_codebase(
|
|
directory=directory,
|
|
output_dir=Path(args.output),
|
|
depth=args.depth,
|
|
languages=languages,
|
|
file_patterns=file_patterns,
|
|
build_api_reference=not args.skip_api_reference,
|
|
extract_comments=not args.no_comments,
|
|
build_dependency_graph=not args.skip_dependency_graph,
|
|
detect_patterns=not args.skip_patterns,
|
|
extract_test_examples=not args.skip_test_examples,
|
|
build_how_to_guides=not args.skip_how_to_guides,
|
|
extract_config_patterns=not args.skip_config_patterns,
|
|
enhance_with_ai=True, # Auto-disables if no API key present
|
|
ai_mode=args.ai_mode # NEW: AI enhancement mode for how-to guides
|
|
)
|
|
|
|
# Print summary
|
|
print(f"\n{'='*60}")
|
|
print(f"CODEBASE ANALYSIS COMPLETE")
|
|
print(f"{'='*60}")
|
|
print(f"Files analyzed: {len(results['files'])}")
|
|
print(f"Output directory: {args.output}")
|
|
if args.build_api_reference:
|
|
print(f"API reference: {Path(args.output) / 'api_reference'}")
|
|
print(f"{'='*60}\n")
|
|
|
|
return 0
|
|
|
|
except KeyboardInterrupt:
|
|
logger.error("\nAnalysis interrupted by user")
|
|
return 130
|
|
except Exception as e:
|
|
logger.error(f"Analysis failed: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
return 1
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|