skill-seekers-reference/tests/test_real_world_fastmcp.py

"""
Real-World Integration Test: FastMCP GitHub Repository

Tests the complete three-stream GitHub architecture pipeline on a real repository:
- https://github.com/jlowin/fastmcp

Validates:
1. GitHub three-stream fetcher works with real repo
2. All 3 streams populated (Code, Docs, Insights)
3. C3.x analysis produces ACTUAL results (not placeholders)
4. Router generation includes GitHub metadata
5. Quality metrics meet targets
6. Generated skills are production-quality

This is a comprehensive E2E test that exercises the entire system.
"""

import os
import json
import tempfile
import pytest
from pathlib import Path
from datetime import datetime

# Mark as integration test (slow)
pytestmark = pytest.mark.integration


class TestRealWorldFastMCP:
    """
    Real-world integration test using FastMCP repository.

    This test requires:
    - Internet connection
    - GitHub API access (optional GITHUB_TOKEN for higher rate limits)
    - 20-60 minutes for C3.x analysis

    Run with: pytest tests/test_real_world_fastmcp.py -v -s
    """

    @pytest.fixture(scope="class")
    def github_token(self):
        """Get GitHub token from environment (optional)."""
        token = os.getenv('GITHUB_TOKEN')
        if token:
            print(f"\n✅ GitHub token found - using authenticated API")
        else:
            print(f"\n⚠️  No GitHub token - using public API (lower rate limits)")
            print(f"   Set GITHUB_TOKEN environment variable for higher rate limits")
        return token

    @pytest.fixture(scope="class")
    def output_dir(self, tmp_path_factory):
        """Create output directory for test results."""
        output = tmp_path_factory.mktemp("fastmcp_real_test")
        print(f"\n📁 Test output directory: {output}")
        return output

    @pytest.fixture(scope="class")
    def fastmcp_analysis(self, github_token, output_dir):
        """
        Perform complete FastMCP analysis.

        This fixture runs the full pipeline and caches the result
        for all tests in this class.
        """
        from skill_seekers.cli.unified_codebase_analyzer import UnifiedCodebaseAnalyzer

        print(f"\n{'='*80}")
        print(f"🚀 REAL-WORLD TEST: FastMCP GitHub Repository")
        print(f"{'='*80}")
        print(f"Repository: https://github.com/jlowin/fastmcp")
        print(f"Test started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
        print(f"Output: {output_dir}")
        print(f"{'='*80}\n")

        # Run unified analyzer with C3.x depth
        analyzer = UnifiedCodebaseAnalyzer(github_token=github_token)

        try:
            # Start with basic analysis (fast) to verify three-stream architecture
            # Can be changed to "c3x" for full analysis (20-60 minutes)
            depth_mode = os.getenv('TEST_DEPTH', 'basic')  # Use 'basic' for quick test, 'c3x' for full

            print(f"📊 Analysis depth: {depth_mode}")
            if depth_mode == 'basic':
                print("   (Set TEST_DEPTH=c3x environment variable for full C3.x analysis)")
            print()

            result = analyzer.analyze(
                source="https://github.com/jlowin/fastmcp",
                depth=depth_mode,
                fetch_github_metadata=True,
                output_dir=output_dir
            )

            print(f"\n✅ Analysis complete!")
            print(f"{'='*80}\n")

            return result

        except Exception as e:
            pytest.fail(f"Analysis failed: {e}")

    def test_01_three_streams_present(self, fastmcp_analysis):
        """Test that all 3 streams are present and populated."""
        print("\n" + "="*80)
        print("TEST 1: Verify All 3 Streams Present")
        print("="*80)

        result = fastmcp_analysis

        # Verify result structure
        assert result is not None, "Analysis result is None"
        assert result.source_type == 'github', f"Expected source_type 'github', got '{result.source_type}'"
        # Depth can be 'basic' or 'c3x' depending on TEST_DEPTH env var
        assert result.analysis_depth in ['basic', 'c3x'], f"Invalid depth '{result.analysis_depth}'"
        print(f"\n📊 Analysis depth: {result.analysis_depth}")

        # STREAM 1: Code Analysis
        print("\n📊 STREAM 1: Code Analysis")
        assert result.code_analysis is not None, "Code analysis missing"
        assert 'files' in result.code_analysis, "Files list missing from code analysis"
        files = result.code_analysis['files']
        print(f"   ✅ Files analyzed: {len(files)}")
        assert len(files) > 0, "No files found in code analysis"

        # STREAM 2: GitHub Docs
        print("\n📄 STREAM 2: GitHub Documentation")
        assert result.github_docs is not None, "GitHub docs missing"

        readme = result.github_docs.get('readme')
        assert readme is not None, "README missing from GitHub docs"
        print(f"   ✅ README length: {len(readme)} chars")
        assert len(readme) > 100, "README too short (< 100 chars)"
        assert 'fastmcp' in readme.lower() or 'mcp' in readme.lower(), "README doesn't mention FastMCP/MCP"

        contributing = result.github_docs.get('contributing')
        if contributing:
            print(f"   ✅ CONTRIBUTING.md length: {len(contributing)} chars")

        docs_files = result.github_docs.get('docs_files', [])
        print(f"   ✅ Additional docs files: {len(docs_files)}")

        # STREAM 3: GitHub Insights
        print("\n🐛 STREAM 3: GitHub Insights")
        assert result.github_insights is not None, "GitHub insights missing"

        metadata = result.github_insights.get('metadata', {})
        assert metadata, "Metadata missing from GitHub insights"

        stars = metadata.get('stars', 0)
        language = metadata.get('language', 'Unknown')
        description = metadata.get('description', '')

        print(f"   ✅ Stars: {stars}")
        print(f"   ✅ Language: {language}")
        print(f"   ✅ Description: {description}")

        assert stars >= 0, "Stars count invalid"
        assert language, "Language not detected"

        common_problems = result.github_insights.get('common_problems', [])
        known_solutions = result.github_insights.get('known_solutions', [])
        top_labels = result.github_insights.get('top_labels', [])

        print(f"   ✅ Common problems: {len(common_problems)}")
        print(f"   ✅ Known solutions: {len(known_solutions)}")
        print(f"   ✅ Top labels: {len(top_labels)}")

        print("\n✅ All 3 streams verified!\n")

    def test_02_c3x_components_populated(self, fastmcp_analysis):
        """Test that C3.x components have ACTUAL data (not placeholders)."""
        print("\n" + "="*80)
        print("TEST 2: Verify C3.x Components Populated (NOT Placeholders)")
        print("="*80)

        result = fastmcp_analysis
        code_analysis = result.code_analysis

        # Skip C3.x checks if running in basic mode
        if result.analysis_depth == 'basic':
            print("\n⚠️  Skipping C3.x component checks (running in basic mode)")
            print("   Set TEST_DEPTH=c3x to run full C3.x analysis")
            pytest.skip("C3.x analysis not run in basic mode")

        # This is the CRITICAL test - verify actual C3.x integration
        print("\n🔍 Checking C3.x Components:")

        # C3.1: Design Patterns
        c3_1 = code_analysis.get('c3_1_patterns', [])
        print(f"\n   C3.1 - Design Patterns:")
        print(f"   ✅ Count: {len(c3_1)}")
        if len(c3_1) > 0:
            print(f"   ✅ Sample: {c3_1[0].get('name', 'N/A')} ({c3_1[0].get('count', 0)} instances)")
            # Verify it's not empty/placeholder
            assert c3_1[0].get('name'), "Pattern has no name"
            assert c3_1[0].get('count', 0) > 0, "Pattern has zero count"
        else:
            print(f"   ⚠️  No patterns detected (may be valid for small repos)")

        # C3.2: Test Examples
        c3_2 = code_analysis.get('c3_2_examples', [])
        c3_2_count = code_analysis.get('c3_2_examples_count', 0)
        print(f"\n   C3.2 - Test Examples:")
        print(f"   ✅ Count: {c3_2_count}")
        if len(c3_2) > 0:
            # C3.2 examples use 'test_name' and 'file_path' fields
            test_name = c3_2[0].get('test_name', c3_2[0].get('name', 'N/A'))
            file_path = c3_2[0].get('file_path', c3_2[0].get('file', 'N/A'))
            print(f"   ✅ Sample: {test_name} from {file_path}")
            # Verify it's not empty/placeholder
            assert test_name and test_name != 'N/A', "Example has no test_name"
            assert file_path and file_path != 'N/A', "Example has no file_path"
        else:
            print(f"   ⚠️  No test examples found")

        # C3.3: How-to Guides
        c3_3 = code_analysis.get('c3_3_guides', [])
        print(f"\n   C3.3 - How-to Guides:")
        print(f"   ✅ Count: {len(c3_3)}")
        if len(c3_3) > 0:
            print(f"   ✅ Sample: {c3_3[0].get('title', 'N/A')}")

        # C3.4: Config Patterns
        c3_4 = code_analysis.get('c3_4_configs', [])
        print(f"\n   C3.4 - Config Patterns:")
        print(f"   ✅ Count: {len(c3_4)}")
        if len(c3_4) > 0:
            print(f"   ✅ Sample: {c3_4[0].get('file', 'N/A')}")

        # C3.7: Architecture
        c3_7 = code_analysis.get('c3_7_architecture', [])
        print(f"\n   C3.7 - Architecture:")
        print(f"   ✅ Count: {len(c3_7)}")
        if len(c3_7) > 0:
            print(f"   ✅ Sample: {c3_7[0].get('pattern', 'N/A')}")

        # CRITICAL: Verify at least SOME C3.x components have data
        # Not all repos will have all components, but should have at least one
        total_c3x_items = len(c3_1) + len(c3_2) + len(c3_3) + len(c3_4) + len(c3_7)

        print(f"\n📊 Total C3.x items: {total_c3x_items}")

        assert total_c3x_items > 0, \
            "❌ CRITICAL: No C3.x data found! This suggests placeholders are being used instead of actual analysis."

        print("\n✅ C3.x components verified - ACTUAL data present (not placeholders)!\n")

    def test_03_router_generation(self, fastmcp_analysis, output_dir):
        """Test router generation with GitHub integration."""
        print("\n" + "="*80)
        print("TEST 3: Router Generation with GitHub Integration")
        print("="*80)

        from skill_seekers.cli.generate_router import RouterGenerator
        from skill_seekers.cli.github_fetcher import ThreeStreamData, CodeStream, DocsStream, InsightsStream

        result = fastmcp_analysis

        # Create mock sub-skill configs
        config1 = output_dir / "fastmcp-oauth.json"
        config1.write_text(json.dumps({
            "name": "fastmcp-oauth",
            "description": "OAuth authentication for FastMCP",
            "categories": {
                "oauth": ["oauth", "auth", "provider", "google", "azure"]
            }
        }))

        config2 = output_dir / "fastmcp-async.json"
        config2.write_text(json.dumps({
            "name": "fastmcp-async",
            "description": "Async patterns for FastMCP",
            "categories": {
                "async": ["async", "await", "asyncio"]
            }
        }))

        # Reconstruct ThreeStreamData from result
        github_streams = ThreeStreamData(
            code_stream=CodeStream(
                directory=Path(output_dir),
                files=[]
            ),
            docs_stream=DocsStream(
                readme=result.github_docs.get('readme'),
                contributing=result.github_docs.get('contributing'),
                docs_files=result.github_docs.get('docs_files', [])
            ),
            insights_stream=InsightsStream(
                metadata=result.github_insights.get('metadata', {}),
                common_problems=result.github_insights.get('common_problems', []),
                known_solutions=result.github_insights.get('known_solutions', []),
                top_labels=result.github_insights.get('top_labels', [])
            )
        )

        # Generate router
        print("\n🧭 Generating router...")
        generator = RouterGenerator(
            config_paths=[str(config1), str(config2)],
            router_name="fastmcp",
            github_streams=github_streams
        )

        skill_md = generator.generate_skill_md()

        # Save router for inspection
        router_file = output_dir / "fastmcp_router_SKILL.md"
        router_file.write_text(skill_md)
        print(f"   ✅ Router saved to: {router_file}")

        # Verify router content
        print("\n📝 Router Content Analysis:")

        # Check basic structure
        assert "fastmcp" in skill_md.lower(), "Router doesn't mention FastMCP"
        print(f"   ✅ Contains 'fastmcp'")

        # Check GitHub metadata
        if "Repository:" in skill_md or "github.com" in skill_md:
            print(f"   ✅ Contains repository URL")

        if "⭐" in skill_md or "Stars:" in skill_md:
            print(f"   ✅ Contains star count")

        if "Python" in skill_md or result.github_insights['metadata'].get('language') in skill_md:
            print(f"   ✅ Contains language")

        # Check README content
        if "Quick Start" in skill_md or "README" in skill_md:
            print(f"   ✅ Contains README quick start")

        # Check common issues
        if "Common Issues" in skill_md or "Issue #" in skill_md:
            issue_count = skill_md.count("Issue #")
            print(f"   ✅ Contains {issue_count} GitHub issues")

        # Check routing
        if "fastmcp-oauth" in skill_md:
            print(f"   ✅ Contains sub-skill routing")

        # Measure router size
        router_lines = len(skill_md.split('\n'))
        print(f"\n📏 Router size: {router_lines} lines")

        # Architecture target: 60-250 lines
        # With GitHub integration: expect higher end of range
        if router_lines < 60:
            print(f"   ⚠️  Router smaller than target (60-250 lines)")
        elif router_lines > 250:
            print(f"   ⚠️  Router larger than target (60-250 lines)")
        else:
            print(f"   ✅ Router size within target range")

        print("\n✅ Router generation verified!\n")

    def test_04_quality_metrics(self, fastmcp_analysis, output_dir):
        """Test that quality metrics meet architecture targets."""
        print("\n" + "="*80)
        print("TEST 4: Quality Metrics Validation")
        print("="*80)

        result = fastmcp_analysis

        # Metric 1: GitHub Overhead
        print("\n📊 Metric 1: GitHub Overhead")
        print("   Target: 20-60 lines")

        # Estimate GitHub overhead from insights
        metadata_lines = 3  # Repository, Stars, Language
        readme_estimate = 10  # Quick start section
        issue_count = len(result.github_insights.get('common_problems', []))
        issue_lines = min(issue_count * 3, 25)  # Max 5 issues shown

        total_overhead = metadata_lines + readme_estimate + issue_lines
        print(f"   Estimated: {total_overhead} lines")

        if 20 <= total_overhead <= 60:
            print(f"   ✅ Within target range")
        else:
            print(f"   ⚠️  Outside target range (may be acceptable)")

        # Metric 2: Data Quality
        print("\n📊 Metric 2: Data Quality")

        code_files = len(result.code_analysis.get('files', []))
        print(f"   Code files: {code_files}")
        assert code_files > 0, "No code files found"
        print(f"   ✅ Code files present")

        readme_len = len(result.github_docs.get('readme', ''))
        print(f"   README length: {readme_len} chars")
        assert readme_len > 100, "README too short"
        print(f"   ✅ README has content")

        stars = result.github_insights['metadata'].get('stars', 0)
        print(f"   Repository stars: {stars}")
        print(f"   ✅ Metadata present")

        # Metric 3: C3.x Coverage
        print("\n📊 Metric 3: C3.x Coverage")

        if result.analysis_depth == 'basic':
            print("   ⚠️  Running in basic mode - C3.x components not analyzed")
            print("   Set TEST_DEPTH=c3x to enable C3.x analysis")
        else:
            c3x_components = {
                'Patterns': len(result.code_analysis.get('c3_1_patterns', [])),
                'Examples': result.code_analysis.get('c3_2_examples_count', 0),
                'Guides': len(result.code_analysis.get('c3_3_guides', [])),
                'Configs': len(result.code_analysis.get('c3_4_configs', [])),
                'Architecture': len(result.code_analysis.get('c3_7_architecture', []))
            }

            for name, count in c3x_components.items():
                status = "✅" if count > 0 else "⚠️ "
                print(f"   {status} {name}: {count}")

            total_c3x = sum(c3x_components.values())
            print(f"   Total C3.x items: {total_c3x}")
            assert total_c3x > 0, "No C3.x data extracted"
            print(f"   ✅ C3.x analysis successful")

        print("\n✅ Quality metrics validated!\n")

    def test_05_skill_quality_assessment(self, output_dir):
        """Manual quality assessment of generated router skill."""
        print("\n" + "="*80)
        print("TEST 5: Skill Quality Assessment")
        print("="*80)

        router_file = output_dir / "fastmcp_router_SKILL.md"

        if not router_file.exists():
            pytest.skip("Router file not generated yet")

        content = router_file.read_text()

        print("\n📝 Quality Checklist:")

        # 1. Has frontmatter
        has_frontmatter = content.startswith('---')
        print(f"   {'✅' if has_frontmatter else '❌'} Has YAML frontmatter")

        # 2. Has main heading
        has_heading = '# ' in content
        print(f"   {'✅' if has_heading else '❌'} Has main heading")

        # 3. Has sections
        section_count = content.count('## ')
        print(f"   {'✅' if section_count >= 3 else '❌'} Has {section_count} sections (need 3+)")

        # 4. Has code blocks
        code_block_count = content.count('```')
        has_code = code_block_count >= 2
        print(f"   {'✅' if has_code else '⚠️ '} Has {code_block_count // 2} code blocks")

        # 5. No placeholders
        no_todos = 'TODO' not in content and '[Add' not in content
        print(f"   {'✅' if no_todos else '❌'} No TODO placeholders")

        # 6. Has GitHub content
        has_github = any(marker in content for marker in ['Repository:', '⭐', 'Issue #', 'github.com'])
        print(f"   {'✅' if has_github else '⚠️ '} Has GitHub integration")

        # 7. Has routing
        has_routing = 'skill' in content.lower() and 'use' in content.lower()
        print(f"   {'✅' if has_routing else '⚠️ '} Has routing guidance")

        # Calculate quality score
        checks = [has_frontmatter, has_heading, section_count >= 3, has_code, no_todos, has_github, has_routing]
        score = sum(checks) / len(checks) * 100

        print(f"\n📊 Quality Score: {score:.0f}%")

        if score >= 85:
            print(f"   ✅ Excellent quality")
        elif score >= 70:
            print(f"   ✅ Good quality")
        elif score >= 50:
            print(f"   ⚠️  Acceptable quality")
        else:
            print(f"   ❌ Poor quality")

        assert score >= 50, f"Quality score too low: {score}%"

        print("\n✅ Skill quality assessed!\n")

    def test_06_final_report(self, fastmcp_analysis, output_dir):
        """Generate final test report."""
        print("\n" + "="*80)
        print("FINAL REPORT: Real-World FastMCP Test")
        print("="*80)

        result = fastmcp_analysis

        print("\n📊 Summary:")
        print(f"   Repository: https://github.com/jlowin/fastmcp")
        print(f"   Analysis: {result.analysis_depth}")
        print(f"   Source type: {result.source_type}")
        print(f"   Test completed: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")

        print("\n✅ Stream Verification:")
        print(f"   ✅ Code Stream: {len(result.code_analysis.get('files', []))} files")
        print(f"   ✅ Docs Stream: {len(result.github_docs.get('readme', ''))} char README")
        print(f"   ✅ Insights Stream: {result.github_insights['metadata'].get('stars', 0)} stars")

        print("\n✅ C3.x Components:")
        print(f"   ✅ Patterns: {len(result.code_analysis.get('c3_1_patterns', []))}")
        print(f"   ✅ Examples: {result.code_analysis.get('c3_2_examples_count', 0)}")
        print(f"   ✅ Guides: {len(result.code_analysis.get('c3_3_guides', []))}")
        print(f"   ✅ Configs: {len(result.code_analysis.get('c3_4_configs', []))}")
        print(f"   ✅ Architecture: {len(result.code_analysis.get('c3_7_architecture', []))}")

        print("\n✅ Quality Metrics:")
        print(f"   ✅ All 3 streams present and populated")
        print(f"   ✅ C3.x actual data (not placeholders)")
        print(f"   ✅ Router generated with GitHub integration")
        print(f"   ✅ Quality metrics within targets")

        print("\n🎉 SUCCESS: System working correctly with real repository!")
        print(f"\n📁 Test artifacts saved to: {output_dir}")
        print(f"   - Router: {output_dir}/fastmcp_router_SKILL.md")

        print(f"\n{'='*80}\n")


if __name__ == '__main__':
    pytest.main([__file__, '-v', '-s', '--tb=short'])