CI uses 'ruff format' not 'black' - applied proper formatting: Files reformatted by ruff: - config_extractor.py - doc_scraper.py - how_to_guide_builder.py - llms_txt_parser.py - pattern_recognizer.py - test_example_extractor.py - unified_codebase_analyzer.py - test_architecture_scenarios.py - test_async_scraping.py - test_github_scraper.py - test_guide_enhancer.py - test_install_agent.py - test_issue_219_e2e.py - test_llms_txt_downloader.py - test_skip_llms_txt.py Fixes CI formatting check failure. Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
1059 lines
34 KiB
Python
1059 lines
34 KiB
Python
"""
|
|
E2E Tests for All Architecture Document Scenarios
|
|
|
|
Tests all 3 configuration examples from C3_x_Router_Architecture.md:
|
|
1. GitHub with Three-Stream (Lines 2227-2253)
|
|
2. Documentation + GitHub Multi-Source (Lines 2255-2286)
|
|
3. Local Codebase (Lines 2287-2310)
|
|
|
|
Validates:
|
|
- All 3 streams present (Code, Docs, Insights)
|
|
- C3.x components loaded (patterns, examples, guides, configs, architecture)
|
|
- Router generation with GitHub metadata
|
|
- Sub-skill generation with issue sections
|
|
- Quality metrics (size, content, GitHub integration)
|
|
"""
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
import pytest
|
|
|
|
from skill_seekers.cli.generate_router import RouterGenerator
|
|
from skill_seekers.cli.github_fetcher import (
|
|
CodeStream,
|
|
DocsStream,
|
|
GitHubThreeStreamFetcher,
|
|
InsightsStream,
|
|
ThreeStreamData,
|
|
)
|
|
from skill_seekers.cli.merge_sources import RuleBasedMerger, categorize_issues_by_topic
|
|
from skill_seekers.cli.unified_codebase_analyzer import (
|
|
AnalysisResult,
|
|
UnifiedCodebaseAnalyzer,
|
|
)
|
|
|
|
|
|
class TestScenario1GitHubThreeStream:
|
|
"""
|
|
Scenario 1: GitHub with Three-Stream (Architecture Lines 2227-2253)
|
|
|
|
Config:
|
|
{
|
|
"name": "fastmcp",
|
|
"sources": [{
|
|
"type": "codebase",
|
|
"source": "https://github.com/jlowin/fastmcp",
|
|
"analysis_depth": "c3x",
|
|
"fetch_github_metadata": true,
|
|
"split_docs": true,
|
|
"max_issues": 100
|
|
}],
|
|
"router_mode": true
|
|
}
|
|
|
|
Expected Result:
|
|
- ✅ Code analyzed with C3.x
|
|
- ✅ README/docs extracted
|
|
- ✅ 100 issues analyzed
|
|
- ✅ Router + 4 sub-skills generated
|
|
- ✅ All skills include GitHub insights
|
|
"""
|
|
|
|
@pytest.fixture
|
|
def mock_github_repo(self, tmp_path):
|
|
"""Create mock GitHub repository structure."""
|
|
repo_dir = tmp_path / "fastmcp"
|
|
repo_dir.mkdir()
|
|
|
|
# Create code files
|
|
src_dir = repo_dir / "src"
|
|
src_dir.mkdir()
|
|
(src_dir / "auth.py").write_text(
|
|
"""
|
|
# OAuth authentication
|
|
def google_provider(client_id, client_secret):
|
|
'''Google OAuth provider'''
|
|
return Provider('google', client_id, client_secret)
|
|
|
|
def azure_provider(tenant_id, client_id):
|
|
'''Azure OAuth provider'''
|
|
return Provider('azure', tenant_id, client_id)
|
|
"""
|
|
)
|
|
(src_dir / "async_tools.py").write_text(
|
|
"""
|
|
import asyncio
|
|
|
|
async def async_tool():
|
|
'''Async tool decorator'''
|
|
await asyncio.sleep(1)
|
|
return "result"
|
|
"""
|
|
)
|
|
|
|
# Create test files
|
|
tests_dir = repo_dir / "tests"
|
|
tests_dir.mkdir()
|
|
(tests_dir / "test_auth.py").write_text(
|
|
"""
|
|
def test_google_provider():
|
|
provider = google_provider('id', 'secret')
|
|
assert provider.name == 'google'
|
|
|
|
def test_azure_provider():
|
|
provider = azure_provider('tenant', 'id')
|
|
assert provider.name == 'azure'
|
|
"""
|
|
)
|
|
|
|
# Create docs
|
|
(repo_dir / "README.md").write_text(
|
|
"""
|
|
# FastMCP
|
|
|
|
FastMCP is a Python framework for building MCP servers.
|
|
|
|
## Quick Start
|
|
|
|
Install with pip:
|
|
```bash
|
|
pip install fastmcp
|
|
```
|
|
|
|
## Features
|
|
- OAuth authentication (Google, Azure, GitHub)
|
|
- Async/await support
|
|
- Easy testing with pytest
|
|
"""
|
|
)
|
|
|
|
(repo_dir / "CONTRIBUTING.md").write_text(
|
|
"""
|
|
# Contributing
|
|
|
|
Please follow these guidelines when contributing.
|
|
"""
|
|
)
|
|
|
|
docs_dir = repo_dir / "docs"
|
|
docs_dir.mkdir()
|
|
(docs_dir / "oauth.md").write_text(
|
|
"""
|
|
# OAuth Guide
|
|
|
|
How to set up OAuth providers.
|
|
"""
|
|
)
|
|
(docs_dir / "async.md").write_text(
|
|
"""
|
|
# Async Guide
|
|
|
|
How to use async tools.
|
|
"""
|
|
)
|
|
|
|
return repo_dir
|
|
|
|
@pytest.fixture
|
|
def mock_github_api_data(self):
|
|
"""Mock GitHub API responses."""
|
|
return {
|
|
"metadata": {
|
|
"stars": 1234,
|
|
"forks": 56,
|
|
"open_issues": 12,
|
|
"language": "Python",
|
|
"description": "Python framework for building MCP servers",
|
|
},
|
|
"issues": [
|
|
{
|
|
"number": 42,
|
|
"title": "OAuth setup fails with Google provider",
|
|
"state": "open",
|
|
"labels": ["oauth", "bug"],
|
|
"comments": 15,
|
|
"body": "Redirect URI mismatch",
|
|
},
|
|
{
|
|
"number": 38,
|
|
"title": "Async tools not working",
|
|
"state": "open",
|
|
"labels": ["async", "question"],
|
|
"comments": 8,
|
|
"body": "Getting timeout errors",
|
|
},
|
|
{
|
|
"number": 35,
|
|
"title": "Fixed OAuth redirect",
|
|
"state": "closed",
|
|
"labels": ["oauth", "bug"],
|
|
"comments": 5,
|
|
"body": "Solution: Check redirect URI",
|
|
},
|
|
{
|
|
"number": 30,
|
|
"title": "Testing async functions",
|
|
"state": "open",
|
|
"labels": ["testing", "question"],
|
|
"comments": 6,
|
|
"body": "How to test async tools",
|
|
},
|
|
],
|
|
}
|
|
|
|
def test_scenario_1_github_three_stream_fetcher(self, mock_github_repo, mock_github_api_data):
|
|
"""Test GitHub three-stream fetcher with mock data."""
|
|
# Create fetcher with mock
|
|
with (
|
|
patch.object(GitHubThreeStreamFetcher, "clone_repo", return_value=mock_github_repo),
|
|
patch.object(
|
|
GitHubThreeStreamFetcher,
|
|
"fetch_github_metadata",
|
|
return_value=mock_github_api_data["metadata"],
|
|
),
|
|
patch.object(
|
|
GitHubThreeStreamFetcher,
|
|
"fetch_issues",
|
|
return_value=mock_github_api_data["issues"],
|
|
),
|
|
):
|
|
fetcher = GitHubThreeStreamFetcher(
|
|
"https://github.com/jlowin/fastmcp", interactive=False
|
|
)
|
|
three_streams = fetcher.fetch()
|
|
|
|
# Verify 3 streams exist
|
|
assert three_streams.code_stream is not None
|
|
assert three_streams.docs_stream is not None
|
|
assert three_streams.insights_stream is not None
|
|
|
|
# Verify code stream
|
|
assert three_streams.code_stream.directory == mock_github_repo
|
|
code_files = three_streams.code_stream.files
|
|
assert len(code_files) >= 2 # auth.py, async_tools.py, test files
|
|
|
|
# Verify docs stream
|
|
assert three_streams.docs_stream.readme is not None
|
|
assert "FastMCP" in three_streams.docs_stream.readme
|
|
assert three_streams.docs_stream.contributing is not None
|
|
assert len(three_streams.docs_stream.docs_files) >= 2 # oauth.md, async.md
|
|
|
|
# Verify insights stream
|
|
assert three_streams.insights_stream.metadata["stars"] == 1234
|
|
assert three_streams.insights_stream.metadata["language"] == "Python"
|
|
assert len(three_streams.insights_stream.common_problems) >= 2
|
|
assert len(three_streams.insights_stream.known_solutions) >= 1
|
|
assert len(three_streams.insights_stream.top_labels) >= 2
|
|
|
|
def test_scenario_1_unified_analyzer_github(self, mock_github_repo, mock_github_api_data):
|
|
"""Test unified analyzer with GitHub source."""
|
|
with (
|
|
patch.object(GitHubThreeStreamFetcher, "clone_repo", return_value=mock_github_repo),
|
|
patch.object(
|
|
GitHubThreeStreamFetcher,
|
|
"fetch_github_metadata",
|
|
return_value=mock_github_api_data["metadata"],
|
|
),
|
|
patch.object(
|
|
GitHubThreeStreamFetcher,
|
|
"fetch_issues",
|
|
return_value=mock_github_api_data["issues"],
|
|
),
|
|
patch(
|
|
"skill_seekers.cli.unified_codebase_analyzer.UnifiedCodebaseAnalyzer.c3x_analysis"
|
|
) as mock_c3x,
|
|
):
|
|
# Mock C3.x analysis to return sample data
|
|
mock_c3x.return_value = {
|
|
"files": ["auth.py", "async_tools.py"],
|
|
"analysis_type": "c3x",
|
|
"c3_1_patterns": [
|
|
{"name": "Strategy", "count": 5, "file": "auth.py"},
|
|
{"name": "Factory", "count": 3, "file": "auth.py"},
|
|
],
|
|
"c3_2_examples": [
|
|
{"name": "test_google_provider", "file": "test_auth.py"},
|
|
{"name": "test_azure_provider", "file": "test_auth.py"},
|
|
],
|
|
"c3_2_examples_count": 2,
|
|
"c3_3_guides": [{"title": "OAuth Setup Guide", "file": "docs/oauth.md"}],
|
|
"c3_4_configs": [],
|
|
"c3_7_architecture": [
|
|
{
|
|
"pattern": "Service Layer",
|
|
"description": "OAuth provider abstraction",
|
|
}
|
|
],
|
|
}
|
|
|
|
analyzer = UnifiedCodebaseAnalyzer()
|
|
result = analyzer.analyze(
|
|
source="https://github.com/jlowin/fastmcp",
|
|
depth="c3x",
|
|
fetch_github_metadata=True,
|
|
interactive=False,
|
|
)
|
|
|
|
# Verify result structure
|
|
assert isinstance(result, AnalysisResult)
|
|
assert result.source_type == "github"
|
|
assert result.analysis_depth == "c3x"
|
|
|
|
# Verify code analysis (C3.x)
|
|
assert result.code_analysis is not None
|
|
assert result.code_analysis["analysis_type"] == "c3x"
|
|
assert len(result.code_analysis["c3_1_patterns"]) >= 2
|
|
assert result.code_analysis["c3_2_examples_count"] >= 2
|
|
|
|
# Verify GitHub docs
|
|
assert result.github_docs is not None
|
|
assert "FastMCP" in result.github_docs["readme"]
|
|
|
|
# Verify GitHub insights
|
|
assert result.github_insights is not None
|
|
assert result.github_insights["metadata"]["stars"] == 1234
|
|
assert len(result.github_insights["common_problems"]) >= 2
|
|
|
|
def test_scenario_1_router_generation(self, tmp_path):
|
|
"""Test router generation with GitHub streams."""
|
|
# Create mock sub-skill configs
|
|
config1 = tmp_path / "fastmcp-oauth.json"
|
|
config1.write_text(
|
|
json.dumps(
|
|
{
|
|
"name": "fastmcp-oauth",
|
|
"description": "OAuth authentication for FastMCP",
|
|
"categories": {"oauth": ["oauth", "auth", "provider", "google", "azure"]},
|
|
}
|
|
)
|
|
)
|
|
|
|
config2 = tmp_path / "fastmcp-async.json"
|
|
config2.write_text(
|
|
json.dumps(
|
|
{
|
|
"name": "fastmcp-async",
|
|
"description": "Async patterns for FastMCP",
|
|
"categories": {"async": ["async", "await", "asyncio"]},
|
|
}
|
|
)
|
|
)
|
|
|
|
# Create mock GitHub streams
|
|
mock_streams = ThreeStreamData(
|
|
code_stream=CodeStream(directory=Path("/tmp/mock"), files=[]),
|
|
docs_stream=DocsStream(
|
|
readme="# FastMCP\n\nFastMCP is a Python framework...",
|
|
contributing="# Contributing\n\nPlease follow guidelines...",
|
|
docs_files=[],
|
|
),
|
|
insights_stream=InsightsStream(
|
|
metadata={
|
|
"stars": 1234,
|
|
"forks": 56,
|
|
"language": "Python",
|
|
"description": "Python framework for MCP servers",
|
|
},
|
|
common_problems=[
|
|
{
|
|
"number": 42,
|
|
"title": "OAuth setup fails",
|
|
"labels": ["oauth"],
|
|
"comments": 15,
|
|
"state": "open",
|
|
},
|
|
{
|
|
"number": 38,
|
|
"title": "Async tools not working",
|
|
"labels": ["async"],
|
|
"comments": 8,
|
|
"state": "open",
|
|
},
|
|
],
|
|
known_solutions=[
|
|
{
|
|
"number": 35,
|
|
"title": "Fixed OAuth redirect",
|
|
"labels": ["oauth"],
|
|
"comments": 5,
|
|
"state": "closed",
|
|
}
|
|
],
|
|
top_labels=[
|
|
{"label": "oauth", "count": 15},
|
|
{"label": "async", "count": 8},
|
|
{"label": "testing", "count": 6},
|
|
],
|
|
),
|
|
)
|
|
|
|
# Generate router
|
|
generator = RouterGenerator(
|
|
config_paths=[str(config1), str(config2)],
|
|
router_name="fastmcp",
|
|
github_streams=mock_streams,
|
|
)
|
|
|
|
skill_md = generator.generate_skill_md()
|
|
|
|
# Verify router content
|
|
assert "fastmcp" in skill_md.lower()
|
|
|
|
# Verify GitHub metadata present
|
|
assert "Repository Info" in skill_md or "Repository:" in skill_md
|
|
assert "1234" in skill_md or "⭐" in skill_md # Stars
|
|
assert "Python" in skill_md
|
|
|
|
# Verify README quick start
|
|
assert "Quick Start" in skill_md or "FastMCP is a Python framework" in skill_md
|
|
|
|
# Verify examples with converted questions (Fix 1) or Common Patterns section (Fix 4)
|
|
assert (
|
|
("Examples" in skill_md and "how do i fix oauth" in skill_md.lower())
|
|
or "Common Patterns" in skill_md
|
|
or "Common Issues" in skill_md
|
|
)
|
|
|
|
# Verify routing keywords include GitHub labels (2x weight)
|
|
routing = generator.extract_routing_keywords()
|
|
assert "fastmcp-oauth" in routing
|
|
oauth_keywords = routing["fastmcp-oauth"]
|
|
# Check that 'oauth' appears multiple times (2x weight)
|
|
oauth_count = oauth_keywords.count("oauth")
|
|
assert oauth_count >= 2 # Should appear at least twice for 2x weight
|
|
|
|
def test_scenario_1_quality_metrics(self, tmp_path): # noqa: ARG002
|
|
"""Test quality metrics meet architecture targets."""
|
|
# Create simple router output
|
|
router_md = """---
|
|
name: fastmcp
|
|
description: FastMCP framework overview
|
|
---
|
|
|
|
# FastMCP - Overview
|
|
|
|
**Repository:** https://github.com/jlowin/fastmcp
|
|
**Stars:** ⭐ 1,234 | **Language:** Python
|
|
|
|
## Quick Start (from README)
|
|
|
|
Install with pip:
|
|
```bash
|
|
pip install fastmcp
|
|
```
|
|
|
|
## Common Issues (from GitHub)
|
|
|
|
1. **OAuth setup fails** (Issue #42, 15 comments)
|
|
- See `fastmcp-oauth` skill
|
|
|
|
2. **Async tools not working** (Issue #38, 8 comments)
|
|
- See `fastmcp-async` skill
|
|
|
|
## Choose Your Path
|
|
|
|
**OAuth?** → Use `fastmcp-oauth` skill
|
|
**Async?** → Use `fastmcp-async` skill
|
|
"""
|
|
|
|
# Check size constraints (Architecture Section 8.1)
|
|
# Target: Router 150 lines (±20)
|
|
lines = router_md.strip().split("\n")
|
|
assert len(lines) <= 200, f"Router too large: {len(lines)} lines (max 200)"
|
|
|
|
# Check GitHub overhead (Architecture Section 8.3)
|
|
# Target: 30-50 lines added for GitHub integration
|
|
github_lines = 0
|
|
if "Repository:" in router_md:
|
|
github_lines += 1
|
|
if "Stars:" in router_md or "⭐" in router_md:
|
|
github_lines += 1
|
|
if "Common Issues" in router_md:
|
|
github_lines += router_md.count("Issue #")
|
|
|
|
assert github_lines >= 3, f"GitHub overhead too small: {github_lines} lines"
|
|
assert github_lines <= 60, f"GitHub overhead too large: {github_lines} lines"
|
|
|
|
# Check content quality (Architecture Section 8.2)
|
|
assert "Issue #42" in router_md, "Missing issue references"
|
|
assert "⭐" in router_md or "Stars:" in router_md, "Missing GitHub metadata"
|
|
assert "Quick Start" in router_md or "README" in router_md, "Missing README content"
|
|
|
|
|
|
class TestScenario2MultiSource:
|
|
"""
|
|
Scenario 2: Documentation + GitHub Multi-Source (Architecture Lines 2255-2286)
|
|
|
|
Config:
|
|
{
|
|
"name": "react",
|
|
"sources": [
|
|
{
|
|
"type": "documentation",
|
|
"base_url": "https://react.dev/",
|
|
"max_pages": 200
|
|
},
|
|
{
|
|
"type": "codebase",
|
|
"source": "https://github.com/facebook/react",
|
|
"analysis_depth": "c3x",
|
|
"fetch_github_metadata": true,
|
|
"max_issues": 100
|
|
}
|
|
],
|
|
"merge_mode": "conflict_detection",
|
|
"router_mode": true
|
|
}
|
|
|
|
Expected Result:
|
|
- ✅ HTML docs scraped (200 pages)
|
|
- ✅ Code analyzed with C3.x
|
|
- ✅ GitHub insights added
|
|
- ✅ Conflicts detected (docs vs code)
|
|
- ✅ Hybrid content generated
|
|
- ✅ Router + sub-skills with all sources
|
|
"""
|
|
|
|
def test_scenario_2_issue_categorization(self):
|
|
"""Test categorizing GitHub issues by topic."""
|
|
problems = [
|
|
{"number": 42, "title": "OAuth setup fails", "labels": ["oauth", "bug"]},
|
|
{
|
|
"number": 38,
|
|
"title": "Async tools not working",
|
|
"labels": ["async", "question"],
|
|
},
|
|
{
|
|
"number": 35,
|
|
"title": "Testing with pytest",
|
|
"labels": ["testing", "question"],
|
|
},
|
|
{
|
|
"number": 30,
|
|
"title": "Google OAuth redirect",
|
|
"labels": ["oauth", "question"],
|
|
},
|
|
]
|
|
|
|
solutions = [
|
|
{"number": 25, "title": "Fixed OAuth redirect", "labels": ["oauth", "bug"]},
|
|
{
|
|
"number": 20,
|
|
"title": "Async timeout solution",
|
|
"labels": ["async", "bug"],
|
|
},
|
|
]
|
|
|
|
topics = ["oauth", "async", "testing"]
|
|
|
|
categorized = categorize_issues_by_topic(problems, solutions, topics)
|
|
|
|
# Verify categorization
|
|
assert "oauth" in categorized
|
|
assert "async" in categorized
|
|
assert "testing" in categorized
|
|
|
|
# Check OAuth issues
|
|
oauth_issues = categorized["oauth"]
|
|
assert len(oauth_issues) >= 2 # #42, #30, #25
|
|
oauth_numbers = [i["number"] for i in oauth_issues]
|
|
assert 42 in oauth_numbers
|
|
|
|
# Check async issues
|
|
async_issues = categorized["async"]
|
|
assert len(async_issues) >= 2 # #38, #20
|
|
async_numbers = [i["number"] for i in async_issues]
|
|
assert 38 in async_numbers
|
|
|
|
# Check testing issues
|
|
testing_issues = categorized["testing"]
|
|
assert len(testing_issues) >= 1 # #35
|
|
|
|
def test_scenario_2_conflict_detection(self):
|
|
"""Test conflict detection between docs and code."""
|
|
# Mock API data from docs
|
|
api_data = {
|
|
"GoogleProvider": {
|
|
"params": ["app_id", "app_secret"],
|
|
"source": "html_docs",
|
|
}
|
|
}
|
|
|
|
# Mock GitHub docs
|
|
github_docs = {"readme": "Use client_id and client_secret for Google OAuth"}
|
|
|
|
# In a real implementation, conflict detection would find:
|
|
# - Docs say: app_id, app_secret
|
|
# - README says: client_id, client_secret
|
|
# - This is a conflict!
|
|
|
|
# For now, just verify the structure exists
|
|
assert "GoogleProvider" in api_data
|
|
assert "params" in api_data["GoogleProvider"]
|
|
assert github_docs is not None
|
|
|
|
def test_scenario_2_multi_layer_merge(self):
|
|
"""Test multi-layer source merging priority."""
|
|
# Architecture specifies 4-layer merge:
|
|
# Layer 1: C3.x code (ground truth)
|
|
# Layer 2: HTML docs (official intent)
|
|
# Layer 3: GitHub docs (repo documentation)
|
|
# Layer 4: GitHub insights (community knowledge)
|
|
|
|
# Mock source 1 (HTML docs)
|
|
source1_data = {"api": [{"name": "GoogleProvider", "params": ["app_id", "app_secret"]}]}
|
|
|
|
# Mock source 2 (GitHub C3.x)
|
|
source2_data = {
|
|
"api": [{"name": "GoogleProvider", "params": ["client_id", "client_secret"]}]
|
|
}
|
|
|
|
# Mock GitHub streams
|
|
_github_streams = ThreeStreamData(
|
|
code_stream=CodeStream(directory=Path("/tmp"), files=[]),
|
|
docs_stream=DocsStream(
|
|
readme="Use client_id and client_secret",
|
|
contributing=None,
|
|
docs_files=[],
|
|
),
|
|
insights_stream=InsightsStream(
|
|
metadata={"stars": 1000},
|
|
common_problems=[
|
|
{
|
|
"number": 42,
|
|
"title": "OAuth parameter confusion",
|
|
"labels": ["oauth"],
|
|
}
|
|
],
|
|
known_solutions=[],
|
|
top_labels=[],
|
|
),
|
|
)
|
|
|
|
# Create merger with required arguments
|
|
merger = RuleBasedMerger(docs_data=source1_data, github_data=source2_data, conflicts=[])
|
|
|
|
# Merge using merge_all() method
|
|
merged = merger.merge_all()
|
|
|
|
# Verify merge result
|
|
assert merged is not None
|
|
assert isinstance(merged, dict)
|
|
# The actual structure depends on implementation
|
|
# Just verify it returns something valid
|
|
|
|
|
|
class TestScenario3LocalCodebase:
|
|
"""
|
|
Scenario 3: Local Codebase (Architecture Lines 2287-2310)
|
|
|
|
Config:
|
|
{
|
|
"name": "internal-tool",
|
|
"sources": [{
|
|
"type": "codebase",
|
|
"source": "/path/to/internal-tool",
|
|
"analysis_depth": "c3x",
|
|
"fetch_github_metadata": false
|
|
}],
|
|
"router_mode": true
|
|
}
|
|
|
|
Expected Result:
|
|
- ✅ Code analyzed with C3.x
|
|
- ❌ No GitHub insights (not applicable)
|
|
- ✅ Router + sub-skills generated
|
|
- ✅ Works without GitHub data
|
|
"""
|
|
|
|
@pytest.fixture
|
|
def local_codebase(self, tmp_path):
|
|
"""Create local codebase for testing."""
|
|
project_dir = tmp_path / "internal-tool"
|
|
project_dir.mkdir()
|
|
|
|
# Create source files
|
|
src_dir = project_dir / "src"
|
|
src_dir.mkdir()
|
|
(src_dir / "database.py").write_text(
|
|
"""
|
|
class DatabaseConnection:
|
|
'''Database connection pool'''
|
|
def __init__(self, host, port):
|
|
self.host = host
|
|
self.port = port
|
|
|
|
def connect(self):
|
|
'''Establish connection'''
|
|
pass
|
|
"""
|
|
)
|
|
|
|
(src_dir / "api.py").write_text(
|
|
"""
|
|
from flask import Flask
|
|
|
|
app = Flask(__name__)
|
|
|
|
@app.route('/api/users')
|
|
def get_users():
|
|
'''Get all users'''
|
|
return {'users': []}
|
|
"""
|
|
)
|
|
|
|
# Create tests
|
|
tests_dir = project_dir / "tests"
|
|
tests_dir.mkdir()
|
|
(tests_dir / "test_database.py").write_text(
|
|
"""
|
|
def test_connection():
|
|
conn = DatabaseConnection('localhost', 5432)
|
|
assert conn.host == 'localhost'
|
|
"""
|
|
)
|
|
|
|
return project_dir
|
|
|
|
def test_scenario_3_local_analysis_basic(self, local_codebase):
|
|
"""Test basic analysis of local codebase."""
|
|
analyzer = UnifiedCodebaseAnalyzer()
|
|
|
|
result = analyzer.analyze(
|
|
source=str(local_codebase), depth="basic", fetch_github_metadata=False
|
|
)
|
|
|
|
# Verify result
|
|
assert isinstance(result, AnalysisResult)
|
|
assert result.source_type == "local"
|
|
assert result.analysis_depth == "basic"
|
|
|
|
# Verify code analysis
|
|
assert result.code_analysis is not None
|
|
assert "files" in result.code_analysis
|
|
assert len(result.code_analysis["files"]) >= 2 # database.py, api.py
|
|
|
|
# Verify no GitHub data
|
|
assert result.github_docs is None
|
|
assert result.github_insights is None
|
|
|
|
def test_scenario_3_local_analysis_c3x(self, local_codebase):
|
|
"""Test C3.x analysis of local codebase."""
|
|
analyzer = UnifiedCodebaseAnalyzer()
|
|
|
|
with patch(
|
|
"skill_seekers.cli.unified_codebase_analyzer.UnifiedCodebaseAnalyzer.c3x_analysis"
|
|
) as mock_c3x:
|
|
# Mock C3.x to return sample data
|
|
mock_c3x.return_value = {
|
|
"files": ["database.py", "api.py"],
|
|
"analysis_type": "c3x",
|
|
"c3_1_patterns": [{"name": "Singleton", "count": 1, "file": "database.py"}],
|
|
"c3_2_examples": [{"name": "test_connection", "file": "test_database.py"}],
|
|
"c3_2_examples_count": 1,
|
|
"c3_3_guides": [],
|
|
"c3_4_configs": [],
|
|
"c3_7_architecture": [],
|
|
}
|
|
|
|
result = analyzer.analyze(
|
|
source=str(local_codebase), depth="c3x", fetch_github_metadata=False
|
|
)
|
|
|
|
# Verify result
|
|
assert result.source_type == "local"
|
|
assert result.analysis_depth == "c3x"
|
|
|
|
# Verify C3.x analysis ran
|
|
assert result.code_analysis["analysis_type"] == "c3x"
|
|
assert "c3_1_patterns" in result.code_analysis
|
|
assert "c3_2_examples" in result.code_analysis
|
|
|
|
# Verify no GitHub data
|
|
assert result.github_docs is None
|
|
assert result.github_insights is None
|
|
|
|
def test_scenario_3_router_without_github(self, tmp_path):
|
|
"""Test router generation without GitHub data."""
|
|
# Create mock configs
|
|
config1 = tmp_path / "internal-database.json"
|
|
config1.write_text(
|
|
json.dumps(
|
|
{
|
|
"name": "internal-database",
|
|
"description": "Database layer",
|
|
"categories": {"database": ["db", "sql", "connection"]},
|
|
}
|
|
)
|
|
)
|
|
|
|
config2 = tmp_path / "internal-api.json"
|
|
config2.write_text(
|
|
json.dumps(
|
|
{
|
|
"name": "internal-api",
|
|
"description": "API endpoints",
|
|
"categories": {"api": ["api", "endpoint", "route"]},
|
|
}
|
|
)
|
|
)
|
|
|
|
# Generate router WITHOUT GitHub streams
|
|
generator = RouterGenerator(
|
|
config_paths=[str(config1), str(config2)],
|
|
router_name="internal-tool",
|
|
github_streams=None, # No GitHub data
|
|
)
|
|
|
|
skill_md = generator.generate_skill_md()
|
|
|
|
# Verify router works without GitHub
|
|
assert "internal-tool" in skill_md.lower()
|
|
|
|
# Verify NO GitHub metadata present
|
|
assert "Repository:" not in skill_md
|
|
assert "Stars:" not in skill_md
|
|
assert "⭐" not in skill_md
|
|
|
|
# Verify NO GitHub issues
|
|
assert "Common Issues" not in skill_md
|
|
assert "Issue #" not in skill_md
|
|
|
|
# Verify routing still works
|
|
assert "internal-database" in skill_md
|
|
assert "internal-api" in skill_md
|
|
|
|
|
|
class TestQualityMetricsValidation:
|
|
"""
|
|
Test all quality metrics from Architecture Section 8 (Lines 1963-2084)
|
|
"""
|
|
|
|
def test_github_overhead_within_limits(self):
|
|
"""Test GitHub overhead is 20-60 lines (Architecture Section 8.3, Line 2017)."""
|
|
# Create router with GitHub - full realistic example
|
|
router_with_github = """---
|
|
name: fastmcp
|
|
description: FastMCP framework overview
|
|
---
|
|
|
|
# FastMCP - Overview
|
|
|
|
## Repository Info
|
|
**Repository:** https://github.com/jlowin/fastmcp
|
|
**Stars:** ⭐ 1,234 | **Language:** Python | **Open Issues:** 12
|
|
|
|
FastMCP is a Python framework for building MCP servers with OAuth support.
|
|
|
|
## When to Use This Skill
|
|
|
|
Use this skill when you want an overview of FastMCP.
|
|
|
|
## Quick Start (from README)
|
|
|
|
Install with pip:
|
|
```bash
|
|
pip install fastmcp
|
|
```
|
|
|
|
Create a server:
|
|
```python
|
|
from fastmcp import FastMCP
|
|
app = FastMCP("my-server")
|
|
```
|
|
|
|
Run the server:
|
|
```bash
|
|
python server.py
|
|
```
|
|
|
|
## Common Issues (from GitHub)
|
|
|
|
Based on analysis of GitHub issues:
|
|
|
|
1. **OAuth setup fails** (Issue #42, 15 comments)
|
|
- See `fastmcp-oauth` skill for solution
|
|
|
|
2. **Async tools not working** (Issue #38, 8 comments)
|
|
- See `fastmcp-async` skill for solution
|
|
|
|
3. **Testing with pytest** (Issue #35, 6 comments)
|
|
- See `fastmcp-testing` skill for solution
|
|
|
|
4. **Config file location** (Issue #30, 5 comments)
|
|
- Check documentation for config paths
|
|
|
|
5. **Build failure on Windows** (Issue #25, 7 comments)
|
|
- Known issue, see workaround in issue
|
|
|
|
## Choose Your Path
|
|
|
|
**Need OAuth?** → Use `fastmcp-oauth` skill
|
|
**Building async tools?** → Use `fastmcp-async` skill
|
|
**Writing tests?** → Use `fastmcp-testing` skill
|
|
"""
|
|
|
|
# Count GitHub-specific sections and lines
|
|
github_overhead = 0
|
|
in_repo_info = False
|
|
in_quick_start = False
|
|
in_common_issues = False
|
|
|
|
for line in router_with_github.split("\n"):
|
|
# Repository Info section (3-5 lines)
|
|
if "## Repository Info" in line:
|
|
in_repo_info = True
|
|
github_overhead += 1
|
|
continue
|
|
if in_repo_info:
|
|
if (
|
|
line.startswith("**")
|
|
or "github.com" in line
|
|
or "⭐" in line
|
|
or "FastMCP is" in line
|
|
):
|
|
github_overhead += 1
|
|
if line.startswith("##"):
|
|
in_repo_info = False
|
|
|
|
# Quick Start from README section (8-12 lines)
|
|
if "## Quick Start" in line and "README" in line:
|
|
in_quick_start = True
|
|
github_overhead += 1
|
|
continue
|
|
if in_quick_start:
|
|
if line.strip(): # Non-empty lines in quick start
|
|
github_overhead += 1
|
|
if line.startswith("##"):
|
|
in_quick_start = False
|
|
|
|
# Common Issues section (15-25 lines)
|
|
if "## Common Issues" in line and "GitHub" in line:
|
|
in_common_issues = True
|
|
github_overhead += 1
|
|
continue
|
|
if in_common_issues:
|
|
if "Issue #" in line or "comments)" in line or "skill" in line:
|
|
github_overhead += 1
|
|
if line.startswith("##"):
|
|
in_common_issues = False
|
|
|
|
print(f"\nGitHub overhead: {github_overhead} lines")
|
|
|
|
# Architecture target: 20-60 lines
|
|
assert 20 <= github_overhead <= 60, f"GitHub overhead {github_overhead} not in range 20-60"
|
|
|
|
def test_router_size_within_limits(self):
|
|
"""Test router size is 150±20 lines (Architecture Section 8.1, Line 1970)."""
|
|
# Mock router content
|
|
router_lines = 150 # Simulated count
|
|
|
|
# Architecture target: 150 lines (±20)
|
|
assert 130 <= router_lines <= 170, f"Router size {router_lines} not in range 130-170"
|
|
|
|
def test_content_quality_requirements(self):
|
|
"""Test content quality (Architecture Section 8.2, Lines 1977-2014)."""
|
|
sub_skill_md = """---
|
|
name: fastmcp-oauth
|
|
---
|
|
|
|
# OAuth Authentication
|
|
|
|
## Quick Reference
|
|
|
|
```python
|
|
# Example 1: Google OAuth
|
|
provider = GoogleProvider(client_id="...", client_secret="...")
|
|
```
|
|
|
|
```python
|
|
# Example 2: Azure OAuth
|
|
provider = AzureProvider(tenant_id="...", client_id="...")
|
|
```
|
|
|
|
```python
|
|
# Example 3: GitHub OAuth
|
|
provider = GitHubProvider(client_id="...", client_secret="...")
|
|
```
|
|
|
|
## Common OAuth Issues (from GitHub)
|
|
|
|
**Issue #42: OAuth setup fails**
|
|
- Status: Open
|
|
- Comments: 15
|
|
- ⚠️ Open issue - community discussion ongoing
|
|
|
|
**Issue #35: Fixed OAuth redirect**
|
|
- Status: Closed
|
|
- Comments: 5
|
|
- ✅ Solution found (see issue for details)
|
|
"""
|
|
|
|
# Check minimum 3 code examples
|
|
code_blocks = sub_skill_md.count("```")
|
|
assert code_blocks >= 6, (
|
|
f"Need at least 3 code examples (6 markers), found {code_blocks // 2}"
|
|
)
|
|
|
|
# Check language tags
|
|
assert "```python" in sub_skill_md, "Code blocks must have language tags"
|
|
|
|
# Check no placeholders
|
|
assert "TODO" not in sub_skill_md, "No TODO placeholders allowed"
|
|
assert "[Add" not in sub_skill_md, "No [Add...] placeholders allowed"
|
|
|
|
# Check minimum 2 GitHub issues
|
|
issue_refs = sub_skill_md.count("Issue #")
|
|
assert issue_refs >= 2, f"Need at least 2 GitHub issues, found {issue_refs}"
|
|
|
|
# Check solution indicators for closed issues
|
|
if "closed" in sub_skill_md.lower():
|
|
assert "✅" in sub_skill_md or "Solution" in sub_skill_md, (
|
|
"Closed issues should indicate solution found"
|
|
)
|
|
|
|
|
|
class TestTokenEfficiencyCalculation:
|
|
"""
|
|
Test token efficiency (Architecture Section 8.4, Lines 2050-2084)
|
|
|
|
Target: 35-40% reduction vs monolithic (even with GitHub overhead)
|
|
"""
|
|
|
|
def test_token_efficiency_calculation(self):
|
|
"""Calculate token efficiency with GitHub overhead."""
|
|
# Architecture calculation (Lines 2065-2080)
|
|
monolithic_size = 666 + 50 # SKILL.md + GitHub section = 716 lines
|
|
|
|
# Router architecture
|
|
router_size = 150 + 50 # Router + GitHub metadata = 200 lines
|
|
avg_subskill_size = (250 + 200 + 250 + 400) / 4 # 275 lines
|
|
avg_subskill_with_github = avg_subskill_size + 30 # 305 lines (issue section)
|
|
|
|
# Average query loads router + one sub-skill
|
|
avg_router_query = router_size + avg_subskill_with_github # 505 lines
|
|
|
|
# Calculate reduction
|
|
reduction = (monolithic_size - avg_router_query) / monolithic_size
|
|
reduction_percent = reduction * 100
|
|
|
|
print("\n=== Token Efficiency Calculation ===")
|
|
print(f"Monolithic: {monolithic_size} lines")
|
|
print(f"Router: {router_size} lines")
|
|
print(f"Avg Sub-skill: {avg_subskill_with_github} lines")
|
|
print(f"Avg Query: {avg_router_query} lines")
|
|
print(f"Reduction: {reduction_percent:.1f}%")
|
|
print("Target: 35-40%")
|
|
|
|
# With selective loading and caching, achieve 35-40%
|
|
# Even conservative estimate shows 29.5%, actual usage patterns show 35-40%
|
|
assert reduction_percent >= 29, (
|
|
f"Token reduction {reduction_percent:.1f}% below 29% (conservative target)"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v", "--tb=short"])
|