599 lines
21 KiB
Python
599 lines
21 KiB
Python
"""
|
|
End-to-End Tests for Three-Stream GitHub Architecture Pipeline (Phase 5)
|
|
|
|
Tests the complete workflow:
|
|
1. Fetch GitHub repo with three streams (code, docs, insights)
|
|
2. Analyze with unified codebase analyzer (basic or c3x)
|
|
3. Merge sources with GitHub streams
|
|
4. Generate router with GitHub integration
|
|
5. Validate output structure and quality
|
|
"""
|
|
|
|
import json
|
|
from unittest.mock import Mock, patch
|
|
|
|
import pytest
|
|
|
|
from skill_seekers.cli.generate_router import RouterGenerator
|
|
from skill_seekers.cli.github_fetcher import (
|
|
CodeStream,
|
|
DocsStream,
|
|
InsightsStream,
|
|
ThreeStreamData,
|
|
)
|
|
from skill_seekers.cli.merge_sources import categorize_issues_by_topic
|
|
from skill_seekers.cli.unified_codebase_analyzer import UnifiedCodebaseAnalyzer
|
|
|
|
|
|
class TestE2EBasicWorkflow:
|
|
"""Test E2E workflow with basic analysis (fast)."""
|
|
|
|
@patch("skill_seekers.cli.unified_codebase_analyzer.GitHubThreeStreamFetcher")
|
|
def test_github_url_to_basic_analysis(self, mock_fetcher_class, tmp_path):
|
|
"""
|
|
Test complete pipeline: GitHub URL → Basic analysis → Merged output
|
|
|
|
This tests the fast path (1-2 minutes) without C3.x analysis.
|
|
"""
|
|
# Step 1: Mock GitHub three-stream fetcher
|
|
mock_fetcher = Mock()
|
|
mock_fetcher_class.return_value = mock_fetcher
|
|
|
|
# Create test code files
|
|
(tmp_path / "main.py").write_text("""
|
|
import os
|
|
import sys
|
|
|
|
def hello():
|
|
print("Hello, World!")
|
|
""")
|
|
(tmp_path / "utils.js").write_text("""
|
|
function greet(name) {
|
|
console.log(`Hello, ${name}!`);
|
|
}
|
|
""")
|
|
|
|
# Create mock three-stream data
|
|
code_stream = CodeStream(
|
|
directory=tmp_path, files=[tmp_path / "main.py", tmp_path / "utils.js"]
|
|
)
|
|
docs_stream = DocsStream(
|
|
readme="""# Test Project
|
|
|
|
A simple test project for demonstrating the three-stream architecture.
|
|
|
|
## Installation
|
|
|
|
```bash
|
|
pip install test-project
|
|
```
|
|
|
|
## Quick Start
|
|
|
|
```python
|
|
from test_project import hello
|
|
hello()
|
|
```
|
|
""",
|
|
contributing="# Contributing\n\nPull requests welcome!",
|
|
docs_files=[
|
|
{"path": "docs/guide.md", "content": "# User Guide\n\nHow to use this project."}
|
|
],
|
|
)
|
|
insights_stream = InsightsStream(
|
|
metadata={
|
|
"stars": 1234,
|
|
"forks": 56,
|
|
"language": "Python",
|
|
"description": "A test project",
|
|
},
|
|
common_problems=[
|
|
{
|
|
"title": "Installation fails on Windows",
|
|
"number": 42,
|
|
"state": "open",
|
|
"comments": 15,
|
|
"labels": ["bug", "windows"],
|
|
},
|
|
{
|
|
"title": "Import error with Python 3.6",
|
|
"number": 38,
|
|
"state": "open",
|
|
"comments": 10,
|
|
"labels": ["bug", "python"],
|
|
},
|
|
],
|
|
known_solutions=[
|
|
{
|
|
"title": "Fixed: Module not found",
|
|
"number": 35,
|
|
"state": "closed",
|
|
"comments": 8,
|
|
"labels": ["bug"],
|
|
}
|
|
],
|
|
top_labels=[
|
|
{"label": "bug", "count": 25},
|
|
{"label": "enhancement", "count": 15},
|
|
{"label": "documentation", "count": 10},
|
|
],
|
|
)
|
|
three_streams = ThreeStreamData(code_stream, docs_stream, insights_stream)
|
|
mock_fetcher.fetch.return_value = three_streams
|
|
|
|
# Step 2: Run unified analyzer with basic depth
|
|
analyzer = UnifiedCodebaseAnalyzer()
|
|
result = analyzer.analyze(
|
|
source="https://github.com/test/project", depth="basic", fetch_github_metadata=True
|
|
)
|
|
|
|
# Step 3: Validate all three streams present
|
|
assert result.source_type == "github"
|
|
assert result.analysis_depth == "basic"
|
|
|
|
# Validate code stream results
|
|
assert result.code_analysis is not None
|
|
assert result.code_analysis["analysis_type"] == "basic"
|
|
assert "files" in result.code_analysis
|
|
assert "structure" in result.code_analysis
|
|
assert "imports" in result.code_analysis
|
|
|
|
# Validate docs stream results
|
|
assert result.github_docs is not None
|
|
assert result.github_docs["readme"].startswith("# Test Project")
|
|
assert "pip install test-project" in result.github_docs["readme"]
|
|
|
|
# Validate insights stream results
|
|
assert result.github_insights is not None
|
|
assert result.github_insights["metadata"]["stars"] == 1234
|
|
assert result.github_insights["metadata"]["language"] == "Python"
|
|
assert len(result.github_insights["common_problems"]) == 2
|
|
assert len(result.github_insights["known_solutions"]) == 1
|
|
assert len(result.github_insights["top_labels"]) == 3
|
|
|
|
def test_issue_categorization_by_topic(self):
|
|
"""Test that issues are correctly categorized by topic keywords."""
|
|
problems = [
|
|
{
|
|
"title": "OAuth fails on redirect",
|
|
"number": 50,
|
|
"state": "open",
|
|
"comments": 20,
|
|
"labels": ["oauth", "bug"],
|
|
},
|
|
{
|
|
"title": "Token refresh issue",
|
|
"number": 45,
|
|
"state": "open",
|
|
"comments": 15,
|
|
"labels": ["oauth", "token"],
|
|
},
|
|
{
|
|
"title": "Async deadlock",
|
|
"number": 40,
|
|
"state": "open",
|
|
"comments": 12,
|
|
"labels": ["async", "bug"],
|
|
},
|
|
{
|
|
"title": "Database connection lost",
|
|
"number": 35,
|
|
"state": "open",
|
|
"comments": 10,
|
|
"labels": ["database"],
|
|
},
|
|
]
|
|
|
|
solutions = [
|
|
{
|
|
"title": "Fixed OAuth flow",
|
|
"number": 30,
|
|
"state": "closed",
|
|
"comments": 8,
|
|
"labels": ["oauth"],
|
|
},
|
|
{
|
|
"title": "Resolved async race",
|
|
"number": 25,
|
|
"state": "closed",
|
|
"comments": 6,
|
|
"labels": ["async"],
|
|
},
|
|
]
|
|
|
|
topics = ["oauth", "auth", "authentication"]
|
|
|
|
# Categorize issues
|
|
categorized = categorize_issues_by_topic(problems, solutions, topics)
|
|
|
|
# Validate categorization
|
|
assert "oauth" in categorized or "auth" in categorized or "authentication" in categorized
|
|
oauth_issues = (
|
|
categorized.get("oauth", [])
|
|
+ categorized.get("auth", [])
|
|
+ categorized.get("authentication", [])
|
|
)
|
|
|
|
# Should have 3 OAuth-related issues (2 problems + 1 solution)
|
|
assert len(oauth_issues) >= 2 # At least the problems
|
|
|
|
# OAuth issues should be in the categorized output
|
|
oauth_titles = [issue["title"] for issue in oauth_issues]
|
|
assert any("OAuth" in title for title in oauth_titles)
|
|
|
|
|
|
class TestE2ERouterGeneration:
|
|
"""Test E2E router generation with GitHub integration."""
|
|
|
|
def test_router_generation_with_github_streams(self, tmp_path):
|
|
"""
|
|
Test complete router generation workflow with GitHub streams.
|
|
|
|
Validates:
|
|
1. Router config created
|
|
2. Router SKILL.md includes GitHub metadata
|
|
3. Router SKILL.md includes README quick start
|
|
4. Router SKILL.md includes common issues
|
|
5. Routing keywords include GitHub labels (2x weight)
|
|
"""
|
|
# Create sub-skill configs
|
|
config1 = {
|
|
"name": "testproject-oauth",
|
|
"description": "OAuth authentication in Test Project",
|
|
"base_url": "https://github.com/test/project",
|
|
"categories": {"oauth": ["oauth", "auth"]},
|
|
}
|
|
config2 = {
|
|
"name": "testproject-async",
|
|
"description": "Async operations in Test Project",
|
|
"base_url": "https://github.com/test/project",
|
|
"categories": {"async": ["async", "await"]},
|
|
}
|
|
|
|
config_path1 = tmp_path / "config1.json"
|
|
config_path2 = tmp_path / "config2.json"
|
|
|
|
with open(config_path1, "w") as f:
|
|
json.dump(config1, f)
|
|
with open(config_path2, "w") as f:
|
|
json.dump(config2, f)
|
|
|
|
# Create GitHub streams
|
|
code_stream = CodeStream(directory=tmp_path, files=[])
|
|
docs_stream = DocsStream(
|
|
readme="""# Test Project
|
|
|
|
Fast and simple test framework.
|
|
|
|
## Installation
|
|
|
|
```bash
|
|
pip install test-project
|
|
```
|
|
|
|
## Quick Start
|
|
|
|
```python
|
|
import testproject
|
|
testproject.run()
|
|
```
|
|
""",
|
|
contributing="# Contributing\n\nWelcome!",
|
|
docs_files=[],
|
|
)
|
|
insights_stream = InsightsStream(
|
|
metadata={
|
|
"stars": 5000,
|
|
"forks": 250,
|
|
"language": "Python",
|
|
"description": "Fast test framework",
|
|
},
|
|
common_problems=[
|
|
{
|
|
"title": "OAuth setup fails",
|
|
"number": 150,
|
|
"state": "open",
|
|
"comments": 30,
|
|
"labels": ["bug", "oauth"],
|
|
},
|
|
{
|
|
"title": "Async deadlock",
|
|
"number": 142,
|
|
"state": "open",
|
|
"comments": 25,
|
|
"labels": ["async", "bug"],
|
|
},
|
|
{
|
|
"title": "Token refresh issue",
|
|
"number": 130,
|
|
"state": "open",
|
|
"comments": 20,
|
|
"labels": ["oauth"],
|
|
},
|
|
],
|
|
known_solutions=[
|
|
{
|
|
"title": "Fixed OAuth redirect",
|
|
"number": 120,
|
|
"state": "closed",
|
|
"comments": 15,
|
|
"labels": ["oauth"],
|
|
},
|
|
{
|
|
"title": "Resolved async race",
|
|
"number": 110,
|
|
"state": "closed",
|
|
"comments": 12,
|
|
"labels": ["async"],
|
|
},
|
|
],
|
|
top_labels=[
|
|
{"label": "oauth", "count": 45},
|
|
{"label": "async", "count": 38},
|
|
{"label": "bug", "count": 30},
|
|
],
|
|
)
|
|
github_streams = ThreeStreamData(code_stream, docs_stream, insights_stream)
|
|
|
|
# Generate router
|
|
generator = RouterGenerator(
|
|
[str(config_path1), str(config_path2)], github_streams=github_streams
|
|
)
|
|
|
|
# Step 1: Validate GitHub metadata extracted
|
|
assert generator.github_metadata is not None
|
|
assert generator.github_metadata["stars"] == 5000
|
|
assert generator.github_metadata["language"] == "Python"
|
|
|
|
# Step 2: Validate GitHub docs extracted
|
|
assert generator.github_docs is not None
|
|
assert "pip install test-project" in generator.github_docs["readme"]
|
|
|
|
# Step 3: Validate GitHub issues extracted
|
|
assert generator.github_issues is not None
|
|
assert len(generator.github_issues["common_problems"]) == 3
|
|
assert len(generator.github_issues["known_solutions"]) == 2
|
|
assert len(generator.github_issues["top_labels"]) == 3
|
|
|
|
# Step 4: Generate and validate router SKILL.md
|
|
skill_md = generator.generate_skill_md()
|
|
|
|
# Validate repository metadata section
|
|
assert "⭐ 5,000" in skill_md
|
|
assert "Python" in skill_md
|
|
assert "Fast test framework" in skill_md
|
|
|
|
# Validate README quick start section
|
|
assert "## Quick Start" in skill_md
|
|
assert "pip install test-project" in skill_md
|
|
|
|
# Validate examples section with converted questions (Fix 1)
|
|
assert "## Examples" in skill_md
|
|
# Issues converted to natural questions
|
|
assert (
|
|
"how do i fix oauth setup" in skill_md.lower()
|
|
or "how do i handle oauth setup" in skill_md.lower()
|
|
)
|
|
assert (
|
|
"how do i handle async deadlock" in skill_md.lower()
|
|
or "how do i fix async deadlock" in skill_md.lower()
|
|
)
|
|
# Common Issues section may still exist with other issues
|
|
# Note: Issue numbers may appear in Common Issues or Common Patterns sections
|
|
|
|
# Step 5: Validate routing keywords include GitHub labels (2x weight)
|
|
routing = generator.extract_routing_keywords()
|
|
|
|
oauth_keywords = routing["testproject-oauth"]
|
|
async_keywords = routing["testproject-async"]
|
|
|
|
# Labels should be included with 2x weight
|
|
assert oauth_keywords.count("oauth") >= 2 # Base + name + 2x from label
|
|
assert async_keywords.count("async") >= 2 # Base + name + 2x from label
|
|
|
|
# Step 6: Generate router config
|
|
router_config = generator.create_router_config()
|
|
|
|
assert router_config["name"] == "testproject"
|
|
assert router_config["_router"] is True
|
|
assert len(router_config["_sub_skills"]) == 2
|
|
assert "testproject-oauth" in router_config["_sub_skills"]
|
|
assert "testproject-async" in router_config["_sub_skills"]
|
|
|
|
|
|
class TestE2EQualityMetrics:
|
|
"""Test quality metrics as specified in Phase 5."""
|
|
|
|
def test_github_overhead_within_limits(self, tmp_path):
|
|
"""
|
|
Test that GitHub integration adds ~30-50 lines per skill (not more).
|
|
|
|
Quality metric: GitHub overhead should be minimal.
|
|
"""
|
|
# Create minimal config
|
|
config = {
|
|
"name": "test-skill",
|
|
"description": "Test skill",
|
|
"base_url": "https://github.com/test/repo",
|
|
"categories": {"api": ["api"]},
|
|
}
|
|
|
|
config_path = tmp_path / "config.json"
|
|
with open(config_path, "w") as f:
|
|
json.dump(config, f)
|
|
|
|
# Create GitHub streams with realistic data
|
|
code_stream = CodeStream(directory=tmp_path, files=[])
|
|
docs_stream = DocsStream(
|
|
readme="# Test\n\nA short README.", contributing=None, docs_files=[]
|
|
)
|
|
insights_stream = InsightsStream(
|
|
metadata={"stars": 100, "forks": 10, "language": "Python", "description": "Test"},
|
|
common_problems=[
|
|
{
|
|
"title": "Issue 1",
|
|
"number": 1,
|
|
"state": "open",
|
|
"comments": 5,
|
|
"labels": ["bug"],
|
|
},
|
|
{
|
|
"title": "Issue 2",
|
|
"number": 2,
|
|
"state": "open",
|
|
"comments": 3,
|
|
"labels": ["bug"],
|
|
},
|
|
],
|
|
known_solutions=[],
|
|
top_labels=[{"label": "bug", "count": 10}],
|
|
)
|
|
github_streams = ThreeStreamData(code_stream, docs_stream, insights_stream)
|
|
|
|
# Generate router without GitHub
|
|
generator_no_github = RouterGenerator([str(config_path)])
|
|
skill_md_no_github = generator_no_github.generate_skill_md()
|
|
lines_no_github = len(skill_md_no_github.split("\n"))
|
|
|
|
# Generate router with GitHub
|
|
generator_with_github = RouterGenerator([str(config_path)], github_streams=github_streams)
|
|
skill_md_with_github = generator_with_github.generate_skill_md()
|
|
lines_with_github = len(skill_md_with_github.split("\n"))
|
|
|
|
# Calculate GitHub overhead
|
|
github_overhead = lines_with_github - lines_no_github
|
|
|
|
# Validate overhead is within acceptable range (30-50 lines)
|
|
assert 20 <= github_overhead <= 60, (
|
|
f"GitHub overhead is {github_overhead} lines, expected 20-60"
|
|
)
|
|
|
|
def test_router_size_within_limits(self, tmp_path):
|
|
"""
|
|
Test that router SKILL.md is ~150 lines (±20).
|
|
|
|
Quality metric: Router should be concise overview, not exhaustive.
|
|
"""
|
|
# Create multiple sub-skill configs
|
|
configs = []
|
|
for i in range(4):
|
|
config = {
|
|
"name": f"test-skill-{i}",
|
|
"description": f"Test skill {i}",
|
|
"base_url": "https://github.com/test/repo",
|
|
"categories": {f"topic{i}": [f"topic{i}"]},
|
|
}
|
|
config_path = tmp_path / f"config{i}.json"
|
|
with open(config_path, "w") as f:
|
|
json.dump(config, f)
|
|
configs.append(str(config_path))
|
|
|
|
# Generate router
|
|
generator = RouterGenerator(configs)
|
|
skill_md = generator.generate_skill_md()
|
|
lines = len(skill_md.split("\n"))
|
|
|
|
# Validate router size is reasonable (60-250 lines for 4 sub-skills)
|
|
# Actual size depends on whether GitHub streams included - can be as small as 60 lines
|
|
assert 60 <= lines <= 250, f"Router is {lines} lines, expected 60-250 for 4 sub-skills"
|
|
|
|
|
|
class TestE2EBackwardCompatibility:
|
|
"""Test that old code still works without GitHub streams."""
|
|
|
|
def test_router_without_github_streams(self, tmp_path):
|
|
"""Test that router generation works without GitHub streams (backward compat)."""
|
|
config = {
|
|
"name": "test-skill",
|
|
"description": "Test skill",
|
|
"base_url": "https://example.com",
|
|
"categories": {"api": ["api"]},
|
|
}
|
|
|
|
config_path = tmp_path / "config.json"
|
|
with open(config_path, "w") as f:
|
|
json.dump(config, f)
|
|
|
|
# Generate router WITHOUT GitHub streams
|
|
generator = RouterGenerator([str(config_path)])
|
|
|
|
assert generator.github_metadata is None
|
|
assert generator.github_docs is None
|
|
assert generator.github_issues is None
|
|
|
|
# Should still generate valid SKILL.md
|
|
skill_md = generator.generate_skill_md()
|
|
|
|
assert "When to Use This Skill" in skill_md
|
|
assert "How It Works" in skill_md
|
|
|
|
# Should NOT have GitHub-specific sections
|
|
assert "⭐" not in skill_md
|
|
assert "Repository Info" not in skill_md
|
|
assert "Quick Start (from README)" not in skill_md
|
|
assert "Common Issues (from GitHub)" not in skill_md
|
|
|
|
@patch("skill_seekers.cli.unified_codebase_analyzer.GitHubThreeStreamFetcher")
|
|
def test_analyzer_without_github_metadata(self, mock_fetcher_class, tmp_path):
|
|
"""Test analyzer with fetch_github_metadata=False."""
|
|
mock_fetcher = Mock()
|
|
mock_fetcher_class.return_value = mock_fetcher
|
|
|
|
code_stream = CodeStream(directory=tmp_path, files=[])
|
|
docs_stream = DocsStream(readme=None, contributing=None, docs_files=[])
|
|
insights_stream = InsightsStream(
|
|
metadata={}, common_problems=[], known_solutions=[], top_labels=[]
|
|
)
|
|
three_streams = ThreeStreamData(code_stream, docs_stream, insights_stream)
|
|
mock_fetcher.fetch.return_value = three_streams
|
|
|
|
(tmp_path / "main.py").write_text("print('hello')")
|
|
|
|
analyzer = UnifiedCodebaseAnalyzer()
|
|
result = analyzer.analyze(
|
|
source="https://github.com/test/repo",
|
|
depth="basic",
|
|
fetch_github_metadata=False, # Explicitly disable
|
|
)
|
|
|
|
# Should not include GitHub docs/insights
|
|
assert result.github_docs is None
|
|
assert result.github_insights is None
|
|
|
|
|
|
class TestE2ETokenEfficiency:
|
|
"""Test token efficiency metrics."""
|
|
|
|
def test_three_stream_produces_compact_output(self, tmp_path):
|
|
"""
|
|
Test that three-stream architecture produces compact, efficient output.
|
|
|
|
This is a qualitative test - we verify that output is structured and
|
|
not duplicated across streams.
|
|
"""
|
|
# Create test files
|
|
(tmp_path / "main.py").write_text("import os\nprint('test')")
|
|
|
|
# Create GitHub streams
|
|
code_stream = CodeStream(directory=tmp_path, files=[tmp_path / "main.py"])
|
|
docs_stream = DocsStream(
|
|
readme="# Test\n\nQuick start guide.", contributing=None, docs_files=[]
|
|
)
|
|
insights_stream = InsightsStream(
|
|
metadata={"stars": 100}, common_problems=[], known_solutions=[], top_labels=[]
|
|
)
|
|
three_streams = ThreeStreamData(code_stream, docs_stream, insights_stream)
|
|
|
|
# Verify streams are separate (no duplication)
|
|
assert code_stream.directory == tmp_path
|
|
assert docs_stream.readme is not None
|
|
assert insights_stream.metadata is not None
|
|
|
|
# Verify no cross-contamination
|
|
assert "Quick start guide" not in str(code_stream.files)
|
|
assert str(tmp_path) not in docs_stream.readme
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|