#!/usr/bin/env python3
"""
End-to-End Tests for Multi-LLM Adaptors
Tests complete workflows without real API uploads:
- Scrape → Package → Verify for all platforms
- Same scraped data works for all platforms
- Package structure validation
- Enhancement workflow (mocked)
"""
import json
import tarfile
import tempfile
import unittest
import zipfile
from pathlib import Path
from skill_seekers.cli.adaptors import get_adaptor
from skill_seekers.cli.adaptors.base import SkillMetadata
class TestAdaptorsE2E(unittest.TestCase):
"""End-to-end tests for all platform adaptors"""
def setUp(self):
"""Set up test environment with sample skill directory"""
self.temp_dir = tempfile.TemporaryDirectory()
self.skill_dir = Path(self.temp_dir.name) / "test-skill"
self.skill_dir.mkdir()
# Create realistic skill structure
self._create_sample_skill()
self.output_dir = Path(self.temp_dir.name) / "output"
self.output_dir.mkdir()
def tearDown(self):
"""Clean up temporary directory"""
self.temp_dir.cleanup()
def _create_sample_skill(self):
"""Create a sample skill directory with realistic content"""
# Create SKILL.md
skill_md_content = """# React Framework
React is a JavaScript library for building user interfaces.
## Quick Reference
```javascript
// Create a component
function Welcome(props) {
return
Hello, {props.name}
;
}
```
## Key Concepts
- Components
- Props
- State
- Hooks
"""
(self.skill_dir / "SKILL.md").write_text(skill_md_content)
# Create references directory
refs_dir = self.skill_dir / "references"
refs_dir.mkdir()
# Create sample reference files
(refs_dir / "getting_started.md").write_text("""# Getting Started
Install React:
```bash
npm install react
```
Create your first component:
```javascript
function App() {
return Hello World
;
}
```
""")
(refs_dir / "hooks.md").write_text("""# React Hooks
## useState
```javascript
const [count, setCount] = useState(0);
```
## useEffect
```javascript
useEffect(() => {
document.title = `Count: ${count}`;
}, [count]);
```
""")
(refs_dir / "components.md").write_text("""# Components
## Functional Components
```javascript
function Greeting({ name }) {
return Hello {name}
;
}
```
## Props
Pass data to components:
```javascript
```
""")
# Create empty scripts and assets directories
(self.skill_dir / "scripts").mkdir()
(self.skill_dir / "assets").mkdir()
def test_e2e_all_platforms_from_same_skill(self):
"""Test that all platforms can package the same skill"""
platforms = ["claude", "gemini", "openai", "markdown"]
packages = {}
for platform in platforms:
adaptor = get_adaptor(platform)
# Package for this platform
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Verify package was created
self.assertTrue(package_path.exists(), f"Package not created for {platform}")
# Store for later verification
packages[platform] = package_path
# Verify all packages were created
self.assertEqual(len(packages), 4)
# Verify correct extensions
self.assertTrue(str(packages["claude"]).endswith(".zip"))
self.assertTrue(str(packages["gemini"]).endswith(".tar.gz"))
self.assertTrue(str(packages["openai"]).endswith(".zip"))
self.assertTrue(str(packages["markdown"]).endswith(".zip"))
def test_e2e_claude_workflow(self):
"""Test complete Claude workflow: package + verify structure"""
adaptor = get_adaptor("claude")
# Package
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Verify package
self.assertTrue(package_path.exists())
self.assertTrue(str(package_path).endswith(".zip"))
# Verify contents
with zipfile.ZipFile(package_path, "r") as zf:
names = zf.namelist()
# Should have SKILL.md
self.assertIn("SKILL.md", names)
# Should have references
self.assertTrue(any("references/" in name for name in names))
# Verify SKILL.md content (should have YAML frontmatter)
skill_content = zf.read("SKILL.md").decode("utf-8")
# Claude uses YAML frontmatter (but current implementation doesn't add it in package)
# Just verify content exists
self.assertGreater(len(skill_content), 0)
def test_e2e_gemini_workflow(self):
"""Test complete Gemini workflow: package + verify structure"""
adaptor = get_adaptor("gemini")
# Package
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Verify package
self.assertTrue(package_path.exists())
self.assertTrue(str(package_path).endswith(".tar.gz"))
# Verify contents
with tarfile.open(package_path, "r:gz") as tar:
names = tar.getnames()
# Should have system_instructions.md (not SKILL.md)
self.assertIn("system_instructions.md", names)
# Should have references
self.assertTrue(any("references/" in name for name in names))
# Should have metadata
self.assertIn("gemini_metadata.json", names)
# Verify metadata content
metadata_member = tar.getmember("gemini_metadata.json")
metadata_file = tar.extractfile(metadata_member)
metadata = json.loads(metadata_file.read().decode("utf-8"))
self.assertEqual(metadata["platform"], "gemini")
self.assertEqual(metadata["name"], "test-skill")
self.assertIn("created_with", metadata)
def test_e2e_openai_workflow(self):
"""Test complete OpenAI workflow: package + verify structure"""
adaptor = get_adaptor("openai")
# Package
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Verify package
self.assertTrue(package_path.exists())
self.assertTrue(str(package_path).endswith(".zip"))
# Verify contents
with zipfile.ZipFile(package_path, "r") as zf:
names = zf.namelist()
# Should have assistant_instructions.txt
self.assertIn("assistant_instructions.txt", names)
# Should have vector store files
self.assertTrue(any("vector_store_files/" in name for name in names))
# Should have metadata
self.assertIn("openai_metadata.json", names)
# Verify metadata content
metadata_content = zf.read("openai_metadata.json").decode("utf-8")
metadata = json.loads(metadata_content)
self.assertEqual(metadata["platform"], "openai")
self.assertEqual(metadata["name"], "test-skill")
self.assertEqual(metadata["model"], "gpt-4o")
self.assertIn("file_search", metadata["tools"])
def test_e2e_markdown_workflow(self):
"""Test complete Markdown workflow: package + verify structure"""
adaptor = get_adaptor("markdown")
# Package
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Verify package
self.assertTrue(package_path.exists())
self.assertTrue(str(package_path).endswith(".zip"))
# Verify contents
with zipfile.ZipFile(package_path, "r") as zf:
names = zf.namelist()
# Should have README.md
self.assertIn("README.md", names)
# Should have DOCUMENTATION.md (combined)
self.assertIn("DOCUMENTATION.md", names)
# Should have references
self.assertTrue(any("references/" in name for name in names))
# Should have metadata
self.assertIn("metadata.json", names)
# Verify combined documentation
doc_content = zf.read("DOCUMENTATION.md").decode("utf-8")
# Should contain content from all references
self.assertIn("Getting Started", doc_content)
self.assertIn("React Hooks", doc_content)
self.assertIn("Components", doc_content)
def test_e2e_package_format_validation(self):
"""Test that each platform creates correct package format"""
test_cases = [
("claude", ".zip"),
("gemini", ".tar.gz"),
("openai", ".zip"),
("markdown", ".zip"),
]
for platform, expected_ext in test_cases:
adaptor = get_adaptor(platform)
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Verify extension
if expected_ext == ".tar.gz":
self.assertTrue(
str(package_path).endswith(".tar.gz"), f"{platform} should create .tar.gz file"
)
else:
self.assertTrue(
str(package_path).endswith(".zip"), f"{platform} should create .zip file"
)
def test_e2e_package_filename_convention(self):
"""Test that package filenames follow convention"""
test_cases = [
("claude", "test-skill.zip"),
("gemini", "test-skill-gemini.tar.gz"),
("openai", "test-skill-openai.zip"),
("markdown", "test-skill-markdown.zip"),
]
for platform, expected_name in test_cases:
adaptor = get_adaptor(platform)
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Verify filename
self.assertEqual(
package_path.name, expected_name, f"{platform} package filename incorrect"
)
def test_e2e_all_platforms_preserve_references(self):
"""Test that all platforms preserve reference files"""
ref_files = ["getting_started.md", "hooks.md", "components.md"]
for platform in ["claude", "gemini", "openai", "markdown"]:
adaptor = get_adaptor(platform)
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Check references are preserved
if platform == "gemini":
with tarfile.open(package_path, "r:gz") as tar:
names = tar.getnames()
for ref_file in ref_files:
self.assertTrue(
any(ref_file in name for name in names),
f"{platform}: {ref_file} not found in package",
)
else:
with zipfile.ZipFile(package_path, "r") as zf:
names = zf.namelist()
for ref_file in ref_files:
# OpenAI moves to vector_store_files/
if platform == "openai":
self.assertTrue(
any(f"vector_store_files/{ref_file}" in name for name in names),
f"{platform}: {ref_file} not found in vector_store_files/",
)
else:
self.assertTrue(
any(ref_file in name for name in names),
f"{platform}: {ref_file} not found in package",
)
def test_e2e_metadata_consistency(self):
"""Test that metadata is consistent across platforms"""
platforms_with_metadata = ["gemini", "openai", "markdown"]
for platform in platforms_with_metadata:
adaptor = get_adaptor(platform)
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Extract and verify metadata
if platform == "gemini":
with tarfile.open(package_path, "r:gz") as tar:
metadata_member = tar.getmember("gemini_metadata.json")
metadata_file = tar.extractfile(metadata_member)
metadata = json.loads(metadata_file.read().decode("utf-8"))
else:
with zipfile.ZipFile(package_path, "r") as zf:
metadata_filename = (
f"{platform}_metadata.json" if platform == "openai" else "metadata.json"
)
metadata_content = zf.read(metadata_filename).decode("utf-8")
metadata = json.loads(metadata_content)
# Verify required fields
self.assertEqual(metadata["platform"], platform)
self.assertEqual(metadata["name"], "test-skill")
self.assertIn("created_with", metadata)
def test_e2e_format_skill_md_differences(self):
"""Test that each platform formats SKILL.md differently"""
metadata = SkillMetadata(name="test-skill", description="Test skill for E2E testing")
formats = {}
for platform in ["claude", "gemini", "openai", "markdown"]:
adaptor = get_adaptor(platform)
formatted = adaptor.format_skill_md(self.skill_dir, metadata)
formats[platform] = formatted
# Claude should have YAML frontmatter
self.assertTrue(formats["claude"].startswith("---"))
# Gemini and Markdown should NOT have YAML frontmatter
self.assertFalse(formats["gemini"].startswith("---"))
self.assertFalse(formats["markdown"].startswith("---"))
# All should contain content from existing SKILL.md (React Framework)
for platform, formatted in formats.items():
# Check for content from existing SKILL.md
self.assertIn("react", formatted.lower(), f"{platform} should contain skill content")
# All should have non-empty content
self.assertGreater(len(formatted), 100, f"{platform} should have substantial content")
def test_e2e_upload_without_api_key(self):
"""Test upload behavior without API keys (should fail gracefully)"""
platforms_with_upload = ["claude", "gemini", "openai"]
for platform in platforms_with_upload:
adaptor = get_adaptor(platform)
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Try upload without API key
result = adaptor.upload(package_path, "")
# Should fail
self.assertFalse(result["success"], f"{platform} should fail without API key")
self.assertIsNone(result["skill_id"])
self.assertIn("message", result)
def test_e2e_markdown_no_upload_support(self):
"""Test that markdown adaptor doesn't support upload"""
adaptor = get_adaptor("markdown")
package_path = adaptor.package(self.skill_dir, self.output_dir)
# Try upload (should return informative message)
result = adaptor.upload(package_path, "not-used")
# Should indicate no upload support
self.assertFalse(result["success"])
self.assertIsNone(result["skill_id"])
self.assertIn("not support", result["message"].lower())
# URL should point to local file
self.assertIn(str(package_path.absolute()), result["url"])
class TestAdaptorsWorkflowIntegration(unittest.TestCase):
"""Integration tests for common workflow patterns"""
def test_workflow_export_to_all_platforms(self):
"""Test exporting same skill to all platforms"""
with tempfile.TemporaryDirectory() as temp_dir:
skill_dir = Path(temp_dir) / "react"
skill_dir.mkdir()
# Create minimal skill
(skill_dir / "SKILL.md").write_text("# React\n\nReact documentation")
refs_dir = skill_dir / "references"
refs_dir.mkdir()
(refs_dir / "guide.md").write_text("# Guide\n\nContent")
output_dir = Path(temp_dir) / "output"
output_dir.mkdir()
# Export to all platforms
packages = {}
for platform in ["claude", "gemini", "openai", "markdown"]:
adaptor = get_adaptor(platform)
package_path = adaptor.package(skill_dir, output_dir)
packages[platform] = package_path
# Verify all packages exist and are distinct
self.assertEqual(len(packages), 4)
self.assertEqual(len(set(packages.values())), 4) # All unique
def test_workflow_package_to_custom_path(self):
"""Test packaging to custom output paths"""
with tempfile.TemporaryDirectory() as temp_dir:
skill_dir = Path(temp_dir) / "skill"
skill_dir.mkdir()
(skill_dir / "SKILL.md").write_text("# Test")
(skill_dir / "references").mkdir()
# Test custom output paths
custom_output = Path(temp_dir) / "custom" / "my-package.zip"
adaptor = get_adaptor("claude")
package_path = adaptor.package(skill_dir, custom_output)
# Should respect custom path
self.assertTrue(package_path.exists())
self.assertTrue(
"my-package" in package_path.name or package_path.parent.name == "custom"
)
def test_workflow_api_key_validation(self):
"""Test API key validation for each platform"""
test_cases = [
("claude", "sk-ant-test123", True),
("claude", "invalid-key", False),
("gemini", "AIzaSyTest123", True),
("gemini", "sk-ant-test", False),
("openai", "sk-proj-test123", True),
("openai", "sk-test123", True),
("openai", "AIzaSy123", False),
("markdown", "any-key", False), # Never uses keys
]
for platform, api_key, expected in test_cases:
adaptor = get_adaptor(platform)
result = adaptor.validate_api_key(api_key)
self.assertEqual(
result, expected, f"{platform}: validate_api_key('{api_key}') should be {expected}"
)
class TestAdaptorsErrorHandling(unittest.TestCase):
"""Test error handling in adaptors"""
def test_error_invalid_skill_directory(self):
"""Test packaging with invalid skill directory"""
with tempfile.TemporaryDirectory() as temp_dir:
# Empty directory (no SKILL.md)
empty_dir = Path(temp_dir) / "empty"
empty_dir.mkdir()
output_dir = Path(temp_dir) / "output"
output_dir.mkdir()
# Should handle gracefully (may create package but with empty content)
for platform in ["claude", "gemini", "openai", "markdown"]:
adaptor = get_adaptor(platform)
# Should not crash
try:
package_path = adaptor.package(empty_dir, output_dir)
# Package may be created but should exist
self.assertTrue(package_path.exists())
except Exception as e:
# If it raises, should be clear error
self.assertIn("SKILL.md", str(e).lower() or "reference" in str(e).lower())
def test_error_upload_nonexistent_file(self):
"""Test upload with nonexistent file"""
for platform in ["claude", "gemini", "openai"]:
adaptor = get_adaptor(platform)
result = adaptor.upload(Path("/nonexistent/file.zip"), "test-key")
self.assertFalse(result["success"])
self.assertIn("not found", result["message"].lower())
def test_error_upload_wrong_format(self):
"""Test upload with wrong file format"""
with tempfile.NamedTemporaryFile(suffix=".txt") as tmp:
# Try uploading .txt file
for platform in ["claude", "gemini", "openai"]:
adaptor = get_adaptor(platform)
result = adaptor.upload(Path(tmp.name), "test-key")
self.assertFalse(result["success"])
if __name__ == "__main__":
unittest.main()