ae924a9d05
Major restructure to support both CLI usage and MCP integration: **Repository Structure:** - cli/ - All CLI tools (doc_scraper, estimate_pages, enhance_skill, etc.) - mcp/ - New MCP server for Claude Code integration - configs/ - Shared configuration files - tests/ - Updated to import from cli/ - docs/ - Shared documentation **MCP Server (NEW):** - mcp/server.py - Full MCP server implementation - 6 tools available: * generate_config - Create config from URL * estimate_pages - Fast page count estimation * scrape_docs - Full documentation scraping * package_skill - Package to .zip * list_configs - Show available presets * validate_config - Validate config files - mcp/README.md - Complete MCP documentation - mcp/requirements.txt - MCP dependencies **CLI Tools (Moved to cli/):** - All existing functionality preserved - Same commands, same behavior - Tests updated to import from cli.doc_scraper **Tests:** - 68/71 passing (95.8%) - Updated imports from doc_scraper to cli.doc_scraper - Fixed validate_config() tuple unpacking (errors, warnings) - 3 minor test failures (checking warnings instead of errors) **Benefits:** - Use as CLI tool: python3 cli/doc_scraper.py - Use via MCP: Integrated with Claude Code - Shared code and configs - Single source of truth 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
360 lines
13 KiB
Python
360 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Test suite for doc_scraper core features
|
|
Tests URL validation, language detection, pattern extraction, and categorization
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
import unittest
|
|
from unittest.mock import Mock, MagicMock
|
|
from bs4 import BeautifulSoup
|
|
|
|
# Add parent directory to path
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
|
|
from cli.doc_scraper import DocToSkillConverter
|
|
|
|
|
|
class TestURLValidation(unittest.TestCase):
|
|
"""Test URL validation logic"""
|
|
|
|
def setUp(self):
|
|
"""Set up test converter"""
|
|
self.config = {
|
|
'name': 'test',
|
|
'base_url': 'https://docs.example.com/',
|
|
'url_patterns': {
|
|
'include': ['/guide/', '/api/'],
|
|
'exclude': ['/blog/', '/about/']
|
|
},
|
|
'selectors': {
|
|
'main_content': 'article',
|
|
'title': 'h1',
|
|
'code_blocks': 'pre code'
|
|
},
|
|
'rate_limit': 0.1,
|
|
'max_pages': 10
|
|
}
|
|
self.converter = DocToSkillConverter(self.config, dry_run=True)
|
|
|
|
def test_valid_url_with_include_pattern(self):
|
|
"""Test URL matching include pattern"""
|
|
url = 'https://docs.example.com/guide/getting-started'
|
|
self.assertTrue(self.converter.is_valid_url(url))
|
|
|
|
def test_valid_url_with_api_pattern(self):
|
|
"""Test URL matching API pattern"""
|
|
url = 'https://docs.example.com/api/reference'
|
|
self.assertTrue(self.converter.is_valid_url(url))
|
|
|
|
def test_invalid_url_with_exclude_pattern(self):
|
|
"""Test URL matching exclude pattern"""
|
|
url = 'https://docs.example.com/blog/announcement'
|
|
self.assertFalse(self.converter.is_valid_url(url))
|
|
|
|
def test_invalid_url_different_domain(self):
|
|
"""Test URL from different domain"""
|
|
url = 'https://other-site.com/guide/tutorial'
|
|
self.assertFalse(self.converter.is_valid_url(url))
|
|
|
|
def test_invalid_url_no_include_match(self):
|
|
"""Test URL not matching any include pattern"""
|
|
url = 'https://docs.example.com/download/installer'
|
|
self.assertFalse(self.converter.is_valid_url(url))
|
|
|
|
def test_url_validation_no_patterns(self):
|
|
"""Test URL validation with no include/exclude patterns"""
|
|
config = {
|
|
'name': 'test',
|
|
'base_url': 'https://docs.example.com/',
|
|
'url_patterns': {
|
|
'include': [],
|
|
'exclude': []
|
|
},
|
|
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
|
'rate_limit': 0.1,
|
|
'max_pages': 10
|
|
}
|
|
converter = DocToSkillConverter(config, dry_run=True)
|
|
|
|
# Should accept any URL under base_url
|
|
self.assertTrue(converter.is_valid_url('https://docs.example.com/anything'))
|
|
self.assertFalse(converter.is_valid_url('https://other.com/anything'))
|
|
|
|
|
|
class TestLanguageDetection(unittest.TestCase):
|
|
"""Test language detection from code blocks"""
|
|
|
|
def setUp(self):
|
|
"""Set up test converter"""
|
|
config = {
|
|
'name': 'test',
|
|
'base_url': 'https://example.com/',
|
|
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
|
'rate_limit': 0.1,
|
|
'max_pages': 10
|
|
}
|
|
self.converter = DocToSkillConverter(config, dry_run=True)
|
|
|
|
def test_detect_language_from_class(self):
|
|
"""Test language detection from CSS class"""
|
|
html = '<code class="language-python">print("hello")</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
lang = self.converter.detect_language(elem, 'print("hello")')
|
|
self.assertEqual(lang, 'python')
|
|
|
|
def test_detect_language_from_lang_class(self):
|
|
"""Test language detection from lang- prefix"""
|
|
html = '<code class="lang-javascript">console.log("hello")</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
lang = self.converter.detect_language(elem, 'console.log("hello")')
|
|
self.assertEqual(lang, 'javascript')
|
|
|
|
def test_detect_language_from_parent(self):
|
|
"""Test language detection from parent pre element"""
|
|
html = '<pre class="language-cpp"><code>int main() {}</code></pre>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
lang = self.converter.detect_language(elem, 'int main() {}')
|
|
self.assertEqual(lang, 'cpp')
|
|
|
|
def test_detect_python_from_heuristics(self):
|
|
"""Test Python detection from code content"""
|
|
html = '<code>import os\nfrom pathlib import Path</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
code = elem.get_text()
|
|
lang = self.converter.detect_language(elem, code)
|
|
self.assertEqual(lang, 'python')
|
|
|
|
def test_detect_python_from_def(self):
|
|
"""Test Python detection from def keyword"""
|
|
html = '<code>def my_function():\n pass</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
code = elem.get_text()
|
|
lang = self.converter.detect_language(elem, code)
|
|
self.assertEqual(lang, 'python')
|
|
|
|
def test_detect_javascript_from_const(self):
|
|
"""Test JavaScript detection from const keyword"""
|
|
html = '<code>const myVar = 10;</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
code = elem.get_text()
|
|
lang = self.converter.detect_language(elem, code)
|
|
self.assertEqual(lang, 'javascript')
|
|
|
|
def test_detect_javascript_from_arrow(self):
|
|
"""Test JavaScript detection from arrow function"""
|
|
html = '<code>const add = (a, b) => a + b;</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
code = elem.get_text()
|
|
lang = self.converter.detect_language(elem, code)
|
|
self.assertEqual(lang, 'javascript')
|
|
|
|
def test_detect_gdscript(self):
|
|
"""Test GDScript detection"""
|
|
html = '<code>func _ready():\n var x = 5</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
code = elem.get_text()
|
|
lang = self.converter.detect_language(elem, code)
|
|
self.assertEqual(lang, 'gdscript')
|
|
|
|
def test_detect_cpp(self):
|
|
"""Test C++ detection"""
|
|
html = '<code>#include <iostream>\nint main() { return 0; }</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
code = elem.get_text()
|
|
lang = self.converter.detect_language(elem, code)
|
|
self.assertEqual(lang, 'cpp')
|
|
|
|
def test_detect_unknown(self):
|
|
"""Test unknown language detection"""
|
|
html = '<code>some random text without clear indicators</code>'
|
|
elem = BeautifulSoup(html, 'html.parser').find('code')
|
|
code = elem.get_text()
|
|
lang = self.converter.detect_language(elem, code)
|
|
self.assertEqual(lang, 'unknown')
|
|
|
|
|
|
class TestPatternExtraction(unittest.TestCase):
|
|
"""Test pattern extraction from documentation"""
|
|
|
|
def setUp(self):
|
|
"""Set up test converter"""
|
|
config = {
|
|
'name': 'test',
|
|
'base_url': 'https://example.com/',
|
|
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
|
'rate_limit': 0.1,
|
|
'max_pages': 10
|
|
}
|
|
self.converter = DocToSkillConverter(config, dry_run=True)
|
|
|
|
def test_extract_pattern_with_example_marker(self):
|
|
"""Test pattern extraction with 'Example:' marker"""
|
|
html = '''
|
|
<article>
|
|
<p>Example: Here's how to use it</p>
|
|
<pre><code>print("hello")</code></pre>
|
|
</article>
|
|
'''
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
main = soup.find('article')
|
|
patterns = self.converter.extract_patterns(main, [])
|
|
|
|
self.assertGreater(len(patterns), 0)
|
|
self.assertIn('example', patterns[0]['description'].lower())
|
|
|
|
def test_extract_pattern_with_usage_marker(self):
|
|
"""Test pattern extraction with 'Usage:' marker"""
|
|
html = '''
|
|
<article>
|
|
<p>Usage: Call this function like so</p>
|
|
<pre><code>my_function(arg)</code></pre>
|
|
</article>
|
|
'''
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
main = soup.find('article')
|
|
patterns = self.converter.extract_patterns(main, [])
|
|
|
|
self.assertGreater(len(patterns), 0)
|
|
self.assertIn('usage', patterns[0]['description'].lower())
|
|
|
|
def test_extract_pattern_limit(self):
|
|
"""Test pattern extraction limits to 5 patterns"""
|
|
html = '<article>'
|
|
for i in range(10):
|
|
html += f'<p>Example {i}: Test</p><pre><code>code_{i}</code></pre>'
|
|
html += '</article>'
|
|
|
|
soup = BeautifulSoup(html, 'html.parser')
|
|
main = soup.find('article')
|
|
patterns = self.converter.extract_patterns(main, [])
|
|
|
|
self.assertLessEqual(len(patterns), 5, "Should limit to 5 patterns max")
|
|
|
|
|
|
class TestCategorization(unittest.TestCase):
|
|
"""Test smart categorization logic"""
|
|
|
|
def setUp(self):
|
|
"""Set up test converter"""
|
|
config = {
|
|
'name': 'test',
|
|
'base_url': 'https://example.com/',
|
|
'categories': {
|
|
'getting_started': ['intro', 'tutorial', 'getting-started'],
|
|
'api': ['api', 'reference', 'class'],
|
|
'guides': ['guide', 'how-to']
|
|
},
|
|
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
|
'rate_limit': 0.1,
|
|
'max_pages': 10
|
|
}
|
|
self.converter = DocToSkillConverter(config, dry_run=True)
|
|
|
|
def test_categorize_by_url(self):
|
|
"""Test categorization based on URL"""
|
|
pages = [{
|
|
'url': 'https://example.com/api/reference',
|
|
'title': 'Some Title',
|
|
'content': 'Some content'
|
|
}]
|
|
categories = self.converter.smart_categorize(pages)
|
|
|
|
# Should categorize to 'api' based on URL containing 'api'
|
|
self.assertIn('api', categories)
|
|
self.assertEqual(len(categories['api']), 1)
|
|
|
|
def test_categorize_by_title(self):
|
|
"""Test categorization based on title"""
|
|
pages = [{
|
|
'url': 'https://example.com/docs/page',
|
|
'title': 'API Reference Documentation',
|
|
'content': 'Some content'
|
|
}]
|
|
categories = self.converter.smart_categorize(pages)
|
|
|
|
self.assertIn('api', categories)
|
|
self.assertEqual(len(categories['api']), 1)
|
|
|
|
def test_categorize_by_content(self):
|
|
"""Test categorization based on content (lower priority)"""
|
|
pages = [{
|
|
'url': 'https://example.com/docs/page',
|
|
'title': 'Some Page',
|
|
'content': 'This is a tutorial for beginners. An intro to the system.'
|
|
}]
|
|
categories = self.converter.smart_categorize(pages)
|
|
|
|
# Should categorize based on 'tutorial' and 'intro' in content
|
|
self.assertIn('getting_started', categories)
|
|
|
|
def test_categorize_to_other(self):
|
|
"""Test pages that don't match any category go to 'other'"""
|
|
pages = [{
|
|
'url': 'https://example.com/random/page',
|
|
'title': 'Random Page',
|
|
'content': 'Random content with no keywords'
|
|
}]
|
|
categories = self.converter.smart_categorize(pages)
|
|
|
|
self.assertIn('other', categories)
|
|
self.assertEqual(len(categories['other']), 1)
|
|
|
|
def test_empty_categories_removed(self):
|
|
"""Test empty categories are removed"""
|
|
pages = [{
|
|
'url': 'https://example.com/api/reference',
|
|
'title': 'API Reference',
|
|
'content': 'API documentation'
|
|
}]
|
|
categories = self.converter.smart_categorize(pages)
|
|
|
|
# Only 'api' should exist, not empty 'guides' or 'getting_started'
|
|
# (categories with no pages are removed)
|
|
self.assertIn('api', categories)
|
|
self.assertNotIn('guides', categories)
|
|
|
|
|
|
class TestTextCleaning(unittest.TestCase):
|
|
"""Test text cleaning utility"""
|
|
|
|
def setUp(self):
|
|
"""Set up test converter"""
|
|
config = {
|
|
'name': 'test',
|
|
'base_url': 'https://example.com/',
|
|
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
|
'rate_limit': 0.1,
|
|
'max_pages': 10
|
|
}
|
|
self.converter = DocToSkillConverter(config, dry_run=True)
|
|
|
|
def test_clean_multiple_spaces(self):
|
|
"""Test cleaning multiple spaces"""
|
|
text = "Hello world test"
|
|
cleaned = self.converter.clean_text(text)
|
|
self.assertEqual(cleaned, "Hello world test")
|
|
|
|
def test_clean_newlines(self):
|
|
"""Test cleaning newlines"""
|
|
text = "Hello\n\nworld\ntest"
|
|
cleaned = self.converter.clean_text(text)
|
|
self.assertEqual(cleaned, "Hello world test")
|
|
|
|
def test_clean_tabs(self):
|
|
"""Test cleaning tabs"""
|
|
text = "Hello\t\tworld\ttest"
|
|
cleaned = self.converter.clean_text(text)
|
|
self.assertEqual(cleaned, "Hello world test")
|
|
|
|
def test_clean_strip_whitespace(self):
|
|
"""Test stripping leading/trailing whitespace"""
|
|
text = " Hello world "
|
|
cleaned = self.converter.clean_text(text)
|
|
self.assertEqual(cleaned, "Hello world")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|