Add comprehensive test system with 71 tests (100% pass rate)
Test Framework: - Created tests/ directory structure - Added __init__.py for test package - Implemented 71 comprehensive tests across 3 test suites Test Suites: 1. test_config_validation.py (25 tests) - Valid/invalid config structure - Required fields validation - Name format validation - URL format validation - Selectors validation - URL patterns validation - Categories validation - Rate limit validation (0-10 range) - Max pages validation (1-10000 range) - Start URLs validation 2. test_scraper_features.py (28 tests) - URL validation (include/exclude patterns) - Language detection (Python, JavaScript, GDScript, C++, etc.) - Pattern extraction from documentation - Smart categorization (by URL, title, content) - Text cleaning utilities 3. test_integration.py (18 tests) - Dry-run mode functionality - Config loading and validation - Real config files validation (godot, react, vue, django, fastapi, steam) - URL processing and normalization - Content extraction Test Runner (run_tests.py): - Custom colored test runner with ANSI colors - Detailed test summary with breakdown by category - Success rate calculation - Command-line options: --suite: Run specific test suite --verbose: Show each test name --quiet: Minimal output --failfast: Stop on first failure --list: List all available tests - Execution time: ~1 second for full suite Documentation: - Added comprehensive TESTING.md guide - Test writing templates - Best practices - Coverage information - Troubleshooting guide .gitignore: - Added Python cache files - Added output directory - Added IDE and OS files Test Results: ✅ 71/71 tests passing (100% pass rate) ✅ All existing configs validated ✅ Fast execution (<1 second) ✅ Ready for CI/CD integration 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
yusyus
1
tests/__init__.py
Normal file
1
tests/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
# Test package for Skill Seeker
|
||||
301
tests/test_config_validation.py
Normal file
301
tests/test_config_validation.py
Normal file
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test suite for configuration validation
|
||||
Tests the validate_config() function with various valid and invalid configs
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from doc_scraper import validate_config
|
||||
|
||||
|
||||
class TestConfigValidation(unittest.TestCase):
|
||||
"""Test configuration validation"""
|
||||
|
||||
def test_valid_minimal_config(self):
|
||||
"""Test valid minimal configuration"""
|
||||
config = {
|
||||
'name': 'test-skill',
|
||||
'base_url': 'https://example.com/'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
# Should have warnings about missing selectors, but no critical errors
|
||||
self.assertIsInstance(errors, list)
|
||||
|
||||
def test_valid_complete_config(self):
|
||||
"""Test valid complete configuration"""
|
||||
config = {
|
||||
'name': 'godot',
|
||||
'base_url': 'https://docs.godotengine.org/en/stable/',
|
||||
'description': 'Godot Engine documentation',
|
||||
'selectors': {
|
||||
'main_content': 'div[role="main"]',
|
||||
'title': 'title',
|
||||
'code_blocks': 'pre code'
|
||||
},
|
||||
'url_patterns': {
|
||||
'include': ['/guide/', '/api/'],
|
||||
'exclude': ['/blog/']
|
||||
},
|
||||
'categories': {
|
||||
'getting_started': ['intro', 'tutorial'],
|
||||
'api': ['api', 'reference']
|
||||
},
|
||||
'rate_limit': 0.5,
|
||||
'max_pages': 500
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertEqual(len(errors), 0, f"Valid config should have no errors, got: {errors}")
|
||||
|
||||
def test_missing_name(self):
|
||||
"""Test missing required field 'name'"""
|
||||
config = {
|
||||
'base_url': 'https://example.com/'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('name' in error.lower() for error in errors))
|
||||
|
||||
def test_missing_base_url(self):
|
||||
"""Test missing required field 'base_url'"""
|
||||
config = {
|
||||
'name': 'test'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('base_url' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_name_special_chars(self):
|
||||
"""Test invalid name with special characters"""
|
||||
config = {
|
||||
'name': 'test@skill!',
|
||||
'base_url': 'https://example.com/'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('invalid name' in error.lower() for error in errors))
|
||||
|
||||
def test_valid_name_formats(self):
|
||||
"""Test various valid name formats"""
|
||||
valid_names = ['test', 'test-skill', 'test_skill', 'TestSkill123', 'my-awesome-skill_v2']
|
||||
for name in valid_names:
|
||||
config = {
|
||||
'name': name,
|
||||
'base_url': 'https://example.com/'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
name_errors = [e for e in errors if 'invalid name' in e.lower()]
|
||||
self.assertEqual(len(name_errors), 0, f"Name '{name}' should be valid")
|
||||
|
||||
def test_invalid_base_url_no_protocol(self):
|
||||
"""Test invalid base_url without protocol"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'example.com'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('base_url' in error.lower() for error in errors))
|
||||
|
||||
def test_valid_url_protocols(self):
|
||||
"""Test valid URL protocols"""
|
||||
for protocol in ['http://', 'https://']:
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': f'{protocol}example.com/'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
url_errors = [e for e in errors if 'base_url' in e.lower() and 'invalid' in e.lower()]
|
||||
self.assertEqual(len(url_errors), 0, f"Protocol '{protocol}' should be valid")
|
||||
|
||||
def test_invalid_selectors_not_dict(self):
|
||||
"""Test invalid selectors (not a dictionary)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': 'invalid'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('selectors' in error.lower() and 'dictionary' in error.lower() for error in errors))
|
||||
|
||||
def test_missing_recommended_selectors(self):
|
||||
"""Test warning for missing recommended selectors"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {
|
||||
'main_content': 'article'
|
||||
# Missing 'title' and 'code_blocks'
|
||||
}
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('title' in error.lower() for error in errors))
|
||||
self.assertTrue(any('code_blocks' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_url_patterns_not_dict(self):
|
||||
"""Test invalid url_patterns (not a dictionary)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'url_patterns': []
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('url_patterns' in error.lower() and 'dictionary' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_url_patterns_include_not_list(self):
|
||||
"""Test invalid url_patterns.include (not a list)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'url_patterns': {
|
||||
'include': 'not-a-list'
|
||||
}
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('include' in error.lower() and 'list' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_categories_not_dict(self):
|
||||
"""Test invalid categories (not a dictionary)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'categories': []
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('categories' in error.lower() and 'dictionary' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_category_keywords_not_list(self):
|
||||
"""Test invalid category keywords (not a list)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'categories': {
|
||||
'getting_started': 'not-a-list'
|
||||
}
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('getting_started' in error.lower() and 'list' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_rate_limit_negative(self):
|
||||
"""Test invalid rate_limit (negative)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'rate_limit': -1
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('rate_limit' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_rate_limit_too_high(self):
|
||||
"""Test invalid rate_limit (too high)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'rate_limit': 20
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('rate_limit' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_rate_limit_not_number(self):
|
||||
"""Test invalid rate_limit (not a number)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'rate_limit': 'fast'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('rate_limit' in error.lower() for error in errors))
|
||||
|
||||
def test_valid_rate_limit_range(self):
|
||||
"""Test valid rate_limit range"""
|
||||
for rate in [0, 0.1, 0.5, 1, 5, 10]:
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'rate_limit': rate
|
||||
}
|
||||
errors = validate_config(config)
|
||||
rate_errors = [e for e in errors if 'rate_limit' in e.lower()]
|
||||
self.assertEqual(len(rate_errors), 0, f"Rate limit {rate} should be valid")
|
||||
|
||||
def test_invalid_max_pages_zero(self):
|
||||
"""Test invalid max_pages (zero)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'max_pages': 0
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('max_pages' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_max_pages_too_high(self):
|
||||
"""Test invalid max_pages (too high)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'max_pages': 20000
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('max_pages' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_max_pages_not_int(self):
|
||||
"""Test invalid max_pages (not an integer)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'max_pages': 'many'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('max_pages' in error.lower() for error in errors))
|
||||
|
||||
def test_valid_max_pages_range(self):
|
||||
"""Test valid max_pages range"""
|
||||
for max_p in [1, 10, 100, 500, 5000, 10000]:
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'max_pages': max_p
|
||||
}
|
||||
errors = validate_config(config)
|
||||
max_errors = [e for e in errors if 'max_pages' in e.lower()]
|
||||
self.assertEqual(len(max_errors), 0, f"Max pages {max_p} should be valid")
|
||||
|
||||
def test_invalid_start_urls_not_list(self):
|
||||
"""Test invalid start_urls (not a list)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'start_urls': 'https://example.com/page1'
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('start_urls' in error.lower() and 'list' in error.lower() for error in errors))
|
||||
|
||||
def test_invalid_start_urls_bad_protocol(self):
|
||||
"""Test invalid start_urls (bad protocol)"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'start_urls': ['ftp://example.com/page1']
|
||||
}
|
||||
errors = validate_config(config)
|
||||
self.assertTrue(any('start_url' in error.lower() for error in errors))
|
||||
|
||||
def test_valid_start_urls(self):
|
||||
"""Test valid start_urls"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'start_urls': [
|
||||
'https://example.com/page1',
|
||||
'http://example.com/page2',
|
||||
'https://example.com/api/docs'
|
||||
]
|
||||
}
|
||||
errors = validate_config(config)
|
||||
url_errors = [e for e in errors if 'start_url' in e.lower()]
|
||||
self.assertEqual(len(url_errors), 0, "Valid start_urls should pass validation")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
309
tests/test_integration.py
Normal file
309
tests/test_integration.py
Normal file
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Integration tests for doc_scraper
|
||||
Tests complete workflows and dry-run mode
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
import json
|
||||
import tempfile
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from doc_scraper import DocToSkillConverter, load_config, validate_config
|
||||
|
||||
|
||||
class TestDryRunMode(unittest.TestCase):
|
||||
"""Test dry-run mode functionality"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test configuration"""
|
||||
self.config = {
|
||||
'name': 'test-dry-run',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {
|
||||
'main_content': 'article',
|
||||
'title': 'h1',
|
||||
'code_blocks': 'pre code'
|
||||
},
|
||||
'url_patterns': {
|
||||
'include': [],
|
||||
'exclude': []
|
||||
},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
|
||||
def test_dry_run_no_directories_created(self):
|
||||
"""Test that dry-run mode doesn't create directories"""
|
||||
converter = DocToSkillConverter(self.config, dry_run=True)
|
||||
|
||||
# Check directories were NOT created
|
||||
data_dir = Path(f"output/{self.config['name']}_data")
|
||||
skill_dir = Path(f"output/{self.config['name']}")
|
||||
|
||||
self.assertFalse(data_dir.exists(), "Dry-run should not create data directory")
|
||||
self.assertFalse(skill_dir.exists(), "Dry-run should not create skill directory")
|
||||
|
||||
def test_dry_run_flag_set(self):
|
||||
"""Test that dry_run flag is properly set"""
|
||||
converter = DocToSkillConverter(self.config, dry_run=True)
|
||||
self.assertTrue(converter.dry_run)
|
||||
|
||||
converter_normal = DocToSkillConverter(self.config, dry_run=False)
|
||||
self.assertFalse(converter_normal.dry_run)
|
||||
|
||||
# Clean up
|
||||
shutil.rmtree(f"output/{self.config['name']}_data", ignore_errors=True)
|
||||
shutil.rmtree(f"output/{self.config['name']}", ignore_errors=True)
|
||||
|
||||
def test_normal_mode_creates_directories(self):
|
||||
"""Test that normal mode creates directories"""
|
||||
converter = DocToSkillConverter(self.config, dry_run=False)
|
||||
|
||||
# Check directories WERE created
|
||||
data_dir = Path(f"output/{self.config['name']}_data")
|
||||
skill_dir = Path(f"output/{self.config['name']}")
|
||||
|
||||
self.assertTrue(data_dir.exists(), "Normal mode should create data directory")
|
||||
self.assertTrue(skill_dir.exists(), "Normal mode should create skill directory")
|
||||
|
||||
# Clean up
|
||||
shutil.rmtree(data_dir, ignore_errors=True)
|
||||
shutil.rmtree(skill_dir, ignore_errors=True)
|
||||
|
||||
|
||||
class TestConfigLoading(unittest.TestCase):
|
||||
"""Test configuration loading and validation"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up temporary directory for test configs"""
|
||||
self.temp_dir = tempfile.mkdtemp()
|
||||
|
||||
def tearDown(self):
|
||||
"""Clean up temporary directory"""
|
||||
shutil.rmtree(self.temp_dir, ignore_errors=True)
|
||||
|
||||
def test_load_valid_config(self):
|
||||
"""Test loading a valid configuration file"""
|
||||
config_data = {
|
||||
'name': 'test-config',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {
|
||||
'main_content': 'article',
|
||||
'title': 'h1',
|
||||
'code_blocks': 'pre code'
|
||||
},
|
||||
'rate_limit': 0.5,
|
||||
'max_pages': 100
|
||||
}
|
||||
|
||||
config_path = Path(self.temp_dir) / 'test.json'
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(config_data, f)
|
||||
|
||||
loaded_config = load_config(str(config_path))
|
||||
self.assertEqual(loaded_config['name'], 'test-config')
|
||||
self.assertEqual(loaded_config['base_url'], 'https://example.com/')
|
||||
|
||||
def test_load_invalid_json(self):
|
||||
"""Test loading an invalid JSON file"""
|
||||
config_path = Path(self.temp_dir) / 'invalid.json'
|
||||
with open(config_path, 'w') as f:
|
||||
f.write('{ invalid json }')
|
||||
|
||||
with self.assertRaises(SystemExit):
|
||||
load_config(str(config_path))
|
||||
|
||||
def test_load_nonexistent_file(self):
|
||||
"""Test loading a nonexistent file"""
|
||||
config_path = Path(self.temp_dir) / 'nonexistent.json'
|
||||
|
||||
with self.assertRaises(SystemExit):
|
||||
load_config(str(config_path))
|
||||
|
||||
def test_load_config_with_validation_errors(self):
|
||||
"""Test loading a config with validation errors"""
|
||||
config_data = {
|
||||
'name': 'invalid@name', # Invalid name
|
||||
'base_url': 'example.com' # Missing protocol
|
||||
}
|
||||
|
||||
config_path = Path(self.temp_dir) / 'invalid_config.json'
|
||||
with open(config_path, 'w') as f:
|
||||
json.dump(config_data, f)
|
||||
|
||||
with self.assertRaises(SystemExit):
|
||||
load_config(str(config_path))
|
||||
|
||||
|
||||
class TestRealConfigFiles(unittest.TestCase):
|
||||
"""Test that real config files in the repository are valid"""
|
||||
|
||||
def test_godot_config(self):
|
||||
"""Test Godot config is valid"""
|
||||
config_path = 'configs/godot.json'
|
||||
if os.path.exists(config_path):
|
||||
config = load_config(config_path)
|
||||
errors = validate_config(config)
|
||||
self.assertEqual(len(errors), 0, f"Godot config should be valid, got errors: {errors}")
|
||||
|
||||
def test_react_config(self):
|
||||
"""Test React config is valid"""
|
||||
config_path = 'configs/react.json'
|
||||
if os.path.exists(config_path):
|
||||
config = load_config(config_path)
|
||||
errors = validate_config(config)
|
||||
self.assertEqual(len(errors), 0, f"React config should be valid, got errors: {errors}")
|
||||
|
||||
def test_vue_config(self):
|
||||
"""Test Vue config is valid"""
|
||||
config_path = 'configs/vue.json'
|
||||
if os.path.exists(config_path):
|
||||
config = load_config(config_path)
|
||||
errors = validate_config(config)
|
||||
self.assertEqual(len(errors), 0, f"Vue config should be valid, got errors: {errors}")
|
||||
|
||||
def test_django_config(self):
|
||||
"""Test Django config is valid"""
|
||||
config_path = 'configs/django.json'
|
||||
if os.path.exists(config_path):
|
||||
config = load_config(config_path)
|
||||
errors = validate_config(config)
|
||||
self.assertEqual(len(errors), 0, f"Django config should be valid, got errors: {errors}")
|
||||
|
||||
def test_fastapi_config(self):
|
||||
"""Test FastAPI config is valid"""
|
||||
config_path = 'configs/fastapi.json'
|
||||
if os.path.exists(config_path):
|
||||
config = load_config(config_path)
|
||||
errors = validate_config(config)
|
||||
self.assertEqual(len(errors), 0, f"FastAPI config should be valid, got errors: {errors}")
|
||||
|
||||
def test_steam_economy_config(self):
|
||||
"""Test Steam Economy config is valid"""
|
||||
config_path = 'configs/steam-economy-complete.json'
|
||||
if os.path.exists(config_path):
|
||||
config = load_config(config_path)
|
||||
errors = validate_config(config)
|
||||
self.assertEqual(len(errors), 0, f"Steam Economy config should be valid, got errors: {errors}")
|
||||
|
||||
|
||||
class TestURLProcessing(unittest.TestCase):
|
||||
"""Test URL processing and validation"""
|
||||
|
||||
def test_url_normalization(self):
|
||||
"""Test URL normalization in converter"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'url_patterns': {'include': [], 'exclude': []},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
# Base URL should be stored correctly
|
||||
self.assertEqual(converter.base_url, 'https://example.com/')
|
||||
|
||||
def test_start_urls_fallback(self):
|
||||
"""Test that start_urls defaults to base_url"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
# Should have base_url in pending_urls
|
||||
self.assertEqual(len(converter.pending_urls), 1)
|
||||
self.assertEqual(converter.pending_urls[0], 'https://example.com/')
|
||||
|
||||
def test_multiple_start_urls(self):
|
||||
"""Test multiple start URLs"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'start_urls': [
|
||||
'https://example.com/guide/',
|
||||
'https://example.com/api/',
|
||||
'https://example.com/tutorial/'
|
||||
],
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
# Should have all start URLs in pending_urls
|
||||
self.assertEqual(len(converter.pending_urls), 3)
|
||||
|
||||
|
||||
class TestContentExtraction(unittest.TestCase):
|
||||
"""Test content extraction functionality"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test converter"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {
|
||||
'main_content': 'article',
|
||||
'title': 'h1',
|
||||
'code_blocks': 'pre code'
|
||||
},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
self.converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
def test_extract_empty_content(self):
|
||||
"""Test extracting from empty HTML"""
|
||||
from bs4 import BeautifulSoup
|
||||
html = '<html><body></body></html>'
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
page = self.converter.extract_content(soup, 'https://example.com/test')
|
||||
|
||||
self.assertEqual(page['url'], 'https://example.com/test')
|
||||
self.assertEqual(page['title'], '')
|
||||
self.assertEqual(page['content'], '')
|
||||
self.assertEqual(len(page['code_samples']), 0)
|
||||
|
||||
def test_extract_basic_content(self):
|
||||
"""Test extracting basic content"""
|
||||
from bs4 import BeautifulSoup
|
||||
html = '''
|
||||
<html>
|
||||
<head><title>Test Page</title></head>
|
||||
<body>
|
||||
<article>
|
||||
<h1>Page Title</h1>
|
||||
<p>This is some content.</p>
|
||||
<p>This is more content with sufficient length to be included.</p>
|
||||
<pre><code class="language-python">print("hello")</code></pre>
|
||||
</article>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
|
||||
page = self.converter.extract_content(soup, 'https://example.com/test')
|
||||
|
||||
self.assertEqual(page['url'], 'https://example.com/test')
|
||||
self.assertIn('Page Title', page['title'])
|
||||
self.assertIn('content', page['content'].lower())
|
||||
self.assertGreater(len(page['code_samples']), 0)
|
||||
self.assertEqual(page['code_samples'][0]['language'], 'python')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
359
tests/test_scraper_features.py
Normal file
359
tests/test_scraper_features.py
Normal file
@@ -0,0 +1,359 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test suite for doc_scraper core features
|
||||
Tests URL validation, language detection, pattern extraction, and categorization
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import Mock, MagicMock
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Add parent directory to path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from doc_scraper import DocToSkillConverter
|
||||
|
||||
|
||||
class TestURLValidation(unittest.TestCase):
|
||||
"""Test URL validation logic"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test converter"""
|
||||
self.config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://docs.example.com/',
|
||||
'url_patterns': {
|
||||
'include': ['/guide/', '/api/'],
|
||||
'exclude': ['/blog/', '/about/']
|
||||
},
|
||||
'selectors': {
|
||||
'main_content': 'article',
|
||||
'title': 'h1',
|
||||
'code_blocks': 'pre code'
|
||||
},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
self.converter = DocToSkillConverter(self.config, dry_run=True)
|
||||
|
||||
def test_valid_url_with_include_pattern(self):
|
||||
"""Test URL matching include pattern"""
|
||||
url = 'https://docs.example.com/guide/getting-started'
|
||||
self.assertTrue(self.converter.is_valid_url(url))
|
||||
|
||||
def test_valid_url_with_api_pattern(self):
|
||||
"""Test URL matching API pattern"""
|
||||
url = 'https://docs.example.com/api/reference'
|
||||
self.assertTrue(self.converter.is_valid_url(url))
|
||||
|
||||
def test_invalid_url_with_exclude_pattern(self):
|
||||
"""Test URL matching exclude pattern"""
|
||||
url = 'https://docs.example.com/blog/announcement'
|
||||
self.assertFalse(self.converter.is_valid_url(url))
|
||||
|
||||
def test_invalid_url_different_domain(self):
|
||||
"""Test URL from different domain"""
|
||||
url = 'https://other-site.com/guide/tutorial'
|
||||
self.assertFalse(self.converter.is_valid_url(url))
|
||||
|
||||
def test_invalid_url_no_include_match(self):
|
||||
"""Test URL not matching any include pattern"""
|
||||
url = 'https://docs.example.com/download/installer'
|
||||
self.assertFalse(self.converter.is_valid_url(url))
|
||||
|
||||
def test_url_validation_no_patterns(self):
|
||||
"""Test URL validation with no include/exclude patterns"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://docs.example.com/',
|
||||
'url_patterns': {
|
||||
'include': [],
|
||||
'exclude': []
|
||||
},
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
# Should accept any URL under base_url
|
||||
self.assertTrue(converter.is_valid_url('https://docs.example.com/anything'))
|
||||
self.assertFalse(converter.is_valid_url('https://other.com/anything'))
|
||||
|
||||
|
||||
class TestLanguageDetection(unittest.TestCase):
|
||||
"""Test language detection from code blocks"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test converter"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
self.converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
def test_detect_language_from_class(self):
|
||||
"""Test language detection from CSS class"""
|
||||
html = '<code class="language-python">print("hello")</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
lang = self.converter.detect_language(elem, 'print("hello")')
|
||||
self.assertEqual(lang, 'python')
|
||||
|
||||
def test_detect_language_from_lang_class(self):
|
||||
"""Test language detection from lang- prefix"""
|
||||
html = '<code class="lang-javascript">console.log("hello")</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
lang = self.converter.detect_language(elem, 'console.log("hello")')
|
||||
self.assertEqual(lang, 'javascript')
|
||||
|
||||
def test_detect_language_from_parent(self):
|
||||
"""Test language detection from parent pre element"""
|
||||
html = '<pre class="language-cpp"><code>int main() {}</code></pre>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
lang = self.converter.detect_language(elem, 'int main() {}')
|
||||
self.assertEqual(lang, 'cpp')
|
||||
|
||||
def test_detect_python_from_heuristics(self):
|
||||
"""Test Python detection from code content"""
|
||||
html = '<code>import os\nfrom pathlib import Path</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
code = elem.get_text()
|
||||
lang = self.converter.detect_language(elem, code)
|
||||
self.assertEqual(lang, 'python')
|
||||
|
||||
def test_detect_python_from_def(self):
|
||||
"""Test Python detection from def keyword"""
|
||||
html = '<code>def my_function():\n pass</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
code = elem.get_text()
|
||||
lang = self.converter.detect_language(elem, code)
|
||||
self.assertEqual(lang, 'python')
|
||||
|
||||
def test_detect_javascript_from_const(self):
|
||||
"""Test JavaScript detection from const keyword"""
|
||||
html = '<code>const myVar = 10;</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
code = elem.get_text()
|
||||
lang = self.converter.detect_language(elem, code)
|
||||
self.assertEqual(lang, 'javascript')
|
||||
|
||||
def test_detect_javascript_from_arrow(self):
|
||||
"""Test JavaScript detection from arrow function"""
|
||||
html = '<code>const add = (a, b) => a + b;</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
code = elem.get_text()
|
||||
lang = self.converter.detect_language(elem, code)
|
||||
self.assertEqual(lang, 'javascript')
|
||||
|
||||
def test_detect_gdscript(self):
|
||||
"""Test GDScript detection"""
|
||||
html = '<code>func _ready():\n var x = 5</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
code = elem.get_text()
|
||||
lang = self.converter.detect_language(elem, code)
|
||||
self.assertEqual(lang, 'gdscript')
|
||||
|
||||
def test_detect_cpp(self):
|
||||
"""Test C++ detection"""
|
||||
html = '<code>#include <iostream>\nint main() { return 0; }</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
code = elem.get_text()
|
||||
lang = self.converter.detect_language(elem, code)
|
||||
self.assertEqual(lang, 'cpp')
|
||||
|
||||
def test_detect_unknown(self):
|
||||
"""Test unknown language detection"""
|
||||
html = '<code>some random text without clear indicators</code>'
|
||||
elem = BeautifulSoup(html, 'html.parser').find('code')
|
||||
code = elem.get_text()
|
||||
lang = self.converter.detect_language(elem, code)
|
||||
self.assertEqual(lang, 'unknown')
|
||||
|
||||
|
||||
class TestPatternExtraction(unittest.TestCase):
|
||||
"""Test pattern extraction from documentation"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test converter"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
self.converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
def test_extract_pattern_with_example_marker(self):
|
||||
"""Test pattern extraction with 'Example:' marker"""
|
||||
html = '''
|
||||
<article>
|
||||
<p>Example: Here's how to use it</p>
|
||||
<pre><code>print("hello")</code></pre>
|
||||
</article>
|
||||
'''
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
main = soup.find('article')
|
||||
patterns = self.converter.extract_patterns(main, [])
|
||||
|
||||
self.assertGreater(len(patterns), 0)
|
||||
self.assertIn('example', patterns[0]['description'].lower())
|
||||
|
||||
def test_extract_pattern_with_usage_marker(self):
|
||||
"""Test pattern extraction with 'Usage:' marker"""
|
||||
html = '''
|
||||
<article>
|
||||
<p>Usage: Call this function like so</p>
|
||||
<pre><code>my_function(arg)</code></pre>
|
||||
</article>
|
||||
'''
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
main = soup.find('article')
|
||||
patterns = self.converter.extract_patterns(main, [])
|
||||
|
||||
self.assertGreater(len(patterns), 0)
|
||||
self.assertIn('usage', patterns[0]['description'].lower())
|
||||
|
||||
def test_extract_pattern_limit(self):
|
||||
"""Test pattern extraction limits to 5 patterns"""
|
||||
html = '<article>'
|
||||
for i in range(10):
|
||||
html += f'<p>Example {i}: Test</p><pre><code>code_{i}</code></pre>'
|
||||
html += '</article>'
|
||||
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
main = soup.find('article')
|
||||
patterns = self.converter.extract_patterns(main, [])
|
||||
|
||||
self.assertLessEqual(len(patterns), 5, "Should limit to 5 patterns max")
|
||||
|
||||
|
||||
class TestCategorization(unittest.TestCase):
|
||||
"""Test smart categorization logic"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test converter"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'categories': {
|
||||
'getting_started': ['intro', 'tutorial', 'getting-started'],
|
||||
'api': ['api', 'reference', 'class'],
|
||||
'guides': ['guide', 'how-to']
|
||||
},
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
self.converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
def test_categorize_by_url(self):
|
||||
"""Test categorization based on URL"""
|
||||
pages = [{
|
||||
'url': 'https://example.com/api/reference',
|
||||
'title': 'Some Title',
|
||||
'content': 'Some content'
|
||||
}]
|
||||
categories = self.converter.smart_categorize(pages)
|
||||
|
||||
# Should categorize to 'api' based on URL containing 'api'
|
||||
self.assertIn('api', categories)
|
||||
self.assertEqual(len(categories['api']), 1)
|
||||
|
||||
def test_categorize_by_title(self):
|
||||
"""Test categorization based on title"""
|
||||
pages = [{
|
||||
'url': 'https://example.com/docs/page',
|
||||
'title': 'API Reference Documentation',
|
||||
'content': 'Some content'
|
||||
}]
|
||||
categories = self.converter.smart_categorize(pages)
|
||||
|
||||
self.assertIn('api', categories)
|
||||
self.assertEqual(len(categories['api']), 1)
|
||||
|
||||
def test_categorize_by_content(self):
|
||||
"""Test categorization based on content (lower priority)"""
|
||||
pages = [{
|
||||
'url': 'https://example.com/docs/page',
|
||||
'title': 'Some Page',
|
||||
'content': 'This is a tutorial for beginners. An intro to the system.'
|
||||
}]
|
||||
categories = self.converter.smart_categorize(pages)
|
||||
|
||||
# Should categorize based on 'tutorial' and 'intro' in content
|
||||
self.assertIn('getting_started', categories)
|
||||
|
||||
def test_categorize_to_other(self):
|
||||
"""Test pages that don't match any category go to 'other'"""
|
||||
pages = [{
|
||||
'url': 'https://example.com/random/page',
|
||||
'title': 'Random Page',
|
||||
'content': 'Random content with no keywords'
|
||||
}]
|
||||
categories = self.converter.smart_categorize(pages)
|
||||
|
||||
self.assertIn('other', categories)
|
||||
self.assertEqual(len(categories['other']), 1)
|
||||
|
||||
def test_empty_categories_removed(self):
|
||||
"""Test empty categories are removed"""
|
||||
pages = [{
|
||||
'url': 'https://example.com/api/reference',
|
||||
'title': 'API Reference',
|
||||
'content': 'API documentation'
|
||||
}]
|
||||
categories = self.converter.smart_categorize(pages)
|
||||
|
||||
# Only 'api' should exist, not empty 'guides' or 'getting_started'
|
||||
# (categories with no pages are removed)
|
||||
self.assertIn('api', categories)
|
||||
self.assertNotIn('guides', categories)
|
||||
|
||||
|
||||
class TestTextCleaning(unittest.TestCase):
|
||||
"""Test text cleaning utility"""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test converter"""
|
||||
config = {
|
||||
'name': 'test',
|
||||
'base_url': 'https://example.com/',
|
||||
'selectors': {'main_content': 'article', 'title': 'h1', 'code_blocks': 'pre'},
|
||||
'rate_limit': 0.1,
|
||||
'max_pages': 10
|
||||
}
|
||||
self.converter = DocToSkillConverter(config, dry_run=True)
|
||||
|
||||
def test_clean_multiple_spaces(self):
|
||||
"""Test cleaning multiple spaces"""
|
||||
text = "Hello world test"
|
||||
cleaned = self.converter.clean_text(text)
|
||||
self.assertEqual(cleaned, "Hello world test")
|
||||
|
||||
def test_clean_newlines(self):
|
||||
"""Test cleaning newlines"""
|
||||
text = "Hello\n\nworld\ntest"
|
||||
cleaned = self.converter.clean_text(text)
|
||||
self.assertEqual(cleaned, "Hello world test")
|
||||
|
||||
def test_clean_tabs(self):
|
||||
"""Test cleaning tabs"""
|
||||
text = "Hello\t\tworld\ttest"
|
||||
cleaned = self.converter.clean_text(text)
|
||||
self.assertEqual(cleaned, "Hello world test")
|
||||
|
||||
def test_clean_strip_whitespace(self):
|
||||
"""Test stripping leading/trailing whitespace"""
|
||||
text = " Hello world "
|
||||
cleaned = self.converter.clean_text(text)
|
||||
self.assertEqual(cleaned, "Hello world")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user