fix(#300): centralize selector fallback, fix dry-run link discovery, and smart --config routing
- Add FALLBACK_MAIN_SELECTORS constant and _find_main_content() helper to eliminate 3 duplicated fallback loops in doc_scraper.py - Move link extraction before early return in extract_content() so links are always discovered from the full page, not just main content - Fix single-threaded dry-run to extract links from soup (full page) instead of main element only — fixes reactflow.dev finding only 1 page - Add link extraction to async dry-run path (was completely missing) - Remove main_content from get_configuration() defaults so fallback logic kicks in instead of a broad CSS comma selector matching body - Smart create --config routing: peek at JSON to determine unified (sources array → unified_scraper) vs simple (base_url → doc_scraper) - Update docs/user-guide/02-scraping.md and docs/reference/CONFIG_FORMAT.md to use unified config format (legacy format rejected since v2.11.0) - Fix test_auto_fetch_enabled and test_mcp_validate_legacy_config Closes #300 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
69
configs/react.json
Normal file
69
configs/react.json
Normal file
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"name": "react",
|
||||
"description": "Complete React knowledge base combining official documentation and React codebase insights. Use when working with React, understanding API changes, or debugging React internals.",
|
||||
"version": "1.1.0",
|
||||
"merge_mode": "rule-based",
|
||||
"sources": [
|
||||
{
|
||||
"type": "documentation",
|
||||
"base_url": "https://react.dev/",
|
||||
"extract_api": true,
|
||||
"selectors": {
|
||||
"main_content": "article",
|
||||
"title": "h1",
|
||||
"code_blocks": "pre code"
|
||||
},
|
||||
"url_patterns": {
|
||||
"include": [],
|
||||
"exclude": [
|
||||
"/blog/",
|
||||
"/community/"
|
||||
]
|
||||
},
|
||||
"categories": {
|
||||
"getting_started": [
|
||||
"learn",
|
||||
"installation",
|
||||
"quick-start"
|
||||
],
|
||||
"components": [
|
||||
"components",
|
||||
"props",
|
||||
"state"
|
||||
],
|
||||
"hooks": [
|
||||
"hooks",
|
||||
"usestate",
|
||||
"useeffect",
|
||||
"usecontext"
|
||||
],
|
||||
"api": [
|
||||
"api",
|
||||
"reference"
|
||||
],
|
||||
"advanced": [
|
||||
"context",
|
||||
"refs",
|
||||
"portals",
|
||||
"suspense"
|
||||
]
|
||||
},
|
||||
"rate_limit": 0.5
|
||||
},
|
||||
{
|
||||
"type": "github",
|
||||
"repo": "facebook/react",
|
||||
"enable_codebase_analysis": true,
|
||||
"code_analysis_depth": "deep",
|
||||
"fetch_issues": true,
|
||||
"max_issues": 100,
|
||||
"fetch_changelog": true,
|
||||
"fetch_releases": true,
|
||||
"file_patterns": [
|
||||
"packages/react/src/**/*.js",
|
||||
"packages/react-dom/src/**/*.js"
|
||||
]
|
||||
}
|
||||
],
|
||||
"base_url": "https://react.dev/"
|
||||
}
|
||||
Reference in New Issue
Block a user