Optimize all framework configs with start_urls for better coverage

All configs now follow the steam-economy-complete.json pattern with:
- Multiple start_urls for comprehensive entry points
- Improved include patterns for better targeting
- Enhanced exclude patterns to skip irrelevant pages

Godot Config:
- Added 7 start_urls covering getting started, scripting, 2D, 3D, physics, animation, and classes
- Added include patterns: /getting_started/, /tutorials/, /classes/
- More focused scraping of core documentation

React Config:
- Added 6 start_urls covering learn, quick-start, reference, and hooks
- Existing patterns maintained (already well-optimized)

Vue Config:
- Added 6 start_urls covering introduction, essentials, components, composables, and API
- Fixed base_url from https://vuejs.org/guide/ to https://vuejs.org/
- Added /partners/ to exclude list

Django Config:
- Added 7 start_urls covering intro, models, views, templates, forms, auth, and reference
- Added /intro/ to include patterns
- Added /releases/ to exclude list (changelog not needed)

FastAPI Config:
- Added 7 start_urls covering tutorial, first-steps, path-params, body, dependencies, advanced, and reference
- Added /deployment/ to exclude list

Benefits:
- Better initial page discovery
- More comprehensive documentation coverage
- Faster scraping (direct entry to important sections)
- Reduced unnecessary page crawling
- Consistent pattern across all configs

All configs tested and validated:
 71/71 tests passing
 All 6 configs validated successfully

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
yusyus
2025-10-19 02:24:56 +03:00
parent f9c8f1d610
commit 59c2f9126d
5 changed files with 53 additions and 6 deletions

View File

@@ -2,14 +2,23 @@
"name": "django",
"description": "Django web framework for Python. Use for Django models, views, templates, ORM, authentication, and web development.",
"base_url": "https://docs.djangoproject.com/en/stable/",
"start_urls": [
"https://docs.djangoproject.com/en/stable/intro/",
"https://docs.djangoproject.com/en/stable/topics/db/models/",
"https://docs.djangoproject.com/en/stable/topics/http/views/",
"https://docs.djangoproject.com/en/stable/topics/templates/",
"https://docs.djangoproject.com/en/stable/topics/forms/",
"https://docs.djangoproject.com/en/stable/topics/auth/",
"https://docs.djangoproject.com/en/stable/ref/models/"
],
"selectors": {
"main_content": "div.document",
"title": "h1",
"code_blocks": "pre"
},
"url_patterns": {
"include": ["/topics/", "/ref/", "/howto/"],
"exclude": ["/faq/", "/misc/"]
"include": ["/intro/", "/topics/", "/ref/", "/howto/"],
"exclude": ["/faq/", "/misc/", "/releases/"]
},
"categories": {
"getting_started": ["intro", "tutorial", "install"],

View File

@@ -2,6 +2,15 @@
"name": "fastapi",
"description": "FastAPI modern Python web framework. Use for building APIs, async endpoints, dependency injection, and Python backend development.",
"base_url": "https://fastapi.tiangolo.com/",
"start_urls": [
"https://fastapi.tiangolo.com/tutorial/",
"https://fastapi.tiangolo.com/tutorial/first-steps/",
"https://fastapi.tiangolo.com/tutorial/path-params/",
"https://fastapi.tiangolo.com/tutorial/body/",
"https://fastapi.tiangolo.com/tutorial/dependencies/",
"https://fastapi.tiangolo.com/advanced/",
"https://fastapi.tiangolo.com/reference/"
],
"selectors": {
"main_content": "article",
"title": "h1",
@@ -9,7 +18,7 @@
},
"url_patterns": {
"include": ["/tutorial/", "/advanced/", "/reference/"],
"exclude": ["/help/", "/external-links/"]
"exclude": ["/help/", "/external-links/", "/deployment/"]
},
"categories": {
"getting_started": ["first-steps", "tutorial", "intro"],

View File

@@ -2,13 +2,26 @@
"name": "godot",
"description": "Godot Engine game development. Use for Godot projects, GDScript/C# coding, scene setup, node systems, 2D/3D development, physics, animation, UI, shaders, or any Godot-specific questions.",
"base_url": "https://docs.godotengine.org/en/stable/",
"start_urls": [
"https://docs.godotengine.org/en/stable/getting_started/introduction/index.html",
"https://docs.godotengine.org/en/stable/tutorials/scripting/gdscript/index.html",
"https://docs.godotengine.org/en/stable/tutorials/2d/index.html",
"https://docs.godotengine.org/en/stable/tutorials/3d/index.html",
"https://docs.godotengine.org/en/stable/tutorials/physics/index.html",
"https://docs.godotengine.org/en/stable/tutorials/animation/index.html",
"https://docs.godotengine.org/en/stable/classes/index.html"
],
"selectors": {
"main_content": "div[role='main']",
"title": "title",
"code_blocks": "pre"
},
"url_patterns": {
"include": [],
"include": [
"/getting_started/",
"/tutorials/",
"/classes/"
],
"exclude": [
"/genindex.html",
"/search.html",

View File

@@ -2,6 +2,14 @@
"name": "react",
"description": "React framework for building user interfaces. Use for React components, hooks, state management, JSX, and modern frontend development.",
"base_url": "https://react.dev/",
"start_urls": [
"https://react.dev/learn",
"https://react.dev/learn/quick-start",
"https://react.dev/learn/thinking-in-react",
"https://react.dev/reference/react",
"https://react.dev/reference/react-dom",
"https://react.dev/reference/react/hooks"
],
"selectors": {
"main_content": "article",
"title": "h1",

View File

@@ -1,7 +1,15 @@
{
"name": "vue",
"description": "Vue.js progressive JavaScript framework. Use for Vue components, reactivity, composition API, and frontend development.",
"base_url": "https://vuejs.org/guide/",
"base_url": "https://vuejs.org/",
"start_urls": [
"https://vuejs.org/guide/introduction.html",
"https://vuejs.org/guide/quick-start.html",
"https://vuejs.org/guide/essentials/application.html",
"https://vuejs.org/guide/components/registration.html",
"https://vuejs.org/guide/reusability/composables.html",
"https://vuejs.org/api/"
],
"selectors": {
"main_content": "main",
"title": "h1",
@@ -9,7 +17,7 @@
},
"url_patterns": {
"include": ["/guide/", "/api/", "/examples/"],
"exclude": ["/about/", "/sponsor/"]
"exclude": ["/about/", "/sponsor/", "/partners/"]
},
"categories": {
"getting_started": ["quick-start", "introduction", "essentials"],