From 59c2f9126d40fa34385c813f1aca2979a983cbcc Mon Sep 17 00:00:00 2001 From: yusyus Date: Sun, 19 Oct 2025 02:24:56 +0300 Subject: [PATCH] Optimize all framework configs with start_urls for better coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All configs now follow the steam-economy-complete.json pattern with: - Multiple start_urls for comprehensive entry points - Improved include patterns for better targeting - Enhanced exclude patterns to skip irrelevant pages Godot Config: - Added 7 start_urls covering getting started, scripting, 2D, 3D, physics, animation, and classes - Added include patterns: /getting_started/, /tutorials/, /classes/ - More focused scraping of core documentation React Config: - Added 6 start_urls covering learn, quick-start, reference, and hooks - Existing patterns maintained (already well-optimized) Vue Config: - Added 6 start_urls covering introduction, essentials, components, composables, and API - Fixed base_url from https://vuejs.org/guide/ to https://vuejs.org/ - Added /partners/ to exclude list Django Config: - Added 7 start_urls covering intro, models, views, templates, forms, auth, and reference - Added /intro/ to include patterns - Added /releases/ to exclude list (changelog not needed) FastAPI Config: - Added 7 start_urls covering tutorial, first-steps, path-params, body, dependencies, advanced, and reference - Added /deployment/ to exclude list Benefits: - Better initial page discovery - More comprehensive documentation coverage - Faster scraping (direct entry to important sections) - Reduced unnecessary page crawling - Consistent pattern across all configs All configs tested and validated: ✅ 71/71 tests passing ✅ All 6 configs validated successfully 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- configs/django.json | 13 +++++++++++-- configs/fastapi.json | 11 ++++++++++- configs/godot.json | 15 ++++++++++++++- configs/react.json | 8 ++++++++ configs/vue.json | 12 ++++++++++-- 5 files changed, 53 insertions(+), 6 deletions(-) diff --git a/configs/django.json b/configs/django.json index b9fd865..b1a66bc 100644 --- a/configs/django.json +++ b/configs/django.json @@ -2,14 +2,23 @@ "name": "django", "description": "Django web framework for Python. Use for Django models, views, templates, ORM, authentication, and web development.", "base_url": "https://docs.djangoproject.com/en/stable/", + "start_urls": [ + "https://docs.djangoproject.com/en/stable/intro/", + "https://docs.djangoproject.com/en/stable/topics/db/models/", + "https://docs.djangoproject.com/en/stable/topics/http/views/", + "https://docs.djangoproject.com/en/stable/topics/templates/", + "https://docs.djangoproject.com/en/stable/topics/forms/", + "https://docs.djangoproject.com/en/stable/topics/auth/", + "https://docs.djangoproject.com/en/stable/ref/models/" + ], "selectors": { "main_content": "div.document", "title": "h1", "code_blocks": "pre" }, "url_patterns": { - "include": ["/topics/", "/ref/", "/howto/"], - "exclude": ["/faq/", "/misc/"] + "include": ["/intro/", "/topics/", "/ref/", "/howto/"], + "exclude": ["/faq/", "/misc/", "/releases/"] }, "categories": { "getting_started": ["intro", "tutorial", "install"], diff --git a/configs/fastapi.json b/configs/fastapi.json index 3dcbbfb..f08a08c 100644 --- a/configs/fastapi.json +++ b/configs/fastapi.json @@ -2,6 +2,15 @@ "name": "fastapi", "description": "FastAPI modern Python web framework. Use for building APIs, async endpoints, dependency injection, and Python backend development.", "base_url": "https://fastapi.tiangolo.com/", + "start_urls": [ + "https://fastapi.tiangolo.com/tutorial/", + "https://fastapi.tiangolo.com/tutorial/first-steps/", + "https://fastapi.tiangolo.com/tutorial/path-params/", + "https://fastapi.tiangolo.com/tutorial/body/", + "https://fastapi.tiangolo.com/tutorial/dependencies/", + "https://fastapi.tiangolo.com/advanced/", + "https://fastapi.tiangolo.com/reference/" + ], "selectors": { "main_content": "article", "title": "h1", @@ -9,7 +18,7 @@ }, "url_patterns": { "include": ["/tutorial/", "/advanced/", "/reference/"], - "exclude": ["/help/", "/external-links/"] + "exclude": ["/help/", "/external-links/", "/deployment/"] }, "categories": { "getting_started": ["first-steps", "tutorial", "intro"], diff --git a/configs/godot.json b/configs/godot.json index 962d66d..acd49f2 100644 --- a/configs/godot.json +++ b/configs/godot.json @@ -2,13 +2,26 @@ "name": "godot", "description": "Godot Engine game development. Use for Godot projects, GDScript/C# coding, scene setup, node systems, 2D/3D development, physics, animation, UI, shaders, or any Godot-specific questions.", "base_url": "https://docs.godotengine.org/en/stable/", + "start_urls": [ + "https://docs.godotengine.org/en/stable/getting_started/introduction/index.html", + "https://docs.godotengine.org/en/stable/tutorials/scripting/gdscript/index.html", + "https://docs.godotengine.org/en/stable/tutorials/2d/index.html", + "https://docs.godotengine.org/en/stable/tutorials/3d/index.html", + "https://docs.godotengine.org/en/stable/tutorials/physics/index.html", + "https://docs.godotengine.org/en/stable/tutorials/animation/index.html", + "https://docs.godotengine.org/en/stable/classes/index.html" + ], "selectors": { "main_content": "div[role='main']", "title": "title", "code_blocks": "pre" }, "url_patterns": { - "include": [], + "include": [ + "/getting_started/", + "/tutorials/", + "/classes/" + ], "exclude": [ "/genindex.html", "/search.html", diff --git a/configs/react.json b/configs/react.json index 9cd2231..e6f4c92 100644 --- a/configs/react.json +++ b/configs/react.json @@ -2,6 +2,14 @@ "name": "react", "description": "React framework for building user interfaces. Use for React components, hooks, state management, JSX, and modern frontend development.", "base_url": "https://react.dev/", + "start_urls": [ + "https://react.dev/learn", + "https://react.dev/learn/quick-start", + "https://react.dev/learn/thinking-in-react", + "https://react.dev/reference/react", + "https://react.dev/reference/react-dom", + "https://react.dev/reference/react/hooks" + ], "selectors": { "main_content": "article", "title": "h1", diff --git a/configs/vue.json b/configs/vue.json index 9a7ff3a..dc39d13 100644 --- a/configs/vue.json +++ b/configs/vue.json @@ -1,7 +1,15 @@ { "name": "vue", "description": "Vue.js progressive JavaScript framework. Use for Vue components, reactivity, composition API, and frontend development.", - "base_url": "https://vuejs.org/guide/", + "base_url": "https://vuejs.org/", + "start_urls": [ + "https://vuejs.org/guide/introduction.html", + "https://vuejs.org/guide/quick-start.html", + "https://vuejs.org/guide/essentials/application.html", + "https://vuejs.org/guide/components/registration.html", + "https://vuejs.org/guide/reusability/composables.html", + "https://vuejs.org/api/" + ], "selectors": { "main_content": "main", "title": "h1", @@ -9,7 +17,7 @@ }, "url_patterns": { "include": ["/guide/", "/api/", "/examples/"], - "exclude": ["/about/", "/sponsor/"] + "exclude": ["/about/", "/sponsor/", "/partners/"] }, "categories": { "getting_started": ["quick-start", "introduction", "essentials"],