feat(v2.7.0): Smart Rate Limit Management & Multi-Token Configuration
Major Features: - Multi-profile GitHub token system with secure storage - Smart rate limit handler with 4 strategies (prompt/wait/switch/fail) - Interactive configuration wizard with browser integration - Configurable timeout (default 30 min) per profile - Automatic profile switching on rate limits - Live countdown timers with real-time progress - Non-interactive mode for CI/CD (--non-interactive flag) - Progress tracking and resume capability (skeleton) - Comprehensive test suite (16 tests, all passing) Solves: - Indefinite waiting on GitHub rate limits - Confusing GitHub token setup Files Added: - src/skill_seekers/cli/config_manager.py (~490 lines) - src/skill_seekers/cli/config_command.py (~400 lines) - src/skill_seekers/cli/rate_limit_handler.py (~450 lines) - src/skill_seekers/cli/resume_command.py (~150 lines) - tests/test_rate_limit_handler.py (16 tests) Files Modified: - src/skill_seekers/cli/github_fetcher.py (rate limit integration) - src/skill_seekers/cli/github_scraper.py (--non-interactive, --profile flags) - src/skill_seekers/cli/main.py (config, resume subcommands) - pyproject.toml (version 2.7.0) - CHANGELOG.md, README.md, CLAUDE.md (documentation) Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,6 +8,7 @@ Usage:
|
||||
skill-seekers <command> [options]
|
||||
|
||||
Commands:
|
||||
config Configure GitHub tokens, API keys, and settings
|
||||
scrape Scrape documentation website
|
||||
github Scrape GitHub repository
|
||||
pdf Extract from PDF file
|
||||
@@ -19,6 +20,7 @@ Commands:
|
||||
estimate Estimate page count before scraping
|
||||
extract-test-examples Extract usage examples from test files
|
||||
install-agent Install skill to AI agent directories
|
||||
resume Resume interrupted scraping job
|
||||
|
||||
Examples:
|
||||
skill-seekers scrape --config configs/react.json
|
||||
@@ -65,7 +67,7 @@ For more information: https://github.com/yusufkaraaslan/Skill_Seekers
|
||||
parser.add_argument(
|
||||
"--version",
|
||||
action="version",
|
||||
version="%(prog)s 2.5.1"
|
||||
version="%(prog)s 2.7.0"
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(
|
||||
@@ -75,6 +77,33 @@ For more information: https://github.com/yusufkaraaslan/Skill_Seekers
|
||||
help="Command to run"
|
||||
)
|
||||
|
||||
# === config subcommand ===
|
||||
config_parser = subparsers.add_parser(
|
||||
"config",
|
||||
help="Configure GitHub tokens, API keys, and settings",
|
||||
description="Interactive configuration wizard"
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--github",
|
||||
action="store_true",
|
||||
help="Go directly to GitHub token setup"
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--api-keys",
|
||||
action="store_true",
|
||||
help="Go directly to API keys setup"
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--show",
|
||||
action="store_true",
|
||||
help="Show current configuration and exit"
|
||||
)
|
||||
config_parser.add_argument(
|
||||
"--test",
|
||||
action="store_true",
|
||||
help="Test connections and exit"
|
||||
)
|
||||
|
||||
# === scrape subcommand ===
|
||||
scrape_parser = subparsers.add_parser(
|
||||
"scrape",
|
||||
@@ -105,6 +134,8 @@ For more information: https://github.com/yusufkaraaslan/Skill_Seekers
|
||||
github_parser.add_argument("--enhance", action="store_true", help="AI enhancement (API)")
|
||||
github_parser.add_argument("--enhance-local", action="store_true", help="AI enhancement (local)")
|
||||
github_parser.add_argument("--api-key", type=str, help="Anthropic API key for --enhance")
|
||||
github_parser.add_argument("--non-interactive", action="store_true", help="Non-interactive mode (fail fast on rate limits)")
|
||||
github_parser.add_argument("--profile", type=str, help="GitHub profile name from config")
|
||||
|
||||
# === pdf subcommand ===
|
||||
pdf_parser = subparsers.add_parser(
|
||||
@@ -280,6 +311,28 @@ For more information: https://github.com/yusufkaraaslan/Skill_Seekers
|
||||
help="Preview workflow without executing"
|
||||
)
|
||||
|
||||
# === resume subcommand ===
|
||||
resume_parser = subparsers.add_parser(
|
||||
"resume",
|
||||
help="Resume interrupted scraping job",
|
||||
description="Continue from saved progress checkpoint"
|
||||
)
|
||||
resume_parser.add_argument(
|
||||
"job_id",
|
||||
nargs="?",
|
||||
help="Job ID to resume (or use --list to see available jobs)"
|
||||
)
|
||||
resume_parser.add_argument(
|
||||
"--list",
|
||||
action="store_true",
|
||||
help="List all resumable jobs"
|
||||
)
|
||||
resume_parser.add_argument(
|
||||
"--clean",
|
||||
action="store_true",
|
||||
help="Clean up old progress files"
|
||||
)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
@@ -301,7 +354,20 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
|
||||
# Delegate to the appropriate tool
|
||||
try:
|
||||
if args.command == "scrape":
|
||||
if args.command == "config":
|
||||
from skill_seekers.cli.config_command import main as config_main
|
||||
sys.argv = ["config_command.py"]
|
||||
if args.github:
|
||||
sys.argv.append("--github")
|
||||
if args.api_keys:
|
||||
sys.argv.append("--api-keys")
|
||||
if args.show:
|
||||
sys.argv.append("--show")
|
||||
if args.test:
|
||||
sys.argv.append("--test")
|
||||
return config_main() or 0
|
||||
|
||||
elif args.command == "scrape":
|
||||
from skill_seekers.cli.doc_scraper import main as scrape_main
|
||||
# Convert args namespace to sys.argv format for doc_scraper
|
||||
sys.argv = ["doc_scraper.py"]
|
||||
@@ -344,6 +410,10 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
sys.argv.append("--enhance-local")
|
||||
if args.api_key:
|
||||
sys.argv.extend(["--api-key", args.api_key])
|
||||
if args.non_interactive:
|
||||
sys.argv.append("--non-interactive")
|
||||
if args.profile:
|
||||
sys.argv.extend(["--profile", args.profile])
|
||||
return github_main() or 0
|
||||
|
||||
elif args.command == "pdf":
|
||||
@@ -464,6 +534,17 @@ def main(argv: Optional[List[str]] = None) -> int:
|
||||
sys.argv.append("--dry-run")
|
||||
return install_main() or 0
|
||||
|
||||
elif args.command == "resume":
|
||||
from skill_seekers.cli.resume_command import main as resume_main
|
||||
sys.argv = ["resume_command.py"]
|
||||
if args.job_id:
|
||||
sys.argv.append(args.job_id)
|
||||
if args.list:
|
||||
sys.argv.append("--list")
|
||||
if args.clean:
|
||||
sys.argv.append("--clean")
|
||||
return resume_main() or 0
|
||||
|
||||
else:
|
||||
print(f"Error: Unknown command '{args.command}'", file=sys.stderr)
|
||||
parser.print_help()
|
||||
|
||||
Reference in New Issue
Block a user