docs: update README, CHANGELOG, plugin JSONs with batch 2 skills (59→65)
This commit is contained in:
Leo
@@ -4,11 +4,11 @@
|
||||
"name": "Alireza Rezvani",
|
||||
"url": "https://alirezarezvani.com"
|
||||
},
|
||||
"description": "Production-ready skill packages for Claude AI - 53 expert skills across marketing, engineering, product, C-level advisory, project management, regulatory compliance, business growth, and finance",
|
||||
"description": "Production-ready skill packages for Claude AI - 65 expert skills across marketing, engineering, product, C-level advisory, project management, regulatory compliance, business growth, and finance",
|
||||
"homepage": "https://github.com/alirezarezvani/claude-skills",
|
||||
"repository": "https://github.com/alirezarezvani/claude-skills",
|
||||
"metadata": {
|
||||
"description": "53 production-ready skill packages across 8 domains: marketing, engineering, product, C-level advisory, project management, regulatory compliance, business growth, and finance",
|
||||
"description": "65 production-ready skill packages across 8 domains: marketing, engineering, product, C-level advisory, project management, regulatory compliance, business growth, and finance",
|
||||
"version": "1.0.0"
|
||||
},
|
||||
"plugins": [
|
||||
@@ -26,7 +26,7 @@
|
||||
{
|
||||
"name": "engineering-skills",
|
||||
"source": "./engineering-team",
|
||||
"description": "18 engineering skills: architecture, frontend, backend, fullstack, QA, DevOps, security, AI/ML, data engineering",
|
||||
"description": "30 engineering skills: architecture, frontend, backend, fullstack, QA, DevOps, security, AI/ML, data engineering",
|
||||
"version": "1.0.0",
|
||||
"author": {
|
||||
"name": "Alireza Rezvani"
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"name": "claude-code-skills",
|
||||
"description": "Production-ready skill packages for AI agents - Marketing, Engineering, Product, C-Level, PM, and RA/QM",
|
||||
"repository": "https://github.com/alirezarezvani/claude-skills",
|
||||
"total_skills": 59,
|
||||
"total_skills": 65,
|
||||
"skills": [
|
||||
{
|
||||
"name": "customer-success-manager",
|
||||
@@ -366,6 +366,42 @@
|
||||
],
|
||||
"references": true,
|
||||
"assets": true
|
||||
},
|
||||
{
|
||||
"name": "dependency-auditor",
|
||||
"source": "../../engineering/dependency-auditor",
|
||||
"category": "engineering",
|
||||
"description": "Multi-language dependency scanning, license compliance, and upgrade planning for modern software projects"
|
||||
},
|
||||
{
|
||||
"name": "release-manager",
|
||||
"source": "../../engineering/release-manager",
|
||||
"category": "engineering",
|
||||
"description": "Automated changelog generation, semantic version bumping, and release readiness planning for production deployments"
|
||||
},
|
||||
{
|
||||
"name": "database-designer",
|
||||
"source": "../../engineering/database-designer",
|
||||
"category": "engineering",
|
||||
"description": "Schema analysis with ERD generation, index optimization, and migration generation for database architecture"
|
||||
},
|
||||
{
|
||||
"name": "rag-architect",
|
||||
"source": "../../engineering/rag-architect",
|
||||
"category": "engineering",
|
||||
"description": "RAG pipeline design with chunking optimization, retrieval evaluation, and architecture generation for AI systems"
|
||||
},
|
||||
{
|
||||
"name": "agent-designer",
|
||||
"source": "../../engineering/agent-designer",
|
||||
"category": "engineering",
|
||||
"description": "Multi-agent system architecture, tool schema generation, and agent performance evaluation for agentic AI"
|
||||
},
|
||||
{
|
||||
"name": "skill-tester",
|
||||
"source": "../../engineering/skill-tester",
|
||||
"category": "engineering",
|
||||
"description": "Meta-skill for automated skill validation, script testing, and quality scoring for skill development workflows"
|
||||
}
|
||||
],
|
||||
"categories": {
|
||||
@@ -380,7 +416,7 @@
|
||||
"description": "Executive leadership and advisory skills"
|
||||
},
|
||||
"engineering": {
|
||||
"count": 19,
|
||||
"count": 25,
|
||||
"source": "../../engineering-team",
|
||||
"description": "Software engineering and technical skills"
|
||||
},
|
||||
@@ -410,5 +446,5 @@
|
||||
"description": "Regulatory affairs and quality management skills"
|
||||
}
|
||||
},
|
||||
"total": 59
|
||||
"total": 65
|
||||
}
|
||||
@@ -14,6 +14,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **interview-system-designer** (POWERFUL tier) — Interview loop designer, question bank generator, and hiring calibrator
|
||||
- **migration-architect** (POWERFUL tier) — Migration planner, compatibility checker, and rollback generator
|
||||
- **observability-designer** (POWERFUL tier) — SLO designer, alert optimizer, and dashboard generator
|
||||
- **dependency-auditor** (POWERFUL tier) — Multi-language dependency scanner, license compliance checker, and upgrade planner
|
||||
- **release-manager** (POWERFUL tier) — Automated changelog generator, semantic version bumper, and release readiness checker
|
||||
- **database-designer** (POWERFUL tier) — Schema analyzer with ERD generation, index optimizer, and migration generator
|
||||
- **rag-architect** (POWERFUL tier) — RAG pipeline builder, chunking optimizer, and retrieval evaluator
|
||||
- **agent-designer** (POWERFUL tier) — Multi-agent architect, tool schema generator, and agent performance evaluator
|
||||
- **skill-tester** (POWERFUL tier) — Meta-skill validator, script tester, and quality scorer
|
||||
- `campaign-analytics` - Multi-touch attribution, funnel conversion, campaign ROI (3 Python tools)
|
||||
- `customer-success-manager` - Onboarding, retention, expansion, health scoring (2 Python tools)
|
||||
- `sales-engineer` - Technical sales, solution design, RFP responses (2 Python tools)
|
||||
@@ -22,7 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- New `business-growth` domain with 3 skills
|
||||
- New `finance` domain with 1 skill
|
||||
- 92+ Python automation tools (up from 87+, including 17 new POWERFUL-tier tools)
|
||||
- 58 total skills across 8 domains (up from 53)
|
||||
- 64 total skills across 8 domains (up from 53)
|
||||
|
||||
### Fixed
|
||||
- CI workflows (smart-sync.yml, pr-issue-auto-close.yml) — PR #193
|
||||
|
||||
106
README.md
106
README.md
@@ -6,7 +6,7 @@
|
||||
[](https://claude.ai)
|
||||
[](https://claude.ai/code)
|
||||
[](https://github.com/skillcreatorai/Ai-Agent-Skills)
|
||||
[](#-available-skills)
|
||||
[](#-available-skills)
|
||||
[](https://getskillcheck.com)
|
||||
|
||||
---
|
||||
@@ -533,7 +533,7 @@ Template and file creation/modification specialist.
|
||||
|
||||
### Engineering Team Skills
|
||||
|
||||
**Complete engineering skills suite with 13 specialized roles** covering architecture, development, testing, security, operations, cloud infrastructure, and enterprise systems.
|
||||
**Complete engineering skills suite with 19 specialized roles** covering architecture, development, testing, security, operations, cloud infrastructure, and enterprise systems.
|
||||
|
||||
#### 🏗️ Senior Software Architect
|
||||
**Status:** ✅ Production Ready | **Version:** 1.0
|
||||
@@ -762,6 +762,108 @@ Comprehensive technology evaluation with TCO analysis, security assessment, and
|
||||
|
||||
---
|
||||
|
||||
#### 📦 Dependency Auditor
|
||||
**Status:** ✅ Production Ready | **Version:** 1.0
|
||||
|
||||
Multi-language dependency scanning, license compliance, and upgrade planning for modern software projects.
|
||||
|
||||
**What's Included:**
|
||||
- **Dependency Scanner** - Multi-language dependency analysis and vulnerability detection (Python CLI)
|
||||
- **License Compliance Checker** - License compatibility and compliance validation (Python CLI)
|
||||
- **Upgrade Planner** - Strategic dependency upgrade planning with risk assessment (Python CLI)
|
||||
- **Security Vulnerability Assessment** - CVE analysis and remediation recommendations
|
||||
- **License Compatibility Matrix** - Legal compliance checking across multiple licenses
|
||||
- **Upgrade Impact Analysis** - Breaking changes and migration effort estimation
|
||||
|
||||
**Learn More:** [engineering/dependency-auditor/SKILL.md](engineering/dependency-auditor/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
#### 🚀 Release Manager
|
||||
**Status:** ✅ Production Ready | **Version:** 1.0
|
||||
|
||||
Automated changelog generation, semantic version bumping, and release readiness planning for production deployments.
|
||||
|
||||
**What's Included:**
|
||||
- **Changelog Generator** - Automated changelog creation from commit history (Python CLI)
|
||||
- **Version Bumper** - Semantic versioning with automated version increment (Python CLI)
|
||||
- **Release Readiness Checker** - Pre-release validation and readiness assessment (Python CLI)
|
||||
- **Release Notes Automation** - Generate comprehensive release documentation
|
||||
- **Semantic Versioning Guide** - Best practices for version management
|
||||
- **Release Pipeline Integration** - CI/CD release workflow optimization
|
||||
|
||||
**Learn More:** [engineering/release-manager/SKILL.md](engineering/release-manager/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
#### 🗄️ Database Designer
|
||||
**Status:** ✅ Production Ready | **Version:** 1.0
|
||||
|
||||
Schema analysis with ERD generation, index optimization, and migration generation for database architecture.
|
||||
|
||||
**What's Included:**
|
||||
- **Schema Analyzer** - Database schema analysis and optimization recommendations (Python CLI)
|
||||
- **ERD Generator** - Entity Relationship Diagram generation from schema (Python CLI)
|
||||
- **Index Optimizer** - Query performance optimization through strategic indexing (Python CLI)
|
||||
- **Migration Generator** - Automated database migration script creation (Python CLI)
|
||||
- **Schema Validation** - Database integrity and constraint validation
|
||||
- **Performance Tuning Guide** - Query optimization and database performance patterns
|
||||
|
||||
**Learn More:** [engineering/database-designer/SKILL.md](engineering/database-designer/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
#### 🧠 RAG Architect
|
||||
**Status:** ✅ Production Ready | **Version:** 1.0
|
||||
|
||||
RAG pipeline design with chunking optimization, retrieval evaluation, and architecture generation for AI systems.
|
||||
|
||||
**What's Included:**
|
||||
- **RAG Pipeline Builder** - Complete RAG system architecture and implementation (Python CLI)
|
||||
- **Chunking Optimizer** - Document chunking strategy optimization for retrieval (Python CLI)
|
||||
- **Retrieval Evaluator** - RAG system performance evaluation and tuning (Python CLI)
|
||||
- **Vector Database Integration** - Multi-provider vector database setup and optimization
|
||||
- **Embedding Strategy Guide** - Embedding model selection and fine-tuning
|
||||
- **RAG Performance Patterns** - Architecture patterns for scalable RAG systems
|
||||
|
||||
**Learn More:** [engineering/rag-architect/SKILL.md](engineering/rag-architect/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
#### 🤖 Agent Designer
|
||||
**Status:** ✅ Production Ready | **Version:** 1.0
|
||||
|
||||
Multi-agent system architecture, tool schema generation, and agent performance evaluation for agentic AI.
|
||||
|
||||
**What's Included:**
|
||||
- **Multi-Agent Architect** - Design and implement multi-agent system architecture (Python CLI)
|
||||
- **Tool Schema Generator** - Generate standardized tool schemas for agent integration (Python CLI)
|
||||
- **Agent Performance Evaluator** - Comprehensive agent performance analysis and optimization (Python CLI)
|
||||
- **Agent Communication Patterns** - Inter-agent communication and coordination strategies
|
||||
- **Tool Integration Framework** - Standardized tool integration patterns for agents
|
||||
- **Agent Orchestration Guide** - Best practices for agent system orchestration
|
||||
|
||||
**Learn More:** [engineering/agent-designer/SKILL.md](engineering/agent-designer/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
#### 🧪 Skill Tester
|
||||
**Status:** ✅ Production Ready | **Version:** 1.0
|
||||
|
||||
Meta-skill for automated skill validation, script testing, and quality scoring for skill development workflows.
|
||||
|
||||
**What's Included:**
|
||||
- **Skill Validator** - Automated skill functionality validation and testing (Python CLI)
|
||||
- **Script Tester** - Comprehensive testing framework for skill scripts (Python CLI)
|
||||
- **Quality Scorer** - Skill quality assessment and scoring system (Python CLI)
|
||||
- **Skill CI/CD Integration** - Automated testing in skill development workflows
|
||||
- **Quality Metrics Framework** - Standardized quality assessment criteria
|
||||
- **Skill Performance Benchmarking** - Performance testing and optimization guidance
|
||||
|
||||
**Learn More:** [engineering/skill-tester/SKILL.md](engineering/skill-tester/SKILL.md)
|
||||
|
||||
---
|
||||
|
||||
### AI/ML/Data Team Skills
|
||||
|
||||
**5 specialized AI/ML and data engineering skills** for building modern data-driven and AI-powered products.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "engineering-skills",
|
||||
"description": "18 production-ready engineering skills covering architecture, frontend, backend, fullstack, QA, DevOps, security, AI/ML, and data engineering",
|
||||
"description": "30 production-ready engineering skills covering architecture, frontend, backend, fullstack, QA, DevOps, security, AI/ML, and data engineering",
|
||||
"version": "1.0.0",
|
||||
"author": {
|
||||
"name": "Alireza Rezvani",
|
||||
|
||||
430
engineering/agent-designer/README.md
Normal file
430
engineering/agent-designer/README.md
Normal file
@@ -0,0 +1,430 @@
|
||||
# Agent Designer - Multi-Agent System Architecture Toolkit
|
||||
|
||||
**Tier:** POWERFUL
|
||||
**Category:** Engineering
|
||||
**Tags:** AI agents, architecture, system design, orchestration, multi-agent systems
|
||||
|
||||
A comprehensive toolkit for designing, architecting, and evaluating multi-agent systems. Provides structured approaches to agent architecture patterns, tool design principles, communication strategies, and performance evaluation frameworks.
|
||||
|
||||
## Overview
|
||||
|
||||
The Agent Designer skill includes three core components:
|
||||
|
||||
1. **Agent Planner** (`agent_planner.py`) - Designs multi-agent system architectures
|
||||
2. **Tool Schema Generator** (`tool_schema_generator.py`) - Creates structured tool schemas
|
||||
3. **Agent Evaluator** (`agent_evaluator.py`) - Evaluates system performance and identifies optimizations
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Design a Multi-Agent Architecture
|
||||
|
||||
```bash
|
||||
# Use sample requirements or create your own
|
||||
python agent_planner.py assets/sample_system_requirements.json -o my_architecture
|
||||
|
||||
# This generates:
|
||||
# - my_architecture.json (complete architecture)
|
||||
# - my_architecture_diagram.mmd (Mermaid diagram)
|
||||
# - my_architecture_roadmap.json (implementation plan)
|
||||
```
|
||||
|
||||
### 2. Generate Tool Schemas
|
||||
|
||||
```bash
|
||||
# Use sample tool descriptions or create your own
|
||||
python tool_schema_generator.py assets/sample_tool_descriptions.json -o my_tools
|
||||
|
||||
# This generates:
|
||||
# - my_tools.json (complete schemas)
|
||||
# - my_tools_openai.json (OpenAI format)
|
||||
# - my_tools_anthropic.json (Anthropic format)
|
||||
# - my_tools_validation.json (validation rules)
|
||||
# - my_tools_examples.json (usage examples)
|
||||
```
|
||||
|
||||
### 3. Evaluate System Performance
|
||||
|
||||
```bash
|
||||
# Use sample execution logs or your own
|
||||
python agent_evaluator.py assets/sample_execution_logs.json -o evaluation
|
||||
|
||||
# This generates:
|
||||
# - evaluation.json (complete report)
|
||||
# - evaluation_summary.json (executive summary)
|
||||
# - evaluation_recommendations.json (optimization suggestions)
|
||||
# - evaluation_errors.json (error analysis)
|
||||
```
|
||||
|
||||
## Detailed Usage
|
||||
|
||||
### Agent Planner
|
||||
|
||||
The Agent Planner designs multi-agent architectures based on system requirements.
|
||||
|
||||
#### Input Format
|
||||
|
||||
Create a JSON file with system requirements:
|
||||
|
||||
```json
|
||||
{
|
||||
"goal": "Your system's primary objective",
|
||||
"description": "Detailed system description",
|
||||
"tasks": ["List", "of", "required", "tasks"],
|
||||
"constraints": {
|
||||
"max_response_time": 30000,
|
||||
"budget_per_task": 1.0,
|
||||
"quality_threshold": 0.9
|
||||
},
|
||||
"team_size": 6,
|
||||
"performance_requirements": {
|
||||
"high_throughput": true,
|
||||
"fault_tolerance": true,
|
||||
"low_latency": false
|
||||
},
|
||||
"safety_requirements": [
|
||||
"Input validation and sanitization",
|
||||
"Output content filtering"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Command Line Options
|
||||
|
||||
```bash
|
||||
python agent_planner.py <input_file> [OPTIONS]
|
||||
|
||||
Options:
|
||||
-o, --output PREFIX Output file prefix (default: agent_architecture)
|
||||
--format FORMAT Output format: json, both (default: both)
|
||||
```
|
||||
|
||||
#### Output Files
|
||||
|
||||
- **Architecture JSON**: Complete system design with agents, communication topology, and scaling strategy
|
||||
- **Mermaid Diagram**: Visual representation of the agent architecture
|
||||
- **Implementation Roadmap**: Phased implementation plan with timelines and risks
|
||||
|
||||
#### Architecture Patterns
|
||||
|
||||
The planner automatically selects from these patterns based on requirements:
|
||||
|
||||
- **Single Agent**: Simple, focused tasks (1 agent)
|
||||
- **Supervisor**: Hierarchical delegation (2-8 agents)
|
||||
- **Swarm**: Peer-to-peer collaboration (3-20 agents)
|
||||
- **Hierarchical**: Multi-level management (5-50 agents)
|
||||
- **Pipeline**: Sequential processing (3-15 agents)
|
||||
|
||||
### Tool Schema Generator
|
||||
|
||||
Generates structured tool schemas compatible with OpenAI and Anthropic formats.
|
||||
|
||||
#### Input Format
|
||||
|
||||
Create a JSON file with tool descriptions:
|
||||
|
||||
```json
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"name": "tool_name",
|
||||
"purpose": "What the tool does",
|
||||
"category": "Tool category (search, data, api, etc.)",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "parameter_name",
|
||||
"type": "string",
|
||||
"description": "Parameter description",
|
||||
"required": true,
|
||||
"examples": ["example1", "example2"]
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "result_field",
|
||||
"type": "object",
|
||||
"description": "Output description"
|
||||
}
|
||||
],
|
||||
"error_conditions": ["List of possible errors"],
|
||||
"side_effects": ["List of side effects"],
|
||||
"idempotent": true,
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 60
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Command Line Options
|
||||
|
||||
```bash
|
||||
python tool_schema_generator.py <input_file> [OPTIONS]
|
||||
|
||||
Options:
|
||||
-o, --output PREFIX Output file prefix (default: tool_schemas)
|
||||
--format FORMAT Output format: json, both (default: both)
|
||||
--validate Validate generated schemas
|
||||
```
|
||||
|
||||
#### Output Files
|
||||
|
||||
- **Complete Schemas**: All schemas with validation and examples
|
||||
- **OpenAI Format**: Schemas compatible with OpenAI function calling
|
||||
- **Anthropic Format**: Schemas compatible with Anthropic tool use
|
||||
- **Validation Rules**: Input validation specifications
|
||||
- **Usage Examples**: Example calls and responses
|
||||
|
||||
#### Schema Features
|
||||
|
||||
- **Input Validation**: Comprehensive parameter validation rules
|
||||
- **Error Handling**: Structured error response formats
|
||||
- **Rate Limiting**: Configurable rate limit specifications
|
||||
- **Documentation**: Auto-generated usage examples
|
||||
- **Security**: Built-in security considerations
|
||||
|
||||
### Agent Evaluator
|
||||
|
||||
Analyzes agent execution logs to identify performance issues and optimization opportunities.
|
||||
|
||||
#### Input Format
|
||||
|
||||
Create a JSON file with execution logs:
|
||||
|
||||
```json
|
||||
{
|
||||
"execution_logs": [
|
||||
{
|
||||
"task_id": "unique_task_identifier",
|
||||
"agent_id": "agent_identifier",
|
||||
"task_type": "task_category",
|
||||
"start_time": "2024-01-15T09:00:00Z",
|
||||
"end_time": "2024-01-15T09:02:34Z",
|
||||
"duration_ms": 154000,
|
||||
"status": "success",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 2300,
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"summary": "Task results",
|
||||
"quality_score": 0.92
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 1250,
|
||||
"output_tokens": 2800,
|
||||
"total_tokens": 4050
|
||||
},
|
||||
"cost_usd": 0.081,
|
||||
"error_details": null,
|
||||
"tools_used": ["web_search"],
|
||||
"retry_count": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
#### Command Line Options
|
||||
|
||||
```bash
|
||||
python agent_evaluator.py <input_file> [OPTIONS]
|
||||
|
||||
Options:
|
||||
-o, --output PREFIX Output file prefix (default: evaluation_report)
|
||||
--format FORMAT Output format: json, both (default: both)
|
||||
--detailed Include detailed analysis in output
|
||||
```
|
||||
|
||||
#### Output Files
|
||||
|
||||
- **Complete Report**: Comprehensive performance analysis
|
||||
- **Executive Summary**: High-level metrics and health assessment
|
||||
- **Optimization Recommendations**: Prioritized improvement suggestions
|
||||
- **Error Analysis**: Detailed error patterns and solutions
|
||||
|
||||
#### Evaluation Metrics
|
||||
|
||||
**Performance Metrics**:
|
||||
- Task success rate and completion times
|
||||
- Token usage and cost efficiency
|
||||
- Error rates and retry patterns
|
||||
- Throughput and latency distributions
|
||||
|
||||
**System Health**:
|
||||
- Overall health score (poor/fair/good/excellent)
|
||||
- SLA compliance tracking
|
||||
- Resource utilization analysis
|
||||
- Trend identification
|
||||
|
||||
**Bottleneck Analysis**:
|
||||
- Agent performance bottlenecks
|
||||
- Tool usage inefficiencies
|
||||
- Communication overhead
|
||||
- Resource constraints
|
||||
|
||||
## Architecture Patterns Guide
|
||||
|
||||
### When to Use Each Pattern
|
||||
|
||||
#### Single Agent
|
||||
- **Best for**: Simple, focused tasks with clear boundaries
|
||||
- **Team size**: 1 agent
|
||||
- **Complexity**: Low
|
||||
- **Examples**: Personal assistant, document summarizer, simple automation
|
||||
|
||||
#### Supervisor
|
||||
- **Best for**: Hierarchical task decomposition with quality control
|
||||
- **Team size**: 2-8 agents
|
||||
- **Complexity**: Medium
|
||||
- **Examples**: Research coordinator with specialists, content review workflow
|
||||
|
||||
#### Swarm
|
||||
- **Best for**: Distributed problem solving with high fault tolerance
|
||||
- **Team size**: 3-20 agents
|
||||
- **Complexity**: High
|
||||
- **Examples**: Parallel data processing, distributed research, competitive analysis
|
||||
|
||||
#### Hierarchical
|
||||
- **Best for**: Large-scale operations with organizational structure
|
||||
- **Team size**: 5-50 agents
|
||||
- **Complexity**: Very High
|
||||
- **Examples**: Enterprise workflows, complex business processes
|
||||
|
||||
#### Pipeline
|
||||
- **Best for**: Sequential processing with specialized stages
|
||||
- **Team size**: 3-15 agents
|
||||
- **Complexity**: Medium
|
||||
- **Examples**: Data ETL pipelines, content processing workflows
|
||||
|
||||
## Best Practices
|
||||
|
||||
### System Design
|
||||
|
||||
1. **Start Simple**: Begin with simpler patterns and evolve
|
||||
2. **Clear Responsibilities**: Define distinct roles for each agent
|
||||
3. **Robust Communication**: Design reliable message passing
|
||||
4. **Error Handling**: Plan for failures and recovery
|
||||
5. **Monitor Everything**: Implement comprehensive observability
|
||||
|
||||
### Tool Design
|
||||
|
||||
1. **Single Responsibility**: Each tool should have one clear purpose
|
||||
2. **Input Validation**: Validate all inputs thoroughly
|
||||
3. **Idempotency**: Design operations to be safely repeatable
|
||||
4. **Error Recovery**: Provide clear error messages and recovery paths
|
||||
5. **Documentation**: Include comprehensive usage examples
|
||||
|
||||
### Performance Optimization
|
||||
|
||||
1. **Measure First**: Use the evaluator to identify actual bottlenecks
|
||||
2. **Optimize Bottlenecks**: Focus on highest-impact improvements
|
||||
3. **Cache Strategically**: Cache expensive operations and results
|
||||
4. **Parallel Processing**: Identify opportunities for parallelization
|
||||
5. **Resource Management**: Monitor and optimize resource usage
|
||||
|
||||
## Sample Files
|
||||
|
||||
The `assets/` directory contains sample files to help you get started:
|
||||
|
||||
- **`sample_system_requirements.json`**: Example system requirements for a research platform
|
||||
- **`sample_tool_descriptions.json`**: Example tool descriptions for common operations
|
||||
- **`sample_execution_logs.json`**: Example execution logs from a running system
|
||||
|
||||
The `expected_outputs/` directory shows expected results from processing these samples.
|
||||
|
||||
## References
|
||||
|
||||
See the `references/` directory for detailed documentation:
|
||||
|
||||
- **`agent_architecture_patterns.md`**: Comprehensive catalog of architecture patterns
|
||||
- **`tool_design_best_practices.md`**: Best practices for tool design and implementation
|
||||
- **`evaluation_methodology.md`**: Detailed methodology for system evaluation
|
||||
|
||||
## Integration Examples
|
||||
|
||||
### With OpenAI
|
||||
|
||||
```python
|
||||
import json
|
||||
import openai
|
||||
|
||||
# Load generated OpenAI schemas
|
||||
with open('my_tools_openai.json') as f:
|
||||
schemas = json.load(f)
|
||||
|
||||
# Use with OpenAI function calling
|
||||
response = openai.ChatCompletion.create(
|
||||
model="gpt-4",
|
||||
messages=[{"role": "user", "content": "Search for AI news"}],
|
||||
functions=schemas['functions']
|
||||
)
|
||||
```
|
||||
|
||||
### With Anthropic Claude
|
||||
|
||||
```python
|
||||
import json
|
||||
import anthropic
|
||||
|
||||
# Load generated Anthropic schemas
|
||||
with open('my_tools_anthropic.json') as f:
|
||||
schemas = json.load(f)
|
||||
|
||||
# Use with Anthropic tool use
|
||||
client = anthropic.Anthropic()
|
||||
response = client.messages.create(
|
||||
model="claude-3-opus-20240229",
|
||||
messages=[{"role": "user", "content": "Search for AI news"}],
|
||||
tools=schemas['tools']
|
||||
)
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**"No valid architecture pattern found"**
|
||||
- Check that team_size is reasonable (1-50)
|
||||
- Ensure tasks list is not empty
|
||||
- Verify performance_requirements are valid
|
||||
|
||||
**"Tool schema validation failed"**
|
||||
- Check that all required fields are present
|
||||
- Ensure parameter types are valid
|
||||
- Verify enum values are provided as arrays
|
||||
|
||||
**"Insufficient execution logs"**
|
||||
- Ensure logs contain required fields (task_id, agent_id, status)
|
||||
- Check that timestamps are in ISO 8601 format
|
||||
- Verify token usage fields are numeric
|
||||
|
||||
### Performance Tips
|
||||
|
||||
1. **Large Systems**: For systems with >20 agents, consider breaking into subsystems
|
||||
2. **Complex Tools**: Tools with >10 parameters may need simplification
|
||||
3. **Log Volume**: For >1000 log entries, consider sampling for faster analysis
|
||||
|
||||
## Contributing
|
||||
|
||||
This skill is part of the claude-skills repository. To contribute:
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Add tests and documentation
|
||||
5. Submit a pull request
|
||||
|
||||
## License
|
||||
|
||||
This project is licensed under the MIT License - see the main repository for details.
|
||||
|
||||
## Support
|
||||
|
||||
For issues and questions:
|
||||
- Check the troubleshooting section above
|
||||
- Review the reference documentation in `references/`
|
||||
- Create an issue in the claude-skills repository
|
||||
274
engineering/agent-designer/SKILL.md
Normal file
274
engineering/agent-designer/SKILL.md
Normal file
@@ -0,0 +1,274 @@
|
||||
# Agent Designer - Multi-Agent System Architecture
|
||||
|
||||
**Tier:** POWERFUL
|
||||
**Category:** Engineering
|
||||
**Tags:** AI agents, architecture, system design, orchestration, multi-agent systems
|
||||
|
||||
## Overview
|
||||
|
||||
Agent Designer is a comprehensive toolkit for designing, architecting, and evaluating multi-agent systems. It provides structured approaches to agent architecture patterns, tool design principles, communication strategies, and performance evaluation frameworks for building robust, scalable AI agent systems.
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
### 1. Agent Architecture Patterns
|
||||
|
||||
#### Single Agent Pattern
|
||||
- **Use Case:** Simple, focused tasks with clear boundaries
|
||||
- **Pros:** Minimal complexity, easy debugging, predictable behavior
|
||||
- **Cons:** Limited scalability, single point of failure
|
||||
- **Implementation:** Direct user-agent interaction with comprehensive tool access
|
||||
|
||||
#### Supervisor Pattern
|
||||
- **Use Case:** Hierarchical task decomposition with centralized control
|
||||
- **Architecture:** One supervisor agent coordinating multiple specialist agents
|
||||
- **Pros:** Clear command structure, centralized decision making
|
||||
- **Cons:** Supervisor bottleneck, complex coordination logic
|
||||
- **Implementation:** Supervisor receives tasks, delegates to specialists, aggregates results
|
||||
|
||||
#### Swarm Pattern
|
||||
- **Use Case:** Distributed problem solving with peer-to-peer collaboration
|
||||
- **Architecture:** Multiple autonomous agents with shared objectives
|
||||
- **Pros:** High parallelism, fault tolerance, emergent intelligence
|
||||
- **Cons:** Complex coordination, potential conflicts, harder to predict
|
||||
- **Implementation:** Agent discovery, consensus mechanisms, distributed task allocation
|
||||
|
||||
#### Hierarchical Pattern
|
||||
- **Use Case:** Complex systems with multiple organizational layers
|
||||
- **Architecture:** Tree structure with managers and workers at different levels
|
||||
- **Pros:** Natural organizational mapping, clear responsibilities
|
||||
- **Cons:** Communication overhead, potential bottlenecks at each level
|
||||
- **Implementation:** Multi-level delegation with feedback loops
|
||||
|
||||
#### Pipeline Pattern
|
||||
- **Use Case:** Sequential processing with specialized stages
|
||||
- **Architecture:** Agents arranged in processing pipeline
|
||||
- **Pros:** Clear data flow, specialized optimization per stage
|
||||
- **Cons:** Sequential bottlenecks, rigid processing order
|
||||
- **Implementation:** Message queues between stages, state handoffs
|
||||
|
||||
### 2. Agent Role Definition
|
||||
|
||||
#### Role Specification Framework
|
||||
- **Identity:** Name, purpose statement, core competencies
|
||||
- **Responsibilities:** Primary tasks, decision boundaries, success criteria
|
||||
- **Capabilities:** Required tools, knowledge domains, processing limits
|
||||
- **Interfaces:** Input/output formats, communication protocols
|
||||
- **Constraints:** Security boundaries, resource limits, operational guidelines
|
||||
|
||||
#### Common Agent Archetypes
|
||||
|
||||
**Coordinator Agent**
|
||||
- Orchestrates multi-agent workflows
|
||||
- Makes high-level decisions and resource allocation
|
||||
- Monitors system health and performance
|
||||
- Handles escalations and conflict resolution
|
||||
|
||||
**Specialist Agent**
|
||||
- Deep expertise in specific domain (code, data, research)
|
||||
- Optimized tools and knowledge for specialized tasks
|
||||
- High-quality output within narrow scope
|
||||
- Clear handoff protocols for out-of-scope requests
|
||||
|
||||
**Interface Agent**
|
||||
- Handles external interactions (users, APIs, systems)
|
||||
- Protocol translation and format conversion
|
||||
- Authentication and authorization management
|
||||
- User experience optimization
|
||||
|
||||
**Monitor Agent**
|
||||
- System health monitoring and alerting
|
||||
- Performance metrics collection and analysis
|
||||
- Anomaly detection and reporting
|
||||
- Compliance and audit trail maintenance
|
||||
|
||||
### 3. Tool Design Principles
|
||||
|
||||
#### Schema Design
|
||||
- **Input Validation:** Strong typing, required vs optional parameters
|
||||
- **Output Consistency:** Standardized response formats, error handling
|
||||
- **Documentation:** Clear descriptions, usage examples, edge cases
|
||||
- **Versioning:** Backward compatibility, migration paths
|
||||
|
||||
#### Error Handling Patterns
|
||||
- **Graceful Degradation:** Partial functionality when dependencies fail
|
||||
- **Retry Logic:** Exponential backoff, circuit breakers, max attempts
|
||||
- **Error Propagation:** Structured error responses, error classification
|
||||
- **Recovery Strategies:** Fallback methods, alternative approaches
|
||||
|
||||
#### Idempotency Requirements
|
||||
- **Safe Operations:** Read operations with no side effects
|
||||
- **Idempotent Writes:** Same operation can be safely repeated
|
||||
- **State Management:** Version tracking, conflict resolution
|
||||
- **Atomicity:** All-or-nothing operation completion
|
||||
|
||||
### 4. Communication Patterns
|
||||
|
||||
#### Message Passing
|
||||
- **Asynchronous Messaging:** Decoupled agents, message queues
|
||||
- **Message Format:** Structured payloads with metadata
|
||||
- **Delivery Guarantees:** At-least-once, exactly-once semantics
|
||||
- **Routing:** Direct messaging, publish-subscribe, broadcast
|
||||
|
||||
#### Shared State
|
||||
- **State Stores:** Centralized data repositories
|
||||
- **Consistency Models:** Strong, eventual, weak consistency
|
||||
- **Access Patterns:** Read-heavy, write-heavy, mixed workloads
|
||||
- **Conflict Resolution:** Last-writer-wins, merge strategies
|
||||
|
||||
#### Event-Driven Architecture
|
||||
- **Event Sourcing:** Immutable event logs, state reconstruction
|
||||
- **Event Types:** Domain events, system events, integration events
|
||||
- **Event Processing:** Real-time, batch, stream processing
|
||||
- **Event Schema:** Versioned event formats, backward compatibility
|
||||
|
||||
### 5. Guardrails and Safety
|
||||
|
||||
#### Input Validation
|
||||
- **Schema Enforcement:** Required fields, type checking, format validation
|
||||
- **Content Filtering:** Harmful content detection, PII scrubbing
|
||||
- **Rate Limiting:** Request throttling, resource quotas
|
||||
- **Authentication:** Identity verification, authorization checks
|
||||
|
||||
#### Output Filtering
|
||||
- **Content Moderation:** Harmful content removal, quality checks
|
||||
- **Consistency Validation:** Logic checks, constraint verification
|
||||
- **Formatting:** Standardized output formats, clean presentation
|
||||
- **Audit Logging:** Decision trails, compliance records
|
||||
|
||||
#### Human-in-the-Loop
|
||||
- **Approval Workflows:** Critical decision checkpoints
|
||||
- **Escalation Triggers:** Confidence thresholds, risk assessment
|
||||
- **Override Mechanisms:** Human judgment precedence
|
||||
- **Feedback Loops:** Human corrections improve system behavior
|
||||
|
||||
### 6. Evaluation Frameworks
|
||||
|
||||
#### Task Completion Metrics
|
||||
- **Success Rate:** Percentage of tasks completed successfully
|
||||
- **Partial Completion:** Progress measurement for complex tasks
|
||||
- **Task Classification:** Success criteria by task type
|
||||
- **Failure Analysis:** Root cause identification and categorization
|
||||
|
||||
#### Quality Assessment
|
||||
- **Output Quality:** Accuracy, relevance, completeness measures
|
||||
- **Consistency:** Response variability across similar inputs
|
||||
- **Coherence:** Logical flow and internal consistency
|
||||
- **User Satisfaction:** Feedback scores, usage patterns
|
||||
|
||||
#### Cost Analysis
|
||||
- **Token Usage:** Input/output token consumption per task
|
||||
- **API Costs:** External service usage and charges
|
||||
- **Compute Resources:** CPU, memory, storage utilization
|
||||
- **Time-to-Value:** Cost per successful task completion
|
||||
|
||||
#### Latency Distribution
|
||||
- **Response Time:** End-to-end task completion time
|
||||
- **Processing Stages:** Bottleneck identification per stage
|
||||
- **Queue Times:** Wait times in processing pipelines
|
||||
- **Resource Contention:** Impact of concurrent operations
|
||||
|
||||
### 7. Orchestration Strategies
|
||||
|
||||
#### Centralized Orchestration
|
||||
- **Workflow Engine:** Central coordinator manages all agents
|
||||
- **State Management:** Centralized workflow state tracking
|
||||
- **Decision Logic:** Complex routing and branching rules
|
||||
- **Monitoring:** Comprehensive visibility into all operations
|
||||
|
||||
#### Decentralized Orchestration
|
||||
- **Peer-to-Peer:** Agents coordinate directly with each other
|
||||
- **Service Discovery:** Dynamic agent registration and lookup
|
||||
- **Consensus Protocols:** Distributed decision making
|
||||
- **Fault Tolerance:** No single point of failure
|
||||
|
||||
#### Hybrid Approaches
|
||||
- **Domain Boundaries:** Centralized within domains, federated across
|
||||
- **Hierarchical Coordination:** Multiple orchestration levels
|
||||
- **Context-Dependent:** Strategy selection based on task type
|
||||
- **Load Balancing:** Distribute coordination responsibility
|
||||
|
||||
### 8. Memory Patterns
|
||||
|
||||
#### Short-Term Memory
|
||||
- **Context Windows:** Working memory for current tasks
|
||||
- **Session State:** Temporary data for ongoing interactions
|
||||
- **Cache Management:** Performance optimization strategies
|
||||
- **Memory Pressure:** Handling capacity constraints
|
||||
|
||||
#### Long-Term Memory
|
||||
- **Persistent Storage:** Durable data across sessions
|
||||
- **Knowledge Base:** Accumulated domain knowledge
|
||||
- **Experience Replay:** Learning from past interactions
|
||||
- **Memory Consolidation:** Transferring from short to long-term
|
||||
|
||||
#### Shared Memory
|
||||
- **Collaborative Knowledge:** Shared learning across agents
|
||||
- **Synchronization:** Consistency maintenance strategies
|
||||
- **Access Control:** Permission-based memory access
|
||||
- **Memory Partitioning:** Isolation between agent groups
|
||||
|
||||
### 9. Scaling Considerations
|
||||
|
||||
#### Horizontal Scaling
|
||||
- **Agent Replication:** Multiple instances of same agent type
|
||||
- **Load Distribution:** Request routing across agent instances
|
||||
- **Resource Pooling:** Shared compute and storage resources
|
||||
- **Geographic Distribution:** Multi-region deployments
|
||||
|
||||
#### Vertical Scaling
|
||||
- **Capability Enhancement:** More powerful individual agents
|
||||
- **Tool Expansion:** Broader tool access per agent
|
||||
- **Context Expansion:** Larger working memory capacity
|
||||
- **Processing Power:** Higher throughput per agent
|
||||
|
||||
#### Performance Optimization
|
||||
- **Caching Strategies:** Response caching, tool result caching
|
||||
- **Parallel Processing:** Concurrent task execution
|
||||
- **Resource Optimization:** Efficient resource utilization
|
||||
- **Bottleneck Elimination:** Systematic performance tuning
|
||||
|
||||
### 10. Failure Handling
|
||||
|
||||
#### Retry Mechanisms
|
||||
- **Exponential Backoff:** Increasing delays between retries
|
||||
- **Jitter:** Random delay variation to prevent thundering herd
|
||||
- **Maximum Attempts:** Bounded retry behavior
|
||||
- **Retry Conditions:** Transient vs permanent failure classification
|
||||
|
||||
#### Fallback Strategies
|
||||
- **Graceful Degradation:** Reduced functionality when systems fail
|
||||
- **Alternative Approaches:** Different methods for same goals
|
||||
- **Default Responses:** Safe fallback behaviors
|
||||
- **User Communication:** Clear failure messaging
|
||||
|
||||
#### Circuit Breakers
|
||||
- **Failure Detection:** Monitoring failure rates and response times
|
||||
- **State Management:** Open, closed, half-open circuit states
|
||||
- **Recovery Testing:** Gradual return to normal operation
|
||||
- **Cascading Failure Prevention:** Protecting upstream systems
|
||||
|
||||
## Implementation Guidelines
|
||||
|
||||
### Architecture Decision Process
|
||||
1. **Requirements Analysis:** Understand system goals, constraints, scale
|
||||
2. **Pattern Selection:** Choose appropriate architecture pattern
|
||||
3. **Agent Design:** Define roles, responsibilities, interfaces
|
||||
4. **Tool Architecture:** Design tool schemas and error handling
|
||||
5. **Communication Design:** Select message patterns and protocols
|
||||
6. **Safety Implementation:** Build guardrails and validation
|
||||
7. **Evaluation Planning:** Define success metrics and monitoring
|
||||
8. **Deployment Strategy:** Plan scaling and failure handling
|
||||
|
||||
### Quality Assurance
|
||||
- **Testing Strategy:** Unit, integration, and system testing approaches
|
||||
- **Monitoring:** Real-time system health and performance tracking
|
||||
- **Documentation:** Architecture documentation and runbooks
|
||||
- **Security Review:** Threat modeling and security assessments
|
||||
|
||||
### Continuous Improvement
|
||||
- **Performance Monitoring:** Ongoing system performance analysis
|
||||
- **User Feedback:** Incorporating user experience improvements
|
||||
- **A/B Testing:** Controlled experiments for system improvements
|
||||
- **Knowledge Base Updates:** Continuous learning and adaptation
|
||||
|
||||
This skill provides the foundation for designing robust, scalable multi-agent systems that can handle complex tasks while maintaining safety, reliability, and performance at scale.
|
||||
1223
engineering/agent-designer/agent_evaluator.py
Normal file
1223
engineering/agent-designer/agent_evaluator.py
Normal file
File diff suppressed because it is too large
Load Diff
911
engineering/agent-designer/agent_planner.py
Normal file
911
engineering/agent-designer/agent_planner.py
Normal file
@@ -0,0 +1,911 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Agent Planner - Multi-Agent System Architecture Designer
|
||||
|
||||
Given a system description (goal, tasks, constraints, team size), designs a multi-agent
|
||||
architecture: defines agent roles, responsibilities, capabilities needed, communication
|
||||
topology, tool requirements. Generates architecture diagram (Mermaid).
|
||||
|
||||
Input: system requirements JSON
|
||||
Output: agent architecture + role definitions + Mermaid diagram + implementation roadmap
|
||||
"""
|
||||
|
||||
import json
|
||||
import argparse
|
||||
import sys
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class AgentArchitecturePattern(Enum):
|
||||
"""Supported agent architecture patterns"""
|
||||
SINGLE_AGENT = "single_agent"
|
||||
SUPERVISOR = "supervisor"
|
||||
SWARM = "swarm"
|
||||
HIERARCHICAL = "hierarchical"
|
||||
PIPELINE = "pipeline"
|
||||
|
||||
|
||||
class CommunicationPattern(Enum):
|
||||
"""Agent communication patterns"""
|
||||
DIRECT_MESSAGE = "direct_message"
|
||||
SHARED_STATE = "shared_state"
|
||||
EVENT_DRIVEN = "event_driven"
|
||||
MESSAGE_QUEUE = "message_queue"
|
||||
|
||||
|
||||
class AgentRole(Enum):
|
||||
"""Standard agent role archetypes"""
|
||||
COORDINATOR = "coordinator"
|
||||
SPECIALIST = "specialist"
|
||||
INTERFACE = "interface"
|
||||
MONITOR = "monitor"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Tool:
|
||||
"""Tool definition for agents"""
|
||||
name: str
|
||||
description: str
|
||||
input_schema: Dict[str, Any]
|
||||
output_schema: Dict[str, Any]
|
||||
capabilities: List[str]
|
||||
reliability: str = "high" # high, medium, low
|
||||
latency: str = "low" # low, medium, high
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentDefinition:
|
||||
"""Complete agent definition"""
|
||||
name: str
|
||||
role: str
|
||||
archetype: AgentRole
|
||||
responsibilities: List[str]
|
||||
capabilities: List[str]
|
||||
tools: List[Tool]
|
||||
communication_interfaces: List[str]
|
||||
constraints: Dict[str, Any]
|
||||
success_criteria: List[str]
|
||||
dependencies: List[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommunicationLink:
|
||||
"""Communication link between agents"""
|
||||
from_agent: str
|
||||
to_agent: str
|
||||
pattern: CommunicationPattern
|
||||
data_format: str
|
||||
frequency: str
|
||||
criticality: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemRequirements:
|
||||
"""Input system requirements"""
|
||||
goal: str
|
||||
description: str
|
||||
tasks: List[str]
|
||||
constraints: Dict[str, Any]
|
||||
team_size: int
|
||||
performance_requirements: Dict[str, Any]
|
||||
safety_requirements: List[str]
|
||||
integration_requirements: List[str]
|
||||
scale_requirements: Dict[str, Any]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchitectureDesign:
|
||||
"""Complete architecture design output"""
|
||||
pattern: AgentArchitecturePattern
|
||||
agents: List[AgentDefinition]
|
||||
communication_topology: List[CommunicationLink]
|
||||
shared_resources: List[Dict[str, Any]]
|
||||
guardrails: List[Dict[str, Any]]
|
||||
scaling_strategy: Dict[str, Any]
|
||||
failure_handling: Dict[str, Any]
|
||||
|
||||
|
||||
class AgentPlanner:
|
||||
"""Multi-agent system architecture planner"""
|
||||
|
||||
def __init__(self):
|
||||
self.common_tools = self._define_common_tools()
|
||||
self.pattern_heuristics = self._define_pattern_heuristics()
|
||||
|
||||
def _define_common_tools(self) -> Dict[str, Tool]:
|
||||
"""Define commonly used tools across agents"""
|
||||
return {
|
||||
"web_search": Tool(
|
||||
name="web_search",
|
||||
description="Search the web for information",
|
||||
input_schema={"type": "object", "properties": {"query": {"type": "string"}}},
|
||||
output_schema={"type": "object", "properties": {"results": {"type": "array"}}},
|
||||
capabilities=["research", "information_gathering"],
|
||||
reliability="high",
|
||||
latency="medium"
|
||||
),
|
||||
"code_executor": Tool(
|
||||
name="code_executor",
|
||||
description="Execute code in various languages",
|
||||
input_schema={"type": "object", "properties": {"language": {"type": "string"}, "code": {"type": "string"}}},
|
||||
output_schema={"type": "object", "properties": {"result": {"type": "string"}, "error": {"type": "string"}}},
|
||||
capabilities=["code_execution", "testing", "automation"],
|
||||
reliability="high",
|
||||
latency="low"
|
||||
),
|
||||
"file_manager": Tool(
|
||||
name="file_manager",
|
||||
description="Manage files and directories",
|
||||
input_schema={"type": "object", "properties": {"action": {"type": "string"}, "path": {"type": "string"}}},
|
||||
output_schema={"type": "object", "properties": {"success": {"type": "boolean"}, "content": {"type": "string"}}},
|
||||
capabilities=["file_operations", "data_management"],
|
||||
reliability="high",
|
||||
latency="low"
|
||||
),
|
||||
"data_analyzer": Tool(
|
||||
name="data_analyzer",
|
||||
description="Analyze and process data",
|
||||
input_schema={"type": "object", "properties": {"data": {"type": "object"}, "analysis_type": {"type": "string"}}},
|
||||
output_schema={"type": "object", "properties": {"insights": {"type": "array"}, "metrics": {"type": "object"}}},
|
||||
capabilities=["data_analysis", "statistics", "visualization"],
|
||||
reliability="high",
|
||||
latency="medium"
|
||||
),
|
||||
"api_client": Tool(
|
||||
name="api_client",
|
||||
description="Make API calls to external services",
|
||||
input_schema={"type": "object", "properties": {"url": {"type": "string"}, "method": {"type": "string"}, "data": {"type": "object"}}},
|
||||
output_schema={"type": "object", "properties": {"response": {"type": "object"}, "status": {"type": "integer"}}},
|
||||
capabilities=["integration", "external_services"],
|
||||
reliability="medium",
|
||||
latency="medium"
|
||||
)
|
||||
}
|
||||
|
||||
def _define_pattern_heuristics(self) -> Dict[AgentArchitecturePattern, Dict[str, Any]]:
|
||||
"""Define heuristics for selecting architecture patterns"""
|
||||
return {
|
||||
AgentArchitecturePattern.SINGLE_AGENT: {
|
||||
"team_size_range": (1, 1),
|
||||
"task_complexity": "simple",
|
||||
"coordination_overhead": "none",
|
||||
"suitable_for": ["simple tasks", "prototyping", "single domain"],
|
||||
"scaling_limit": "low"
|
||||
},
|
||||
AgentArchitecturePattern.SUPERVISOR: {
|
||||
"team_size_range": (2, 8),
|
||||
"task_complexity": "medium",
|
||||
"coordination_overhead": "low",
|
||||
"suitable_for": ["hierarchical tasks", "clear delegation", "quality control"],
|
||||
"scaling_limit": "medium"
|
||||
},
|
||||
AgentArchitecturePattern.SWARM: {
|
||||
"team_size_range": (3, 20),
|
||||
"task_complexity": "high",
|
||||
"coordination_overhead": "high",
|
||||
"suitable_for": ["parallel processing", "distributed problem solving", "fault tolerance"],
|
||||
"scaling_limit": "high"
|
||||
},
|
||||
AgentArchitecturePattern.HIERARCHICAL: {
|
||||
"team_size_range": (5, 50),
|
||||
"task_complexity": "very high",
|
||||
"coordination_overhead": "medium",
|
||||
"suitable_for": ["large organizations", "complex workflows", "enterprise systems"],
|
||||
"scaling_limit": "very high"
|
||||
},
|
||||
AgentArchitecturePattern.PIPELINE: {
|
||||
"team_size_range": (3, 15),
|
||||
"task_complexity": "medium",
|
||||
"coordination_overhead": "low",
|
||||
"suitable_for": ["sequential processing", "data pipelines", "assembly line tasks"],
|
||||
"scaling_limit": "medium"
|
||||
}
|
||||
}
|
||||
|
||||
def select_architecture_pattern(self, requirements: SystemRequirements) -> AgentArchitecturePattern:
|
||||
"""Select the most appropriate architecture pattern based on requirements"""
|
||||
team_size = requirements.team_size
|
||||
task_count = len(requirements.tasks)
|
||||
performance_reqs = requirements.performance_requirements
|
||||
|
||||
# Score each pattern based on requirements
|
||||
pattern_scores = {}
|
||||
|
||||
for pattern, heuristics in self.pattern_heuristics.items():
|
||||
score = 0
|
||||
|
||||
# Team size fit
|
||||
min_size, max_size = heuristics["team_size_range"]
|
||||
if min_size <= team_size <= max_size:
|
||||
score += 3
|
||||
elif abs(team_size - min_size) <= 2 or abs(team_size - max_size) <= 2:
|
||||
score += 1
|
||||
|
||||
# Task complexity assessment
|
||||
complexity_indicators = [
|
||||
"parallel" in requirements.description.lower(),
|
||||
"sequential" in requirements.description.lower(),
|
||||
"hierarchical" in requirements.description.lower(),
|
||||
"distributed" in requirements.description.lower(),
|
||||
task_count > 5,
|
||||
len(requirements.constraints) > 3
|
||||
]
|
||||
|
||||
complexity_score = sum(complexity_indicators)
|
||||
|
||||
if pattern == AgentArchitecturePattern.SINGLE_AGENT and complexity_score <= 2:
|
||||
score += 2
|
||||
elif pattern == AgentArchitecturePattern.SUPERVISOR and 2 <= complexity_score <= 4:
|
||||
score += 2
|
||||
elif pattern == AgentArchitecturePattern.PIPELINE and "sequential" in requirements.description.lower():
|
||||
score += 3
|
||||
elif pattern == AgentArchitecturePattern.SWARM and "parallel" in requirements.description.lower():
|
||||
score += 3
|
||||
elif pattern == AgentArchitecturePattern.HIERARCHICAL and complexity_score >= 4:
|
||||
score += 2
|
||||
|
||||
# Performance requirements
|
||||
if performance_reqs.get("high_throughput", False) and pattern in [AgentArchitecturePattern.SWARM, AgentArchitecturePattern.PIPELINE]:
|
||||
score += 2
|
||||
if performance_reqs.get("fault_tolerance", False) and pattern == AgentArchitecturePattern.SWARM:
|
||||
score += 2
|
||||
if performance_reqs.get("low_latency", False) and pattern in [AgentArchitecturePattern.SINGLE_AGENT, AgentArchitecturePattern.PIPELINE]:
|
||||
score += 1
|
||||
|
||||
pattern_scores[pattern] = score
|
||||
|
||||
# Select the highest scoring pattern
|
||||
best_pattern = max(pattern_scores.items(), key=lambda x: x[1])[0]
|
||||
return best_pattern
|
||||
|
||||
def design_agents(self, requirements: SystemRequirements, pattern: AgentArchitecturePattern) -> List[AgentDefinition]:
|
||||
"""Design individual agents based on requirements and architecture pattern"""
|
||||
agents = []
|
||||
|
||||
if pattern == AgentArchitecturePattern.SINGLE_AGENT:
|
||||
agents = self._design_single_agent(requirements)
|
||||
elif pattern == AgentArchitecturePattern.SUPERVISOR:
|
||||
agents = self._design_supervisor_agents(requirements)
|
||||
elif pattern == AgentArchitecturePattern.SWARM:
|
||||
agents = self._design_swarm_agents(requirements)
|
||||
elif pattern == AgentArchitecturePattern.HIERARCHICAL:
|
||||
agents = self._design_hierarchical_agents(requirements)
|
||||
elif pattern == AgentArchitecturePattern.PIPELINE:
|
||||
agents = self._design_pipeline_agents(requirements)
|
||||
|
||||
return agents
|
||||
|
||||
def _design_single_agent(self, requirements: SystemRequirements) -> List[AgentDefinition]:
|
||||
"""Design a single general-purpose agent"""
|
||||
all_tools = list(self.common_tools.values())
|
||||
|
||||
agent = AgentDefinition(
|
||||
name="universal_agent",
|
||||
role="Universal Task Handler",
|
||||
archetype=AgentRole.SPECIALIST,
|
||||
responsibilities=requirements.tasks,
|
||||
capabilities=["general_purpose", "multi_domain", "adaptable"],
|
||||
tools=all_tools,
|
||||
communication_interfaces=["direct_user_interface"],
|
||||
constraints={
|
||||
"max_concurrent_tasks": 1,
|
||||
"memory_limit": "high",
|
||||
"response_time": "fast"
|
||||
},
|
||||
success_criteria=["complete all assigned tasks", "maintain quality standards", "respond within time limits"],
|
||||
dependencies=[]
|
||||
)
|
||||
|
||||
return [agent]
|
||||
|
||||
def _design_supervisor_agents(self, requirements: SystemRequirements) -> List[AgentDefinition]:
|
||||
"""Design supervisor pattern agents"""
|
||||
agents = []
|
||||
|
||||
# Create supervisor agent
|
||||
supervisor = AgentDefinition(
|
||||
name="supervisor_agent",
|
||||
role="Task Coordinator and Quality Controller",
|
||||
archetype=AgentRole.COORDINATOR,
|
||||
responsibilities=[
|
||||
"task_decomposition",
|
||||
"delegation",
|
||||
"progress_monitoring",
|
||||
"quality_assurance",
|
||||
"result_aggregation"
|
||||
],
|
||||
capabilities=["planning", "coordination", "evaluation", "decision_making"],
|
||||
tools=[self.common_tools["file_manager"], self.common_tools["data_analyzer"]],
|
||||
communication_interfaces=["user_interface", "agent_messaging"],
|
||||
constraints={
|
||||
"max_concurrent_supervisions": 5,
|
||||
"decision_timeout": "30s"
|
||||
},
|
||||
success_criteria=["successful task completion", "optimal resource utilization", "quality standards met"],
|
||||
dependencies=[]
|
||||
)
|
||||
agents.append(supervisor)
|
||||
|
||||
# Create specialist agents based on task domains
|
||||
task_domains = self._identify_task_domains(requirements.tasks)
|
||||
for i, domain in enumerate(task_domains[:requirements.team_size - 1]):
|
||||
specialist = AgentDefinition(
|
||||
name=f"{domain}_specialist",
|
||||
role=f"{domain.title()} Specialist",
|
||||
archetype=AgentRole.SPECIALIST,
|
||||
responsibilities=[task for task in requirements.tasks if domain in task.lower()],
|
||||
capabilities=[f"{domain}_expertise", "specialized_tools", "domain_knowledge"],
|
||||
tools=self._select_tools_for_domain(domain),
|
||||
communication_interfaces=["supervisor_messaging"],
|
||||
constraints={
|
||||
"domain_scope": domain,
|
||||
"task_queue_size": 10
|
||||
},
|
||||
success_criteria=[f"excel in {domain} tasks", "maintain domain expertise", "provide quality output"],
|
||||
dependencies=["supervisor_agent"]
|
||||
)
|
||||
agents.append(specialist)
|
||||
|
||||
return agents
|
||||
|
||||
def _design_swarm_agents(self, requirements: SystemRequirements) -> List[AgentDefinition]:
|
||||
"""Design swarm pattern agents"""
|
||||
agents = []
|
||||
|
||||
# Create peer agents with overlapping capabilities
|
||||
agent_count = min(requirements.team_size, 10) # Reasonable swarm size
|
||||
base_capabilities = ["collaboration", "consensus", "adaptation", "peer_communication"]
|
||||
|
||||
for i in range(agent_count):
|
||||
agent = AgentDefinition(
|
||||
name=f"swarm_agent_{i+1}",
|
||||
role=f"Collaborative Worker #{i+1}",
|
||||
archetype=AgentRole.SPECIALIST,
|
||||
responsibilities=requirements.tasks, # All agents can handle all tasks
|
||||
capabilities=base_capabilities + [f"specialization_{i%3}"], # Some specialization
|
||||
tools=list(self.common_tools.values()),
|
||||
communication_interfaces=["peer_messaging", "broadcast", "consensus_protocol"],
|
||||
constraints={
|
||||
"peer_discovery_timeout": "10s",
|
||||
"consensus_threshold": 0.6,
|
||||
"max_retries": 3
|
||||
},
|
||||
success_criteria=["contribute to group goals", "maintain peer relationships", "adapt to failures"],
|
||||
dependencies=[f"swarm_agent_{j+1}" for j in range(agent_count) if j != i]
|
||||
)
|
||||
agents.append(agent)
|
||||
|
||||
return agents
|
||||
|
||||
def _design_hierarchical_agents(self, requirements: SystemRequirements) -> List[AgentDefinition]:
|
||||
"""Design hierarchical pattern agents"""
|
||||
agents = []
|
||||
|
||||
# Create management hierarchy
|
||||
levels = min(3, requirements.team_size // 3) # Reasonable hierarchy depth
|
||||
agents_per_level = requirements.team_size // levels
|
||||
|
||||
# Top level manager
|
||||
manager = AgentDefinition(
|
||||
name="executive_manager",
|
||||
role="Executive Manager",
|
||||
archetype=AgentRole.COORDINATOR,
|
||||
responsibilities=["strategic_planning", "resource_allocation", "performance_monitoring"],
|
||||
capabilities=["leadership", "strategy", "resource_management", "oversight"],
|
||||
tools=[self.common_tools["data_analyzer"], self.common_tools["file_manager"]],
|
||||
communication_interfaces=["executive_dashboard", "management_messaging"],
|
||||
constraints={"management_span": 5, "decision_authority": "high"},
|
||||
success_criteria=["achieve system goals", "optimize resource usage", "maintain quality"],
|
||||
dependencies=[]
|
||||
)
|
||||
agents.append(manager)
|
||||
|
||||
# Middle managers
|
||||
for i in range(agents_per_level - 1):
|
||||
middle_manager = AgentDefinition(
|
||||
name=f"team_manager_{i+1}",
|
||||
role=f"Team Manager #{i+1}",
|
||||
archetype=AgentRole.COORDINATOR,
|
||||
responsibilities=["team_coordination", "task_distribution", "progress_tracking"],
|
||||
capabilities=["team_management", "coordination", "reporting"],
|
||||
tools=[self.common_tools["file_manager"]],
|
||||
communication_interfaces=["management_messaging", "team_messaging"],
|
||||
constraints={"team_size": 3, "reporting_frequency": "hourly"},
|
||||
success_criteria=["team performance", "task completion", "team satisfaction"],
|
||||
dependencies=["executive_manager"]
|
||||
)
|
||||
agents.append(middle_manager)
|
||||
|
||||
# Workers
|
||||
remaining_agents = requirements.team_size - len(agents)
|
||||
for i in range(remaining_agents):
|
||||
worker = AgentDefinition(
|
||||
name=f"worker_agent_{i+1}",
|
||||
role=f"Task Worker #{i+1}",
|
||||
archetype=AgentRole.SPECIALIST,
|
||||
responsibilities=["task_execution", "result_delivery", "status_reporting"],
|
||||
capabilities=["task_execution", "specialized_skills", "reliability"],
|
||||
tools=self._select_diverse_tools(),
|
||||
communication_interfaces=["team_messaging"],
|
||||
constraints={"task_focus": "single", "reporting_interval": "30min"},
|
||||
success_criteria=["complete assigned tasks", "maintain quality", "meet deadlines"],
|
||||
dependencies=[f"team_manager_{(i // 3) + 1}"]
|
||||
)
|
||||
agents.append(worker)
|
||||
|
||||
return agents
|
||||
|
||||
def _design_pipeline_agents(self, requirements: SystemRequirements) -> List[AgentDefinition]:
|
||||
"""Design pipeline pattern agents"""
|
||||
agents = []
|
||||
|
||||
# Create sequential processing stages
|
||||
pipeline_stages = self._identify_pipeline_stages(requirements.tasks)
|
||||
|
||||
for i, stage in enumerate(pipeline_stages):
|
||||
agent = AgentDefinition(
|
||||
name=f"pipeline_stage_{i+1}_{stage}",
|
||||
role=f"Pipeline Stage {i+1}: {stage.title()}",
|
||||
archetype=AgentRole.SPECIALIST,
|
||||
responsibilities=[f"process_{stage}", f"validate_{stage}_output", "handoff_to_next_stage"],
|
||||
capabilities=[f"{stage}_processing", "quality_control", "data_transformation"],
|
||||
tools=self._select_tools_for_stage(stage),
|
||||
communication_interfaces=["pipeline_queue", "stage_messaging"],
|
||||
constraints={
|
||||
"processing_order": i + 1,
|
||||
"batch_size": 10,
|
||||
"stage_timeout": "5min"
|
||||
},
|
||||
success_criteria=[f"successfully process {stage}", "maintain data integrity", "meet throughput targets"],
|
||||
dependencies=[f"pipeline_stage_{i}_{pipeline_stages[i-1]}"] if i > 0 else []
|
||||
)
|
||||
agents.append(agent)
|
||||
|
||||
return agents
|
||||
|
||||
def _identify_task_domains(self, tasks: List[str]) -> List[str]:
|
||||
"""Identify distinct domains from task list"""
|
||||
domains = []
|
||||
domain_keywords = {
|
||||
"research": ["research", "search", "find", "investigate", "analyze"],
|
||||
"development": ["code", "build", "develop", "implement", "program"],
|
||||
"data": ["data", "process", "analyze", "calculate", "compute"],
|
||||
"communication": ["write", "send", "message", "communicate", "report"],
|
||||
"file": ["file", "document", "save", "load", "manage"]
|
||||
}
|
||||
|
||||
for domain, keywords in domain_keywords.items():
|
||||
if any(keyword in " ".join(tasks).lower() for keyword in keywords):
|
||||
domains.append(domain)
|
||||
|
||||
return domains[:5] # Limit to 5 domains
|
||||
|
||||
def _identify_pipeline_stages(self, tasks: List[str]) -> List[str]:
|
||||
"""Identify pipeline stages from task list"""
|
||||
# Common pipeline patterns
|
||||
common_stages = ["input", "process", "transform", "validate", "output"]
|
||||
|
||||
# Try to infer stages from tasks
|
||||
stages = []
|
||||
task_text = " ".join(tasks).lower()
|
||||
|
||||
if "collect" in task_text or "gather" in task_text:
|
||||
stages.append("collection")
|
||||
if "process" in task_text or "transform" in task_text:
|
||||
stages.append("processing")
|
||||
if "analyze" in task_text or "evaluate" in task_text:
|
||||
stages.append("analysis")
|
||||
if "validate" in task_text or "check" in task_text:
|
||||
stages.append("validation")
|
||||
if "output" in task_text or "deliver" in task_text or "report" in task_text:
|
||||
stages.append("output")
|
||||
|
||||
# Default to common stages if none identified
|
||||
return stages if stages else common_stages[:min(5, len(tasks))]
|
||||
|
||||
def _select_tools_for_domain(self, domain: str) -> List[Tool]:
|
||||
"""Select appropriate tools for a specific domain"""
|
||||
domain_tools = {
|
||||
"research": [self.common_tools["web_search"], self.common_tools["data_analyzer"]],
|
||||
"development": [self.common_tools["code_executor"], self.common_tools["file_manager"]],
|
||||
"data": [self.common_tools["data_analyzer"], self.common_tools["file_manager"]],
|
||||
"communication": [self.common_tools["api_client"], self.common_tools["file_manager"]],
|
||||
"file": [self.common_tools["file_manager"]]
|
||||
}
|
||||
|
||||
return domain_tools.get(domain, [self.common_tools["api_client"]])
|
||||
|
||||
def _select_tools_for_stage(self, stage: str) -> List[Tool]:
|
||||
"""Select appropriate tools for a pipeline stage"""
|
||||
stage_tools = {
|
||||
"input": [self.common_tools["api_client"], self.common_tools["file_manager"]],
|
||||
"collection": [self.common_tools["web_search"], self.common_tools["api_client"]],
|
||||
"process": [self.common_tools["code_executor"], self.common_tools["data_analyzer"]],
|
||||
"processing": [self.common_tools["data_analyzer"], self.common_tools["code_executor"]],
|
||||
"transform": [self.common_tools["data_analyzer"], self.common_tools["code_executor"]],
|
||||
"analysis": [self.common_tools["data_analyzer"]],
|
||||
"validate": [self.common_tools["data_analyzer"]],
|
||||
"validation": [self.common_tools["data_analyzer"]],
|
||||
"output": [self.common_tools["file_manager"], self.common_tools["api_client"]]
|
||||
}
|
||||
|
||||
return stage_tools.get(stage, [self.common_tools["file_manager"]])
|
||||
|
||||
def _select_diverse_tools(self) -> List[Tool]:
|
||||
"""Select a diverse set of tools for general purpose agents"""
|
||||
return [
|
||||
self.common_tools["file_manager"],
|
||||
self.common_tools["code_executor"],
|
||||
self.common_tools["data_analyzer"]
|
||||
]
|
||||
|
||||
def design_communication_topology(self, agents: List[AgentDefinition], pattern: AgentArchitecturePattern) -> List[CommunicationLink]:
|
||||
"""Design communication links between agents"""
|
||||
links = []
|
||||
|
||||
if pattern == AgentArchitecturePattern.SINGLE_AGENT:
|
||||
# No inter-agent communication needed
|
||||
return []
|
||||
|
||||
elif pattern == AgentArchitecturePattern.SUPERVISOR:
|
||||
supervisor = next(agent for agent in agents if agent.archetype == AgentRole.COORDINATOR)
|
||||
specialists = [agent for agent in agents if agent.archetype == AgentRole.SPECIALIST]
|
||||
|
||||
for specialist in specialists:
|
||||
# Bidirectional communication with supervisor
|
||||
links.append(CommunicationLink(
|
||||
from_agent=supervisor.name,
|
||||
to_agent=specialist.name,
|
||||
pattern=CommunicationPattern.DIRECT_MESSAGE,
|
||||
data_format="json",
|
||||
frequency="on_demand",
|
||||
criticality="high"
|
||||
))
|
||||
links.append(CommunicationLink(
|
||||
from_agent=specialist.name,
|
||||
to_agent=supervisor.name,
|
||||
pattern=CommunicationPattern.DIRECT_MESSAGE,
|
||||
data_format="json",
|
||||
frequency="on_completion",
|
||||
criticality="high"
|
||||
))
|
||||
|
||||
elif pattern == AgentArchitecturePattern.SWARM:
|
||||
# All-to-all communication for swarm
|
||||
for i, agent1 in enumerate(agents):
|
||||
for j, agent2 in enumerate(agents):
|
||||
if i != j:
|
||||
links.append(CommunicationLink(
|
||||
from_agent=agent1.name,
|
||||
to_agent=agent2.name,
|
||||
pattern=CommunicationPattern.EVENT_DRIVEN,
|
||||
data_format="json",
|
||||
frequency="periodic",
|
||||
criticality="medium"
|
||||
))
|
||||
|
||||
elif pattern == AgentArchitecturePattern.HIERARCHICAL:
|
||||
# Hierarchical communication based on dependencies
|
||||
for agent in agents:
|
||||
if agent.dependencies:
|
||||
for dependency in agent.dependencies:
|
||||
links.append(CommunicationLink(
|
||||
from_agent=dependency,
|
||||
to_agent=agent.name,
|
||||
pattern=CommunicationPattern.DIRECT_MESSAGE,
|
||||
data_format="json",
|
||||
frequency="scheduled",
|
||||
criticality="high"
|
||||
))
|
||||
links.append(CommunicationLink(
|
||||
from_agent=agent.name,
|
||||
to_agent=dependency,
|
||||
pattern=CommunicationPattern.DIRECT_MESSAGE,
|
||||
data_format="json",
|
||||
frequency="on_completion",
|
||||
criticality="high"
|
||||
))
|
||||
|
||||
elif pattern == AgentArchitecturePattern.PIPELINE:
|
||||
# Sequential pipeline communication
|
||||
for i in range(len(agents) - 1):
|
||||
links.append(CommunicationLink(
|
||||
from_agent=agents[i].name,
|
||||
to_agent=agents[i + 1].name,
|
||||
pattern=CommunicationPattern.MESSAGE_QUEUE,
|
||||
data_format="json",
|
||||
frequency="continuous",
|
||||
criticality="high"
|
||||
))
|
||||
|
||||
return links
|
||||
|
||||
def generate_mermaid_diagram(self, design: ArchitectureDesign) -> str:
|
||||
"""Generate Mermaid diagram for the architecture"""
|
||||
diagram = ["graph TD"]
|
||||
|
||||
# Add agent nodes
|
||||
for agent in design.agents:
|
||||
node_style = self._get_node_style(agent.archetype)
|
||||
diagram.append(f" {agent.name}[{agent.role}]{node_style}")
|
||||
|
||||
# Add communication links
|
||||
for link in design.communication_topology:
|
||||
arrow_style = self._get_arrow_style(link.pattern, link.criticality)
|
||||
diagram.append(f" {link.from_agent} {arrow_style} {link.to_agent}")
|
||||
|
||||
# Add styling
|
||||
diagram.extend([
|
||||
"",
|
||||
" classDef coordinator fill:#e1f5fe,stroke:#01579b,stroke-width:2px",
|
||||
" classDef specialist fill:#f3e5f5,stroke:#4a148c,stroke-width:2px",
|
||||
" classDef interface fill:#e8f5e8,stroke:#1b5e20,stroke-width:2px",
|
||||
" classDef monitor fill:#fff3e0,stroke:#e65100,stroke-width:2px"
|
||||
])
|
||||
|
||||
# Apply classes to nodes
|
||||
for agent in design.agents:
|
||||
class_name = agent.archetype.value
|
||||
diagram.append(f" class {agent.name} {class_name}")
|
||||
|
||||
return "\n".join(diagram)
|
||||
|
||||
def _get_node_style(self, archetype: AgentRole) -> str:
|
||||
"""Get node styling based on archetype"""
|
||||
styles = {
|
||||
AgentRole.COORDINATOR: ":::coordinator",
|
||||
AgentRole.SPECIALIST: ":::specialist",
|
||||
AgentRole.INTERFACE: ":::interface",
|
||||
AgentRole.MONITOR: ":::monitor"
|
||||
}
|
||||
return styles.get(archetype, "")
|
||||
|
||||
def _get_arrow_style(self, pattern: CommunicationPattern, criticality: str) -> str:
|
||||
"""Get arrow styling based on communication pattern and criticality"""
|
||||
base_arrows = {
|
||||
CommunicationPattern.DIRECT_MESSAGE: "-->",
|
||||
CommunicationPattern.SHARED_STATE: "-.->",
|
||||
CommunicationPattern.EVENT_DRIVEN: "===>",
|
||||
CommunicationPattern.MESSAGE_QUEUE: "==="
|
||||
}
|
||||
|
||||
arrow = base_arrows.get(pattern, "-->")
|
||||
|
||||
# Modify for criticality
|
||||
if criticality == "high":
|
||||
return arrow
|
||||
elif criticality == "medium":
|
||||
return arrow.replace("-", ".")
|
||||
else:
|
||||
return arrow.replace("-", ":")
|
||||
|
||||
def generate_implementation_roadmap(self, design: ArchitectureDesign, requirements: SystemRequirements) -> Dict[str, Any]:
|
||||
"""Generate implementation roadmap"""
|
||||
phases = []
|
||||
|
||||
# Phase 1: Core Infrastructure
|
||||
phases.append({
|
||||
"phase": 1,
|
||||
"name": "Core Infrastructure",
|
||||
"duration": "2-3 weeks",
|
||||
"tasks": [
|
||||
"Set up development environment",
|
||||
"Implement basic agent framework",
|
||||
"Create communication infrastructure",
|
||||
"Set up monitoring and logging",
|
||||
"Implement basic tools"
|
||||
],
|
||||
"deliverables": [
|
||||
"Agent runtime framework",
|
||||
"Communication layer",
|
||||
"Basic monitoring dashboard"
|
||||
]
|
||||
})
|
||||
|
||||
# Phase 2: Agent Implementation
|
||||
phases.append({
|
||||
"phase": 2,
|
||||
"name": "Agent Implementation",
|
||||
"duration": "3-4 weeks",
|
||||
"tasks": [
|
||||
"Implement individual agent logic",
|
||||
"Create agent-specific tools",
|
||||
"Implement communication protocols",
|
||||
"Add error handling and recovery",
|
||||
"Create agent configuration system"
|
||||
],
|
||||
"deliverables": [
|
||||
"Functional agent implementations",
|
||||
"Tool integration",
|
||||
"Configuration management"
|
||||
]
|
||||
})
|
||||
|
||||
# Phase 3: Integration and Testing
|
||||
phases.append({
|
||||
"phase": 3,
|
||||
"name": "Integration and Testing",
|
||||
"duration": "2-3 weeks",
|
||||
"tasks": [
|
||||
"Integrate all agents",
|
||||
"End-to-end testing",
|
||||
"Performance optimization",
|
||||
"Security implementation",
|
||||
"Documentation creation"
|
||||
],
|
||||
"deliverables": [
|
||||
"Integrated system",
|
||||
"Test suite",
|
||||
"Performance benchmarks",
|
||||
"Security audit report"
|
||||
]
|
||||
})
|
||||
|
||||
# Phase 4: Deployment and Monitoring
|
||||
phases.append({
|
||||
"phase": 4,
|
||||
"name": "Deployment and Monitoring",
|
||||
"duration": "1-2 weeks",
|
||||
"tasks": [
|
||||
"Production deployment",
|
||||
"Monitoring setup",
|
||||
"Alerting configuration",
|
||||
"User training",
|
||||
"Go-live support"
|
||||
],
|
||||
"deliverables": [
|
||||
"Production system",
|
||||
"Monitoring dashboard",
|
||||
"Operational runbooks",
|
||||
"Training materials"
|
||||
]
|
||||
})
|
||||
|
||||
return {
|
||||
"total_duration": "8-12 weeks",
|
||||
"phases": phases,
|
||||
"critical_path": [
|
||||
"Agent framework implementation",
|
||||
"Communication layer development",
|
||||
"Integration testing",
|
||||
"Production deployment"
|
||||
],
|
||||
"risks": [
|
||||
{
|
||||
"risk": "Communication complexity",
|
||||
"impact": "high",
|
||||
"mitigation": "Start with simple protocols, iterate"
|
||||
},
|
||||
{
|
||||
"risk": "Agent coordination failures",
|
||||
"impact": "medium",
|
||||
"mitigation": "Implement robust error handling and fallbacks"
|
||||
},
|
||||
{
|
||||
"risk": "Performance bottlenecks",
|
||||
"impact": "medium",
|
||||
"mitigation": "Early performance testing and optimization"
|
||||
}
|
||||
],
|
||||
"success_criteria": requirements.safety_requirements + [
|
||||
"All agents operational",
|
||||
"Communication working reliably",
|
||||
"Performance targets met",
|
||||
"Error rate below 1%"
|
||||
]
|
||||
}
|
||||
|
||||
def plan_system(self, requirements: SystemRequirements) -> Tuple[ArchitectureDesign, str, Dict[str, Any]]:
|
||||
"""Main planning function"""
|
||||
# Select architecture pattern
|
||||
pattern = self.select_architecture_pattern(requirements)
|
||||
|
||||
# Design agents
|
||||
agents = self.design_agents(requirements, pattern)
|
||||
|
||||
# Design communication topology
|
||||
communication_topology = self.design_communication_topology(agents, pattern)
|
||||
|
||||
# Create complete design
|
||||
design = ArchitectureDesign(
|
||||
pattern=pattern,
|
||||
agents=agents,
|
||||
communication_topology=communication_topology,
|
||||
shared_resources=[
|
||||
{"type": "message_queue", "capacity": 1000},
|
||||
{"type": "shared_memory", "size": "1GB"},
|
||||
{"type": "event_store", "retention": "30 days"}
|
||||
],
|
||||
guardrails=[
|
||||
{"type": "input_validation", "rules": "strict_schema_enforcement"},
|
||||
{"type": "rate_limiting", "limit": "100_requests_per_minute"},
|
||||
{"type": "output_filtering", "rules": "content_safety_check"}
|
||||
],
|
||||
scaling_strategy={
|
||||
"horizontal_scaling": True,
|
||||
"auto_scaling_triggers": ["cpu > 80%", "queue_depth > 100"],
|
||||
"max_instances_per_agent": 5
|
||||
},
|
||||
failure_handling={
|
||||
"retry_policy": "exponential_backoff",
|
||||
"circuit_breaker": True,
|
||||
"fallback_strategies": ["graceful_degradation", "human_escalation"]
|
||||
}
|
||||
)
|
||||
|
||||
# Generate Mermaid diagram
|
||||
mermaid_diagram = self.generate_mermaid_diagram(design)
|
||||
|
||||
# Generate implementation roadmap
|
||||
roadmap = self.generate_implementation_roadmap(design, requirements)
|
||||
|
||||
return design, mermaid_diagram, roadmap
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Multi-Agent System Architecture Planner")
|
||||
parser.add_argument("input_file", help="JSON file with system requirements")
|
||||
parser.add_argument("-o", "--output", help="Output file prefix (default: agent_architecture)")
|
||||
parser.add_argument("--format", choices=["json", "yaml", "both"], default="both",
|
||||
help="Output format")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Load requirements
|
||||
with open(args.input_file, 'r') as f:
|
||||
requirements_data = json.load(f)
|
||||
|
||||
requirements = SystemRequirements(**requirements_data)
|
||||
|
||||
# Plan the system
|
||||
planner = AgentPlanner()
|
||||
design, mermaid_diagram, roadmap = planner.plan_system(requirements)
|
||||
|
||||
# Prepare output
|
||||
output_data = {
|
||||
"architecture_design": asdict(design),
|
||||
"mermaid_diagram": mermaid_diagram,
|
||||
"implementation_roadmap": roadmap,
|
||||
"metadata": {
|
||||
"generated_by": "agent_planner.py",
|
||||
"requirements_file": args.input_file,
|
||||
"architecture_pattern": design.pattern.value,
|
||||
"agent_count": len(design.agents)
|
||||
}
|
||||
}
|
||||
|
||||
# Output files
|
||||
output_prefix = args.output or "agent_architecture"
|
||||
|
||||
if args.format in ["json", "both"]:
|
||||
with open(f"{output_prefix}.json", 'w') as f:
|
||||
json.dump(output_data, f, indent=2, default=str)
|
||||
print(f"JSON output written to {output_prefix}.json")
|
||||
|
||||
if args.format in ["both"]:
|
||||
# Also create separate files for key components
|
||||
with open(f"{output_prefix}_diagram.mmd", 'w') as f:
|
||||
f.write(mermaid_diagram)
|
||||
print(f"Mermaid diagram written to {output_prefix}_diagram.mmd")
|
||||
|
||||
with open(f"{output_prefix}_roadmap.json", 'w') as f:
|
||||
json.dump(roadmap, f, indent=2)
|
||||
print(f"Implementation roadmap written to {output_prefix}_roadmap.json")
|
||||
|
||||
# Print summary
|
||||
print(f"\nArchitecture Summary:")
|
||||
print(f"Pattern: {design.pattern.value}")
|
||||
print(f"Agents: {len(design.agents)}")
|
||||
print(f"Communication Links: {len(design.communication_topology)}")
|
||||
print(f"Estimated Duration: {roadmap['total_duration']}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
543
engineering/agent-designer/assets/sample_execution_logs.json
Normal file
543
engineering/agent-designer/assets/sample_execution_logs.json
Normal file
@@ -0,0 +1,543 @@
|
||||
{
|
||||
"execution_logs": [
|
||||
{
|
||||
"task_id": "task_001",
|
||||
"agent_id": "research_agent_1",
|
||||
"task_type": "web_research",
|
||||
"task_description": "Research recent developments in artificial intelligence",
|
||||
"start_time": "2024-01-15T09:00:00Z",
|
||||
"end_time": "2024-01-15T09:02:34Z",
|
||||
"duration_ms": 154000,
|
||||
"status": "success",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 2300,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"query": "artificial intelligence developments 2024",
|
||||
"limit": 10
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 2100,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"query": "machine learning breakthroughs recent",
|
||||
"limit": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "analysis",
|
||||
"description": "Synthesize search results",
|
||||
"duration_ms": 149600,
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"summary": "Found 15 relevant sources covering recent AI developments including GPT-4 improvements, autonomous vehicle progress, and medical AI applications.",
|
||||
"sources_found": 15,
|
||||
"quality_score": 0.92
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 1250,
|
||||
"output_tokens": 2800,
|
||||
"total_tokens": 4050
|
||||
},
|
||||
"cost_usd": 0.081,
|
||||
"error_details": null,
|
||||
"tools_used": ["web_search"],
|
||||
"retry_count": 0,
|
||||
"metadata": {
|
||||
"user_id": "user_123",
|
||||
"session_id": "session_abc",
|
||||
"request_priority": "normal"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_002",
|
||||
"agent_id": "data_agent_1",
|
||||
"task_type": "data_analysis",
|
||||
"task_description": "Analyze sales performance data for Q4 2023",
|
||||
"start_time": "2024-01-15T09:05:00Z",
|
||||
"end_time": "2024-01-15T09:07:45Z",
|
||||
"duration_ms": 165000,
|
||||
"status": "success",
|
||||
"actions": [
|
||||
{
|
||||
"type": "data_ingestion",
|
||||
"description": "Load Q4 sales data",
|
||||
"duration_ms": 5000,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "data_analyzer",
|
||||
"duration_ms": 155000,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"analysis_type": "descriptive",
|
||||
"target_column": "revenue"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "visualization",
|
||||
"description": "Generate charts and graphs",
|
||||
"duration_ms": 5000,
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"insights": [
|
||||
"Revenue increased by 15% compared to Q3",
|
||||
"December was the strongest month",
|
||||
"Product category A led growth"
|
||||
],
|
||||
"charts_generated": 4,
|
||||
"quality_score": 0.88
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 3200,
|
||||
"output_tokens": 1800,
|
||||
"total_tokens": 5000
|
||||
},
|
||||
"cost_usd": 0.095,
|
||||
"error_details": null,
|
||||
"tools_used": ["data_analyzer"],
|
||||
"retry_count": 0,
|
||||
"metadata": {
|
||||
"user_id": "user_456",
|
||||
"session_id": "session_def",
|
||||
"request_priority": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_003",
|
||||
"agent_id": "document_agent_1",
|
||||
"task_type": "document_processing",
|
||||
"task_description": "Extract key information from research paper PDF",
|
||||
"start_time": "2024-01-15T09:10:00Z",
|
||||
"end_time": "2024-01-15T09:12:20Z",
|
||||
"duration_ms": 140000,
|
||||
"status": "partial",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "document_processor",
|
||||
"duration_ms": 135000,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"document_url": "https://example.com/research.pdf",
|
||||
"processing_mode": "key_points"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "validation",
|
||||
"description": "Validate extracted content",
|
||||
"duration_ms": 5000,
|
||||
"success": false,
|
||||
"error": "Content validation failed - missing abstract"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"extracted_content": "Partial content extracted successfully",
|
||||
"pages_processed": 12,
|
||||
"validation_issues": ["Missing abstract section"],
|
||||
"quality_score": 0.65
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 5400,
|
||||
"output_tokens": 3200,
|
||||
"total_tokens": 8600
|
||||
},
|
||||
"cost_usd": 0.172,
|
||||
"error_details": {
|
||||
"error_type": "validation_error",
|
||||
"error_message": "Document structure validation failed",
|
||||
"affected_section": "abstract"
|
||||
},
|
||||
"tools_used": ["document_processor"],
|
||||
"retry_count": 1,
|
||||
"metadata": {
|
||||
"user_id": "user_789",
|
||||
"session_id": "session_ghi",
|
||||
"request_priority": "normal"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_004",
|
||||
"agent_id": "communication_agent_1",
|
||||
"task_type": "notification",
|
||||
"task_description": "Send completion notification to project stakeholders",
|
||||
"start_time": "2024-01-15T09:15:00Z",
|
||||
"end_time": "2024-01-15T09:15:08Z",
|
||||
"duration_ms": 8000,
|
||||
"status": "success",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "notification_sender",
|
||||
"duration_ms": 7500,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"recipients": ["manager@example.com", "team@example.com"],
|
||||
"message": "Project analysis completed successfully",
|
||||
"channel": "email"
|
||||
}
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"notifications_sent": 2,
|
||||
"delivery_confirmations": 2,
|
||||
"quality_score": 1.0
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 200,
|
||||
"output_tokens": 150,
|
||||
"total_tokens": 350
|
||||
},
|
||||
"cost_usd": 0.007,
|
||||
"error_details": null,
|
||||
"tools_used": ["notification_sender"],
|
||||
"retry_count": 0,
|
||||
"metadata": {
|
||||
"user_id": "system",
|
||||
"session_id": "session_jkl",
|
||||
"request_priority": "normal"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_005",
|
||||
"agent_id": "research_agent_2",
|
||||
"task_type": "web_research",
|
||||
"task_description": "Research competitive landscape analysis",
|
||||
"start_time": "2024-01-15T09:20:00Z",
|
||||
"end_time": "2024-01-15T09:25:30Z",
|
||||
"duration_ms": 330000,
|
||||
"status": "failure",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 2800,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"query": "competitive analysis software industry",
|
||||
"limit": 15
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 30000,
|
||||
"success": false,
|
||||
"error": "Rate limit exceeded"
|
||||
},
|
||||
{
|
||||
"type": "retry",
|
||||
"description": "Wait and retry search",
|
||||
"duration_ms": 60000,
|
||||
"success": false
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 30000,
|
||||
"success": false,
|
||||
"error": "Service timeout"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"partial_results": "Initial search completed, subsequent searches failed",
|
||||
"sources_found": 8,
|
||||
"quality_score": 0.3
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 800,
|
||||
"output_tokens": 400,
|
||||
"total_tokens": 1200
|
||||
},
|
||||
"cost_usd": 0.024,
|
||||
"error_details": {
|
||||
"error_type": "service_timeout",
|
||||
"error_message": "Web search service exceeded timeout limit",
|
||||
"retry_attempts": 2
|
||||
},
|
||||
"tools_used": ["web_search"],
|
||||
"retry_count": 2,
|
||||
"metadata": {
|
||||
"user_id": "user_101",
|
||||
"session_id": "session_mno",
|
||||
"request_priority": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_006",
|
||||
"agent_id": "scheduler_agent_1",
|
||||
"task_type": "task_scheduling",
|
||||
"task_description": "Schedule weekly report generation",
|
||||
"start_time": "2024-01-15T09:30:00Z",
|
||||
"end_time": "2024-01-15T09:30:15Z",
|
||||
"duration_ms": 15000,
|
||||
"status": "success",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "task_scheduler",
|
||||
"duration_ms": 12000,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"task_definition": {
|
||||
"action": "generate_report",
|
||||
"parameters": {"report_type": "weekly_summary"}
|
||||
},
|
||||
"schedule": {
|
||||
"type": "recurring",
|
||||
"recurrence_pattern": "weekly"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "validation",
|
||||
"description": "Verify schedule creation",
|
||||
"duration_ms": 3000,
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"task_scheduled": true,
|
||||
"next_execution": "2024-01-22T09:30:00Z",
|
||||
"schedule_id": "sched_789",
|
||||
"quality_score": 1.0
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 300,
|
||||
"output_tokens": 200,
|
||||
"total_tokens": 500
|
||||
},
|
||||
"cost_usd": 0.01,
|
||||
"error_details": null,
|
||||
"tools_used": ["task_scheduler"],
|
||||
"retry_count": 0,
|
||||
"metadata": {
|
||||
"user_id": "user_202",
|
||||
"session_id": "session_pqr",
|
||||
"request_priority": "low"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_007",
|
||||
"agent_id": "data_agent_2",
|
||||
"task_type": "data_analysis",
|
||||
"task_description": "Analyze customer satisfaction survey results",
|
||||
"start_time": "2024-01-15T10:00:00Z",
|
||||
"end_time": "2024-01-15T10:04:25Z",
|
||||
"duration_ms": 265000,
|
||||
"status": "timeout",
|
||||
"actions": [
|
||||
{
|
||||
"type": "data_ingestion",
|
||||
"description": "Load survey response data",
|
||||
"duration_ms": 15000,
|
||||
"success": true
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "data_analyzer",
|
||||
"duration_ms": 250000,
|
||||
"success": false,
|
||||
"error": "Operation timeout after 250 seconds"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"partial_analysis": "Data loaded but analysis incomplete",
|
||||
"records_processed": 5000,
|
||||
"total_records": 15000,
|
||||
"quality_score": 0.2
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 8000,
|
||||
"output_tokens": 1000,
|
||||
"total_tokens": 9000
|
||||
},
|
||||
"cost_usd": 0.18,
|
||||
"error_details": {
|
||||
"error_type": "timeout",
|
||||
"error_message": "Data analysis operation exceeded maximum allowed time",
|
||||
"timeout_limit_ms": 250000
|
||||
},
|
||||
"tools_used": ["data_analyzer"],
|
||||
"retry_count": 0,
|
||||
"metadata": {
|
||||
"user_id": "user_303",
|
||||
"session_id": "session_stu",
|
||||
"request_priority": "normal"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_008",
|
||||
"agent_id": "research_agent_1",
|
||||
"task_type": "web_research",
|
||||
"task_description": "Research industry best practices for remote work",
|
||||
"start_time": "2024-01-15T10:30:00Z",
|
||||
"end_time": "2024-01-15T10:33:15Z",
|
||||
"duration_ms": 195000,
|
||||
"status": "success",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 2200,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"query": "remote work best practices 2024",
|
||||
"limit": 12
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "web_search",
|
||||
"duration_ms": 2400,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"query": "hybrid work policies companies",
|
||||
"limit": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "content_synthesis",
|
||||
"description": "Synthesize findings from multiple sources",
|
||||
"duration_ms": 190400,
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"comprehensive_report": "Detailed analysis of remote work best practices with industry examples",
|
||||
"sources_analyzed": 20,
|
||||
"key_insights": 8,
|
||||
"quality_score": 0.94
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 2800,
|
||||
"output_tokens": 4200,
|
||||
"total_tokens": 7000
|
||||
},
|
||||
"cost_usd": 0.14,
|
||||
"error_details": null,
|
||||
"tools_used": ["web_search"],
|
||||
"retry_count": 0,
|
||||
"metadata": {
|
||||
"user_id": "user_404",
|
||||
"session_id": "session_vwx",
|
||||
"request_priority": "normal"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_009",
|
||||
"agent_id": "document_agent_2",
|
||||
"task_type": "document_processing",
|
||||
"task_description": "Process and summarize quarterly financial report",
|
||||
"start_time": "2024-01-15T11:00:00Z",
|
||||
"end_time": "2024-01-15T11:02:30Z",
|
||||
"duration_ms": 150000,
|
||||
"status": "success",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "document_processor",
|
||||
"duration_ms": 145000,
|
||||
"success": true,
|
||||
"parameters": {
|
||||
"document_url": "https://example.com/q4-financial-report.pdf",
|
||||
"processing_mode": "summary",
|
||||
"output_format": "json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "quality_check",
|
||||
"description": "Validate summary completeness",
|
||||
"duration_ms": 5000,
|
||||
"success": true
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"executive_summary": "Q4 revenue grew 12% YoY with strong performance in all segments",
|
||||
"key_metrics_extracted": 15,
|
||||
"summary_length": 500,
|
||||
"quality_score": 0.91
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 6500,
|
||||
"output_tokens": 2200,
|
||||
"total_tokens": 8700
|
||||
},
|
||||
"cost_usd": 0.174,
|
||||
"error_details": null,
|
||||
"tools_used": ["document_processor"],
|
||||
"retry_count": 0,
|
||||
"metadata": {
|
||||
"user_id": "user_505",
|
||||
"session_id": "session_yzA",
|
||||
"request_priority": "high"
|
||||
}
|
||||
},
|
||||
{
|
||||
"task_id": "task_010",
|
||||
"agent_id": "communication_agent_2",
|
||||
"task_type": "notification",
|
||||
"task_description": "Send urgent system maintenance notification",
|
||||
"start_time": "2024-01-15T11:30:00Z",
|
||||
"end_time": "2024-01-15T11:30:45Z",
|
||||
"duration_ms": 45000,
|
||||
"status": "failure",
|
||||
"actions": [
|
||||
{
|
||||
"type": "tool_call",
|
||||
"tool_name": "notification_sender",
|
||||
"duration_ms": 30000,
|
||||
"success": false,
|
||||
"error": "Authentication failed - invalid API key",
|
||||
"parameters": {
|
||||
"recipients": ["all-users@example.com"],
|
||||
"message": "Scheduled maintenance tonight 11 PM - 2 AM",
|
||||
"channel": "email",
|
||||
"priority": "urgent"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "retry",
|
||||
"description": "Retry with backup credentials",
|
||||
"duration_ms": 15000,
|
||||
"success": false,
|
||||
"error": "Backup authentication also failed"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"notifications_sent": 0,
|
||||
"delivery_failures": 1,
|
||||
"quality_score": 0.0
|
||||
},
|
||||
"tokens_used": {
|
||||
"input_tokens": 150,
|
||||
"output_tokens": 50,
|
||||
"total_tokens": 200
|
||||
},
|
||||
"cost_usd": 0.004,
|
||||
"error_details": {
|
||||
"error_type": "authentication_error",
|
||||
"error_message": "Failed to authenticate with notification service",
|
||||
"retry_attempts": 1
|
||||
},
|
||||
"tools_used": ["notification_sender"],
|
||||
"retry_count": 1,
|
||||
"metadata": {
|
||||
"user_id": "system",
|
||||
"session_id": "session_BcD",
|
||||
"request_priority": "urgent"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
{
|
||||
"goal": "Build a comprehensive research and analysis platform that can gather information from multiple sources, analyze data, and generate detailed reports",
|
||||
"description": "The system needs to handle complex research tasks involving web searches, data analysis, document processing, and collaborative report generation. It should be able to coordinate multiple specialists working in parallel while maintaining quality control and ensuring comprehensive coverage of research topics.",
|
||||
"tasks": [
|
||||
"Conduct multi-source web research on specified topics",
|
||||
"Analyze and synthesize information from various sources",
|
||||
"Perform data processing and statistical analysis",
|
||||
"Generate visualizations and charts from data",
|
||||
"Create comprehensive written reports",
|
||||
"Fact-check and validate information accuracy",
|
||||
"Coordinate parallel research streams",
|
||||
"Handle real-time information updates",
|
||||
"Manage research project timelines",
|
||||
"Provide interactive research assistance"
|
||||
],
|
||||
"constraints": {
|
||||
"max_response_time": 30000,
|
||||
"budget_per_task": 1.0,
|
||||
"quality_threshold": 0.9,
|
||||
"concurrent_tasks": 10,
|
||||
"data_retention_days": 90,
|
||||
"security_level": "standard",
|
||||
"compliance_requirements": ["GDPR", "data_minimization"]
|
||||
},
|
||||
"team_size": 6,
|
||||
"performance_requirements": {
|
||||
"high_throughput": true,
|
||||
"fault_tolerance": true,
|
||||
"low_latency": false,
|
||||
"scalability": "medium",
|
||||
"availability": 0.99
|
||||
},
|
||||
"safety_requirements": [
|
||||
"Input validation and sanitization",
|
||||
"Output content filtering",
|
||||
"Rate limiting for external APIs",
|
||||
"Error handling and graceful degradation",
|
||||
"Human oversight for critical decisions",
|
||||
"Audit logging for all operations"
|
||||
],
|
||||
"integration_requirements": [
|
||||
"REST API endpoints for external systems",
|
||||
"Webhook support for real-time updates",
|
||||
"Database integration for data persistence",
|
||||
"File storage for documents and media",
|
||||
"Email notifications for important events",
|
||||
"Dashboard for monitoring and control"
|
||||
],
|
||||
"scale_requirements": {
|
||||
"initial_users": 50,
|
||||
"peak_concurrent_users": 200,
|
||||
"data_volume_gb": 100,
|
||||
"requests_per_hour": 1000,
|
||||
"geographic_regions": ["US", "EU"],
|
||||
"growth_projection": "50% per year"
|
||||
}
|
||||
}
|
||||
545
engineering/agent-designer/assets/sample_tool_descriptions.json
Normal file
545
engineering/agent-designer/assets/sample_tool_descriptions.json
Normal file
@@ -0,0 +1,545 @@
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"name": "web_search",
|
||||
"purpose": "Search the web for information on specified topics with customizable filters and result limits",
|
||||
"category": "search",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "query",
|
||||
"type": "string",
|
||||
"description": "Search query string to find relevant information",
|
||||
"required": true,
|
||||
"min_length": 1,
|
||||
"max_length": 500,
|
||||
"examples": ["artificial intelligence trends", "climate change impact", "python programming tutorial"]
|
||||
},
|
||||
{
|
||||
"name": "limit",
|
||||
"type": "integer",
|
||||
"description": "Maximum number of search results to return",
|
||||
"required": false,
|
||||
"default": 10,
|
||||
"minimum": 1,
|
||||
"maximum": 100
|
||||
},
|
||||
{
|
||||
"name": "language",
|
||||
"type": "string",
|
||||
"description": "Language code for search results",
|
||||
"required": false,
|
||||
"default": "en",
|
||||
"enum": ["en", "es", "fr", "de", "it", "pt", "zh", "ja"]
|
||||
},
|
||||
{
|
||||
"name": "time_range",
|
||||
"type": "string",
|
||||
"description": "Time range filter for search results",
|
||||
"required": false,
|
||||
"enum": ["any", "day", "week", "month", "year"]
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "results",
|
||||
"type": "array",
|
||||
"description": "Array of search result objects",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"url": {"type": "string"},
|
||||
"snippet": {"type": "string"},
|
||||
"relevance_score": {"type": "number"}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "total_found",
|
||||
"type": "integer",
|
||||
"description": "Total number of results available"
|
||||
}
|
||||
],
|
||||
"error_conditions": [
|
||||
"Invalid query format",
|
||||
"Network timeout",
|
||||
"API rate limit exceeded",
|
||||
"No results found",
|
||||
"Service unavailable"
|
||||
],
|
||||
"side_effects": [
|
||||
"Logs search query for analytics",
|
||||
"May cache results temporarily"
|
||||
],
|
||||
"idempotent": true,
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 60,
|
||||
"requests_per_hour": 1000,
|
||||
"burst_limit": 10
|
||||
},
|
||||
"dependencies": [
|
||||
"search_api_service",
|
||||
"content_filter_service"
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"description": "Basic web search",
|
||||
"input": {
|
||||
"query": "machine learning algorithms",
|
||||
"limit": 5
|
||||
},
|
||||
"expected_output": {
|
||||
"results": [
|
||||
{
|
||||
"title": "Introduction to Machine Learning Algorithms",
|
||||
"url": "https://example.com/ml-intro",
|
||||
"snippet": "Machine learning algorithms are computational methods...",
|
||||
"relevance_score": 0.95
|
||||
}
|
||||
],
|
||||
"total_found": 1250
|
||||
}
|
||||
}
|
||||
],
|
||||
"security_requirements": [
|
||||
"Query sanitization",
|
||||
"Rate limiting by user",
|
||||
"Content filtering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"purpose": "Analyze structured data and generate statistical insights, trends, and visualizations",
|
||||
"category": "data",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "data",
|
||||
"type": "object",
|
||||
"description": "Structured data to analyze in JSON format",
|
||||
"required": true,
|
||||
"properties": {
|
||||
"columns": {"type": "array"},
|
||||
"rows": {"type": "array"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "analysis_type",
|
||||
"type": "string",
|
||||
"description": "Type of analysis to perform",
|
||||
"required": true,
|
||||
"enum": ["descriptive", "correlation", "trend", "distribution", "outlier_detection"]
|
||||
},
|
||||
{
|
||||
"name": "target_column",
|
||||
"type": "string",
|
||||
"description": "Primary column to focus analysis on",
|
||||
"required": false
|
||||
},
|
||||
{
|
||||
"name": "include_visualization",
|
||||
"type": "boolean",
|
||||
"description": "Whether to generate visualization data",
|
||||
"required": false,
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "insights",
|
||||
"type": "array",
|
||||
"description": "Array of analytical insights and findings"
|
||||
},
|
||||
{
|
||||
"name": "statistics",
|
||||
"type": "object",
|
||||
"description": "Statistical measures and metrics"
|
||||
},
|
||||
{
|
||||
"name": "visualization_data",
|
||||
"type": "object",
|
||||
"description": "Data formatted for visualization creation"
|
||||
}
|
||||
],
|
||||
"error_conditions": [
|
||||
"Invalid data format",
|
||||
"Insufficient data points",
|
||||
"Missing required columns",
|
||||
"Data type mismatch",
|
||||
"Analysis timeout"
|
||||
],
|
||||
"side_effects": [
|
||||
"May create temporary analysis files",
|
||||
"Logs analysis parameters for optimization"
|
||||
],
|
||||
"idempotent": true,
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 30,
|
||||
"requests_per_hour": 500,
|
||||
"burst_limit": 5
|
||||
},
|
||||
"dependencies": [
|
||||
"statistics_engine",
|
||||
"visualization_service"
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"description": "Basic descriptive analysis",
|
||||
"input": {
|
||||
"data": {
|
||||
"columns": ["age", "salary", "department"],
|
||||
"rows": [
|
||||
[25, 50000, "engineering"],
|
||||
[30, 60000, "engineering"],
|
||||
[28, 55000, "marketing"]
|
||||
]
|
||||
},
|
||||
"analysis_type": "descriptive",
|
||||
"target_column": "salary"
|
||||
},
|
||||
"expected_output": {
|
||||
"insights": [
|
||||
"Average salary is $55,000",
|
||||
"Salary range: $50,000 - $60,000",
|
||||
"Engineering department has higher average salary"
|
||||
],
|
||||
"statistics": {
|
||||
"mean": 55000,
|
||||
"median": 55000,
|
||||
"std_dev": 5000
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"security_requirements": [
|
||||
"Data anonymization",
|
||||
"Access control validation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "document_processor",
|
||||
"purpose": "Process and extract information from various document formats including PDFs, Word docs, and plain text",
|
||||
"category": "file",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "document_url",
|
||||
"type": "string",
|
||||
"description": "URL or path to the document to process",
|
||||
"required": true,
|
||||
"pattern": "^(https?://|file://|/)"
|
||||
},
|
||||
{
|
||||
"name": "processing_mode",
|
||||
"type": "string",
|
||||
"description": "How to process the document",
|
||||
"required": false,
|
||||
"default": "full_text",
|
||||
"enum": ["full_text", "summary", "key_points", "metadata_only"]
|
||||
},
|
||||
{
|
||||
"name": "output_format",
|
||||
"type": "string",
|
||||
"description": "Desired output format",
|
||||
"required": false,
|
||||
"default": "json",
|
||||
"enum": ["json", "markdown", "plain_text"]
|
||||
},
|
||||
{
|
||||
"name": "language_detection",
|
||||
"type": "boolean",
|
||||
"description": "Whether to detect document language",
|
||||
"required": false,
|
||||
"default": true
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "content",
|
||||
"type": "string",
|
||||
"description": "Extracted and processed document content"
|
||||
},
|
||||
{
|
||||
"name": "metadata",
|
||||
"type": "object",
|
||||
"description": "Document metadata including author, creation date, etc."
|
||||
},
|
||||
{
|
||||
"name": "language",
|
||||
"type": "string",
|
||||
"description": "Detected language of the document"
|
||||
},
|
||||
{
|
||||
"name": "word_count",
|
||||
"type": "integer",
|
||||
"description": "Total word count in the document"
|
||||
}
|
||||
],
|
||||
"error_conditions": [
|
||||
"Document not found",
|
||||
"Unsupported file format",
|
||||
"Document corrupted or unreadable",
|
||||
"Access permission denied",
|
||||
"Document too large"
|
||||
],
|
||||
"side_effects": [
|
||||
"May download and cache documents temporarily",
|
||||
"Creates processing logs for debugging"
|
||||
],
|
||||
"idempotent": true,
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 20,
|
||||
"requests_per_hour": 300,
|
||||
"burst_limit": 3
|
||||
},
|
||||
"dependencies": [
|
||||
"document_parser_service",
|
||||
"language_detection_service",
|
||||
"file_storage_service"
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"description": "Process PDF document for full text extraction",
|
||||
"input": {
|
||||
"document_url": "https://example.com/research-paper.pdf",
|
||||
"processing_mode": "full_text",
|
||||
"output_format": "markdown"
|
||||
},
|
||||
"expected_output": {
|
||||
"content": "# Research Paper Title\n\nAbstract: This paper discusses...",
|
||||
"metadata": {
|
||||
"author": "Dr. Smith",
|
||||
"creation_date": "2024-01-15",
|
||||
"pages": 15
|
||||
},
|
||||
"language": "en",
|
||||
"word_count": 3500
|
||||
}
|
||||
}
|
||||
],
|
||||
"security_requirements": [
|
||||
"URL validation",
|
||||
"File type verification",
|
||||
"Malware scanning",
|
||||
"Access control enforcement"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "notification_sender",
|
||||
"purpose": "Send notifications via multiple channels including email, SMS, and webhooks",
|
||||
"category": "communication",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "recipients",
|
||||
"type": "array",
|
||||
"description": "List of recipient identifiers",
|
||||
"required": true,
|
||||
"min_items": 1,
|
||||
"max_items": 100,
|
||||
"items": {
|
||||
"type": "string",
|
||||
"pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$|^\\+?[1-9]\\d{1,14}$"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "message",
|
||||
"type": "string",
|
||||
"description": "Message content to send",
|
||||
"required": true,
|
||||
"min_length": 1,
|
||||
"max_length": 10000
|
||||
},
|
||||
{
|
||||
"name": "channel",
|
||||
"type": "string",
|
||||
"description": "Communication channel to use",
|
||||
"required": false,
|
||||
"default": "email",
|
||||
"enum": ["email", "sms", "webhook", "push"]
|
||||
},
|
||||
{
|
||||
"name": "priority",
|
||||
"type": "string",
|
||||
"description": "Message priority level",
|
||||
"required": false,
|
||||
"default": "normal",
|
||||
"enum": ["low", "normal", "high", "urgent"]
|
||||
},
|
||||
{
|
||||
"name": "template_id",
|
||||
"type": "string",
|
||||
"description": "Optional template ID for formatting",
|
||||
"required": false
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "delivery_status",
|
||||
"type": "object",
|
||||
"description": "Status of message delivery to each recipient"
|
||||
},
|
||||
{
|
||||
"name": "message_id",
|
||||
"type": "string",
|
||||
"description": "Unique identifier for the sent message"
|
||||
},
|
||||
{
|
||||
"name": "delivery_timestamp",
|
||||
"type": "string",
|
||||
"description": "ISO timestamp when message was sent"
|
||||
}
|
||||
],
|
||||
"error_conditions": [
|
||||
"Invalid recipient format",
|
||||
"Message too long",
|
||||
"Channel service unavailable",
|
||||
"Authentication failure",
|
||||
"Rate limit exceeded for channel"
|
||||
],
|
||||
"side_effects": [
|
||||
"Sends actual notifications to recipients",
|
||||
"Logs delivery attempts and results",
|
||||
"Updates delivery statistics"
|
||||
],
|
||||
"idempotent": false,
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 100,
|
||||
"requests_per_hour": 2000,
|
||||
"burst_limit": 20
|
||||
},
|
||||
"dependencies": [
|
||||
"email_service",
|
||||
"sms_service",
|
||||
"webhook_service"
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"description": "Send email notification",
|
||||
"input": {
|
||||
"recipients": ["user@example.com"],
|
||||
"message": "Your report has been completed and is ready for review.",
|
||||
"channel": "email",
|
||||
"priority": "normal"
|
||||
},
|
||||
"expected_output": {
|
||||
"delivery_status": {
|
||||
"user@example.com": "delivered"
|
||||
},
|
||||
"message_id": "msg_12345",
|
||||
"delivery_timestamp": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
}
|
||||
],
|
||||
"security_requirements": [
|
||||
"Recipient validation",
|
||||
"Message content filtering",
|
||||
"Rate limiting per user",
|
||||
"Delivery confirmation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "task_scheduler",
|
||||
"purpose": "Schedule and manage delayed or recurring tasks within the agent system",
|
||||
"category": "compute",
|
||||
"inputs": [
|
||||
{
|
||||
"name": "task_definition",
|
||||
"type": "object",
|
||||
"description": "Definition of the task to be scheduled",
|
||||
"required": true,
|
||||
"properties": {
|
||||
"action": {"type": "string"},
|
||||
"parameters": {"type": "object"},
|
||||
"retry_policy": {"type": "object"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "schedule",
|
||||
"type": "object",
|
||||
"description": "Scheduling parameters for the task",
|
||||
"required": true,
|
||||
"properties": {
|
||||
"type": {"type": "string", "enum": ["once", "recurring"]},
|
||||
"execute_at": {"type": "string"},
|
||||
"recurrence_pattern": {"type": "string"}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "priority",
|
||||
"type": "integer",
|
||||
"description": "Task priority (1-10, higher is more urgent)",
|
||||
"required": false,
|
||||
"default": 5,
|
||||
"minimum": 1,
|
||||
"maximum": 10
|
||||
}
|
||||
],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "task_id",
|
||||
"type": "string",
|
||||
"description": "Unique identifier for the scheduled task"
|
||||
},
|
||||
{
|
||||
"name": "next_execution",
|
||||
"type": "string",
|
||||
"description": "ISO timestamp of next scheduled execution"
|
||||
},
|
||||
{
|
||||
"name": "status",
|
||||
"type": "string",
|
||||
"description": "Current status of the scheduled task"
|
||||
}
|
||||
],
|
||||
"error_conditions": [
|
||||
"Invalid schedule format",
|
||||
"Past execution time specified",
|
||||
"Task queue full",
|
||||
"Invalid task definition",
|
||||
"Scheduling service unavailable"
|
||||
],
|
||||
"side_effects": [
|
||||
"Creates scheduled tasks in the system",
|
||||
"May consume system resources for task storage",
|
||||
"Updates scheduling metrics"
|
||||
],
|
||||
"idempotent": false,
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 50,
|
||||
"requests_per_hour": 1000,
|
||||
"burst_limit": 10
|
||||
},
|
||||
"dependencies": [
|
||||
"task_scheduler_service",
|
||||
"task_executor_service"
|
||||
],
|
||||
"examples": [
|
||||
{
|
||||
"description": "Schedule a one-time report generation",
|
||||
"input": {
|
||||
"task_definition": {
|
||||
"action": "generate_report",
|
||||
"parameters": {
|
||||
"report_type": "monthly_summary",
|
||||
"recipients": ["manager@example.com"]
|
||||
}
|
||||
},
|
||||
"schedule": {
|
||||
"type": "once",
|
||||
"execute_at": "2024-02-01T09:00:00Z"
|
||||
},
|
||||
"priority": 7
|
||||
},
|
||||
"expected_output": {
|
||||
"task_id": "task_67890",
|
||||
"next_execution": "2024-02-01T09:00:00Z",
|
||||
"status": "scheduled"
|
||||
}
|
||||
}
|
||||
],
|
||||
"security_requirements": [
|
||||
"Task definition validation",
|
||||
"User authorization for scheduling",
|
||||
"Resource limit enforcement"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
{
|
||||
"architecture_design": {
|
||||
"pattern": "supervisor",
|
||||
"agents": [
|
||||
{
|
||||
"name": "supervisor_agent",
|
||||
"role": "Task Coordinator and Quality Controller",
|
||||
"archetype": "coordinator",
|
||||
"responsibilities": [
|
||||
"task_decomposition",
|
||||
"delegation",
|
||||
"progress_monitoring",
|
||||
"quality_assurance",
|
||||
"result_aggregation"
|
||||
],
|
||||
"capabilities": [
|
||||
"planning",
|
||||
"coordination",
|
||||
"evaluation",
|
||||
"decision_making"
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "file_manager",
|
||||
"description": "Manage files and directories",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string"
|
||||
},
|
||||
"path": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"file_operations",
|
||||
"data_management"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "low"
|
||||
},
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze and process data",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"insights": {
|
||||
"type": "array"
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"data_analysis",
|
||||
"statistics",
|
||||
"visualization"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
}
|
||||
],
|
||||
"communication_interfaces": [
|
||||
"user_interface",
|
||||
"agent_messaging"
|
||||
],
|
||||
"constraints": {
|
||||
"max_concurrent_supervisions": 5,
|
||||
"decision_timeout": "30s"
|
||||
},
|
||||
"success_criteria": [
|
||||
"successful task completion",
|
||||
"optimal resource utilization",
|
||||
"quality standards met"
|
||||
],
|
||||
"dependencies": []
|
||||
},
|
||||
{
|
||||
"name": "research_specialist",
|
||||
"role": "Research Specialist",
|
||||
"archetype": "specialist",
|
||||
"responsibilities": [
|
||||
"Conduct multi-source web research on specified topics",
|
||||
"Handle real-time information updates"
|
||||
],
|
||||
"capabilities": [
|
||||
"research_expertise",
|
||||
"specialized_tools",
|
||||
"domain_knowledge"
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"results": {
|
||||
"type": "array"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"research",
|
||||
"information_gathering"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
},
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze and process data",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"insights": {
|
||||
"type": "array"
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"data_analysis",
|
||||
"statistics",
|
||||
"visualization"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
}
|
||||
],
|
||||
"communication_interfaces": [
|
||||
"supervisor_messaging"
|
||||
],
|
||||
"constraints": {
|
||||
"domain_scope": "research",
|
||||
"task_queue_size": 10
|
||||
},
|
||||
"success_criteria": [
|
||||
"excel in research tasks",
|
||||
"maintain domain expertise",
|
||||
"provide quality output"
|
||||
],
|
||||
"dependencies": [
|
||||
"supervisor_agent"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "data_specialist",
|
||||
"role": "Data Specialist",
|
||||
"archetype": "specialist",
|
||||
"responsibilities": [
|
||||
"Analyze and synthesize information from various sources",
|
||||
"Perform data processing and statistical analysis",
|
||||
"Generate visualizations and charts from data"
|
||||
],
|
||||
"capabilities": [
|
||||
"data_expertise",
|
||||
"specialized_tools",
|
||||
"domain_knowledge"
|
||||
],
|
||||
"tools": [
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze and process data",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"insights": {
|
||||
"type": "array"
|
||||
},
|
||||
"metrics": {
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"data_analysis",
|
||||
"statistics",
|
||||
"visualization"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "medium"
|
||||
},
|
||||
{
|
||||
"name": "file_manager",
|
||||
"description": "Manage files and directories",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {
|
||||
"type": "string"
|
||||
},
|
||||
"path": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"output_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"success": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"content": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"capabilities": [
|
||||
"file_operations",
|
||||
"data_management"
|
||||
],
|
||||
"reliability": "high",
|
||||
"latency": "low"
|
||||
}
|
||||
],
|
||||
"communication_interfaces": [
|
||||
"supervisor_messaging"
|
||||
],
|
||||
"constraints": {
|
||||
"domain_scope": "data",
|
||||
"task_queue_size": 10
|
||||
},
|
||||
"success_criteria": [
|
||||
"excel in data tasks",
|
||||
"maintain domain expertise",
|
||||
"provide quality output"
|
||||
],
|
||||
"dependencies": [
|
||||
"supervisor_agent"
|
||||
]
|
||||
}
|
||||
],
|
||||
"communication_topology": [
|
||||
{
|
||||
"from_agent": "supervisor_agent",
|
||||
"to_agent": "research_specialist",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_demand",
|
||||
"criticality": "high"
|
||||
},
|
||||
{
|
||||
"from_agent": "research_specialist",
|
||||
"to_agent": "supervisor_agent",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_completion",
|
||||
"criticality": "high"
|
||||
},
|
||||
{
|
||||
"from_agent": "supervisor_agent",
|
||||
"to_agent": "data_specialist",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_demand",
|
||||
"criticality": "high"
|
||||
},
|
||||
{
|
||||
"from_agent": "data_specialist",
|
||||
"to_agent": "supervisor_agent",
|
||||
"pattern": "direct_message",
|
||||
"data_format": "json",
|
||||
"frequency": "on_completion",
|
||||
"criticality": "high"
|
||||
}
|
||||
],
|
||||
"shared_resources": [
|
||||
{
|
||||
"type": "message_queue",
|
||||
"capacity": 1000
|
||||
},
|
||||
{
|
||||
"type": "shared_memory",
|
||||
"size": "1GB"
|
||||
},
|
||||
{
|
||||
"type": "event_store",
|
||||
"retention": "30 days"
|
||||
}
|
||||
],
|
||||
"guardrails": [
|
||||
{
|
||||
"type": "input_validation",
|
||||
"rules": "strict_schema_enforcement"
|
||||
},
|
||||
{
|
||||
"type": "rate_limiting",
|
||||
"limit": "100_requests_per_minute"
|
||||
},
|
||||
{
|
||||
"type": "output_filtering",
|
||||
"rules": "content_safety_check"
|
||||
}
|
||||
],
|
||||
"scaling_strategy": {
|
||||
"horizontal_scaling": true,
|
||||
"auto_scaling_triggers": [
|
||||
"cpu > 80%",
|
||||
"queue_depth > 100"
|
||||
],
|
||||
"max_instances_per_agent": 5
|
||||
},
|
||||
"failure_handling": {
|
||||
"retry_policy": "exponential_backoff",
|
||||
"circuit_breaker": true,
|
||||
"fallback_strategies": [
|
||||
"graceful_degradation",
|
||||
"human_escalation"
|
||||
]
|
||||
}
|
||||
},
|
||||
"mermaid_diagram": "graph TD\n supervisor_agent[Task Coordinator and Quality Controller]:::coordinator\n research_specialist[Research Specialist]:::specialist\n data_specialist[Data Specialist]:::specialist\n supervisor_agent --> research_specialist\n research_specialist --> supervisor_agent\n supervisor_agent --> data_specialist\n data_specialist --> supervisor_agent\n\n classDef coordinator fill:#e1f5fe,stroke:#01579b,stroke-width:2px\n classDef specialist fill:#f3e5f5,stroke:#4a148c,stroke-width:2px\n classDef interface fill:#e8f5e8,stroke:#1b5e20,stroke-width:2px\n classDef monitor fill:#fff3e0,stroke:#e65100,stroke-width:2px\n class supervisor_agent coordinator\n class research_specialist specialist\n class data_specialist specialist",
|
||||
"implementation_roadmap": {
|
||||
"total_duration": "8-12 weeks",
|
||||
"phases": [
|
||||
{
|
||||
"phase": 1,
|
||||
"name": "Core Infrastructure",
|
||||
"duration": "2-3 weeks",
|
||||
"tasks": [
|
||||
"Set up development environment",
|
||||
"Implement basic agent framework",
|
||||
"Create communication infrastructure",
|
||||
"Set up monitoring and logging",
|
||||
"Implement basic tools"
|
||||
],
|
||||
"deliverables": [
|
||||
"Agent runtime framework",
|
||||
"Communication layer",
|
||||
"Basic monitoring dashboard"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": 2,
|
||||
"name": "Agent Implementation",
|
||||
"duration": "3-4 weeks",
|
||||
"tasks": [
|
||||
"Implement individual agent logic",
|
||||
"Create agent-specific tools",
|
||||
"Implement communication protocols",
|
||||
"Add error handling and recovery",
|
||||
"Create agent configuration system"
|
||||
],
|
||||
"deliverables": [
|
||||
"Functional agent implementations",
|
||||
"Tool integration",
|
||||
"Configuration management"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": 3,
|
||||
"name": "Integration and Testing",
|
||||
"duration": "2-3 weeks",
|
||||
"tasks": [
|
||||
"Integrate all agents",
|
||||
"End-to-end testing",
|
||||
"Performance optimization",
|
||||
"Security implementation",
|
||||
"Documentation creation"
|
||||
],
|
||||
"deliverables": [
|
||||
"Integrated system",
|
||||
"Test suite",
|
||||
"Performance benchmarks",
|
||||
"Security audit report"
|
||||
]
|
||||
},
|
||||
{
|
||||
"phase": 4,
|
||||
"name": "Deployment and Monitoring",
|
||||
"duration": "1-2 weeks",
|
||||
"tasks": [
|
||||
"Production deployment",
|
||||
"Monitoring setup",
|
||||
"Alerting configuration",
|
||||
"User training",
|
||||
"Go-live support"
|
||||
],
|
||||
"deliverables": [
|
||||
"Production system",
|
||||
"Monitoring dashboard",
|
||||
"Operational runbooks",
|
||||
"Training materials"
|
||||
]
|
||||
}
|
||||
],
|
||||
"critical_path": [
|
||||
"Agent framework implementation",
|
||||
"Communication layer development",
|
||||
"Integration testing",
|
||||
"Production deployment"
|
||||
],
|
||||
"risks": [
|
||||
{
|
||||
"risk": "Communication complexity",
|
||||
"impact": "high",
|
||||
"mitigation": "Start with simple protocols, iterate"
|
||||
},
|
||||
{
|
||||
"risk": "Agent coordination failures",
|
||||
"impact": "medium",
|
||||
"mitigation": "Implement robust error handling and fallbacks"
|
||||
},
|
||||
{
|
||||
"risk": "Performance bottlenecks",
|
||||
"impact": "medium",
|
||||
"mitigation": "Early performance testing and optimization"
|
||||
}
|
||||
],
|
||||
"success_criteria": [
|
||||
"Input validation and sanitization",
|
||||
"Output content filtering",
|
||||
"Rate limiting for external APIs",
|
||||
"Error handling and graceful degradation",
|
||||
"Human oversight for critical decisions",
|
||||
"Audit logging for all operations",
|
||||
"All agents operational",
|
||||
"Communication working reliably",
|
||||
"Performance targets met",
|
||||
"Error rate below 1%"
|
||||
]
|
||||
},
|
||||
"metadata": {
|
||||
"generated_by": "agent_planner.py",
|
||||
"requirements_file": "sample_system_requirements.json",
|
||||
"architecture_pattern": "supervisor",
|
||||
"agent_count": 3
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,570 @@
|
||||
{
|
||||
"summary": {
|
||||
"evaluation_period": {
|
||||
"start_time": "2024-01-15T09:00:00Z",
|
||||
"end_time": "2024-01-15T11:30:45Z",
|
||||
"total_duration_hours": 2.51
|
||||
},
|
||||
"overall_health": "good",
|
||||
"key_findings": [
|
||||
"Success rate (80.0%) below target",
|
||||
"High average latency (16.9s)",
|
||||
"2 high-impact error patterns identified"
|
||||
],
|
||||
"critical_issues": 0,
|
||||
"improvement_opportunities": 6
|
||||
},
|
||||
"system_metrics": {
|
||||
"total_tasks": 10,
|
||||
"successful_tasks": 8,
|
||||
"failed_tasks": 2,
|
||||
"partial_tasks": 1,
|
||||
"timeout_tasks": 1,
|
||||
"success_rate": 0.8,
|
||||
"failure_rate": 0.2,
|
||||
"average_duration_ms": 169800.0,
|
||||
"median_duration_ms": 152500.0,
|
||||
"percentile_95_duration_ms": 330000.0,
|
||||
"min_duration_ms": 8000,
|
||||
"max_duration_ms": 330000,
|
||||
"total_tokens_used": 53700,
|
||||
"average_tokens_per_task": 5370.0,
|
||||
"total_cost_usd": 1.074,
|
||||
"average_cost_per_task": 0.1074,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 3.98,
|
||||
"error_rate": 0.3,
|
||||
"retry_rate": 0.3
|
||||
},
|
||||
"agent_metrics": {
|
||||
"research_agent_1": {
|
||||
"total_tasks": 2,
|
||||
"successful_tasks": 2,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 1.0,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 174500.0,
|
||||
"median_duration_ms": 174500.0,
|
||||
"percentile_95_duration_ms": 195000.0,
|
||||
"min_duration_ms": 154000,
|
||||
"max_duration_ms": 195000,
|
||||
"total_tokens_used": 11050,
|
||||
"average_tokens_per_task": 5525.0,
|
||||
"total_cost_usd": 0.221,
|
||||
"average_cost_per_task": 0.1105,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 11.49,
|
||||
"error_rate": 0.0,
|
||||
"retry_rate": 0.0
|
||||
},
|
||||
"data_agent_1": {
|
||||
"total_tasks": 1,
|
||||
"successful_tasks": 1,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 1.0,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 165000.0,
|
||||
"median_duration_ms": 165000.0,
|
||||
"percentile_95_duration_ms": 165000.0,
|
||||
"min_duration_ms": 165000,
|
||||
"max_duration_ms": 165000,
|
||||
"total_tokens_used": 5000,
|
||||
"average_tokens_per_task": 5000.0,
|
||||
"total_cost_usd": 0.095,
|
||||
"average_cost_per_task": 0.095,
|
||||
"cost_per_token": 0.000019,
|
||||
"throughput_tasks_per_hour": 21.82,
|
||||
"error_rate": 0.0,
|
||||
"retry_rate": 0.0
|
||||
},
|
||||
"document_agent_1": {
|
||||
"total_tasks": 1,
|
||||
"successful_tasks": 0,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 1,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 0.0,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 140000.0,
|
||||
"median_duration_ms": 140000.0,
|
||||
"percentile_95_duration_ms": 140000.0,
|
||||
"min_duration_ms": 140000,
|
||||
"max_duration_ms": 140000,
|
||||
"total_tokens_used": 8600,
|
||||
"average_tokens_per_task": 8600.0,
|
||||
"total_cost_usd": 0.172,
|
||||
"average_cost_per_task": 0.172,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 25.71,
|
||||
"error_rate": 1.0,
|
||||
"retry_rate": 1.0
|
||||
}
|
||||
},
|
||||
"task_type_metrics": {
|
||||
"web_research": {
|
||||
"total_tasks": 3,
|
||||
"successful_tasks": 2,
|
||||
"failed_tasks": 1,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 0,
|
||||
"success_rate": 0.667,
|
||||
"failure_rate": 0.333,
|
||||
"average_duration_ms": 226333.33,
|
||||
"median_duration_ms": 195000.0,
|
||||
"percentile_95_duration_ms": 330000.0,
|
||||
"min_duration_ms": 154000,
|
||||
"max_duration_ms": 330000,
|
||||
"total_tokens_used": 12250,
|
||||
"average_tokens_per_task": 4083.33,
|
||||
"total_cost_usd": 0.245,
|
||||
"average_cost_per_task": 0.082,
|
||||
"cost_per_token": 0.00002,
|
||||
"throughput_tasks_per_hour": 2.65,
|
||||
"error_rate": 0.333,
|
||||
"retry_rate": 0.333
|
||||
},
|
||||
"data_analysis": {
|
||||
"total_tasks": 2,
|
||||
"successful_tasks": 1,
|
||||
"failed_tasks": 0,
|
||||
"partial_tasks": 0,
|
||||
"timeout_tasks": 1,
|
||||
"success_rate": 0.5,
|
||||
"failure_rate": 0.0,
|
||||
"average_duration_ms": 215000.0,
|
||||
"median_duration_ms": 215000.0,
|
||||
"percentile_95_duration_ms": 265000.0,
|
||||
"min_duration_ms": 165000,
|
||||
"max_duration_ms": 265000,
|
||||
"total_tokens_used": 14000,
|
||||
"average_tokens_per_task": 7000.0,
|
||||
"total_cost_usd": 0.275,
|
||||
"average_cost_per_task": 0.138,
|
||||
"cost_per_token": 0.0000196,
|
||||
"throughput_tasks_per_hour": 1.86,
|
||||
"error_rate": 0.5,
|
||||
"retry_rate": 0.0
|
||||
}
|
||||
},
|
||||
"tool_usage_analysis": {
|
||||
"web_search": {
|
||||
"usage_count": 3,
|
||||
"error_rate": 0.333,
|
||||
"avg_duration": 126666.67,
|
||||
"affected_workflows": [
|
||||
"web_research"
|
||||
],
|
||||
"retry_count": 2
|
||||
},
|
||||
"data_analyzer": {
|
||||
"usage_count": 2,
|
||||
"error_rate": 0.0,
|
||||
"avg_duration": 205000.0,
|
||||
"affected_workflows": [
|
||||
"data_analysis"
|
||||
],
|
||||
"retry_count": 0
|
||||
},
|
||||
"document_processor": {
|
||||
"usage_count": 2,
|
||||
"error_rate": 0.0,
|
||||
"avg_duration": 140000.0,
|
||||
"affected_workflows": [
|
||||
"document_processing"
|
||||
],
|
||||
"retry_count": 1
|
||||
},
|
||||
"notification_sender": {
|
||||
"usage_count": 2,
|
||||
"error_rate": 0.5,
|
||||
"avg_duration": 18750.0,
|
||||
"affected_workflows": [
|
||||
"notification"
|
||||
],
|
||||
"retry_count": 1
|
||||
},
|
||||
"task_scheduler": {
|
||||
"usage_count": 1,
|
||||
"error_rate": 0.0,
|
||||
"avg_duration": 12000.0,
|
||||
"affected_workflows": [
|
||||
"task_scheduling"
|
||||
],
|
||||
"retry_count": 0
|
||||
}
|
||||
},
|
||||
"error_analysis": [
|
||||
{
|
||||
"error_type": "timeout",
|
||||
"count": 2,
|
||||
"percentage": 20.0,
|
||||
"affected_agents": [
|
||||
"research_agent_2",
|
||||
"data_agent_2"
|
||||
],
|
||||
"affected_task_types": [
|
||||
"web_research",
|
||||
"data_analysis"
|
||||
],
|
||||
"common_patterns": [
|
||||
"timeout",
|
||||
"exceeded",
|
||||
"limit"
|
||||
],
|
||||
"suggested_fixes": [
|
||||
"Increase timeout values",
|
||||
"Optimize slow operations",
|
||||
"Add retry logic with exponential backoff",
|
||||
"Parallelize independent operations"
|
||||
],
|
||||
"impact_level": "high"
|
||||
},
|
||||
{
|
||||
"error_type": "authentication",
|
||||
"count": 1,
|
||||
"percentage": 10.0,
|
||||
"affected_agents": [
|
||||
"communication_agent_2"
|
||||
],
|
||||
"affected_task_types": [
|
||||
"notification"
|
||||
],
|
||||
"common_patterns": [
|
||||
"authentication",
|
||||
"failed",
|
||||
"invalid"
|
||||
],
|
||||
"suggested_fixes": [
|
||||
"Check credential rotation",
|
||||
"Implement token refresh logic",
|
||||
"Add authentication retry",
|
||||
"Verify permission scopes"
|
||||
],
|
||||
"impact_level": "high"
|
||||
},
|
||||
{
|
||||
"error_type": "validation",
|
||||
"count": 1,
|
||||
"percentage": 10.0,
|
||||
"affected_agents": [
|
||||
"document_agent_1"
|
||||
],
|
||||
"affected_task_types": [
|
||||
"document_processing"
|
||||
],
|
||||
"common_patterns": [
|
||||
"validation",
|
||||
"failed",
|
||||
"missing"
|
||||
],
|
||||
"suggested_fixes": [
|
||||
"Strengthen input validation",
|
||||
"Add data sanitization",
|
||||
"Improve error messages",
|
||||
"Add input examples"
|
||||
],
|
||||
"impact_level": "medium"
|
||||
}
|
||||
],
|
||||
"bottleneck_analysis": [
|
||||
{
|
||||
"bottleneck_type": "tool",
|
||||
"location": "notification_sender",
|
||||
"severity": "medium",
|
||||
"description": "Tool notification_sender has high error rate (50.0%)",
|
||||
"impact_on_performance": {
|
||||
"reliability_impact": 1.0,
|
||||
"retry_overhead": 1000
|
||||
},
|
||||
"affected_workflows": [
|
||||
"notification"
|
||||
],
|
||||
"optimization_suggestions": [
|
||||
"Review tool implementation",
|
||||
"Add better error handling for tool",
|
||||
"Implement tool fallbacks",
|
||||
"Consider alternative tools"
|
||||
],
|
||||
"estimated_improvement": {
|
||||
"error_reduction": 0.35,
|
||||
"performance_gain": 1.2
|
||||
}
|
||||
},
|
||||
{
|
||||
"bottleneck_type": "tool",
|
||||
"location": "web_search",
|
||||
"severity": "medium",
|
||||
"description": "Tool web_search has high error rate (33.3%)",
|
||||
"impact_on_performance": {
|
||||
"reliability_impact": 1.0,
|
||||
"retry_overhead": 2000
|
||||
},
|
||||
"affected_workflows": [
|
||||
"web_research"
|
||||
],
|
||||
"optimization_suggestions": [
|
||||
"Review tool implementation",
|
||||
"Add better error handling for tool",
|
||||
"Implement tool fallbacks",
|
||||
"Consider alternative tools"
|
||||
],
|
||||
"estimated_improvement": {
|
||||
"error_reduction": 0.233,
|
||||
"performance_gain": 1.2
|
||||
}
|
||||
}
|
||||
],
|
||||
"optimization_recommendations": [
|
||||
{
|
||||
"category": "reliability",
|
||||
"priority": "high",
|
||||
"title": "Improve System Reliability",
|
||||
"description": "System success rate is 80.0%, below target of 90%",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"success_rate_improvement": 0.1,
|
||||
"cost_reduction": 0.01611
|
||||
},
|
||||
"estimated_cost_savings": 0.1074,
|
||||
"estimated_performance_gain": 1.2,
|
||||
"implementation_steps": [
|
||||
"Identify and fix top error patterns",
|
||||
"Implement better error handling and retries",
|
||||
"Add comprehensive monitoring and alerting",
|
||||
"Implement graceful degradation patterns"
|
||||
],
|
||||
"risks": [
|
||||
"Temporary increase in complexity",
|
||||
"Potential initial performance overhead"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Error analysis completion",
|
||||
"Monitoring infrastructure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "performance",
|
||||
"priority": "high",
|
||||
"title": "Reduce Task Latency",
|
||||
"description": "Average task duration (169.8s) exceeds target",
|
||||
"implementation_effort": "high",
|
||||
"expected_impact": {
|
||||
"latency_reduction": 0.49,
|
||||
"throughput_improvement": 1.5
|
||||
},
|
||||
"estimated_performance_gain": 1.4,
|
||||
"implementation_steps": [
|
||||
"Profile and optimize slow operations",
|
||||
"Implement parallel processing where possible",
|
||||
"Add caching for expensive operations",
|
||||
"Optimize API calls and reduce round trips"
|
||||
],
|
||||
"risks": [
|
||||
"Increased system complexity",
|
||||
"Potential resource usage increase"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Performance profiling tools",
|
||||
"Caching infrastructure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "cost",
|
||||
"priority": "medium",
|
||||
"title": "Optimize Token Usage and Costs",
|
||||
"description": "Average cost per task ($0.107) is above optimal range",
|
||||
"implementation_effort": "low",
|
||||
"expected_impact": {
|
||||
"cost_reduction": 0.032,
|
||||
"efficiency_improvement": 1.15
|
||||
},
|
||||
"estimated_cost_savings": 0.322,
|
||||
"estimated_performance_gain": 1.05,
|
||||
"implementation_steps": [
|
||||
"Implement prompt optimization",
|
||||
"Add response caching for repeated queries",
|
||||
"Use smaller models for simple tasks",
|
||||
"Implement token usage monitoring and alerts"
|
||||
],
|
||||
"risks": [
|
||||
"Potential quality reduction with smaller models"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Token usage analysis",
|
||||
"Caching infrastructure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "reliability",
|
||||
"priority": "high",
|
||||
"title": "Address Timeout Errors",
|
||||
"description": "Timeout errors occur in 20.0% of cases",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"error_reduction": 0.2,
|
||||
"reliability_improvement": 1.1
|
||||
},
|
||||
"estimated_cost_savings": 0.1074,
|
||||
"implementation_steps": [
|
||||
"Increase timeout values",
|
||||
"Optimize slow operations",
|
||||
"Add retry logic with exponential backoff",
|
||||
"Parallelize independent operations"
|
||||
],
|
||||
"risks": [
|
||||
"May require significant code changes"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Root cause analysis",
|
||||
"Testing framework"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "reliability",
|
||||
"priority": "high",
|
||||
"title": "Address Authentication Errors",
|
||||
"description": "Authentication errors occur in 10.0% of cases",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"error_reduction": 0.1,
|
||||
"reliability_improvement": 1.1
|
||||
},
|
||||
"estimated_cost_savings": 0.1074,
|
||||
"implementation_steps": [
|
||||
"Check credential rotation",
|
||||
"Implement token refresh logic",
|
||||
"Add authentication retry",
|
||||
"Verify permission scopes"
|
||||
],
|
||||
"risks": [
|
||||
"May require significant code changes"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Root cause analysis",
|
||||
"Testing framework"
|
||||
]
|
||||
},
|
||||
{
|
||||
"category": "performance",
|
||||
"priority": "medium",
|
||||
"title": "Address Tool Bottleneck",
|
||||
"description": "Tool notification_sender has high error rate (50.0%)",
|
||||
"implementation_effort": "medium",
|
||||
"expected_impact": {
|
||||
"error_reduction": 0.35,
|
||||
"performance_gain": 1.2
|
||||
},
|
||||
"estimated_performance_gain": 1.2,
|
||||
"implementation_steps": [
|
||||
"Review tool implementation",
|
||||
"Add better error handling for tool",
|
||||
"Implement tool fallbacks",
|
||||
"Consider alternative tools"
|
||||
],
|
||||
"risks": [
|
||||
"System downtime during implementation",
|
||||
"Potential cascade effects"
|
||||
],
|
||||
"prerequisites": [
|
||||
"Impact assessment",
|
||||
"Rollback plan"
|
||||
]
|
||||
}
|
||||
],
|
||||
"trends_analysis": {
|
||||
"daily_success_rates": {
|
||||
"2024-01-15": 0.8
|
||||
},
|
||||
"daily_avg_durations": {
|
||||
"2024-01-15": 169800.0
|
||||
},
|
||||
"daily_costs": {
|
||||
"2024-01-15": 1.074
|
||||
},
|
||||
"trend_direction": {
|
||||
"success_rate": "stable",
|
||||
"duration": "stable",
|
||||
"cost": "stable"
|
||||
}
|
||||
},
|
||||
"cost_breakdown": {
|
||||
"total_cost": 1.074,
|
||||
"cost_by_agent": {
|
||||
"research_agent_1": 0.221,
|
||||
"research_agent_2": 0.024,
|
||||
"data_agent_1": 0.095,
|
||||
"data_agent_2": 0.18,
|
||||
"document_agent_1": 0.172,
|
||||
"document_agent_2": 0.174,
|
||||
"communication_agent_1": 0.007,
|
||||
"communication_agent_2": 0.004,
|
||||
"scheduler_agent_1": 0.01
|
||||
},
|
||||
"cost_by_task_type": {
|
||||
"web_research": 0.245,
|
||||
"data_analysis": 0.275,
|
||||
"document_processing": 0.346,
|
||||
"notification": 0.011,
|
||||
"task_scheduling": 0.01
|
||||
},
|
||||
"cost_per_token": 0.00002,
|
||||
"top_cost_drivers": [
|
||||
[
|
||||
"document_processing",
|
||||
0.346
|
||||
],
|
||||
[
|
||||
"data_analysis",
|
||||
0.275
|
||||
],
|
||||
[
|
||||
"web_research",
|
||||
0.245
|
||||
],
|
||||
[
|
||||
"notification",
|
||||
0.011
|
||||
],
|
||||
[
|
||||
"task_scheduling",
|
||||
0.01
|
||||
]
|
||||
]
|
||||
},
|
||||
"sla_compliance": {
|
||||
"overall_compliant": false,
|
||||
"sla_details": {
|
||||
"success_rate": {
|
||||
"target": 0.95,
|
||||
"actual": 0.8,
|
||||
"compliant": false,
|
||||
"gap": 0.15
|
||||
},
|
||||
"average_latency": {
|
||||
"target": 10000,
|
||||
"actual": 169800.0,
|
||||
"compliant": false,
|
||||
"gap": 159800.0
|
||||
},
|
||||
"error_rate": {
|
||||
"target": 0.05,
|
||||
"actual": 0.3,
|
||||
"compliant": false,
|
||||
"gap": 0.25
|
||||
}
|
||||
},
|
||||
"compliance_score": 0.0
|
||||
},
|
||||
"metadata": {
|
||||
"generated_at": "2024-01-15T12:00:00Z",
|
||||
"evaluator_version": "1.0",
|
||||
"total_logs_processed": 10,
|
||||
"agents_analyzed": 9,
|
||||
"task_types_analyzed": 5,
|
||||
"analysis_completeness": "full"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,416 @@
|
||||
{
|
||||
"tool_schemas": [
|
||||
{
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information on specified topics with customizable filters and result limits",
|
||||
"openai_schema": {
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information on specified topics with customizable filters and result limits",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query string to find relevant information",
|
||||
"minLength": 1,
|
||||
"maxLength": 500,
|
||||
"examples": [
|
||||
"artificial intelligence trends",
|
||||
"climate change impact",
|
||||
"python programming tutorial"
|
||||
]
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of search results to return",
|
||||
"minimum": 1,
|
||||
"maximum": 100,
|
||||
"default": 10
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"description": "Language code for search results",
|
||||
"enum": [
|
||||
"en",
|
||||
"es",
|
||||
"fr",
|
||||
"de",
|
||||
"it",
|
||||
"pt",
|
||||
"zh",
|
||||
"ja"
|
||||
],
|
||||
"default": "en"
|
||||
},
|
||||
"time_range": {
|
||||
"type": "string",
|
||||
"description": "Time range filter for search results",
|
||||
"enum": [
|
||||
"any",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"year"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"query"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"anthropic_schema": {
|
||||
"name": "web_search",
|
||||
"description": "Search the web for information on specified topics with customizable filters and result limits",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query string to find relevant information",
|
||||
"minLength": 1,
|
||||
"maxLength": 500
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of search results to return",
|
||||
"minimum": 1,
|
||||
"maximum": 100
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"description": "Language code for search results",
|
||||
"enum": [
|
||||
"en",
|
||||
"es",
|
||||
"fr",
|
||||
"de",
|
||||
"it",
|
||||
"pt",
|
||||
"zh",
|
||||
"ja"
|
||||
]
|
||||
},
|
||||
"time_range": {
|
||||
"type": "string",
|
||||
"description": "Time range filter for search results",
|
||||
"enum": [
|
||||
"any",
|
||||
"day",
|
||||
"week",
|
||||
"month",
|
||||
"year"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"query"
|
||||
]
|
||||
}
|
||||
},
|
||||
"validation_rules": [
|
||||
{
|
||||
"parameter": "query",
|
||||
"rules": {
|
||||
"minLength": 1,
|
||||
"maxLength": 500
|
||||
}
|
||||
},
|
||||
{
|
||||
"parameter": "limit",
|
||||
"rules": {
|
||||
"minimum": 1,
|
||||
"maximum": 100
|
||||
}
|
||||
}
|
||||
],
|
||||
"error_responses": [
|
||||
{
|
||||
"error_code": "invalid_input",
|
||||
"error_message": "Invalid input parameters provided",
|
||||
"http_status": 400,
|
||||
"retry_after": null,
|
||||
"details": {
|
||||
"validation_errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"error_code": "authentication_required",
|
||||
"error_message": "Authentication required to access this tool",
|
||||
"http_status": 401,
|
||||
"retry_after": null,
|
||||
"details": null
|
||||
},
|
||||
{
|
||||
"error_code": "rate_limit_exceeded",
|
||||
"error_message": "Rate limit exceeded. Please try again later",
|
||||
"http_status": 429,
|
||||
"retry_after": 60,
|
||||
"details": null
|
||||
}
|
||||
],
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 60,
|
||||
"requests_per_hour": 1000,
|
||||
"requests_per_day": 10000,
|
||||
"burst_limit": 10,
|
||||
"cooldown_period": 60,
|
||||
"rate_limit_key": "user_id"
|
||||
},
|
||||
"examples": [
|
||||
{
|
||||
"description": "Basic web search",
|
||||
"input": {
|
||||
"query": "machine learning algorithms",
|
||||
"limit": 5
|
||||
},
|
||||
"expected_output": {
|
||||
"results": [
|
||||
{
|
||||
"title": "Introduction to Machine Learning Algorithms",
|
||||
"url": "https://example.com/ml-intro",
|
||||
"snippet": "Machine learning algorithms are computational methods...",
|
||||
"relevance_score": 0.95
|
||||
}
|
||||
],
|
||||
"total_found": 1250
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"category": "search",
|
||||
"idempotent": true,
|
||||
"side_effects": [
|
||||
"Logs search query for analytics",
|
||||
"May cache results temporarily"
|
||||
],
|
||||
"dependencies": [
|
||||
"search_api_service",
|
||||
"content_filter_service"
|
||||
],
|
||||
"security_requirements": [
|
||||
"Query sanitization",
|
||||
"Rate limiting by user",
|
||||
"Content filtering"
|
||||
],
|
||||
"generated_at": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0",
|
||||
"input_parameters": 4,
|
||||
"output_parameters": 2,
|
||||
"required_parameters": 1,
|
||||
"optional_parameters": 3
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze structured data and generate statistical insights, trends, and visualizations",
|
||||
"openai_schema": {
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze structured data and generate statistical insights, trends, and visualizations",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"description": "Structured data to analyze in JSON format",
|
||||
"properties": {
|
||||
"columns": {
|
||||
"type": "array"
|
||||
},
|
||||
"rows": {
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string",
|
||||
"description": "Type of analysis to perform",
|
||||
"enum": [
|
||||
"descriptive",
|
||||
"correlation",
|
||||
"trend",
|
||||
"distribution",
|
||||
"outlier_detection"
|
||||
]
|
||||
},
|
||||
"target_column": {
|
||||
"type": "string",
|
||||
"description": "Primary column to focus analysis on",
|
||||
"maxLength": 1000
|
||||
},
|
||||
"include_visualization": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to generate visualization data",
|
||||
"default": true
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"data",
|
||||
"analysis_type"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"anthropic_schema": {
|
||||
"name": "data_analyzer",
|
||||
"description": "Analyze structured data and generate statistical insights, trends, and visualizations",
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "object",
|
||||
"description": "Structured data to analyze in JSON format"
|
||||
},
|
||||
"analysis_type": {
|
||||
"type": "string",
|
||||
"description": "Type of analysis to perform",
|
||||
"enum": [
|
||||
"descriptive",
|
||||
"correlation",
|
||||
"trend",
|
||||
"distribution",
|
||||
"outlier_detection"
|
||||
]
|
||||
},
|
||||
"target_column": {
|
||||
"type": "string",
|
||||
"description": "Primary column to focus analysis on",
|
||||
"maxLength": 1000
|
||||
},
|
||||
"include_visualization": {
|
||||
"type": "boolean",
|
||||
"description": "Whether to generate visualization data"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"data",
|
||||
"analysis_type"
|
||||
]
|
||||
}
|
||||
},
|
||||
"validation_rules": [
|
||||
{
|
||||
"parameter": "target_column",
|
||||
"rules": {
|
||||
"maxLength": 1000
|
||||
}
|
||||
}
|
||||
],
|
||||
"error_responses": [
|
||||
{
|
||||
"error_code": "invalid_input",
|
||||
"error_message": "Invalid input parameters provided",
|
||||
"http_status": 400,
|
||||
"retry_after": null,
|
||||
"details": {
|
||||
"validation_errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"error_code": "authentication_required",
|
||||
"error_message": "Authentication required to access this tool",
|
||||
"http_status": 401,
|
||||
"retry_after": null,
|
||||
"details": null
|
||||
},
|
||||
{
|
||||
"error_code": "rate_limit_exceeded",
|
||||
"error_message": "Rate limit exceeded. Please try again later",
|
||||
"http_status": 429,
|
||||
"retry_after": 60,
|
||||
"details": null
|
||||
}
|
||||
],
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 30,
|
||||
"requests_per_hour": 500,
|
||||
"requests_per_day": 5000,
|
||||
"burst_limit": 5,
|
||||
"cooldown_period": 60,
|
||||
"rate_limit_key": "user_id"
|
||||
},
|
||||
"examples": [
|
||||
{
|
||||
"description": "Basic descriptive analysis",
|
||||
"input": {
|
||||
"data": {
|
||||
"columns": [
|
||||
"age",
|
||||
"salary",
|
||||
"department"
|
||||
],
|
||||
"rows": [
|
||||
[
|
||||
25,
|
||||
50000,
|
||||
"engineering"
|
||||
],
|
||||
[
|
||||
30,
|
||||
60000,
|
||||
"engineering"
|
||||
],
|
||||
[
|
||||
28,
|
||||
55000,
|
||||
"marketing"
|
||||
]
|
||||
]
|
||||
},
|
||||
"analysis_type": "descriptive",
|
||||
"target_column": "salary"
|
||||
},
|
||||
"expected_output": {
|
||||
"insights": [
|
||||
"Average salary is $55,000",
|
||||
"Salary range: $50,000 - $60,000",
|
||||
"Engineering department has higher average salary"
|
||||
],
|
||||
"statistics": {
|
||||
"mean": 55000,
|
||||
"median": 55000,
|
||||
"std_dev": 5000
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"category": "data",
|
||||
"idempotent": true,
|
||||
"side_effects": [
|
||||
"May create temporary analysis files",
|
||||
"Logs analysis parameters for optimization"
|
||||
],
|
||||
"dependencies": [
|
||||
"statistics_engine",
|
||||
"visualization_service"
|
||||
],
|
||||
"security_requirements": [
|
||||
"Data anonymization",
|
||||
"Access control validation"
|
||||
],
|
||||
"generated_at": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0",
|
||||
"input_parameters": 4,
|
||||
"output_parameters": 3,
|
||||
"required_parameters": 2,
|
||||
"optional_parameters": 2
|
||||
}
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"generated_by": "tool_schema_generator.py",
|
||||
"input_file": "sample_tool_descriptions.json",
|
||||
"tool_count": 2,
|
||||
"generation_timestamp": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0"
|
||||
},
|
||||
"validation_summary": {
|
||||
"total_tools": 2,
|
||||
"total_parameters": 8,
|
||||
"total_validation_rules": 3,
|
||||
"total_examples": 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,445 @@
|
||||
# Agent Architecture Patterns Catalog
|
||||
|
||||
## Overview
|
||||
|
||||
This document provides a comprehensive catalog of multi-agent system architecture patterns, their characteristics, use cases, and implementation considerations.
|
||||
|
||||
## Pattern Categories
|
||||
|
||||
### 1. Single Agent Pattern
|
||||
|
||||
**Description:** One agent handles all system functionality
|
||||
**Structure:** User → Agent ← Tools
|
||||
**Complexity:** Low
|
||||
|
||||
**Characteristics:**
|
||||
- Centralized decision making
|
||||
- No inter-agent communication
|
||||
- Simple state management
|
||||
- Direct user interaction
|
||||
|
||||
**Use Cases:**
|
||||
- Personal assistants
|
||||
- Simple automation tasks
|
||||
- Prototyping and development
|
||||
- Domain-specific applications
|
||||
|
||||
**Advantages:**
|
||||
- Simple to implement and debug
|
||||
- Predictable behavior
|
||||
- Low coordination overhead
|
||||
- Clear responsibility model
|
||||
|
||||
**Disadvantages:**
|
||||
- Limited scalability
|
||||
- Single point of failure
|
||||
- Resource bottlenecks
|
||||
- Difficulty handling complex workflows
|
||||
|
||||
**Implementation Patterns:**
|
||||
```
|
||||
Agent {
|
||||
receive_request()
|
||||
process_task()
|
||||
use_tools()
|
||||
return_response()
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Supervisor Pattern (Hierarchical Delegation)
|
||||
|
||||
**Description:** One supervisor coordinates multiple specialist agents
|
||||
**Structure:** User → Supervisor → Specialists
|
||||
**Complexity:** Medium
|
||||
|
||||
**Characteristics:**
|
||||
- Central coordination
|
||||
- Clear hierarchy
|
||||
- Specialized capabilities
|
||||
- Delegation and aggregation
|
||||
|
||||
**Use Cases:**
|
||||
- Task decomposition scenarios
|
||||
- Quality control workflows
|
||||
- Resource allocation systems
|
||||
- Project management
|
||||
|
||||
**Advantages:**
|
||||
- Clear command structure
|
||||
- Specialized expertise
|
||||
- Centralized quality control
|
||||
- Efficient resource allocation
|
||||
|
||||
**Disadvantages:**
|
||||
- Supervisor bottleneck
|
||||
- Complex coordination logic
|
||||
- Single point of failure
|
||||
- Limited parallelism
|
||||
|
||||
**Implementation Patterns:**
|
||||
```
|
||||
Supervisor {
|
||||
decompose_task()
|
||||
delegate_to_specialists()
|
||||
monitor_progress()
|
||||
aggregate_results()
|
||||
quality_control()
|
||||
}
|
||||
|
||||
Specialist {
|
||||
receive_assignment()
|
||||
execute_specialized_task()
|
||||
report_results()
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Swarm Pattern (Peer-to-Peer)
|
||||
|
||||
**Description:** Multiple autonomous agents collaborate as peers
|
||||
**Structure:** Agent ↔ Agent ↔ Agent (interconnected)
|
||||
**Complexity:** High
|
||||
|
||||
**Characteristics:**
|
||||
- Distributed decision making
|
||||
- Peer-to-peer communication
|
||||
- Emergent behavior
|
||||
- Self-organization
|
||||
|
||||
**Use Cases:**
|
||||
- Distributed problem solving
|
||||
- Parallel processing
|
||||
- Fault-tolerant systems
|
||||
- Research and exploration
|
||||
|
||||
**Advantages:**
|
||||
- High fault tolerance
|
||||
- Scalable parallelism
|
||||
- Emergent intelligence
|
||||
- No single point of failure
|
||||
|
||||
**Disadvantages:**
|
||||
- Complex coordination
|
||||
- Unpredictable behavior
|
||||
- Difficult debugging
|
||||
- Consensus overhead
|
||||
|
||||
**Implementation Patterns:**
|
||||
```
|
||||
SwarmAgent {
|
||||
discover_peers()
|
||||
share_information()
|
||||
negotiate_tasks()
|
||||
collaborate()
|
||||
adapt_behavior()
|
||||
}
|
||||
|
||||
ConsensusProtocol {
|
||||
propose_action()
|
||||
vote()
|
||||
reach_agreement()
|
||||
execute_collective_decision()
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Hierarchical Pattern (Multi-Level Management)
|
||||
|
||||
**Description:** Multiple levels of management and execution
|
||||
**Structure:** Executive → Managers → Workers (tree structure)
|
||||
**Complexity:** Very High
|
||||
|
||||
**Characteristics:**
|
||||
- Multi-level hierarchy
|
||||
- Distributed management
|
||||
- Clear organizational structure
|
||||
- Scalable command structure
|
||||
|
||||
**Use Cases:**
|
||||
- Enterprise systems
|
||||
- Large-scale operations
|
||||
- Complex workflows
|
||||
- Organizational modeling
|
||||
|
||||
**Advantages:**
|
||||
- Natural organizational mapping
|
||||
- Scalable structure
|
||||
- Clear responsibilities
|
||||
- Efficient resource management
|
||||
|
||||
**Disadvantages:**
|
||||
- Communication overhead
|
||||
- Multi-level bottlenecks
|
||||
- Complex coordination
|
||||
- Slower decision making
|
||||
|
||||
**Implementation Patterns:**
|
||||
```
|
||||
Executive {
|
||||
strategic_planning()
|
||||
resource_allocation()
|
||||
performance_monitoring()
|
||||
}
|
||||
|
||||
Manager {
|
||||
tactical_planning()
|
||||
team_coordination()
|
||||
progress_reporting()
|
||||
}
|
||||
|
||||
Worker {
|
||||
task_execution()
|
||||
status_reporting()
|
||||
resource_requests()
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Pipeline Pattern (Sequential Processing)
|
||||
|
||||
**Description:** Agents arranged in processing pipeline
|
||||
**Structure:** Input → Stage1 → Stage2 → Stage3 → Output
|
||||
**Complexity:** Medium
|
||||
|
||||
**Characteristics:**
|
||||
- Sequential processing
|
||||
- Specialized stages
|
||||
- Data flow architecture
|
||||
- Clear processing order
|
||||
|
||||
**Use Cases:**
|
||||
- Data processing pipelines
|
||||
- Manufacturing workflows
|
||||
- Content processing
|
||||
- ETL operations
|
||||
|
||||
**Advantages:**
|
||||
- Clear data flow
|
||||
- Specialized optimization
|
||||
- Predictable processing
|
||||
- Easy to scale stages
|
||||
|
||||
**Disadvantages:**
|
||||
- Sequential bottlenecks
|
||||
- Rigid processing order
|
||||
- Stage coupling
|
||||
- Limited flexibility
|
||||
|
||||
**Implementation Patterns:**
|
||||
```
|
||||
PipelineStage {
|
||||
receive_input()
|
||||
process_data()
|
||||
validate_output()
|
||||
send_to_next_stage()
|
||||
}
|
||||
|
||||
PipelineController {
|
||||
manage_flow()
|
||||
handle_errors()
|
||||
monitor_throughput()
|
||||
optimize_stages()
|
||||
}
|
||||
```
|
||||
|
||||
## Pattern Selection Criteria
|
||||
|
||||
### Team Size Considerations
|
||||
- **1 Agent:** Single Agent Pattern only
|
||||
- **2-5 Agents:** Supervisor, Pipeline
|
||||
- **6-15 Agents:** Swarm, Hierarchical, Pipeline
|
||||
- **15+ Agents:** Hierarchical, Large Swarm
|
||||
|
||||
### Task Complexity
|
||||
- **Simple:** Single Agent
|
||||
- **Medium:** Supervisor, Pipeline
|
||||
- **Complex:** Swarm, Hierarchical
|
||||
- **Very Complex:** Hierarchical
|
||||
|
||||
### Coordination Requirements
|
||||
- **None:** Single Agent
|
||||
- **Low:** Pipeline, Supervisor
|
||||
- **Medium:** Hierarchical
|
||||
- **High:** Swarm
|
||||
|
||||
### Fault Tolerance Requirements
|
||||
- **Low:** Single Agent, Pipeline
|
||||
- **Medium:** Supervisor, Hierarchical
|
||||
- **High:** Swarm
|
||||
|
||||
## Hybrid Patterns
|
||||
|
||||
### Hub-and-Spoke with Clusters
|
||||
Combines supervisor pattern with swarm clusters
|
||||
- Central coordinator
|
||||
- Specialized swarm clusters
|
||||
- Hierarchical communication
|
||||
|
||||
### Pipeline with Parallel Stages
|
||||
Pipeline stages that can process in parallel
|
||||
- Sequential overall flow
|
||||
- Parallel processing within stages
|
||||
- Load balancing across stage instances
|
||||
|
||||
### Hierarchical Swarms
|
||||
Swarm behavior at each hierarchical level
|
||||
- Distributed decision making
|
||||
- Hierarchical coordination
|
||||
- Multi-level autonomy
|
||||
|
||||
## Communication Patterns by Architecture
|
||||
|
||||
### Single Agent
|
||||
- Direct user interface
|
||||
- Tool API calls
|
||||
- No inter-agent communication
|
||||
|
||||
### Supervisor
|
||||
- Command/response with specialists
|
||||
- Progress reporting
|
||||
- Result aggregation
|
||||
|
||||
### Swarm
|
||||
- Broadcast messaging
|
||||
- Peer discovery
|
||||
- Consensus protocols
|
||||
- Information sharing
|
||||
|
||||
### Hierarchical
|
||||
- Upward reporting
|
||||
- Downward delegation
|
||||
- Lateral coordination
|
||||
- Skip-level communication
|
||||
|
||||
### Pipeline
|
||||
- Stage-to-stage data flow
|
||||
- Error propagation
|
||||
- Status monitoring
|
||||
- Flow control
|
||||
|
||||
## Scaling Considerations
|
||||
|
||||
### Horizontal Scaling
|
||||
- **Single Agent:** Scale by replication
|
||||
- **Supervisor:** Scale specialists
|
||||
- **Swarm:** Add more peers
|
||||
- **Hierarchical:** Add at appropriate levels
|
||||
- **Pipeline:** Scale bottleneck stages
|
||||
|
||||
### Vertical Scaling
|
||||
- **Single Agent:** More powerful agent
|
||||
- **Supervisor:** Enhanced supervisor capabilities
|
||||
- **Swarm:** Smarter individual agents
|
||||
- **Hierarchical:** Better management agents
|
||||
- **Pipeline:** Optimize stage processing
|
||||
|
||||
## Error Handling Patterns
|
||||
|
||||
### Single Agent
|
||||
- Retry logic
|
||||
- Fallback behaviors
|
||||
- User notification
|
||||
|
||||
### Supervisor
|
||||
- Specialist failure detection
|
||||
- Task reassignment
|
||||
- Result validation
|
||||
|
||||
### Swarm
|
||||
- Peer failure detection
|
||||
- Consensus recalculation
|
||||
- Self-healing behavior
|
||||
|
||||
### Hierarchical
|
||||
- Escalation procedures
|
||||
- Skip-level communication
|
||||
- Management override
|
||||
|
||||
### Pipeline
|
||||
- Stage failure recovery
|
||||
- Data replay
|
||||
- Circuit breakers
|
||||
|
||||
## Performance Characteristics
|
||||
|
||||
| Pattern | Latency | Throughput | Scalability | Reliability | Complexity |
|
||||
|---------|---------|------------|-------------|-------------|------------|
|
||||
| Single Agent | Low | Low | Poor | Poor | Low |
|
||||
| Supervisor | Medium | Medium | Good | Medium | Medium |
|
||||
| Swarm | High | High | Excellent | Excellent | High |
|
||||
| Hierarchical | Medium | High | Excellent | Good | Very High |
|
||||
| Pipeline | Low | High | Good | Medium | Medium |
|
||||
|
||||
## Best Practices by Pattern
|
||||
|
||||
### Single Agent
|
||||
- Keep scope focused
|
||||
- Implement comprehensive error handling
|
||||
- Use efficient tool selection
|
||||
- Monitor resource usage
|
||||
|
||||
### Supervisor
|
||||
- Design clear delegation rules
|
||||
- Implement progress monitoring
|
||||
- Use timeout mechanisms
|
||||
- Plan for specialist failures
|
||||
|
||||
### Swarm
|
||||
- Design simple interaction protocols
|
||||
- Implement conflict resolution
|
||||
- Monitor emergent behavior
|
||||
- Plan for network partitions
|
||||
|
||||
### Hierarchical
|
||||
- Define clear role boundaries
|
||||
- Implement efficient communication
|
||||
- Plan escalation procedures
|
||||
- Monitor span of control
|
||||
|
||||
### Pipeline
|
||||
- Optimize bottleneck stages
|
||||
- Implement error recovery
|
||||
- Use appropriate buffering
|
||||
- Monitor flow rates
|
||||
|
||||
## Anti-Patterns to Avoid
|
||||
|
||||
### God Agent
|
||||
Single agent that tries to do everything
|
||||
- Violates single responsibility
|
||||
- Creates maintenance nightmare
|
||||
- Poor scalability
|
||||
|
||||
### Chatty Communication
|
||||
Excessive inter-agent messaging
|
||||
- Performance degradation
|
||||
- Network congestion
|
||||
- Poor scalability
|
||||
|
||||
### Circular Dependencies
|
||||
Agents depending on each other cyclically
|
||||
- Deadlock potential
|
||||
- Complex error handling
|
||||
- Difficult debugging
|
||||
|
||||
### Over-Centralization
|
||||
Too much logic in coordinator
|
||||
- Single point of failure
|
||||
- Bottleneck creation
|
||||
- Poor fault tolerance
|
||||
|
||||
### Under-Specification
|
||||
Unclear roles and responsibilities
|
||||
- Coordination failures
|
||||
- Duplicate work
|
||||
- Inconsistent behavior
|
||||
|
||||
## Conclusion
|
||||
|
||||
The choice of agent architecture pattern depends on multiple factors including team size, task complexity, coordination requirements, fault tolerance needs, and performance objectives. Each pattern has distinct trade-offs that must be carefully considered in the context of specific system requirements.
|
||||
|
||||
Success factors include:
|
||||
- Clear role definitions
|
||||
- Appropriate communication patterns
|
||||
- Robust error handling
|
||||
- Scalability planning
|
||||
- Performance monitoring
|
||||
|
||||
The patterns can be combined and customized to meet specific needs, but maintaining clarity and avoiding unnecessary complexity should always be prioritized.
|
||||
749
engineering/agent-designer/references/evaluation_methodology.md
Normal file
749
engineering/agent-designer/references/evaluation_methodology.md
Normal file
@@ -0,0 +1,749 @@
|
||||
# Multi-Agent System Evaluation Methodology
|
||||
|
||||
## Overview
|
||||
|
||||
This document provides a comprehensive methodology for evaluating multi-agent systems across multiple dimensions including performance, reliability, cost-effectiveness, and user satisfaction. The methodology is designed to provide actionable insights for system optimization.
|
||||
|
||||
## Evaluation Framework
|
||||
|
||||
### Evaluation Dimensions
|
||||
|
||||
#### 1. Task Performance
|
||||
- **Success Rate:** Percentage of tasks completed successfully
|
||||
- **Completion Time:** Time from task initiation to completion
|
||||
- **Quality Metrics:** Accuracy, relevance, completeness of results
|
||||
- **Partial Success:** Progress made on incomplete tasks
|
||||
|
||||
#### 2. System Reliability
|
||||
- **Availability:** System uptime and accessibility
|
||||
- **Error Rates:** Frequency and types of errors
|
||||
- **Recovery Time:** Time to recover from failures
|
||||
- **Fault Tolerance:** System behavior under component failures
|
||||
|
||||
#### 3. Cost Efficiency
|
||||
- **Resource Utilization:** CPU, memory, network, storage usage
|
||||
- **Token Consumption:** LLM API usage and costs
|
||||
- **Operational Costs:** Infrastructure and maintenance costs
|
||||
- **Cost per Task:** Economic efficiency per completed task
|
||||
|
||||
#### 4. User Experience
|
||||
- **Response Time:** User-perceived latency
|
||||
- **User Satisfaction:** Qualitative feedback scores
|
||||
- **Usability:** Ease of system interaction
|
||||
- **Predictability:** Consistency of system behavior
|
||||
|
||||
#### 5. Scalability
|
||||
- **Load Handling:** Performance under increasing load
|
||||
- **Resource Scaling:** Ability to scale resources dynamically
|
||||
- **Concurrency:** Handling multiple simultaneous requests
|
||||
- **Degradation Patterns:** Behavior at capacity limits
|
||||
|
||||
#### 6. Security
|
||||
- **Access Control:** Authentication and authorization effectiveness
|
||||
- **Data Protection:** Privacy and confidentiality measures
|
||||
- **Audit Trail:** Logging and monitoring completeness
|
||||
- **Vulnerability Assessment:** Security weakness identification
|
||||
|
||||
## Metrics Collection
|
||||
|
||||
### Core Metrics
|
||||
|
||||
#### Performance Metrics
|
||||
```json
|
||||
{
|
||||
"task_metrics": {
|
||||
"task_id": "string",
|
||||
"agent_id": "string",
|
||||
"task_type": "string",
|
||||
"start_time": "ISO 8601 timestamp",
|
||||
"end_time": "ISO 8601 timestamp",
|
||||
"duration_ms": "integer",
|
||||
"status": "success|failure|partial|timeout",
|
||||
"quality_score": "float 0-1",
|
||||
"steps_completed": "integer",
|
||||
"total_steps": "integer"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Resource Metrics
|
||||
```json
|
||||
{
|
||||
"resource_metrics": {
|
||||
"timestamp": "ISO 8601 timestamp",
|
||||
"agent_id": "string",
|
||||
"cpu_usage_percent": "float",
|
||||
"memory_usage_mb": "integer",
|
||||
"network_bytes_sent": "integer",
|
||||
"network_bytes_received": "integer",
|
||||
"tokens_consumed": "integer",
|
||||
"api_calls_made": "integer"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Error Metrics
|
||||
```json
|
||||
{
|
||||
"error_metrics": {
|
||||
"timestamp": "ISO 8601 timestamp",
|
||||
"error_type": "string",
|
||||
"error_code": "string",
|
||||
"error_message": "string",
|
||||
"agent_id": "string",
|
||||
"task_id": "string",
|
||||
"severity": "critical|high|medium|low",
|
||||
"recovery_action": "string",
|
||||
"resolved": "boolean"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Advanced Metrics
|
||||
|
||||
#### Agent Collaboration Metrics
|
||||
```json
|
||||
{
|
||||
"collaboration_metrics": {
|
||||
"timestamp": "ISO 8601 timestamp",
|
||||
"initiating_agent": "string",
|
||||
"target_agent": "string",
|
||||
"interaction_type": "request|response|broadcast|delegate",
|
||||
"latency_ms": "integer",
|
||||
"success": "boolean",
|
||||
"payload_size_bytes": "integer",
|
||||
"context_shared": "boolean"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Tool Usage Metrics
|
||||
```json
|
||||
{
|
||||
"tool_metrics": {
|
||||
"timestamp": "ISO 8601 timestamp",
|
||||
"agent_id": "string",
|
||||
"tool_name": "string",
|
||||
"invocation_duration_ms": "integer",
|
||||
"success": "boolean",
|
||||
"error_type": "string|null",
|
||||
"input_size_bytes": "integer",
|
||||
"output_size_bytes": "integer",
|
||||
"cached_result": "boolean"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Evaluation Methods
|
||||
|
||||
### 1. Synthetic Benchmarks
|
||||
|
||||
#### Task Complexity Levels
|
||||
- **Level 1 (Simple):** Single-agent, single-tool tasks
|
||||
- **Level 2 (Moderate):** Multi-tool tasks requiring coordination
|
||||
- **Level 3 (Complex):** Multi-agent collaborative tasks
|
||||
- **Level 4 (Advanced):** Long-running, multi-stage workflows
|
||||
- **Level 5 (Expert):** Adaptive tasks requiring learning
|
||||
|
||||
#### Benchmark Task Categories
|
||||
```yaml
|
||||
benchmark_categories:
|
||||
information_retrieval:
|
||||
- simple_web_search
|
||||
- multi_source_research
|
||||
- fact_verification
|
||||
- comparative_analysis
|
||||
|
||||
content_generation:
|
||||
- text_summarization
|
||||
- creative_writing
|
||||
- technical_documentation
|
||||
- multilingual_translation
|
||||
|
||||
data_processing:
|
||||
- data_cleaning
|
||||
- statistical_analysis
|
||||
- visualization_creation
|
||||
- report_generation
|
||||
|
||||
problem_solving:
|
||||
- algorithm_development
|
||||
- optimization_tasks
|
||||
- troubleshooting
|
||||
- decision_support
|
||||
|
||||
workflow_automation:
|
||||
- multi_step_processes
|
||||
- conditional_workflows
|
||||
- exception_handling
|
||||
- resource_coordination
|
||||
```
|
||||
|
||||
#### Benchmark Execution
|
||||
```python
|
||||
def run_benchmark_suite(agents, benchmark_tasks):
|
||||
results = {}
|
||||
|
||||
for category, tasks in benchmark_tasks.items():
|
||||
category_results = []
|
||||
|
||||
for task in tasks:
|
||||
task_result = execute_benchmark_task(
|
||||
agents=agents,
|
||||
task=task,
|
||||
timeout=task.max_duration,
|
||||
repetitions=task.repetitions
|
||||
)
|
||||
category_results.append(task_result)
|
||||
|
||||
results[category] = analyze_category_results(category_results)
|
||||
|
||||
return generate_benchmark_report(results)
|
||||
```
|
||||
|
||||
### 2. A/B Testing
|
||||
|
||||
#### Test Design
|
||||
```yaml
|
||||
ab_test_design:
|
||||
hypothesis: "New agent architecture improves task success rate"
|
||||
success_metrics:
|
||||
primary: "task_success_rate"
|
||||
secondary: ["response_time", "cost_per_task", "user_satisfaction"]
|
||||
|
||||
test_configuration:
|
||||
control_group: "current_architecture"
|
||||
treatment_group: "new_architecture"
|
||||
traffic_split: 50/50
|
||||
duration_days: 14
|
||||
minimum_sample_size: 1000
|
||||
|
||||
statistical_parameters:
|
||||
confidence_level: 0.95
|
||||
minimum_detectable_effect: 0.05
|
||||
statistical_power: 0.8
|
||||
```
|
||||
|
||||
#### Analysis Framework
|
||||
```python
|
||||
def analyze_ab_test(control_data, treatment_data, metrics):
|
||||
results = {}
|
||||
|
||||
for metric in metrics:
|
||||
control_values = extract_metric_values(control_data, metric)
|
||||
treatment_values = extract_metric_values(treatment_data, metric)
|
||||
|
||||
# Statistical significance test
|
||||
stat_result = perform_statistical_test(
|
||||
control_values,
|
||||
treatment_values,
|
||||
test_type=determine_test_type(metric)
|
||||
)
|
||||
|
||||
# Effect size calculation
|
||||
effect_size = calculate_effect_size(
|
||||
control_values,
|
||||
treatment_values
|
||||
)
|
||||
|
||||
results[metric] = {
|
||||
"control_mean": np.mean(control_values),
|
||||
"treatment_mean": np.mean(treatment_values),
|
||||
"p_value": stat_result.p_value,
|
||||
"confidence_interval": stat_result.confidence_interval,
|
||||
"effect_size": effect_size,
|
||||
"practical_significance": assess_practical_significance(
|
||||
effect_size, metric
|
||||
)
|
||||
}
|
||||
|
||||
return results
|
||||
```
|
||||
|
||||
### 3. Load Testing
|
||||
|
||||
#### Load Test Scenarios
|
||||
```yaml
|
||||
load_test_scenarios:
|
||||
baseline_load:
|
||||
concurrent_users: 10
|
||||
ramp_up_time: "5 minutes"
|
||||
duration: "30 minutes"
|
||||
|
||||
normal_load:
|
||||
concurrent_users: 100
|
||||
ramp_up_time: "10 minutes"
|
||||
duration: "1 hour"
|
||||
|
||||
peak_load:
|
||||
concurrent_users: 500
|
||||
ramp_up_time: "15 minutes"
|
||||
duration: "2 hours"
|
||||
|
||||
stress_test:
|
||||
concurrent_users: 1000
|
||||
ramp_up_time: "20 minutes"
|
||||
duration: "1 hour"
|
||||
|
||||
spike_test:
|
||||
phases:
|
||||
- users: 100, duration: "10 minutes"
|
||||
- users: 1000, duration: "5 minutes" # Spike
|
||||
- users: 100, duration: "15 minutes"
|
||||
```
|
||||
|
||||
#### Performance Thresholds
|
||||
```yaml
|
||||
performance_thresholds:
|
||||
response_time:
|
||||
p50: 2000ms # 50th percentile
|
||||
p90: 5000ms # 90th percentile
|
||||
p95: 8000ms # 95th percentile
|
||||
p99: 15000ms # 99th percentile
|
||||
|
||||
throughput:
|
||||
minimum: 10 # requests per second
|
||||
target: 50 # requests per second
|
||||
|
||||
error_rate:
|
||||
maximum: 5% # percentage of failed requests
|
||||
|
||||
resource_utilization:
|
||||
cpu_max: 80%
|
||||
memory_max: 85%
|
||||
network_max: 70%
|
||||
```
|
||||
|
||||
### 4. Real-World Evaluation
|
||||
|
||||
#### Production Monitoring
|
||||
```yaml
|
||||
production_metrics:
|
||||
business_metrics:
|
||||
- task_completion_rate
|
||||
- user_retention_rate
|
||||
- feature_adoption_rate
|
||||
- time_to_value
|
||||
|
||||
technical_metrics:
|
||||
- system_availability
|
||||
- mean_time_to_recovery
|
||||
- resource_efficiency
|
||||
- cost_per_transaction
|
||||
|
||||
user_experience_metrics:
|
||||
- net_promoter_score
|
||||
- user_satisfaction_rating
|
||||
- task_abandonment_rate
|
||||
- help_desk_ticket_volume
|
||||
```
|
||||
|
||||
#### Continuous Evaluation Pipeline
|
||||
```python
|
||||
class ContinuousEvaluationPipeline:
|
||||
def __init__(self, metrics_collector, analyzer, alerting):
|
||||
self.metrics_collector = metrics_collector
|
||||
self.analyzer = analyzer
|
||||
self.alerting = alerting
|
||||
|
||||
def run_evaluation_cycle(self):
|
||||
# Collect recent metrics
|
||||
metrics = self.metrics_collector.collect_recent_metrics(
|
||||
time_window="1 hour"
|
||||
)
|
||||
|
||||
# Analyze performance
|
||||
analysis = self.analyzer.analyze_metrics(metrics)
|
||||
|
||||
# Check for anomalies
|
||||
anomalies = self.analyzer.detect_anomalies(
|
||||
metrics,
|
||||
baseline_window="24 hours"
|
||||
)
|
||||
|
||||
# Generate alerts if needed
|
||||
if anomalies:
|
||||
self.alerting.send_alerts(anomalies)
|
||||
|
||||
# Update performance baselines
|
||||
self.analyzer.update_baselines(metrics)
|
||||
|
||||
return analysis
|
||||
```
|
||||
|
||||
## Analysis Techniques
|
||||
|
||||
### 1. Statistical Analysis
|
||||
|
||||
#### Descriptive Statistics
|
||||
```python
|
||||
def calculate_descriptive_stats(data):
|
||||
return {
|
||||
"count": len(data),
|
||||
"mean": np.mean(data),
|
||||
"median": np.median(data),
|
||||
"std_dev": np.std(data),
|
||||
"min": np.min(data),
|
||||
"max": np.max(data),
|
||||
"percentiles": {
|
||||
"p25": np.percentile(data, 25),
|
||||
"p50": np.percentile(data, 50),
|
||||
"p75": np.percentile(data, 75),
|
||||
"p90": np.percentile(data, 90),
|
||||
"p95": np.percentile(data, 95),
|
||||
"p99": np.percentile(data, 99)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Correlation Analysis
|
||||
```python
|
||||
def analyze_metric_correlations(metrics_df):
|
||||
correlation_matrix = metrics_df.corr()
|
||||
|
||||
# Identify strong correlations
|
||||
strong_correlations = []
|
||||
for i in range(len(correlation_matrix.columns)):
|
||||
for j in range(i + 1, len(correlation_matrix.columns)):
|
||||
corr_value = correlation_matrix.iloc[i, j]
|
||||
if abs(corr_value) > 0.7: # Strong correlation threshold
|
||||
strong_correlations.append({
|
||||
"metric1": correlation_matrix.columns[i],
|
||||
"metric2": correlation_matrix.columns[j],
|
||||
"correlation": corr_value,
|
||||
"strength": "strong" if abs(corr_value) > 0.8 else "moderate"
|
||||
})
|
||||
|
||||
return strong_correlations
|
||||
```
|
||||
|
||||
### 2. Trend Analysis
|
||||
|
||||
#### Time Series Analysis
|
||||
```python
|
||||
def analyze_performance_trends(time_series_data, metric):
|
||||
# Decompose time series
|
||||
decomposition = seasonal_decompose(
|
||||
time_series_data[metric],
|
||||
model='additive',
|
||||
period=24 # Daily seasonality
|
||||
)
|
||||
|
||||
# Trend detection
|
||||
trend_slope = calculate_trend_slope(decomposition.trend)
|
||||
|
||||
# Seasonality detection
|
||||
seasonal_patterns = identify_seasonal_patterns(decomposition.seasonal)
|
||||
|
||||
# Anomaly detection
|
||||
anomalies = detect_anomalies_isolation_forest(time_series_data[metric])
|
||||
|
||||
return {
|
||||
"trend_direction": "increasing" if trend_slope > 0 else "decreasing" if trend_slope < 0 else "stable",
|
||||
"trend_strength": abs(trend_slope),
|
||||
"seasonal_patterns": seasonal_patterns,
|
||||
"anomalies": anomalies,
|
||||
"forecast": generate_forecast(time_series_data[metric], periods=24)
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Comparative Analysis
|
||||
|
||||
#### Multi-System Comparison
|
||||
```python
|
||||
def compare_systems(system_metrics_dict):
|
||||
comparison_results = {}
|
||||
|
||||
metrics_to_compare = [
|
||||
"success_rate", "average_response_time",
|
||||
"cost_per_task", "error_rate"
|
||||
]
|
||||
|
||||
for metric in metrics_to_compare:
|
||||
metric_values = {
|
||||
system: metrics[metric]
|
||||
for system, metrics in system_metrics_dict.items()
|
||||
}
|
||||
|
||||
# Rank systems by metric
|
||||
ranked_systems = sorted(
|
||||
metric_values.items(),
|
||||
key=lambda x: x[1],
|
||||
reverse=(metric in ["success_rate"]) # Higher is better for some metrics
|
||||
)
|
||||
|
||||
# Calculate relative performance
|
||||
best_value = ranked_systems[0][1]
|
||||
relative_performance = {
|
||||
system: value / best_value if best_value > 0 else 0
|
||||
for system, value in metric_values.items()
|
||||
}
|
||||
|
||||
comparison_results[metric] = {
|
||||
"rankings": ranked_systems,
|
||||
"relative_performance": relative_performance,
|
||||
"best_system": ranked_systems[0][0]
|
||||
}
|
||||
|
||||
return comparison_results
|
||||
```
|
||||
|
||||
## Quality Assurance
|
||||
|
||||
### 1. Data Quality Validation
|
||||
|
||||
#### Data Completeness Checks
|
||||
```python
|
||||
def validate_data_completeness(metrics_data):
|
||||
completeness_report = {}
|
||||
|
||||
required_fields = [
|
||||
"timestamp", "task_id", "agent_id",
|
||||
"duration_ms", "status", "success"
|
||||
]
|
||||
|
||||
for field in required_fields:
|
||||
missing_count = metrics_data[field].isnull().sum()
|
||||
total_count = len(metrics_data)
|
||||
completeness_percentage = (total_count - missing_count) / total_count * 100
|
||||
|
||||
completeness_report[field] = {
|
||||
"completeness_percentage": completeness_percentage,
|
||||
"missing_count": missing_count,
|
||||
"status": "pass" if completeness_percentage >= 95 else "fail"
|
||||
}
|
||||
|
||||
return completeness_report
|
||||
```
|
||||
|
||||
#### Data Consistency Checks
|
||||
```python
|
||||
def validate_data_consistency(metrics_data):
|
||||
consistency_issues = []
|
||||
|
||||
# Check timestamp ordering
|
||||
if not metrics_data['timestamp'].is_monotonic_increasing:
|
||||
consistency_issues.append("Timestamps are not in chronological order")
|
||||
|
||||
# Check duration consistency
|
||||
duration_negative = (metrics_data['duration_ms'] < 0).sum()
|
||||
if duration_negative > 0:
|
||||
consistency_issues.append(f"Found {duration_negative} negative durations")
|
||||
|
||||
# Check status-success consistency
|
||||
success_status_mismatch = (
|
||||
(metrics_data['status'] == 'success') != metrics_data['success']
|
||||
).sum()
|
||||
if success_status_mismatch > 0:
|
||||
consistency_issues.append(f"Found {success_status_mismatch} status-success mismatches")
|
||||
|
||||
return consistency_issues
|
||||
```
|
||||
|
||||
### 2. Evaluation Reliability
|
||||
|
||||
#### Reproducibility Framework
|
||||
```python
|
||||
class ReproducibleEvaluation:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.random_seed = config.get('random_seed', 42)
|
||||
|
||||
def setup_environment(self):
|
||||
# Set random seeds
|
||||
random.seed(self.random_seed)
|
||||
np.random.seed(self.random_seed)
|
||||
|
||||
# Configure logging
|
||||
self.setup_evaluation_logging()
|
||||
|
||||
# Snapshot system state
|
||||
self.snapshot_system_state()
|
||||
|
||||
def run_evaluation(self, test_suite):
|
||||
self.setup_environment()
|
||||
|
||||
# Execute evaluation with full logging
|
||||
results = self.execute_test_suite(test_suite)
|
||||
|
||||
# Verify reproducibility
|
||||
self.verify_reproducibility(results)
|
||||
|
||||
return results
|
||||
```
|
||||
|
||||
## Reporting Framework
|
||||
|
||||
### 1. Executive Summary Report
|
||||
|
||||
#### Key Performance Indicators
|
||||
```yaml
|
||||
kpi_dashboard:
|
||||
overall_health_score: 85/100
|
||||
|
||||
performance:
|
||||
task_success_rate: 94.2%
|
||||
average_response_time: 2.3s
|
||||
p95_response_time: 8.1s
|
||||
|
||||
reliability:
|
||||
system_uptime: 99.8%
|
||||
error_rate: 2.1%
|
||||
mean_recovery_time: 45s
|
||||
|
||||
cost_efficiency:
|
||||
cost_per_task: $0.05
|
||||
token_utilization: 78%
|
||||
resource_efficiency: 82%
|
||||
|
||||
user_satisfaction:
|
||||
net_promoter_score: 42
|
||||
task_completion_rate: 89%
|
||||
user_retention_rate: 76%
|
||||
```
|
||||
|
||||
#### Trend Indicators
|
||||
```yaml
|
||||
trend_analysis:
|
||||
performance_trends:
|
||||
success_rate: "↗ +2.3% vs last month"
|
||||
response_time: "↘ -15% vs last month"
|
||||
error_rate: "→ stable vs last month"
|
||||
|
||||
cost_trends:
|
||||
total_cost: "↗ +8% vs last month"
|
||||
cost_per_task: "↘ -5% vs last month"
|
||||
efficiency: "↗ +12% vs last month"
|
||||
```
|
||||
|
||||
### 2. Technical Deep-Dive Report
|
||||
|
||||
#### Performance Analysis
|
||||
```markdown
|
||||
## Performance Analysis
|
||||
|
||||
### Task Success Patterns
|
||||
- **Overall Success Rate**: 94.2% (target: 95%)
|
||||
- **By Task Type**:
|
||||
- Simple tasks: 98.1% success
|
||||
- Complex tasks: 87.4% success
|
||||
- Multi-agent tasks: 91.2% success
|
||||
|
||||
### Response Time Distribution
|
||||
- **Median**: 1.8 seconds
|
||||
- **95th Percentile**: 8.1 seconds
|
||||
- **Peak Hours Impact**: +35% slower during 9-11 AM
|
||||
|
||||
### Error Analysis
|
||||
- **Top Error Types**:
|
||||
1. Timeout errors (34% of failures)
|
||||
2. Rate limit exceeded (28% of failures)
|
||||
3. Invalid input (19% of failures)
|
||||
```
|
||||
|
||||
#### Resource Utilization
|
||||
```markdown
|
||||
## Resource Utilization
|
||||
|
||||
### Compute Resources
|
||||
- **CPU Utilization**: 45% average, 78% peak
|
||||
- **Memory Usage**: 6.2GB average, 12.1GB peak
|
||||
- **Network I/O**: 125 MB/s average
|
||||
|
||||
### API Usage
|
||||
- **Token Consumption**: 2.4M tokens/day
|
||||
- **Cost Breakdown**:
|
||||
- GPT-4: 68% of token costs
|
||||
- GPT-3.5: 28% of token costs
|
||||
- Other models: 4% of token costs
|
||||
```
|
||||
|
||||
### 3. Actionable Recommendations
|
||||
|
||||
#### Performance Optimization
|
||||
```yaml
|
||||
recommendations:
|
||||
high_priority:
|
||||
- title: "Reduce timeout error rate"
|
||||
impact: "Could improve success rate by 2.1%"
|
||||
effort: "Medium"
|
||||
timeline: "2 weeks"
|
||||
|
||||
- title: "Optimize complex task handling"
|
||||
impact: "Could improve complex task success by 5%"
|
||||
effort: "High"
|
||||
timeline: "4 weeks"
|
||||
|
||||
medium_priority:
|
||||
- title: "Implement intelligent caching"
|
||||
impact: "Could reduce costs by 15%"
|
||||
effort: "Medium"
|
||||
timeline: "3 weeks"
|
||||
```
|
||||
|
||||
## Continuous Improvement Process
|
||||
|
||||
### 1. Evaluation Cadence
|
||||
|
||||
#### Regular Evaluation Schedule
|
||||
```yaml
|
||||
evaluation_schedule:
|
||||
real_time:
|
||||
frequency: "continuous"
|
||||
metrics: ["error_rate", "response_time", "system_health"]
|
||||
|
||||
hourly:
|
||||
frequency: "every hour"
|
||||
metrics: ["throughput", "resource_utilization", "user_activity"]
|
||||
|
||||
daily:
|
||||
frequency: "daily at 2 AM UTC"
|
||||
metrics: ["success_rates", "cost_analysis", "user_satisfaction"]
|
||||
|
||||
weekly:
|
||||
frequency: "every Sunday"
|
||||
metrics: ["trend_analysis", "comparative_analysis", "capacity_planning"]
|
||||
|
||||
monthly:
|
||||
frequency: "first Monday of month"
|
||||
metrics: ["comprehensive_evaluation", "benchmark_testing", "strategic_review"]
|
||||
```
|
||||
|
||||
### 2. Performance Baseline Management
|
||||
|
||||
#### Baseline Update Process
|
||||
```python
|
||||
def update_performance_baselines(current_metrics, historical_baselines):
|
||||
updated_baselines = {}
|
||||
|
||||
for metric, current_value in current_metrics.items():
|
||||
historical_values = historical_baselines.get(metric, [])
|
||||
historical_values.append(current_value)
|
||||
|
||||
# Keep rolling window of last 30 days
|
||||
historical_values = historical_values[-30:]
|
||||
|
||||
# Calculate new baseline
|
||||
baseline = {
|
||||
"mean": np.mean(historical_values),
|
||||
"std": np.std(historical_values),
|
||||
"p95": np.percentile(historical_values, 95),
|
||||
"trend": calculate_trend(historical_values)
|
||||
}
|
||||
|
||||
updated_baselines[metric] = baseline
|
||||
|
||||
return updated_baselines
|
||||
```
|
||||
|
||||
## Conclusion
|
||||
|
||||
Effective evaluation of multi-agent systems requires a comprehensive, multi-dimensional approach that combines quantitative metrics with qualitative assessments. The methodology should be:
|
||||
|
||||
1. **Comprehensive**: Cover all aspects of system performance
|
||||
2. **Continuous**: Provide ongoing monitoring and evaluation
|
||||
3. **Actionable**: Generate specific, implementable recommendations
|
||||
4. **Adaptable**: Evolve with system changes and requirements
|
||||
5. **Reliable**: Produce consistent, reproducible results
|
||||
|
||||
Regular evaluation using this methodology will ensure multi-agent systems continue to meet user needs while optimizing for cost, performance, and reliability.
|
||||
@@ -0,0 +1,470 @@
|
||||
# Tool Design Best Practices for Multi-Agent Systems
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines comprehensive best practices for designing tools that work effectively within multi-agent systems. Tools are the primary interface between agents and external capabilities, making their design critical for system success.
|
||||
|
||||
## Core Principles
|
||||
|
||||
### 1. Single Responsibility Principle
|
||||
Each tool should have a clear, focused purpose:
|
||||
- **Do one thing well:** Avoid multi-purpose tools that try to solve many problems
|
||||
- **Clear boundaries:** Well-defined input/output contracts
|
||||
- **Predictable behavior:** Consistent results for similar inputs
|
||||
- **Easy to understand:** Purpose should be obvious from name and description
|
||||
|
||||
### 2. Idempotency
|
||||
Tools should produce consistent results:
|
||||
- **Safe operations:** Read operations should never modify state
|
||||
- **Repeatable operations:** Same input should yield same output (when possible)
|
||||
- **State handling:** Clear semantics for state-modifying operations
|
||||
- **Error recovery:** Failed operations should be safely retryable
|
||||
|
||||
### 3. Composability
|
||||
Tools should work well together:
|
||||
- **Standard interfaces:** Consistent input/output formats
|
||||
- **Minimal assumptions:** Don't assume specific calling contexts
|
||||
- **Chain-friendly:** Output of one tool can be input to another
|
||||
- **Modular design:** Tools can be combined in different ways
|
||||
|
||||
### 4. Robustness
|
||||
Tools should handle edge cases gracefully:
|
||||
- **Input validation:** Comprehensive validation of all inputs
|
||||
- **Error handling:** Graceful degradation on failures
|
||||
- **Resource management:** Proper cleanup and resource management
|
||||
- **Timeout handling:** Operations should have reasonable timeouts
|
||||
|
||||
## Input Schema Design
|
||||
|
||||
### Schema Structure
|
||||
```json
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"parameter_name": {
|
||||
"type": "string",
|
||||
"description": "Clear, specific description",
|
||||
"examples": ["example1", "example2"],
|
||||
"minLength": 1,
|
||||
"maxLength": 1000
|
||||
}
|
||||
},
|
||||
"required": ["parameter_name"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
```
|
||||
|
||||
### Parameter Guidelines
|
||||
|
||||
#### Required vs Optional Parameters
|
||||
- **Required parameters:** Essential for tool function
|
||||
- **Optional parameters:** Provide additional control or customization
|
||||
- **Default values:** Sensible defaults for optional parameters
|
||||
- **Parameter groups:** Related parameters should be grouped logically
|
||||
|
||||
#### Parameter Types
|
||||
- **Primitives:** string, number, boolean for simple values
|
||||
- **Arrays:** For lists of similar items
|
||||
- **Objects:** For complex structured data
|
||||
- **Enums:** For fixed sets of valid values
|
||||
- **Unions:** When multiple types are acceptable
|
||||
|
||||
#### Validation Rules
|
||||
- **String validation:**
|
||||
- Length constraints (minLength, maxLength)
|
||||
- Pattern matching for formats (email, URL, etc.)
|
||||
- Character set restrictions
|
||||
- Content filtering for security
|
||||
|
||||
- **Numeric validation:**
|
||||
- Range constraints (minimum, maximum)
|
||||
- Multiple restrictions (multipleOf)
|
||||
- Precision requirements
|
||||
- Special value handling (NaN, infinity)
|
||||
|
||||
- **Array validation:**
|
||||
- Size constraints (minItems, maxItems)
|
||||
- Item type validation
|
||||
- Uniqueness requirements
|
||||
- Ordering requirements
|
||||
|
||||
- **Object validation:**
|
||||
- Required property enforcement
|
||||
- Additional property policies
|
||||
- Nested validation rules
|
||||
- Dependency validation
|
||||
|
||||
### Input Examples
|
||||
|
||||
#### Good Example:
|
||||
```json
|
||||
{
|
||||
"name": "search_web",
|
||||
"description": "Search the web for information",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "Search query string",
|
||||
"minLength": 1,
|
||||
"maxLength": 500,
|
||||
"examples": ["latest AI developments", "weather forecast"]
|
||||
},
|
||||
"limit": {
|
||||
"type": "integer",
|
||||
"description": "Maximum number of results to return",
|
||||
"minimum": 1,
|
||||
"maximum": 100,
|
||||
"default": 10
|
||||
},
|
||||
"language": {
|
||||
"type": "string",
|
||||
"description": "Language code for search results",
|
||||
"enum": ["en", "es", "fr", "de"],
|
||||
"default": "en"
|
||||
}
|
||||
},
|
||||
"required": ["query"],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Bad Example:
|
||||
```json
|
||||
{
|
||||
"name": "do_stuff",
|
||||
"description": "Does various operations",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "string",
|
||||
"description": "Some data"
|
||||
}
|
||||
},
|
||||
"additionalProperties": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Output Schema Design
|
||||
|
||||
### Response Structure
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {
|
||||
// Actual response data
|
||||
},
|
||||
"metadata": {
|
||||
"timestamp": "2024-01-15T10:30:00Z",
|
||||
"execution_time_ms": 234,
|
||||
"version": "1.0"
|
||||
},
|
||||
"warnings": [],
|
||||
"pagination": {
|
||||
"total": 100,
|
||||
"page": 1,
|
||||
"per_page": 10,
|
||||
"has_next": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Data Consistency
|
||||
- **Predictable structure:** Same structure regardless of success/failure
|
||||
- **Type consistency:** Same data types across different calls
|
||||
- **Null handling:** Clear semantics for missing/null values
|
||||
- **Empty responses:** Consistent handling of empty result sets
|
||||
|
||||
### Metadata Inclusion
|
||||
- **Execution time:** Performance monitoring
|
||||
- **Timestamps:** Audit trails and debugging
|
||||
- **Version information:** Compatibility tracking
|
||||
- **Request identifiers:** Correlation and debugging
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Error Response Structure
|
||||
```json
|
||||
{
|
||||
"success": false,
|
||||
"error": {
|
||||
"code": "INVALID_INPUT",
|
||||
"message": "The provided query is too short",
|
||||
"details": {
|
||||
"field": "query",
|
||||
"provided_length": 0,
|
||||
"minimum_length": 1
|
||||
},
|
||||
"retry_after": null,
|
||||
"documentation_url": "https://docs.example.com/errors#INVALID_INPUT"
|
||||
},
|
||||
"request_id": "req_12345"
|
||||
}
|
||||
```
|
||||
|
||||
### Error Categories
|
||||
|
||||
#### Client Errors (4xx equivalent)
|
||||
- **INVALID_INPUT:** Malformed or invalid parameters
|
||||
- **MISSING_PARAMETER:** Required parameter not provided
|
||||
- **VALIDATION_ERROR:** Parameter fails validation rules
|
||||
- **AUTHENTICATION_ERROR:** Invalid or missing credentials
|
||||
- **PERMISSION_ERROR:** Insufficient permissions
|
||||
- **RATE_LIMIT_ERROR:** Too many requests
|
||||
|
||||
#### Server Errors (5xx equivalent)
|
||||
- **INTERNAL_ERROR:** Unexpected server error
|
||||
- **SERVICE_UNAVAILABLE:** Downstream service unavailable
|
||||
- **TIMEOUT_ERROR:** Operation timed out
|
||||
- **RESOURCE_EXHAUSTED:** Out of resources (memory, disk, etc.)
|
||||
- **DEPENDENCY_ERROR:** External dependency failed
|
||||
|
||||
#### Tool-Specific Errors
|
||||
- **DATA_NOT_FOUND:** Requested data doesn't exist
|
||||
- **FORMAT_ERROR:** Data in unexpected format
|
||||
- **PROCESSING_ERROR:** Error during data processing
|
||||
- **CONFIGURATION_ERROR:** Tool misconfiguration
|
||||
|
||||
### Error Recovery Strategies
|
||||
|
||||
#### Retry Logic
|
||||
```json
|
||||
{
|
||||
"retry_policy": {
|
||||
"max_attempts": 3,
|
||||
"backoff_strategy": "exponential",
|
||||
"base_delay_ms": 1000,
|
||||
"max_delay_ms": 30000,
|
||||
"retryable_errors": [
|
||||
"TIMEOUT_ERROR",
|
||||
"SERVICE_UNAVAILABLE",
|
||||
"RATE_LIMIT_ERROR"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Fallback Behaviors
|
||||
- **Graceful degradation:** Partial results when possible
|
||||
- **Alternative approaches:** Different methods to achieve same goal
|
||||
- **Cached responses:** Return stale data if fresh data unavailable
|
||||
- **Default responses:** Safe default when specific response impossible
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Input Sanitization
|
||||
- **SQL injection prevention:** Parameterized queries
|
||||
- **XSS prevention:** HTML encoding of outputs
|
||||
- **Command injection prevention:** Input validation and sandboxing
|
||||
- **Path traversal prevention:** Path validation and restrictions
|
||||
|
||||
### Authentication and Authorization
|
||||
- **API key management:** Secure storage and rotation
|
||||
- **Token validation:** JWT validation and expiration
|
||||
- **Permission checking:** Role-based access control
|
||||
- **Audit logging:** Security event logging
|
||||
|
||||
### Data Protection
|
||||
- **PII handling:** Detection and protection of personal data
|
||||
- **Encryption:** Data encryption in transit and at rest
|
||||
- **Data retention:** Compliance with retention policies
|
||||
- **Access logging:** Who accessed what data when
|
||||
|
||||
## Performance Optimization
|
||||
|
||||
### Response Time
|
||||
- **Caching strategies:** Result caching for repeated requests
|
||||
- **Connection pooling:** Reuse connections to external services
|
||||
- **Async processing:** Non-blocking operations where possible
|
||||
- **Resource optimization:** Efficient resource utilization
|
||||
|
||||
### Throughput
|
||||
- **Batch operations:** Support for bulk operations
|
||||
- **Parallel processing:** Concurrent execution where safe
|
||||
- **Load balancing:** Distribute load across instances
|
||||
- **Resource scaling:** Auto-scaling based on demand
|
||||
|
||||
### Resource Management
|
||||
- **Memory usage:** Efficient memory allocation and cleanup
|
||||
- **CPU optimization:** Avoid unnecessary computations
|
||||
- **Network efficiency:** Minimize network round trips
|
||||
- **Storage optimization:** Efficient data structures and storage
|
||||
|
||||
## Testing Strategies
|
||||
|
||||
### Unit Testing
|
||||
```python
|
||||
def test_search_web_valid_input():
|
||||
result = search_web("test query", limit=5)
|
||||
assert result["success"] is True
|
||||
assert len(result["data"]["results"]) <= 5
|
||||
|
||||
def test_search_web_invalid_input():
|
||||
result = search_web("", limit=5)
|
||||
assert result["success"] is False
|
||||
assert result["error"]["code"] == "INVALID_INPUT"
|
||||
```
|
||||
|
||||
### Integration Testing
|
||||
- **End-to-end workflows:** Complete user scenarios
|
||||
- **External service mocking:** Mock external dependencies
|
||||
- **Error simulation:** Simulate various error conditions
|
||||
- **Performance testing:** Load and stress testing
|
||||
|
||||
### Contract Testing
|
||||
- **Schema validation:** Validate against defined schemas
|
||||
- **Backward compatibility:** Ensure changes don't break clients
|
||||
- **API versioning:** Test multiple API versions
|
||||
- **Consumer-driven contracts:** Test from consumer perspective
|
||||
|
||||
## Documentation
|
||||
|
||||
### Tool Documentation Template
|
||||
```markdown
|
||||
# Tool Name
|
||||
|
||||
## Description
|
||||
Brief description of what the tool does.
|
||||
|
||||
## Parameters
|
||||
### Required Parameters
|
||||
- `parameter_name` (type): Description
|
||||
|
||||
### Optional Parameters
|
||||
- `optional_param` (type, default: value): Description
|
||||
|
||||
## Response
|
||||
Description of response format and data.
|
||||
|
||||
## Examples
|
||||
### Basic Usage
|
||||
Input:
|
||||
```json
|
||||
{
|
||||
"parameter_name": "value"
|
||||
}
|
||||
```
|
||||
|
||||
Output:
|
||||
```json
|
||||
{
|
||||
"success": true,
|
||||
"data": {...}
|
||||
}
|
||||
```
|
||||
|
||||
## Error Codes
|
||||
- `ERROR_CODE`: Description of when this error occurs
|
||||
```
|
||||
|
||||
### API Documentation
|
||||
- **OpenAPI/Swagger specs:** Machine-readable API documentation
|
||||
- **Interactive examples:** Runnable examples in documentation
|
||||
- **Code samples:** Examples in multiple programming languages
|
||||
- **Changelog:** Version history and breaking changes
|
||||
|
||||
## Versioning Strategy
|
||||
|
||||
### Semantic Versioning
|
||||
- **Major version:** Breaking changes
|
||||
- **Minor version:** New features, backward compatible
|
||||
- **Patch version:** Bug fixes, no new features
|
||||
|
||||
### API Evolution
|
||||
- **Deprecation policy:** How to deprecate old features
|
||||
- **Migration guides:** Help users upgrade to new versions
|
||||
- **Backward compatibility:** Support for old versions
|
||||
- **Feature flags:** Gradual rollout of new features
|
||||
|
||||
## Monitoring and Observability
|
||||
|
||||
### Metrics Collection
|
||||
- **Usage metrics:** Call frequency, success rates
|
||||
- **Performance metrics:** Response times, throughput
|
||||
- **Error metrics:** Error rates by type
|
||||
- **Resource metrics:** CPU, memory, network usage
|
||||
|
||||
### Logging
|
||||
```json
|
||||
{
|
||||
"timestamp": "2024-01-15T10:30:00Z",
|
||||
"tool_name": "search_web",
|
||||
"request_id": "req_12345",
|
||||
"agent_id": "agent_001",
|
||||
"input_hash": "abc123",
|
||||
"execution_time_ms": 234,
|
||||
"success": true,
|
||||
"error_code": null
|
||||
}
|
||||
```
|
||||
|
||||
### Alerting
|
||||
- **Error rate thresholds:** Alert on high error rates
|
||||
- **Performance degradation:** Alert on slow responses
|
||||
- **Resource exhaustion:** Alert on resource limits
|
||||
- **Service availability:** Alert on service downtime
|
||||
|
||||
## Common Anti-Patterns
|
||||
|
||||
### Tool Design Anti-Patterns
|
||||
- **God tools:** Tools that try to do everything
|
||||
- **Chatty tools:** Tools that require many calls for simple tasks
|
||||
- **Stateful tools:** Tools that maintain state between calls
|
||||
- **Inconsistent interfaces:** Tools with different conventions
|
||||
|
||||
### Error Handling Anti-Patterns
|
||||
- **Silent failures:** Failing without proper error reporting
|
||||
- **Generic errors:** Non-descriptive error messages
|
||||
- **Inconsistent error formats:** Different error structures
|
||||
- **No retry guidance:** Not indicating if operation is retryable
|
||||
|
||||
### Performance Anti-Patterns
|
||||
- **Synchronous everything:** Not using async operations where appropriate
|
||||
- **No caching:** Repeatedly fetching same data
|
||||
- **Resource leaks:** Not properly cleaning up resources
|
||||
- **Unbounded operations:** Operations that can run indefinitely
|
||||
|
||||
## Best Practices Checklist
|
||||
|
||||
### Design Phase
|
||||
- [ ] Single, clear purpose
|
||||
- [ ] Well-defined input/output contracts
|
||||
- [ ] Comprehensive input validation
|
||||
- [ ] Idempotent operations where possible
|
||||
- [ ] Error handling strategy defined
|
||||
|
||||
### Implementation Phase
|
||||
- [ ] Robust error handling
|
||||
- [ ] Input sanitization
|
||||
- [ ] Resource management
|
||||
- [ ] Timeout handling
|
||||
- [ ] Logging implementation
|
||||
|
||||
### Testing Phase
|
||||
- [ ] Unit tests for all functionality
|
||||
- [ ] Integration tests with dependencies
|
||||
- [ ] Error condition testing
|
||||
- [ ] Performance testing
|
||||
- [ ] Security testing
|
||||
|
||||
### Documentation Phase
|
||||
- [ ] Complete API documentation
|
||||
- [ ] Usage examples
|
||||
- [ ] Error code documentation
|
||||
- [ ] Performance characteristics
|
||||
- [ ] Security considerations
|
||||
|
||||
### Deployment Phase
|
||||
- [ ] Monitoring setup
|
||||
- [ ] Alerting configuration
|
||||
- [ ] Performance baselines
|
||||
- [ ] Security reviews
|
||||
- [ ] Operational runbooks
|
||||
|
||||
## Conclusion
|
||||
|
||||
Well-designed tools are the foundation of effective multi-agent systems. They should be reliable, secure, performant, and easy to use. Following these best practices will result in tools that agents can effectively compose to solve complex problems while maintaining system reliability and security.
|
||||
978
engineering/agent-designer/tool_schema_generator.py
Normal file
978
engineering/agent-designer/tool_schema_generator.py
Normal file
@@ -0,0 +1,978 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Tool Schema Generator - Generate structured tool schemas for AI agents
|
||||
|
||||
Given a description of desired tools (name, purpose, inputs, outputs), generates
|
||||
structured tool schemas compatible with OpenAI function calling format and
|
||||
Anthropic tool use format. Includes: input validation rules, error response
|
||||
formats, example calls, rate limit suggestions.
|
||||
|
||||
Input: tool descriptions JSON
|
||||
Output: tool schemas (OpenAI + Anthropic format) + validation rules + example usage
|
||||
"""
|
||||
|
||||
import json
|
||||
import argparse
|
||||
import sys
|
||||
import re
|
||||
from typing import Dict, List, Any, Optional, Union, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
from enum import Enum
|
||||
|
||||
|
||||
class ParameterType(Enum):
|
||||
"""Parameter types for tool schemas"""
|
||||
STRING = "string"
|
||||
INTEGER = "integer"
|
||||
NUMBER = "number"
|
||||
BOOLEAN = "boolean"
|
||||
ARRAY = "array"
|
||||
OBJECT = "object"
|
||||
NULL = "null"
|
||||
|
||||
|
||||
class ValidationRule(Enum):
|
||||
"""Validation rule types"""
|
||||
REQUIRED = "required"
|
||||
MIN_LENGTH = "min_length"
|
||||
MAX_LENGTH = "max_length"
|
||||
PATTERN = "pattern"
|
||||
ENUM = "enum"
|
||||
MINIMUM = "minimum"
|
||||
MAXIMUM = "maximum"
|
||||
MIN_ITEMS = "min_items"
|
||||
MAX_ITEMS = "max_items"
|
||||
UNIQUE_ITEMS = "unique_items"
|
||||
FORMAT = "format"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParameterSpec:
|
||||
"""Parameter specification for tool inputs/outputs"""
|
||||
name: str
|
||||
type: ParameterType
|
||||
description: str
|
||||
required: bool = False
|
||||
default: Any = None
|
||||
validation_rules: Dict[str, Any] = None
|
||||
examples: List[Any] = None
|
||||
deprecated: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class ErrorSpec:
|
||||
"""Error specification for tool responses"""
|
||||
error_code: str
|
||||
error_message: str
|
||||
http_status: int
|
||||
retry_after: Optional[int] = None
|
||||
details: Dict[str, Any] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RateLimitSpec:
|
||||
"""Rate limiting specification"""
|
||||
requests_per_minute: int
|
||||
requests_per_hour: int
|
||||
requests_per_day: int
|
||||
burst_limit: int
|
||||
cooldown_period: int
|
||||
rate_limit_key: str = "user_id"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolDescription:
|
||||
"""Input tool description"""
|
||||
name: str
|
||||
purpose: str
|
||||
category: str
|
||||
inputs: List[Dict[str, Any]]
|
||||
outputs: List[Dict[str, Any]]
|
||||
error_conditions: List[str]
|
||||
side_effects: List[str]
|
||||
idempotent: bool
|
||||
rate_limits: Dict[str, Any]
|
||||
dependencies: List[str]
|
||||
examples: List[Dict[str, Any]]
|
||||
security_requirements: List[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ToolSchema:
|
||||
"""Complete tool schema with validation and examples"""
|
||||
name: str
|
||||
description: str
|
||||
openai_schema: Dict[str, Any]
|
||||
anthropic_schema: Dict[str, Any]
|
||||
validation_rules: List[Dict[str, Any]]
|
||||
error_responses: List[ErrorSpec]
|
||||
rate_limits: RateLimitSpec
|
||||
examples: List[Dict[str, Any]]
|
||||
metadata: Dict[str, Any]
|
||||
|
||||
|
||||
class ToolSchemaGenerator:
|
||||
"""Generate structured tool schemas from descriptions"""
|
||||
|
||||
def __init__(self):
|
||||
self.common_patterns = self._define_common_patterns()
|
||||
self.format_validators = self._define_format_validators()
|
||||
self.security_templates = self._define_security_templates()
|
||||
|
||||
def _define_common_patterns(self) -> Dict[str, str]:
|
||||
"""Define common regex patterns for validation"""
|
||||
return {
|
||||
"email": r"^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$",
|
||||
"url": r"^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)$",
|
||||
"uuid": r"^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$",
|
||||
"phone": r"^\+?1?[0-9]{10,15}$",
|
||||
"ip_address": r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$",
|
||||
"date": r"^\d{4}-\d{2}-\d{2}$",
|
||||
"datetime": r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z?$",
|
||||
"slug": r"^[a-z0-9]+(?:-[a-z0-9]+)*$",
|
||||
"semantic_version": r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
|
||||
}
|
||||
|
||||
def _define_format_validators(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Define format validators for common data types"""
|
||||
return {
|
||||
"email": {
|
||||
"type": "string",
|
||||
"format": "email",
|
||||
"pattern": self.common_patterns["email"],
|
||||
"min_length": 5,
|
||||
"max_length": 254
|
||||
},
|
||||
"url": {
|
||||
"type": "string",
|
||||
"format": "uri",
|
||||
"pattern": self.common_patterns["url"],
|
||||
"min_length": 7,
|
||||
"max_length": 2048
|
||||
},
|
||||
"uuid": {
|
||||
"type": "string",
|
||||
"format": "uuid",
|
||||
"pattern": self.common_patterns["uuid"],
|
||||
"min_length": 36,
|
||||
"max_length": 36
|
||||
},
|
||||
"date": {
|
||||
"type": "string",
|
||||
"format": "date",
|
||||
"pattern": self.common_patterns["date"],
|
||||
"min_length": 10,
|
||||
"max_length": 10
|
||||
},
|
||||
"datetime": {
|
||||
"type": "string",
|
||||
"format": "date-time",
|
||||
"pattern": self.common_patterns["datetime"],
|
||||
"min_length": 19,
|
||||
"max_length": 30
|
||||
},
|
||||
"password": {
|
||||
"type": "string",
|
||||
"min_length": 8,
|
||||
"max_length": 128,
|
||||
"pattern": r"^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)(?=.*[@$!%*?&])[A-Za-z\d@$!%*?&]"
|
||||
}
|
||||
}
|
||||
|
||||
def _define_security_templates(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Define security requirement templates"""
|
||||
return {
|
||||
"authentication_required": {
|
||||
"requires_auth": True,
|
||||
"auth_methods": ["bearer_token", "api_key"],
|
||||
"scope_required": ["read", "write"]
|
||||
},
|
||||
"rate_limited": {
|
||||
"rate_limits": {
|
||||
"requests_per_minute": 60,
|
||||
"requests_per_hour": 1000,
|
||||
"burst_limit": 10
|
||||
}
|
||||
},
|
||||
"input_sanitization": {
|
||||
"sanitize_html": True,
|
||||
"validate_sql_injection": True,
|
||||
"escape_special_chars": True
|
||||
},
|
||||
"output_validation": {
|
||||
"validate_response_schema": True,
|
||||
"filter_sensitive_data": True,
|
||||
"content_type_validation": True
|
||||
}
|
||||
}
|
||||
|
||||
def parse_tool_description(self, description: ToolDescription) -> ParameterSpec:
|
||||
"""Parse tool description into structured parameters"""
|
||||
input_params = []
|
||||
output_params = []
|
||||
|
||||
# Parse input parameters
|
||||
for input_spec in description.inputs:
|
||||
param = self._parse_parameter_spec(input_spec)
|
||||
input_params.append(param)
|
||||
|
||||
# Parse output parameters
|
||||
for output_spec in description.outputs:
|
||||
param = self._parse_parameter_spec(output_spec)
|
||||
output_params.append(param)
|
||||
|
||||
return input_params, output_params
|
||||
|
||||
def _parse_parameter_spec(self, param_spec: Dict[str, Any]) -> ParameterSpec:
|
||||
"""Parse individual parameter specification"""
|
||||
name = param_spec.get("name", "")
|
||||
type_str = param_spec.get("type", "string")
|
||||
description = param_spec.get("description", "")
|
||||
required = param_spec.get("required", False)
|
||||
default = param_spec.get("default")
|
||||
examples = param_spec.get("examples", [])
|
||||
|
||||
# Parse parameter type
|
||||
param_type = self._parse_parameter_type(type_str)
|
||||
|
||||
# Generate validation rules
|
||||
validation_rules = self._generate_validation_rules(param_spec, param_type)
|
||||
|
||||
return ParameterSpec(
|
||||
name=name,
|
||||
type=param_type,
|
||||
description=description,
|
||||
required=required,
|
||||
default=default,
|
||||
validation_rules=validation_rules,
|
||||
examples=examples
|
||||
)
|
||||
|
||||
def _parse_parameter_type(self, type_str: str) -> ParameterType:
|
||||
"""Parse parameter type from string"""
|
||||
type_mapping = {
|
||||
"str": ParameterType.STRING,
|
||||
"string": ParameterType.STRING,
|
||||
"text": ParameterType.STRING,
|
||||
"int": ParameterType.INTEGER,
|
||||
"integer": ParameterType.INTEGER,
|
||||
"float": ParameterType.NUMBER,
|
||||
"number": ParameterType.NUMBER,
|
||||
"bool": ParameterType.BOOLEAN,
|
||||
"boolean": ParameterType.BOOLEAN,
|
||||
"list": ParameterType.ARRAY,
|
||||
"array": ParameterType.ARRAY,
|
||||
"dict": ParameterType.OBJECT,
|
||||
"object": ParameterType.OBJECT,
|
||||
"null": ParameterType.NULL,
|
||||
"none": ParameterType.NULL
|
||||
}
|
||||
|
||||
return type_mapping.get(type_str.lower(), ParameterType.STRING)
|
||||
|
||||
def _generate_validation_rules(self, param_spec: Dict[str, Any], param_type: ParameterType) -> Dict[str, Any]:
|
||||
"""Generate validation rules for a parameter"""
|
||||
rules = {}
|
||||
|
||||
# Type-specific validation
|
||||
if param_type == ParameterType.STRING:
|
||||
rules.update(self._generate_string_validation(param_spec))
|
||||
elif param_type == ParameterType.INTEGER:
|
||||
rules.update(self._generate_integer_validation(param_spec))
|
||||
elif param_type == ParameterType.NUMBER:
|
||||
rules.update(self._generate_number_validation(param_spec))
|
||||
elif param_type == ParameterType.ARRAY:
|
||||
rules.update(self._generate_array_validation(param_spec))
|
||||
elif param_type == ParameterType.OBJECT:
|
||||
rules.update(self._generate_object_validation(param_spec))
|
||||
|
||||
# Common validation rules
|
||||
if param_spec.get("required", False):
|
||||
rules["required"] = True
|
||||
|
||||
if "enum" in param_spec:
|
||||
rules["enum"] = param_spec["enum"]
|
||||
|
||||
if "pattern" in param_spec:
|
||||
rules["pattern"] = param_spec["pattern"]
|
||||
elif self._detect_format(param_spec.get("name", ""), param_spec.get("description", "")):
|
||||
format_name = self._detect_format(param_spec.get("name", ""), param_spec.get("description", ""))
|
||||
if format_name in self.format_validators:
|
||||
rules.update(self.format_validators[format_name])
|
||||
|
||||
return rules
|
||||
|
||||
def _generate_string_validation(self, param_spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Generate string-specific validation rules"""
|
||||
rules = {}
|
||||
|
||||
if "min_length" in param_spec:
|
||||
rules["minLength"] = param_spec["min_length"]
|
||||
elif "min_len" in param_spec:
|
||||
rules["minLength"] = param_spec["min_len"]
|
||||
else:
|
||||
# Infer from description
|
||||
desc = param_spec.get("description", "").lower()
|
||||
if "password" in desc:
|
||||
rules["minLength"] = 8
|
||||
elif "email" in desc:
|
||||
rules["minLength"] = 5
|
||||
elif "name" in desc:
|
||||
rules["minLength"] = 1
|
||||
|
||||
if "max_length" in param_spec:
|
||||
rules["maxLength"] = param_spec["max_length"]
|
||||
elif "max_len" in param_spec:
|
||||
rules["maxLength"] = param_spec["max_len"]
|
||||
else:
|
||||
# Reasonable defaults
|
||||
desc = param_spec.get("description", "").lower()
|
||||
if "password" in desc:
|
||||
rules["maxLength"] = 128
|
||||
elif "email" in desc:
|
||||
rules["maxLength"] = 254
|
||||
elif "description" in desc or "content" in desc:
|
||||
rules["maxLength"] = 10000
|
||||
elif "name" in desc or "title" in desc:
|
||||
rules["maxLength"] = 255
|
||||
else:
|
||||
rules["maxLength"] = 1000
|
||||
|
||||
return rules
|
||||
|
||||
def _generate_integer_validation(self, param_spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Generate integer-specific validation rules"""
|
||||
rules = {}
|
||||
|
||||
if "minimum" in param_spec:
|
||||
rules["minimum"] = param_spec["minimum"]
|
||||
elif "min" in param_spec:
|
||||
rules["minimum"] = param_spec["min"]
|
||||
else:
|
||||
# Infer from context
|
||||
name = param_spec.get("name", "").lower()
|
||||
desc = param_spec.get("description", "").lower()
|
||||
if any(word in name + desc for word in ["count", "quantity", "amount", "size", "limit"]):
|
||||
rules["minimum"] = 0
|
||||
elif "page" in name + desc:
|
||||
rules["minimum"] = 1
|
||||
elif "port" in name + desc:
|
||||
rules["minimum"] = 1
|
||||
rules["maximum"] = 65535
|
||||
|
||||
if "maximum" in param_spec:
|
||||
rules["maximum"] = param_spec["maximum"]
|
||||
elif "max" in param_spec:
|
||||
rules["maximum"] = param_spec["max"]
|
||||
|
||||
return rules
|
||||
|
||||
def _generate_number_validation(self, param_spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Generate number-specific validation rules"""
|
||||
rules = {}
|
||||
|
||||
if "minimum" in param_spec:
|
||||
rules["minimum"] = param_spec["minimum"]
|
||||
if "maximum" in param_spec:
|
||||
rules["maximum"] = param_spec["maximum"]
|
||||
if "exclusive_minimum" in param_spec:
|
||||
rules["exclusiveMinimum"] = param_spec["exclusive_minimum"]
|
||||
if "exclusive_maximum" in param_spec:
|
||||
rules["exclusiveMaximum"] = param_spec["exclusive_maximum"]
|
||||
if "multiple_of" in param_spec:
|
||||
rules["multipleOf"] = param_spec["multiple_of"]
|
||||
|
||||
return rules
|
||||
|
||||
def _generate_array_validation(self, param_spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Generate array-specific validation rules"""
|
||||
rules = {}
|
||||
|
||||
if "min_items" in param_spec:
|
||||
rules["minItems"] = param_spec["min_items"]
|
||||
elif "min_length" in param_spec:
|
||||
rules["minItems"] = param_spec["min_length"]
|
||||
else:
|
||||
rules["minItems"] = 0
|
||||
|
||||
if "max_items" in param_spec:
|
||||
rules["maxItems"] = param_spec["max_items"]
|
||||
elif "max_length" in param_spec:
|
||||
rules["maxItems"] = param_spec["max_length"]
|
||||
else:
|
||||
rules["maxItems"] = 1000 # Reasonable default
|
||||
|
||||
if param_spec.get("unique_items", False):
|
||||
rules["uniqueItems"] = True
|
||||
|
||||
if "item_type" in param_spec:
|
||||
rules["items"] = {"type": param_spec["item_type"]}
|
||||
|
||||
return rules
|
||||
|
||||
def _generate_object_validation(self, param_spec: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Generate object-specific validation rules"""
|
||||
rules = {}
|
||||
|
||||
if "properties" in param_spec:
|
||||
rules["properties"] = param_spec["properties"]
|
||||
|
||||
if "required_properties" in param_spec:
|
||||
rules["required"] = param_spec["required_properties"]
|
||||
|
||||
if "additional_properties" in param_spec:
|
||||
rules["additionalProperties"] = param_spec["additional_properties"]
|
||||
else:
|
||||
rules["additionalProperties"] = False
|
||||
|
||||
if "min_properties" in param_spec:
|
||||
rules["minProperties"] = param_spec["min_properties"]
|
||||
|
||||
if "max_properties" in param_spec:
|
||||
rules["maxProperties"] = param_spec["max_properties"]
|
||||
|
||||
return rules
|
||||
|
||||
def _detect_format(self, name: str, description: str) -> Optional[str]:
|
||||
"""Detect parameter format from name and description"""
|
||||
combined = (name + " " + description).lower()
|
||||
|
||||
format_indicators = {
|
||||
"email": ["email", "e-mail", "email_address"],
|
||||
"url": ["url", "uri", "link", "website", "endpoint"],
|
||||
"uuid": ["uuid", "guid", "identifier", "id"],
|
||||
"date": ["date", "birthday", "created_date", "modified_date"],
|
||||
"datetime": ["datetime", "timestamp", "created_at", "updated_at"],
|
||||
"password": ["password", "secret", "token", "api_key"]
|
||||
}
|
||||
|
||||
for format_name, indicators in format_indicators.items():
|
||||
if any(indicator in combined for indicator in indicators):
|
||||
return format_name
|
||||
|
||||
return None
|
||||
|
||||
def generate_openai_schema(self, description: ToolDescription, input_params: List[ParameterSpec]) -> Dict[str, Any]:
|
||||
"""Generate OpenAI function calling schema"""
|
||||
properties = {}
|
||||
required = []
|
||||
|
||||
for param in input_params:
|
||||
prop_def = {
|
||||
"type": param.type.value,
|
||||
"description": param.description
|
||||
}
|
||||
|
||||
# Add validation rules
|
||||
if param.validation_rules:
|
||||
prop_def.update(param.validation_rules)
|
||||
|
||||
# Add examples
|
||||
if param.examples:
|
||||
prop_def["examples"] = param.examples
|
||||
|
||||
# Add default value
|
||||
if param.default is not None:
|
||||
prop_def["default"] = param.default
|
||||
|
||||
properties[param.name] = prop_def
|
||||
|
||||
if param.required:
|
||||
required.append(param.name)
|
||||
|
||||
schema = {
|
||||
"name": description.name,
|
||||
"description": description.purpose,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": properties,
|
||||
"required": required,
|
||||
"additionalProperties": False
|
||||
}
|
||||
}
|
||||
|
||||
return schema
|
||||
|
||||
def generate_anthropic_schema(self, description: ToolDescription, input_params: List[ParameterSpec]) -> Dict[str, Any]:
|
||||
"""Generate Anthropic tool use schema"""
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
"required": []
|
||||
}
|
||||
|
||||
for param in input_params:
|
||||
prop_def = {
|
||||
"type": param.type.value,
|
||||
"description": param.description
|
||||
}
|
||||
|
||||
# Add validation rules (Anthropic uses subset of JSON Schema)
|
||||
if param.validation_rules:
|
||||
# Filter to supported validation rules
|
||||
supported_rules = ["minLength", "maxLength", "minimum", "maximum", "pattern", "enum", "items"]
|
||||
for rule, value in param.validation_rules.items():
|
||||
if rule in supported_rules:
|
||||
prop_def[rule] = value
|
||||
|
||||
input_schema["properties"][param.name] = prop_def
|
||||
|
||||
if param.required:
|
||||
input_schema["required"].append(param.name)
|
||||
|
||||
schema = {
|
||||
"name": description.name,
|
||||
"description": description.purpose,
|
||||
"input_schema": input_schema
|
||||
}
|
||||
|
||||
return schema
|
||||
|
||||
def generate_error_responses(self, description: ToolDescription) -> List[ErrorSpec]:
|
||||
"""Generate error response specifications"""
|
||||
error_specs = []
|
||||
|
||||
# Common errors
|
||||
common_errors = [
|
||||
{
|
||||
"error_code": "invalid_input",
|
||||
"error_message": "Invalid input parameters provided",
|
||||
"http_status": 400,
|
||||
"details": {"validation_errors": []}
|
||||
},
|
||||
{
|
||||
"error_code": "authentication_required",
|
||||
"error_message": "Authentication required to access this tool",
|
||||
"http_status": 401
|
||||
},
|
||||
{
|
||||
"error_code": "insufficient_permissions",
|
||||
"error_message": "Insufficient permissions to perform this operation",
|
||||
"http_status": 403
|
||||
},
|
||||
{
|
||||
"error_code": "rate_limit_exceeded",
|
||||
"error_message": "Rate limit exceeded. Please try again later",
|
||||
"http_status": 429,
|
||||
"retry_after": 60
|
||||
},
|
||||
{
|
||||
"error_code": "internal_error",
|
||||
"error_message": "Internal server error occurred",
|
||||
"http_status": 500
|
||||
},
|
||||
{
|
||||
"error_code": "service_unavailable",
|
||||
"error_message": "Service temporarily unavailable",
|
||||
"http_status": 503,
|
||||
"retry_after": 300
|
||||
}
|
||||
]
|
||||
|
||||
# Add common errors
|
||||
for error in common_errors:
|
||||
error_specs.append(ErrorSpec(**error))
|
||||
|
||||
# Add tool-specific errors based on error conditions
|
||||
for condition in description.error_conditions:
|
||||
if "not found" in condition.lower():
|
||||
error_specs.append(ErrorSpec(
|
||||
error_code="resource_not_found",
|
||||
error_message=f"Requested resource not found: {condition}",
|
||||
http_status=404
|
||||
))
|
||||
elif "timeout" in condition.lower():
|
||||
error_specs.append(ErrorSpec(
|
||||
error_code="operation_timeout",
|
||||
error_message=f"Operation timed out: {condition}",
|
||||
http_status=408,
|
||||
retry_after=30
|
||||
))
|
||||
elif "quota" in condition.lower() or "limit" in condition.lower():
|
||||
error_specs.append(ErrorSpec(
|
||||
error_code="quota_exceeded",
|
||||
error_message=f"Quota or limit exceeded: {condition}",
|
||||
http_status=429,
|
||||
retry_after=3600
|
||||
))
|
||||
elif "dependency" in condition.lower():
|
||||
error_specs.append(ErrorSpec(
|
||||
error_code="dependency_failure",
|
||||
error_message=f"Dependency service failure: {condition}",
|
||||
http_status=502
|
||||
))
|
||||
|
||||
return error_specs
|
||||
|
||||
def generate_rate_limits(self, description: ToolDescription) -> RateLimitSpec:
|
||||
"""Generate rate limiting specification"""
|
||||
rate_limits = description.rate_limits
|
||||
|
||||
# Default rate limits based on tool category
|
||||
defaults = {
|
||||
"search": {"rpm": 60, "rph": 1000, "rpd": 10000, "burst": 10},
|
||||
"data": {"rpm": 30, "rph": 500, "rpd": 5000, "burst": 5},
|
||||
"api": {"rpm": 100, "rph": 2000, "rpd": 20000, "burst": 20},
|
||||
"file": {"rpm": 120, "rph": 3000, "rpd": 30000, "burst": 30},
|
||||
"compute": {"rpm": 10, "rph": 100, "rpd": 1000, "burst": 3},
|
||||
"communication": {"rpm": 30, "rph": 300, "rpd": 3000, "burst": 5}
|
||||
}
|
||||
|
||||
category_defaults = defaults.get(description.category.lower(), defaults["api"])
|
||||
|
||||
return RateLimitSpec(
|
||||
requests_per_minute=rate_limits.get("requests_per_minute", category_defaults["rpm"]),
|
||||
requests_per_hour=rate_limits.get("requests_per_hour", category_defaults["rph"]),
|
||||
requests_per_day=rate_limits.get("requests_per_day", category_defaults["rpd"]),
|
||||
burst_limit=rate_limits.get("burst_limit", category_defaults["burst"]),
|
||||
cooldown_period=rate_limits.get("cooldown_period", 60),
|
||||
rate_limit_key=rate_limits.get("rate_limit_key", "user_id")
|
||||
)
|
||||
|
||||
def generate_examples(self, description: ToolDescription, input_params: List[ParameterSpec]) -> List[Dict[str, Any]]:
|
||||
"""Generate usage examples"""
|
||||
examples = []
|
||||
|
||||
# Use provided examples if available
|
||||
if description.examples:
|
||||
for example in description.examples:
|
||||
examples.append(example)
|
||||
|
||||
# Generate synthetic examples
|
||||
if len(examples) == 0:
|
||||
synthetic_example = self._generate_synthetic_example(description, input_params)
|
||||
if synthetic_example:
|
||||
examples.append(synthetic_example)
|
||||
|
||||
# Ensure we have multiple examples showing different scenarios
|
||||
if len(examples) == 1 and len(input_params) > 1:
|
||||
# Generate minimal example
|
||||
minimal_example = self._generate_minimal_example(description, input_params)
|
||||
if minimal_example and minimal_example != examples[0]:
|
||||
examples.append(minimal_example)
|
||||
|
||||
return examples
|
||||
|
||||
def _generate_synthetic_example(self, description: ToolDescription, input_params: List[ParameterSpec]) -> Dict[str, Any]:
|
||||
"""Generate a synthetic example based on parameter specifications"""
|
||||
example_input = {}
|
||||
|
||||
for param in input_params:
|
||||
if param.examples:
|
||||
example_input[param.name] = param.examples[0]
|
||||
elif param.default is not None:
|
||||
example_input[param.name] = param.default
|
||||
else:
|
||||
example_input[param.name] = self._generate_example_value(param)
|
||||
|
||||
# Generate expected output based on tool purpose
|
||||
expected_output = self._generate_example_output(description)
|
||||
|
||||
return {
|
||||
"description": f"Example usage of {description.name}",
|
||||
"input": example_input,
|
||||
"expected_output": expected_output
|
||||
}
|
||||
|
||||
def _generate_minimal_example(self, description: ToolDescription, input_params: List[ParameterSpec]) -> Dict[str, Any]:
|
||||
"""Generate minimal example with only required parameters"""
|
||||
example_input = {}
|
||||
|
||||
for param in input_params:
|
||||
if param.required:
|
||||
if param.examples:
|
||||
example_input[param.name] = param.examples[0]
|
||||
else:
|
||||
example_input[param.name] = self._generate_example_value(param)
|
||||
|
||||
if not example_input:
|
||||
return None
|
||||
|
||||
expected_output = self._generate_example_output(description)
|
||||
|
||||
return {
|
||||
"description": f"Minimal example of {description.name} with required parameters only",
|
||||
"input": example_input,
|
||||
"expected_output": expected_output
|
||||
}
|
||||
|
||||
def _generate_example_value(self, param: ParameterSpec) -> Any:
|
||||
"""Generate example value for a parameter"""
|
||||
if param.type == ParameterType.STRING:
|
||||
format_examples = {
|
||||
"email": "user@example.com",
|
||||
"url": "https://example.com",
|
||||
"uuid": "123e4567-e89b-12d3-a456-426614174000",
|
||||
"date": "2024-01-15",
|
||||
"datetime": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
|
||||
# Check for format in validation rules
|
||||
if param.validation_rules and "format" in param.validation_rules:
|
||||
format_type = param.validation_rules["format"]
|
||||
if format_type in format_examples:
|
||||
return format_examples[format_type]
|
||||
|
||||
# Check for patterns or enum
|
||||
if param.validation_rules:
|
||||
if "enum" in param.validation_rules:
|
||||
return param.validation_rules["enum"][0]
|
||||
|
||||
# Generate based on name/description
|
||||
name_lower = param.name.lower()
|
||||
if "name" in name_lower:
|
||||
return "example_name"
|
||||
elif "query" in name_lower or "search" in name_lower:
|
||||
return "search query"
|
||||
elif "path" in name_lower:
|
||||
return "/path/to/resource"
|
||||
elif "message" in name_lower:
|
||||
return "Example message"
|
||||
else:
|
||||
return "example_value"
|
||||
|
||||
elif param.type == ParameterType.INTEGER:
|
||||
if param.validation_rules:
|
||||
min_val = param.validation_rules.get("minimum", 0)
|
||||
max_val = param.validation_rules.get("maximum", 100)
|
||||
return min(max(42, min_val), max_val)
|
||||
return 42
|
||||
|
||||
elif param.type == ParameterType.NUMBER:
|
||||
if param.validation_rules:
|
||||
min_val = param.validation_rules.get("minimum", 0.0)
|
||||
max_val = param.validation_rules.get("maximum", 100.0)
|
||||
return min(max(42.5, min_val), max_val)
|
||||
return 42.5
|
||||
|
||||
elif param.type == ParameterType.BOOLEAN:
|
||||
return True
|
||||
|
||||
elif param.type == ParameterType.ARRAY:
|
||||
return ["item1", "item2"]
|
||||
|
||||
elif param.type == ParameterType.OBJECT:
|
||||
return {"key": "value"}
|
||||
|
||||
else:
|
||||
return None
|
||||
|
||||
def _generate_example_output(self, description: ToolDescription) -> Dict[str, Any]:
|
||||
"""Generate example output based on tool description"""
|
||||
category = description.category.lower()
|
||||
|
||||
if category == "search":
|
||||
return {
|
||||
"results": [
|
||||
{"title": "Example Result 1", "url": "https://example.com/1", "snippet": "Example snippet..."},
|
||||
{"title": "Example Result 2", "url": "https://example.com/2", "snippet": "Another snippet..."}
|
||||
],
|
||||
"total_count": 2
|
||||
}
|
||||
elif category == "data":
|
||||
return {
|
||||
"data": [{"id": 1, "value": "example"}, {"id": 2, "value": "another"}],
|
||||
"metadata": {"count": 2, "processed_at": "2024-01-15T10:30:00Z"}
|
||||
}
|
||||
elif category == "file":
|
||||
return {
|
||||
"success": True,
|
||||
"file_path": "/path/to/file.txt",
|
||||
"size": 1024,
|
||||
"modified_at": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
elif category == "api":
|
||||
return {
|
||||
"status": "success",
|
||||
"data": {"result": "operation completed successfully"},
|
||||
"timestamp": "2024-01-15T10:30:00Z"
|
||||
}
|
||||
else:
|
||||
return {
|
||||
"success": True,
|
||||
"message": f"{description.name} executed successfully",
|
||||
"result": "example result"
|
||||
}
|
||||
|
||||
def generate_tool_schema(self, description: ToolDescription) -> ToolSchema:
|
||||
"""Generate complete tool schema"""
|
||||
# Parse parameters
|
||||
input_params, output_params = self.parse_tool_description(description)
|
||||
|
||||
# Generate schemas
|
||||
openai_schema = self.generate_openai_schema(description, input_params)
|
||||
anthropic_schema = self.generate_anthropic_schema(description, input_params)
|
||||
|
||||
# Generate validation rules
|
||||
validation_rules = []
|
||||
for param in input_params:
|
||||
if param.validation_rules:
|
||||
validation_rules.append({
|
||||
"parameter": param.name,
|
||||
"rules": param.validation_rules
|
||||
})
|
||||
|
||||
# Generate error responses
|
||||
error_responses = self.generate_error_responses(description)
|
||||
|
||||
# Generate rate limits
|
||||
rate_limits = self.generate_rate_limits(description)
|
||||
|
||||
# Generate examples
|
||||
examples = self.generate_examples(description, input_params)
|
||||
|
||||
# Generate metadata
|
||||
metadata = {
|
||||
"category": description.category,
|
||||
"idempotent": description.idempotent,
|
||||
"side_effects": description.side_effects,
|
||||
"dependencies": description.dependencies,
|
||||
"security_requirements": description.security_requirements,
|
||||
"generated_at": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0",
|
||||
"input_parameters": len(input_params),
|
||||
"output_parameters": len(output_params),
|
||||
"required_parameters": sum(1 for p in input_params if p.required),
|
||||
"optional_parameters": sum(1 for p in input_params if not p.required)
|
||||
}
|
||||
|
||||
return ToolSchema(
|
||||
name=description.name,
|
||||
description=description.purpose,
|
||||
openai_schema=openai_schema,
|
||||
anthropic_schema=anthropic_schema,
|
||||
validation_rules=validation_rules,
|
||||
error_responses=error_responses,
|
||||
rate_limits=rate_limits,
|
||||
examples=examples,
|
||||
metadata=metadata
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Tool Schema Generator for AI Agents")
|
||||
parser.add_argument("input_file", help="JSON file with tool descriptions")
|
||||
parser.add_argument("-o", "--output", help="Output file prefix (default: tool_schemas)")
|
||||
parser.add_argument("--format", choices=["json", "both"], default="both",
|
||||
help="Output format")
|
||||
parser.add_argument("--validate", action="store_true",
|
||||
help="Validate generated schemas")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Load tool descriptions
|
||||
with open(args.input_file, 'r') as f:
|
||||
tools_data = json.load(f)
|
||||
|
||||
# Parse tool descriptions
|
||||
tool_descriptions = []
|
||||
for tool_data in tools_data.get("tools", []):
|
||||
tool_desc = ToolDescription(**tool_data)
|
||||
tool_descriptions.append(tool_desc)
|
||||
|
||||
# Generate schemas
|
||||
generator = ToolSchemaGenerator()
|
||||
schemas = []
|
||||
|
||||
for description in tool_descriptions:
|
||||
schema = generator.generate_tool_schema(description)
|
||||
schemas.append(schema)
|
||||
print(f"Generated schema for: {schema.name}")
|
||||
|
||||
# Prepare output
|
||||
output_data = {
|
||||
"tool_schemas": [asdict(schema) for schema in schemas],
|
||||
"metadata": {
|
||||
"generated_by": "tool_schema_generator.py",
|
||||
"input_file": args.input_file,
|
||||
"tool_count": len(schemas),
|
||||
"generation_timestamp": "2024-01-15T10:30:00Z",
|
||||
"schema_version": "1.0"
|
||||
},
|
||||
"validation_summary": {
|
||||
"total_tools": len(schemas),
|
||||
"total_parameters": sum(schema.metadata["input_parameters"] for schema in schemas),
|
||||
"total_validation_rules": sum(len(schema.validation_rules) for schema in schemas),
|
||||
"total_examples": sum(len(schema.examples) for schema in schemas)
|
||||
}
|
||||
}
|
||||
|
||||
# Output files
|
||||
output_prefix = args.output or "tool_schemas"
|
||||
|
||||
if args.format in ["json", "both"]:
|
||||
with open(f"{output_prefix}.json", 'w') as f:
|
||||
json.dump(output_data, f, indent=2, default=str)
|
||||
print(f"JSON output written to {output_prefix}.json")
|
||||
|
||||
if args.format == "both":
|
||||
# Generate separate files for different formats
|
||||
|
||||
# OpenAI format
|
||||
openai_schemas = {
|
||||
"functions": [schema.openai_schema for schema in schemas]
|
||||
}
|
||||
with open(f"{output_prefix}_openai.json", 'w') as f:
|
||||
json.dump(openai_schemas, f, indent=2)
|
||||
print(f"OpenAI schemas written to {output_prefix}_openai.json")
|
||||
|
||||
# Anthropic format
|
||||
anthropic_schemas = {
|
||||
"tools": [schema.anthropic_schema for schema in schemas]
|
||||
}
|
||||
with open(f"{output_prefix}_anthropic.json", 'w') as f:
|
||||
json.dump(anthropic_schemas, f, indent=2)
|
||||
print(f"Anthropic schemas written to {output_prefix}_anthropic.json")
|
||||
|
||||
# Validation rules
|
||||
validation_data = {
|
||||
"validation_rules": {schema.name: schema.validation_rules for schema in schemas}
|
||||
}
|
||||
with open(f"{output_prefix}_validation.json", 'w') as f:
|
||||
json.dump(validation_data, f, indent=2)
|
||||
print(f"Validation rules written to {output_prefix}_validation.json")
|
||||
|
||||
# Usage examples
|
||||
examples_data = {
|
||||
"examples": {schema.name: schema.examples for schema in schemas}
|
||||
}
|
||||
with open(f"{output_prefix}_examples.json", 'w') as f:
|
||||
json.dump(examples_data, f, indent=2)
|
||||
print(f"Usage examples written to {output_prefix}_examples.json")
|
||||
|
||||
# Print summary
|
||||
print(f"\nSchema Generation Summary:")
|
||||
print(f"Tools processed: {len(schemas)}")
|
||||
print(f"Total input parameters: {sum(schema.metadata['input_parameters'] for schema in schemas)}")
|
||||
print(f"Total validation rules: {sum(len(schema.validation_rules) for schema in schemas)}")
|
||||
print(f"Total examples generated: {sum(len(schema.examples) for schema in schemas)}")
|
||||
|
||||
# Validation if requested
|
||||
if args.validate:
|
||||
print("\nValidation Results:")
|
||||
for schema in schemas:
|
||||
validation_errors = []
|
||||
|
||||
# Basic validation checks
|
||||
if not schema.openai_schema.get("parameters", {}).get("properties"):
|
||||
validation_errors.append("Missing input parameters")
|
||||
|
||||
if not schema.examples:
|
||||
validation_errors.append("No usage examples")
|
||||
|
||||
if not schema.validation_rules:
|
||||
validation_errors.append("No validation rules defined")
|
||||
|
||||
if validation_errors:
|
||||
print(f" {schema.name}: {', '.join(validation_errors)}")
|
||||
else:
|
||||
print(f" {schema.name}: ✓ Valid")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
388
engineering/database-designer/README.md
Normal file
388
engineering/database-designer/README.md
Normal file
@@ -0,0 +1,388 @@
|
||||
# Database Designer - POWERFUL Tier Skill
|
||||
|
||||
A comprehensive database design and analysis toolkit that provides expert-level schema analysis, index optimization, and migration generation capabilities for modern database systems.
|
||||
|
||||
## Features
|
||||
|
||||
### 🔍 Schema Analyzer
|
||||
- **Normalization Analysis**: Automated detection of 1NF through BCNF violations
|
||||
- **Data Type Optimization**: Identifies antipatterns and inappropriate types
|
||||
- **Constraint Analysis**: Finds missing foreign keys, unique constraints, and checks
|
||||
- **ERD Generation**: Creates Mermaid diagrams from DDL or JSON schema
|
||||
- **Naming Convention Validation**: Ensures consistent naming patterns
|
||||
|
||||
### ⚡ Index Optimizer
|
||||
- **Missing Index Detection**: Identifies indexes needed for query patterns
|
||||
- **Composite Index Design**: Optimizes column ordering for maximum efficiency
|
||||
- **Redundancy Analysis**: Finds duplicate and overlapping indexes
|
||||
- **Performance Modeling**: Estimates selectivity and query performance impact
|
||||
- **Covering Index Recommendations**: Eliminates table lookups
|
||||
|
||||
### 🚀 Migration Generator
|
||||
- **Zero-Downtime Migrations**: Implements expand-contract patterns
|
||||
- **Schema Evolution**: Handles column changes, table renames, constraint updates
|
||||
- **Data Migration Scripts**: Automated data transformation and validation
|
||||
- **Rollback Planning**: Complete reversal capabilities for all changes
|
||||
- **Execution Orchestration**: Dependency-aware migration ordering
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
- Python 3.7+ (no external dependencies required)
|
||||
- Database schema in SQL DDL format or JSON
|
||||
- Query patterns (for index optimization)
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
# Clone or download the database-designer skill
|
||||
cd engineering/database-designer/
|
||||
|
||||
# Make scripts executable
|
||||
chmod +x *.py
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Schema Analysis
|
||||
|
||||
**Analyze SQL DDL file:**
|
||||
```bash
|
||||
python schema_analyzer.py --input assets/sample_schema.sql --output-format text
|
||||
```
|
||||
|
||||
**Generate ERD diagram:**
|
||||
```bash
|
||||
python schema_analyzer.py --input assets/sample_schema.sql --generate-erd --output analysis.txt
|
||||
```
|
||||
|
||||
**JSON schema analysis:**
|
||||
```bash
|
||||
python schema_analyzer.py --input assets/sample_schema.json --output-format json --output results.json
|
||||
```
|
||||
|
||||
### Index Optimization
|
||||
|
||||
**Basic index analysis:**
|
||||
```bash
|
||||
python index_optimizer.py --schema assets/sample_schema.json --queries assets/sample_query_patterns.json
|
||||
```
|
||||
|
||||
**High-priority recommendations only:**
|
||||
```bash
|
||||
python index_optimizer.py --schema assets/sample_schema.json --queries assets/sample_query_patterns.json --min-priority 2
|
||||
```
|
||||
|
||||
**JSON output with existing index analysis:**
|
||||
```bash
|
||||
python index_optimizer.py --schema assets/sample_schema.json --queries assets/sample_query_patterns.json --format json --analyze-existing
|
||||
```
|
||||
|
||||
### Migration Generation
|
||||
|
||||
**Generate migration between schemas:**
|
||||
```bash
|
||||
python migration_generator.py --current assets/current_schema.json --target assets/target_schema.json
|
||||
```
|
||||
|
||||
**Zero-downtime migration:**
|
||||
```bash
|
||||
python migration_generator.py --current current.json --target target.json --zero-downtime --format sql
|
||||
```
|
||||
|
||||
**Include validation queries:**
|
||||
```bash
|
||||
python migration_generator.py --current current.json --target target.json --include-validations --output migration_plan.txt
|
||||
```
|
||||
|
||||
## Tool Documentation
|
||||
|
||||
### Schema Analyzer
|
||||
|
||||
**Input Formats:**
|
||||
- SQL DDL files (.sql)
|
||||
- JSON schema definitions (.json)
|
||||
|
||||
**Key Capabilities:**
|
||||
- Detects 1NF violations (non-atomic values, repeating groups)
|
||||
- Identifies 2NF issues (partial dependencies in composite keys)
|
||||
- Finds 3NF problems (transitive dependencies)
|
||||
- Checks BCNF compliance (determinant key requirements)
|
||||
- Validates data types (VARCHAR(255) antipattern, inappropriate types)
|
||||
- Missing constraints (NOT NULL, UNIQUE, CHECK, foreign keys)
|
||||
- Naming convention adherence
|
||||
|
||||
**Sample Command:**
|
||||
```bash
|
||||
python schema_analyzer.py \
|
||||
--input sample_schema.sql \
|
||||
--generate-erd \
|
||||
--output-format text \
|
||||
--output analysis.txt
|
||||
```
|
||||
|
||||
**Output:**
|
||||
- Comprehensive text or JSON analysis report
|
||||
- Mermaid ERD diagram
|
||||
- Prioritized recommendations
|
||||
- SQL statements for improvements
|
||||
|
||||
### Index Optimizer
|
||||
|
||||
**Input Requirements:**
|
||||
- Schema definition (JSON format)
|
||||
- Query patterns with frequency and selectivity data
|
||||
|
||||
**Analysis Features:**
|
||||
- Selectivity estimation based on column patterns
|
||||
- Composite index column ordering optimization
|
||||
- Covering index recommendations for SELECT queries
|
||||
- Foreign key index validation
|
||||
- Redundancy detection (duplicates, overlaps, unused indexes)
|
||||
- Performance impact modeling
|
||||
|
||||
**Sample Command:**
|
||||
```bash
|
||||
python index_optimizer.py \
|
||||
--schema schema.json \
|
||||
--queries query_patterns.json \
|
||||
--format text \
|
||||
--min-priority 3 \
|
||||
--output recommendations.txt
|
||||
```
|
||||
|
||||
**Output:**
|
||||
- Prioritized index recommendations
|
||||
- CREATE INDEX statements
|
||||
- Drop statements for redundant indexes
|
||||
- Performance impact analysis
|
||||
- Storage size estimates
|
||||
|
||||
### Migration Generator
|
||||
|
||||
**Input Requirements:**
|
||||
- Current schema (JSON format)
|
||||
- Target schema (JSON format)
|
||||
|
||||
**Migration Strategies:**
|
||||
- Standard migrations with ALTER statements
|
||||
- Zero-downtime expand-contract patterns
|
||||
- Data migration and transformation scripts
|
||||
- Constraint management (add/drop in correct order)
|
||||
- Index management with timing estimates
|
||||
|
||||
**Sample Command:**
|
||||
```bash
|
||||
python migration_generator.py \
|
||||
--current current_schema.json \
|
||||
--target target_schema.json \
|
||||
--zero-downtime \
|
||||
--include-validations \
|
||||
--format text
|
||||
```
|
||||
|
||||
**Output:**
|
||||
- Step-by-step migration plan
|
||||
- Forward and rollback SQL statements
|
||||
- Risk assessment for each step
|
||||
- Validation queries
|
||||
- Execution time estimates
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
database-designer/
|
||||
├── README.md # This file
|
||||
├── SKILL.md # Comprehensive database design guide
|
||||
├── schema_analyzer.py # Schema analysis tool
|
||||
├── index_optimizer.py # Index optimization tool
|
||||
├── migration_generator.py # Migration generation tool
|
||||
├── references/ # Reference documentation
|
||||
│ ├── normalization_guide.md # Normalization principles and patterns
|
||||
│ ├── index_strategy_patterns.md # Index design and optimization guide
|
||||
│ └── database_selection_decision_tree.md # Database technology selection
|
||||
├── assets/ # Sample files and test data
|
||||
│ ├── sample_schema.sql # Sample DDL with various issues
|
||||
│ ├── sample_schema.json # JSON schema definition
|
||||
│ └── sample_query_patterns.json # Query patterns for index analysis
|
||||
└── expected_outputs/ # Example tool outputs
|
||||
├── schema_analysis_sample.txt # Sample schema analysis report
|
||||
├── index_optimization_sample.txt # Sample index recommendations
|
||||
└── migration_sample.txt # Sample migration plan
|
||||
```
|
||||
|
||||
## JSON Schema Format
|
||||
|
||||
The tools use a standardized JSON format for schema definitions:
|
||||
|
||||
```json
|
||||
{
|
||||
"tables": {
|
||||
"table_name": {
|
||||
"columns": {
|
||||
"column_name": {
|
||||
"type": "VARCHAR(255)",
|
||||
"nullable": true,
|
||||
"unique": false,
|
||||
"foreign_key": "other_table.column",
|
||||
"default": "default_value",
|
||||
"cardinality_estimate": 1000
|
||||
}
|
||||
},
|
||||
"primary_key": ["id"],
|
||||
"unique_constraints": [["email"], ["username"]],
|
||||
"check_constraints": {
|
||||
"chk_positive_price": "price > 0"
|
||||
},
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_table_column",
|
||||
"columns": ["column_name"],
|
||||
"unique": false,
|
||||
"partial_condition": "status = 'active'"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Query Patterns Format
|
||||
|
||||
For index optimization, provide query patterns in this format:
|
||||
|
||||
```json
|
||||
{
|
||||
"queries": [
|
||||
{
|
||||
"id": "user_lookup",
|
||||
"type": "SELECT",
|
||||
"table": "users",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "email",
|
||||
"operator": "=",
|
||||
"selectivity": 0.95
|
||||
}
|
||||
],
|
||||
"join_conditions": [
|
||||
{
|
||||
"local_column": "user_id",
|
||||
"foreign_table": "orders",
|
||||
"foreign_column": "id",
|
||||
"join_type": "INNER"
|
||||
}
|
||||
],
|
||||
"order_by": [
|
||||
{"column": "created_at", "direction": "DESC"}
|
||||
],
|
||||
"frequency": 1000,
|
||||
"avg_execution_time_ms": 5.2
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Schema Analysis
|
||||
1. **Start with DDL**: Use actual CREATE TABLE statements when possible
|
||||
2. **Include Constraints**: Capture all existing constraints and indexes
|
||||
3. **Consider History**: Some denormalization may be intentional for performance
|
||||
4. **Validate Results**: Review recommendations against business requirements
|
||||
|
||||
### Index Optimization
|
||||
1. **Real Query Patterns**: Use actual application queries, not theoretical ones
|
||||
2. **Include Frequency**: Query frequency is crucial for prioritization
|
||||
3. **Monitor Performance**: Validate recommendations with actual performance testing
|
||||
4. **Gradual Implementation**: Add indexes incrementally and monitor impact
|
||||
|
||||
### Migration Planning
|
||||
1. **Test Migrations**: Always test on non-production environments first
|
||||
2. **Backup First**: Ensure complete backups before running migrations
|
||||
3. **Monitor Progress**: Watch for locks and performance impacts during execution
|
||||
4. **Rollback Ready**: Have rollback procedures tested and ready
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Custom Selectivity Estimation
|
||||
The index optimizer uses pattern-based selectivity estimation. You can improve accuracy by providing cardinality estimates in your schema JSON:
|
||||
|
||||
```json
|
||||
{
|
||||
"columns": {
|
||||
"status": {
|
||||
"type": "VARCHAR(20)",
|
||||
"cardinality_estimate": 5 # Only 5 distinct values
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Zero-Downtime Migration Strategy
|
||||
For production systems, use the zero-downtime flag to generate expand-contract migrations:
|
||||
|
||||
1. **Expand Phase**: Add new columns/tables without constraints
|
||||
2. **Dual Write**: Application writes to both old and new structures
|
||||
3. **Backfill**: Populate new structures with existing data
|
||||
4. **Contract Phase**: Remove old structures after validation
|
||||
|
||||
### Integration with CI/CD
|
||||
Integrate these tools into your deployment pipeline:
|
||||
|
||||
```bash
|
||||
# Schema validation in CI
|
||||
python schema_analyzer.py --input schema.sql --output-format json | \
|
||||
jq '.constraint_analysis.total_issues' | \
|
||||
test $(cat) -eq 0 || exit 1
|
||||
|
||||
# Generate migrations automatically
|
||||
python migration_generator.py \
|
||||
--current prod_schema.json \
|
||||
--target new_schema.json \
|
||||
--zero-downtime \
|
||||
--output migration.sql
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**"No tables found in input file"**
|
||||
- Ensure SQL DDL uses standard CREATE TABLE syntax
|
||||
- Check for syntax errors in DDL
|
||||
- Verify file encoding (UTF-8 recommended)
|
||||
|
||||
**"Invalid JSON schema"**
|
||||
- Validate JSON syntax with a JSON validator
|
||||
- Ensure all required fields are present
|
||||
- Check that foreign key references use "table.column" format
|
||||
|
||||
**"Analysis shows no issues but problems exist"**
|
||||
- Tools use heuristic analysis - review recommendations carefully
|
||||
- Some design decisions may be intentional (denormalization for performance)
|
||||
- Consider domain-specific requirements not captured by general rules
|
||||
|
||||
### Performance Tips
|
||||
|
||||
**Large Schemas:**
|
||||
- Use `--output-format json` for machine processing
|
||||
- Consider analyzing subsets of tables for very large schemas
|
||||
- Provide cardinality estimates for better index recommendations
|
||||
|
||||
**Complex Queries:**
|
||||
- Include actual execution times in query patterns
|
||||
- Provide realistic frequency estimates
|
||||
- Consider seasonal or usage pattern variations
|
||||
|
||||
## Contributing
|
||||
|
||||
This is a self-contained skill with no external dependencies. To extend functionality:
|
||||
|
||||
1. Follow the existing code patterns
|
||||
2. Maintain Python standard library only requirement
|
||||
3. Add comprehensive test cases for new features
|
||||
4. Update documentation and examples
|
||||
|
||||
## License
|
||||
|
||||
This database designer skill is part of the claude-skills collection and follows the same licensing terms.
|
||||
533
engineering/database-designer/SKILL.md
Normal file
533
engineering/database-designer/SKILL.md
Normal file
@@ -0,0 +1,533 @@
|
||||
# Database Designer - POWERFUL Tier Skill
|
||||
|
||||
## Overview
|
||||
|
||||
A comprehensive database design skill that provides expert-level analysis, optimization, and migration capabilities for modern database systems. This skill combines theoretical principles with practical tools to help architects and developers create scalable, performant, and maintainable database schemas.
|
||||
|
||||
## Core Competencies
|
||||
|
||||
### Schema Design & Analysis
|
||||
- **Normalization Analysis**: Automated detection of normalization levels (1NF through BCNF)
|
||||
- **Denormalization Strategy**: Smart recommendations for performance optimization
|
||||
- **Data Type Optimization**: Identification of inappropriate types and size issues
|
||||
- **Constraint Analysis**: Missing foreign keys, unique constraints, and null checks
|
||||
- **Naming Convention Validation**: Consistent table and column naming patterns
|
||||
- **ERD Generation**: Automatic Mermaid diagram creation from DDL
|
||||
|
||||
### Index Optimization
|
||||
- **Index Gap Analysis**: Identification of missing indexes on foreign keys and query patterns
|
||||
- **Composite Index Strategy**: Optimal column ordering for multi-column indexes
|
||||
- **Index Redundancy Detection**: Elimination of overlapping and unused indexes
|
||||
- **Performance Impact Modeling**: Selectivity estimation and query cost analysis
|
||||
- **Index Type Selection**: B-tree, hash, partial, covering, and specialized indexes
|
||||
|
||||
### Migration Management
|
||||
- **Zero-Downtime Migrations**: Expand-contract pattern implementation
|
||||
- **Schema Evolution**: Safe column additions, deletions, and type changes
|
||||
- **Data Migration Scripts**: Automated data transformation and validation
|
||||
- **Rollback Strategy**: Complete reversal capabilities with validation
|
||||
- **Execution Planning**: Ordered migration steps with dependency resolution
|
||||
|
||||
## Database Design Principles
|
||||
|
||||
### Normalization Forms
|
||||
|
||||
#### First Normal Form (1NF)
|
||||
- **Atomic Values**: Each column contains indivisible values
|
||||
- **Unique Column Names**: No duplicate column names within a table
|
||||
- **Uniform Data Types**: Each column contains the same type of data
|
||||
- **Row Uniqueness**: No duplicate rows in the table
|
||||
|
||||
**Example Violation:**
|
||||
```sql
|
||||
-- BAD: Multiple phone numbers in one column
|
||||
CREATE TABLE contacts (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
phones VARCHAR(200) -- "123-456-7890, 098-765-4321"
|
||||
);
|
||||
|
||||
-- GOOD: Separate table for phone numbers
|
||||
CREATE TABLE contacts (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100)
|
||||
);
|
||||
|
||||
CREATE TABLE contact_phones (
|
||||
id INT PRIMARY KEY,
|
||||
contact_id INT REFERENCES contacts(id),
|
||||
phone_number VARCHAR(20),
|
||||
phone_type VARCHAR(10)
|
||||
);
|
||||
```
|
||||
|
||||
#### Second Normal Form (2NF)
|
||||
- **1NF Compliance**: Must satisfy First Normal Form
|
||||
- **Full Functional Dependency**: Non-key attributes depend on the entire primary key
|
||||
- **Partial Dependency Elimination**: Remove attributes that depend on part of a composite key
|
||||
|
||||
**Example Violation:**
|
||||
```sql
|
||||
-- BAD: Student course table with partial dependencies
|
||||
CREATE TABLE student_courses (
|
||||
student_id INT,
|
||||
course_id INT,
|
||||
student_name VARCHAR(100), -- Depends only on student_id
|
||||
course_name VARCHAR(100), -- Depends only on course_id
|
||||
grade CHAR(1),
|
||||
PRIMARY KEY (student_id, course_id)
|
||||
);
|
||||
|
||||
-- GOOD: Separate tables eliminate partial dependencies
|
||||
CREATE TABLE students (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100)
|
||||
);
|
||||
|
||||
CREATE TABLE courses (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100)
|
||||
);
|
||||
|
||||
CREATE TABLE enrollments (
|
||||
student_id INT REFERENCES students(id),
|
||||
course_id INT REFERENCES courses(id),
|
||||
grade CHAR(1),
|
||||
PRIMARY KEY (student_id, course_id)
|
||||
);
|
||||
```
|
||||
|
||||
#### Third Normal Form (3NF)
|
||||
- **2NF Compliance**: Must satisfy Second Normal Form
|
||||
- **Transitive Dependency Elimination**: Non-key attributes should not depend on other non-key attributes
|
||||
- **Direct Dependency**: Non-key attributes depend directly on the primary key
|
||||
|
||||
**Example Violation:**
|
||||
```sql
|
||||
-- BAD: Employee table with transitive dependency
|
||||
CREATE TABLE employees (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
department_id INT,
|
||||
department_name VARCHAR(100), -- Depends on department_id, not employee id
|
||||
department_budget DECIMAL(10,2) -- Transitive dependency
|
||||
);
|
||||
|
||||
-- GOOD: Separate department information
|
||||
CREATE TABLE departments (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
budget DECIMAL(10,2)
|
||||
);
|
||||
|
||||
CREATE TABLE employees (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
department_id INT REFERENCES departments(id)
|
||||
);
|
||||
```
|
||||
|
||||
#### Boyce-Codd Normal Form (BCNF)
|
||||
- **3NF Compliance**: Must satisfy Third Normal Form
|
||||
- **Determinant Key Rule**: Every determinant must be a candidate key
|
||||
- **Stricter 3NF**: Handles anomalies not covered by 3NF
|
||||
|
||||
### Denormalization Strategies
|
||||
|
||||
#### When to Denormalize
|
||||
1. **Read-Heavy Workloads**: High query frequency with acceptable write trade-offs
|
||||
2. **Performance Bottlenecks**: Join operations causing significant latency
|
||||
3. **Aggregation Needs**: Frequent calculation of derived values
|
||||
4. **Caching Requirements**: Pre-computed results for common queries
|
||||
|
||||
#### Common Denormalization Patterns
|
||||
|
||||
**Redundant Storage**
|
||||
```sql
|
||||
-- Store calculated values to avoid expensive joins
|
||||
CREATE TABLE orders (
|
||||
id INT PRIMARY KEY,
|
||||
customer_id INT REFERENCES customers(id),
|
||||
customer_name VARCHAR(100), -- Denormalized from customers table
|
||||
order_total DECIMAL(10,2), -- Denormalized calculation
|
||||
created_at TIMESTAMP
|
||||
);
|
||||
```
|
||||
|
||||
**Materialized Aggregates**
|
||||
```sql
|
||||
-- Pre-computed summary tables
|
||||
CREATE TABLE customer_statistics (
|
||||
customer_id INT PRIMARY KEY,
|
||||
total_orders INT,
|
||||
lifetime_value DECIMAL(12,2),
|
||||
last_order_date DATE,
|
||||
updated_at TIMESTAMP
|
||||
);
|
||||
```
|
||||
|
||||
## Index Optimization Strategies
|
||||
|
||||
### B-Tree Indexes
|
||||
- **Default Choice**: Best for range queries, sorting, and equality matches
|
||||
- **Column Order**: Most selective columns first for composite indexes
|
||||
- **Prefix Matching**: Supports leading column subset queries
|
||||
- **Maintenance Cost**: Balanced tree structure with logarithmic operations
|
||||
|
||||
### Hash Indexes
|
||||
- **Equality Queries**: Optimal for exact match lookups
|
||||
- **Memory Efficiency**: Constant-time access for single-value queries
|
||||
- **Range Limitations**: Cannot support range or partial matches
|
||||
- **Use Cases**: Primary keys, unique constraints, cache keys
|
||||
|
||||
### Composite Indexes
|
||||
```sql
|
||||
-- Query pattern determines optimal column order
|
||||
-- Query: WHERE status = 'active' AND created_date > '2023-01-01' ORDER BY priority DESC
|
||||
CREATE INDEX idx_task_status_date_priority
|
||||
ON tasks (status, created_date, priority DESC);
|
||||
|
||||
-- Query: WHERE user_id = 123 AND category IN ('A', 'B') AND date_field BETWEEN '...' AND '...'
|
||||
CREATE INDEX idx_user_category_date
|
||||
ON user_activities (user_id, category, date_field);
|
||||
```
|
||||
|
||||
### Covering Indexes
|
||||
```sql
|
||||
-- Include additional columns to avoid table lookups
|
||||
CREATE INDEX idx_user_email_covering
|
||||
ON users (email)
|
||||
INCLUDE (first_name, last_name, status);
|
||||
|
||||
-- Query can be satisfied entirely from the index
|
||||
-- SELECT first_name, last_name, status FROM users WHERE email = 'user@example.com';
|
||||
```
|
||||
|
||||
### Partial Indexes
|
||||
```sql
|
||||
-- Index only relevant subset of data
|
||||
CREATE INDEX idx_active_users_email
|
||||
ON users (email)
|
||||
WHERE status = 'active';
|
||||
|
||||
-- Index for recent orders only
|
||||
CREATE INDEX idx_recent_orders_customer
|
||||
ON orders (customer_id, created_at)
|
||||
WHERE created_at > CURRENT_DATE - INTERVAL '30 days';
|
||||
```
|
||||
|
||||
## Query Analysis & Optimization
|
||||
|
||||
### Query Patterns Recognition
|
||||
1. **Equality Filters**: Single-column B-tree indexes
|
||||
2. **Range Queries**: B-tree with proper column ordering
|
||||
3. **Text Search**: Full-text indexes or trigram indexes
|
||||
4. **Join Operations**: Foreign key indexes on both sides
|
||||
5. **Sorting Requirements**: Indexes matching ORDER BY clauses
|
||||
|
||||
### Index Selection Algorithm
|
||||
```
|
||||
1. Identify WHERE clause columns
|
||||
2. Determine most selective columns first
|
||||
3. Consider JOIN conditions
|
||||
4. Include ORDER BY columns if possible
|
||||
5. Evaluate covering index opportunities
|
||||
6. Check for existing overlapping indexes
|
||||
```
|
||||
|
||||
## Data Modeling Patterns
|
||||
|
||||
### Star Schema (Data Warehousing)
|
||||
```sql
|
||||
-- Central fact table
|
||||
CREATE TABLE sales_facts (
|
||||
sale_id BIGINT PRIMARY KEY,
|
||||
product_id INT REFERENCES products(id),
|
||||
customer_id INT REFERENCES customers(id),
|
||||
date_id INT REFERENCES date_dimension(id),
|
||||
store_id INT REFERENCES stores(id),
|
||||
quantity INT,
|
||||
unit_price DECIMAL(8,2),
|
||||
total_amount DECIMAL(10,2)
|
||||
);
|
||||
|
||||
-- Dimension tables
|
||||
CREATE TABLE date_dimension (
|
||||
id INT PRIMARY KEY,
|
||||
date_value DATE,
|
||||
year INT,
|
||||
quarter INT,
|
||||
month INT,
|
||||
day_of_week INT,
|
||||
is_weekend BOOLEAN
|
||||
);
|
||||
```
|
||||
|
||||
### Snowflake Schema
|
||||
```sql
|
||||
-- Normalized dimension tables
|
||||
CREATE TABLE products (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(200),
|
||||
category_id INT REFERENCES product_categories(id),
|
||||
brand_id INT REFERENCES brands(id)
|
||||
);
|
||||
|
||||
CREATE TABLE product_categories (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
parent_category_id INT REFERENCES product_categories(id)
|
||||
);
|
||||
```
|
||||
|
||||
### Document Model (JSON Storage)
|
||||
```sql
|
||||
-- Flexible document storage with indexing
|
||||
CREATE TABLE documents (
|
||||
id UUID PRIMARY KEY,
|
||||
document_type VARCHAR(50),
|
||||
data JSONB,
|
||||
created_at TIMESTAMP DEFAULT NOW(),
|
||||
updated_at TIMESTAMP DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Index on JSON properties
|
||||
CREATE INDEX idx_documents_user_id
|
||||
ON documents USING GIN ((data->>'user_id'));
|
||||
|
||||
CREATE INDEX idx_documents_status
|
||||
ON documents ((data->>'status'))
|
||||
WHERE document_type = 'order';
|
||||
```
|
||||
|
||||
### Graph Data Patterns
|
||||
```sql
|
||||
-- Adjacency list for hierarchical data
|
||||
CREATE TABLE categories (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
parent_id INT REFERENCES categories(id),
|
||||
level INT,
|
||||
path VARCHAR(500) -- Materialized path: "/1/5/12/"
|
||||
);
|
||||
|
||||
-- Many-to-many relationships
|
||||
CREATE TABLE relationships (
|
||||
id UUID PRIMARY KEY,
|
||||
from_entity_id UUID,
|
||||
to_entity_id UUID,
|
||||
relationship_type VARCHAR(50),
|
||||
created_at TIMESTAMP,
|
||||
INDEX (from_entity_id, relationship_type),
|
||||
INDEX (to_entity_id, relationship_type)
|
||||
);
|
||||
```
|
||||
|
||||
## Migration Strategies
|
||||
|
||||
### Zero-Downtime Migration (Expand-Contract Pattern)
|
||||
|
||||
**Phase 1: Expand**
|
||||
```sql
|
||||
-- Add new column without constraints
|
||||
ALTER TABLE users ADD COLUMN new_email VARCHAR(255);
|
||||
|
||||
-- Backfill data in batches
|
||||
UPDATE users SET new_email = email WHERE id BETWEEN 1 AND 1000;
|
||||
-- Continue in batches...
|
||||
|
||||
-- Add constraints after backfill
|
||||
ALTER TABLE users ADD CONSTRAINT users_new_email_unique UNIQUE (new_email);
|
||||
ALTER TABLE users ALTER COLUMN new_email SET NOT NULL;
|
||||
```
|
||||
|
||||
**Phase 2: Contract**
|
||||
```sql
|
||||
-- Update application to use new column
|
||||
-- Deploy application changes
|
||||
-- Verify new column is being used
|
||||
|
||||
-- Remove old column
|
||||
ALTER TABLE users DROP COLUMN email;
|
||||
-- Rename new column
|
||||
ALTER TABLE users RENAME COLUMN new_email TO email;
|
||||
```
|
||||
|
||||
### Data Type Changes
|
||||
```sql
|
||||
-- Safe string to integer conversion
|
||||
ALTER TABLE products ADD COLUMN sku_number INTEGER;
|
||||
UPDATE products SET sku_number = CAST(sku AS INTEGER) WHERE sku ~ '^[0-9]+$';
|
||||
-- Validate conversion success before dropping old column
|
||||
```
|
||||
|
||||
## Partitioning Strategies
|
||||
|
||||
### Horizontal Partitioning (Sharding)
|
||||
```sql
|
||||
-- Range partitioning by date
|
||||
CREATE TABLE sales_2023 PARTITION OF sales
|
||||
FOR VALUES FROM ('2023-01-01') TO ('2024-01-01');
|
||||
|
||||
CREATE TABLE sales_2024 PARTITION OF sales
|
||||
FOR VALUES FROM ('2024-01-01') TO ('2025-01-01');
|
||||
|
||||
-- Hash partitioning by user_id
|
||||
CREATE TABLE user_data_0 PARTITION OF user_data
|
||||
FOR VALUES WITH (MODULUS 4, REMAINDER 0);
|
||||
|
||||
CREATE TABLE user_data_1 PARTITION OF user_data
|
||||
FOR VALUES WITH (MODULUS 4, REMAINDER 1);
|
||||
```
|
||||
|
||||
### Vertical Partitioning
|
||||
```sql
|
||||
-- Separate frequently accessed columns
|
||||
CREATE TABLE users_core (
|
||||
id INT PRIMARY KEY,
|
||||
email VARCHAR(255),
|
||||
status VARCHAR(20),
|
||||
created_at TIMESTAMP
|
||||
);
|
||||
|
||||
-- Less frequently accessed profile data
|
||||
CREATE TABLE users_profile (
|
||||
user_id INT PRIMARY KEY REFERENCES users_core(id),
|
||||
bio TEXT,
|
||||
preferences JSONB,
|
||||
last_login TIMESTAMP
|
||||
);
|
||||
```
|
||||
|
||||
## Connection Management
|
||||
|
||||
### Connection Pooling
|
||||
- **Pool Size**: CPU cores × 2 + effective spindle count
|
||||
- **Connection Lifetime**: Rotate connections to prevent resource leaks
|
||||
- **Timeout Settings**: Connection, idle, and query timeouts
|
||||
- **Health Checks**: Regular connection validation
|
||||
|
||||
### Read Replicas Strategy
|
||||
```sql
|
||||
-- Write queries to primary
|
||||
INSERT INTO users (email, name) VALUES ('user@example.com', 'John Doe');
|
||||
|
||||
-- Read queries to replicas (with appropriate read preference)
|
||||
SELECT * FROM users WHERE status = 'active'; -- Route to read replica
|
||||
|
||||
-- Consistent reads when required
|
||||
SELECT * FROM users WHERE id = LAST_INSERT_ID(); -- Route to primary
|
||||
```
|
||||
|
||||
## Caching Layers
|
||||
|
||||
### Cache-Aside Pattern
|
||||
```python
|
||||
def get_user(user_id):
|
||||
# Try cache first
|
||||
user = cache.get(f"user:{user_id}")
|
||||
if user is None:
|
||||
# Cache miss - query database
|
||||
user = db.query("SELECT * FROM users WHERE id = %s", user_id)
|
||||
# Store in cache
|
||||
cache.set(f"user:{user_id}", user, ttl=3600)
|
||||
return user
|
||||
```
|
||||
|
||||
### Write-Through Cache
|
||||
- **Consistency**: Always keep cache and database in sync
|
||||
- **Write Latency**: Higher due to dual writes
|
||||
- **Data Safety**: No data loss on cache failures
|
||||
|
||||
### Cache Invalidation Strategies
|
||||
1. **TTL-Based**: Time-based expiration
|
||||
2. **Event-Driven**: Invalidate on data changes
|
||||
3. **Version-Based**: Use version numbers for consistency
|
||||
4. **Tag-Based**: Group related cache entries
|
||||
|
||||
## Database Selection Guide
|
||||
|
||||
### SQL Databases
|
||||
**PostgreSQL**
|
||||
- **Strengths**: ACID compliance, complex queries, JSON support, extensibility
|
||||
- **Use Cases**: OLTP applications, data warehousing, geospatial data
|
||||
- **Scale**: Vertical scaling with read replicas
|
||||
|
||||
**MySQL**
|
||||
- **Strengths**: Performance, replication, wide ecosystem support
|
||||
- **Use Cases**: Web applications, content management, e-commerce
|
||||
- **Scale**: Horizontal scaling through sharding
|
||||
|
||||
### NoSQL Databases
|
||||
|
||||
**Document Stores (MongoDB, CouchDB)**
|
||||
- **Strengths**: Flexible schema, horizontal scaling, developer productivity
|
||||
- **Use Cases**: Content management, catalogs, user profiles
|
||||
- **Trade-offs**: Eventual consistency, complex queries limitations
|
||||
|
||||
**Key-Value Stores (Redis, DynamoDB)**
|
||||
- **Strengths**: High performance, simple model, excellent caching
|
||||
- **Use Cases**: Session storage, real-time analytics, gaming leaderboards
|
||||
- **Trade-offs**: Limited query capabilities, data modeling constraints
|
||||
|
||||
**Column-Family (Cassandra, HBase)**
|
||||
- **Strengths**: Write-heavy workloads, linear scalability, fault tolerance
|
||||
- **Use Cases**: Time-series data, IoT applications, messaging systems
|
||||
- **Trade-offs**: Query flexibility, consistency model complexity
|
||||
|
||||
**Graph Databases (Neo4j, Amazon Neptune)**
|
||||
- **Strengths**: Relationship queries, pattern matching, recommendation engines
|
||||
- **Use Cases**: Social networks, fraud detection, knowledge graphs
|
||||
- **Trade-offs**: Specialized use cases, learning curve
|
||||
|
||||
### NewSQL Databases
|
||||
**Distributed SQL (CockroachDB, TiDB, Spanner)**
|
||||
- **Strengths**: SQL compatibility with horizontal scaling
|
||||
- **Use Cases**: Global applications requiring ACID guarantees
|
||||
- **Trade-offs**: Complexity, latency for distributed transactions
|
||||
|
||||
## Tools & Scripts
|
||||
|
||||
### Schema Analyzer
|
||||
- **Input**: SQL DDL files, JSON schema definitions
|
||||
- **Analysis**: Normalization compliance, constraint validation, naming conventions
|
||||
- **Output**: Analysis report, Mermaid ERD, improvement recommendations
|
||||
|
||||
### Index Optimizer
|
||||
- **Input**: Schema definition, query patterns
|
||||
- **Analysis**: Missing indexes, redundancy detection, selectivity estimation
|
||||
- **Output**: Index recommendations, CREATE INDEX statements, performance projections
|
||||
|
||||
### Migration Generator
|
||||
- **Input**: Current and target schemas
|
||||
- **Analysis**: Schema differences, dependency resolution, risk assessment
|
||||
- **Output**: Migration scripts, rollback plans, validation queries
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Schema Design
|
||||
1. **Use meaningful names**: Clear, consistent naming conventions
|
||||
2. **Choose appropriate data types**: Right-sized columns for storage efficiency
|
||||
3. **Define proper constraints**: Foreign keys, check constraints, unique indexes
|
||||
4. **Consider future growth**: Plan for scale from the beginning
|
||||
5. **Document relationships**: Clear foreign key relationships and business rules
|
||||
|
||||
### Performance Optimization
|
||||
1. **Index strategically**: Cover common query patterns without over-indexing
|
||||
2. **Monitor query performance**: Regular analysis of slow queries
|
||||
3. **Partition large tables**: Improve query performance and maintenance
|
||||
4. **Use appropriate isolation levels**: Balance consistency with performance
|
||||
5. **Implement connection pooling**: Efficient resource utilization
|
||||
|
||||
### Security Considerations
|
||||
1. **Principle of least privilege**: Grant minimal necessary permissions
|
||||
2. **Encrypt sensitive data**: At rest and in transit
|
||||
3. **Audit access patterns**: Monitor and log database access
|
||||
4. **Validate inputs**: Prevent SQL injection attacks
|
||||
5. **Regular security updates**: Keep database software current
|
||||
|
||||
## Conclusion
|
||||
|
||||
Effective database design requires balancing multiple competing concerns: performance, scalability, maintainability, and business requirements. This skill provides the tools and knowledge to make informed decisions throughout the database lifecycle, from initial schema design through production optimization and evolution.
|
||||
|
||||
The included tools automate common analysis and optimization tasks, while the comprehensive guides provide the theoretical foundation for making sound architectural decisions. Whether building a new system or optimizing an existing one, these resources provide expert-level guidance for creating robust, scalable database solutions.
|
||||
375
engineering/database-designer/assets/sample_query_patterns.json
Normal file
375
engineering/database-designer/assets/sample_query_patterns.json
Normal file
@@ -0,0 +1,375 @@
|
||||
{
|
||||
"queries": [
|
||||
{
|
||||
"id": "user_login",
|
||||
"type": "SELECT",
|
||||
"table": "users",
|
||||
"description": "User authentication lookup by email",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "email",
|
||||
"operator": "=",
|
||||
"selectivity": 0.95
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [],
|
||||
"group_by": [],
|
||||
"frequency": 5000,
|
||||
"avg_execution_time_ms": 2.5
|
||||
},
|
||||
{
|
||||
"id": "product_search_category",
|
||||
"type": "SELECT",
|
||||
"table": "products",
|
||||
"description": "Product search within category with pagination",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "category_id",
|
||||
"operator": "=",
|
||||
"selectivity": 0.2
|
||||
},
|
||||
{
|
||||
"column": "is_active",
|
||||
"operator": "=",
|
||||
"selectivity": 0.1
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "created_at", "direction": "DESC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 2500,
|
||||
"avg_execution_time_ms": 15.2
|
||||
},
|
||||
{
|
||||
"id": "product_search_price_range",
|
||||
"type": "SELECT",
|
||||
"table": "products",
|
||||
"description": "Product search by price range and brand",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "price",
|
||||
"operator": "BETWEEN",
|
||||
"selectivity": 0.3
|
||||
},
|
||||
{
|
||||
"column": "brand",
|
||||
"operator": "=",
|
||||
"selectivity": 0.05
|
||||
},
|
||||
{
|
||||
"column": "is_active",
|
||||
"operator": "=",
|
||||
"selectivity": 0.1
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "price", "direction": "ASC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 800,
|
||||
"avg_execution_time_ms": 25.7
|
||||
},
|
||||
{
|
||||
"id": "user_orders_history",
|
||||
"type": "SELECT",
|
||||
"table": "orders",
|
||||
"description": "User order history with pagination",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "user_id",
|
||||
"operator": "=",
|
||||
"selectivity": 0.8
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "created_at", "direction": "DESC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 1200,
|
||||
"avg_execution_time_ms": 8.3
|
||||
},
|
||||
{
|
||||
"id": "order_details_with_items",
|
||||
"type": "SELECT",
|
||||
"table": "orders",
|
||||
"description": "Order details with order items (JOIN query)",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "id",
|
||||
"operator": "=",
|
||||
"selectivity": 1.0
|
||||
}
|
||||
],
|
||||
"join_conditions": [
|
||||
{
|
||||
"local_column": "id",
|
||||
"foreign_table": "order_items",
|
||||
"foreign_column": "order_id",
|
||||
"join_type": "INNER"
|
||||
}
|
||||
],
|
||||
"order_by": [],
|
||||
"group_by": [],
|
||||
"frequency": 3000,
|
||||
"avg_execution_time_ms": 12.1
|
||||
},
|
||||
{
|
||||
"id": "pending_orders_processing",
|
||||
"type": "SELECT",
|
||||
"table": "orders",
|
||||
"description": "Processing queue - pending orders by date",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "status",
|
||||
"operator": "=",
|
||||
"selectivity": 0.15
|
||||
},
|
||||
{
|
||||
"column": "created_at",
|
||||
"operator": ">=",
|
||||
"selectivity": 0.3
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "created_at", "direction": "ASC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 150,
|
||||
"avg_execution_time_ms": 45.2
|
||||
},
|
||||
{
|
||||
"id": "user_orders_by_status",
|
||||
"type": "SELECT",
|
||||
"table": "orders",
|
||||
"description": "User orders filtered by status",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "user_id",
|
||||
"operator": "=",
|
||||
"selectivity": 0.8
|
||||
},
|
||||
{
|
||||
"column": "status",
|
||||
"operator": "IN",
|
||||
"selectivity": 0.4
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "created_at", "direction": "DESC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 600,
|
||||
"avg_execution_time_ms": 18.5
|
||||
},
|
||||
{
|
||||
"id": "product_reviews_summary",
|
||||
"type": "SELECT",
|
||||
"table": "product_reviews",
|
||||
"description": "Product review aggregation",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "product_id",
|
||||
"operator": "=",
|
||||
"selectivity": 0.85
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [],
|
||||
"group_by": ["product_id"],
|
||||
"frequency": 1800,
|
||||
"avg_execution_time_ms": 22.3
|
||||
},
|
||||
{
|
||||
"id": "inventory_low_stock",
|
||||
"type": "SELECT",
|
||||
"table": "products",
|
||||
"description": "Low inventory alert query",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "inventory_count",
|
||||
"operator": "<=",
|
||||
"selectivity": 0.1
|
||||
},
|
||||
{
|
||||
"column": "is_active",
|
||||
"operator": "=",
|
||||
"selectivity": 0.1
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "inventory_count", "direction": "ASC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 50,
|
||||
"avg_execution_time_ms": 35.8
|
||||
},
|
||||
{
|
||||
"id": "popular_products_by_category",
|
||||
"type": "SELECT",
|
||||
"table": "order_items",
|
||||
"description": "Popular products analysis with category join",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "created_at",
|
||||
"operator": ">=",
|
||||
"selectivity": 0.2
|
||||
}
|
||||
],
|
||||
"join_conditions": [
|
||||
{
|
||||
"local_column": "product_id",
|
||||
"foreign_table": "products",
|
||||
"foreign_column": "id",
|
||||
"join_type": "INNER"
|
||||
},
|
||||
{
|
||||
"local_column": "category_id",
|
||||
"foreign_table": "categories",
|
||||
"foreign_column": "id",
|
||||
"join_type": "INNER"
|
||||
}
|
||||
],
|
||||
"order_by": [
|
||||
{"column": "total_quantity", "direction": "DESC"}
|
||||
],
|
||||
"group_by": ["product_id", "category_id"],
|
||||
"frequency": 25,
|
||||
"avg_execution_time_ms": 180.5
|
||||
},
|
||||
{
|
||||
"id": "customer_purchase_history",
|
||||
"type": "SELECT",
|
||||
"table": "orders",
|
||||
"description": "Customer analytics - purchase history with items",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "user_id",
|
||||
"operator": "=",
|
||||
"selectivity": 0.8
|
||||
},
|
||||
{
|
||||
"column": "status",
|
||||
"operator": "IN",
|
||||
"selectivity": 0.6
|
||||
}
|
||||
],
|
||||
"join_conditions": [
|
||||
{
|
||||
"local_column": "id",
|
||||
"foreign_table": "order_items",
|
||||
"foreign_column": "order_id",
|
||||
"join_type": "INNER"
|
||||
}
|
||||
],
|
||||
"order_by": [
|
||||
{"column": "created_at", "direction": "DESC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 300,
|
||||
"avg_execution_time_ms": 65.2
|
||||
},
|
||||
{
|
||||
"id": "daily_sales_report",
|
||||
"type": "SELECT",
|
||||
"table": "orders",
|
||||
"description": "Daily sales aggregation report",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "created_at",
|
||||
"operator": ">=",
|
||||
"selectivity": 0.05
|
||||
},
|
||||
{
|
||||
"column": "status",
|
||||
"operator": "IN",
|
||||
"selectivity": 0.6
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "order_date", "direction": "DESC"}
|
||||
],
|
||||
"group_by": ["DATE(created_at)"],
|
||||
"frequency": 10,
|
||||
"avg_execution_time_ms": 250.8
|
||||
},
|
||||
{
|
||||
"id": "category_hierarchy_nav",
|
||||
"type": "SELECT",
|
||||
"table": "categories",
|
||||
"description": "Category navigation - parent-child relationships",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "parent_id",
|
||||
"operator": "=",
|
||||
"selectivity": 0.2
|
||||
},
|
||||
{
|
||||
"column": "is_active",
|
||||
"operator": "=",
|
||||
"selectivity": 0.1
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [
|
||||
{"column": "sort_order", "direction": "ASC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 800,
|
||||
"avg_execution_time_ms": 5.1
|
||||
},
|
||||
{
|
||||
"id": "recent_user_reviews",
|
||||
"type": "SELECT",
|
||||
"table": "product_reviews",
|
||||
"description": "Recent product reviews by user",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "user_id",
|
||||
"operator": "=",
|
||||
"selectivity": 0.95
|
||||
}
|
||||
],
|
||||
"join_conditions": [
|
||||
{
|
||||
"local_column": "product_id",
|
||||
"foreign_table": "products",
|
||||
"foreign_column": "id",
|
||||
"join_type": "INNER"
|
||||
}
|
||||
],
|
||||
"order_by": [
|
||||
{"column": "created_at", "direction": "DESC"}
|
||||
],
|
||||
"group_by": [],
|
||||
"frequency": 200,
|
||||
"avg_execution_time_ms": 12.7
|
||||
},
|
||||
{
|
||||
"id": "product_avg_rating",
|
||||
"type": "SELECT",
|
||||
"table": "product_reviews",
|
||||
"description": "Product average rating calculation",
|
||||
"where_conditions": [
|
||||
{
|
||||
"column": "product_id",
|
||||
"operator": "IN",
|
||||
"selectivity": 0.1
|
||||
}
|
||||
],
|
||||
"join_conditions": [],
|
||||
"order_by": [],
|
||||
"group_by": ["product_id"],
|
||||
"frequency": 400,
|
||||
"avg_execution_time_ms": 35.4
|
||||
}
|
||||
]
|
||||
}
|
||||
372
engineering/database-designer/assets/sample_schema.json
Normal file
372
engineering/database-designer/assets/sample_schema.json
Normal file
@@ -0,0 +1,372 @@
|
||||
{
|
||||
"tables": {
|
||||
"users": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 50000
|
||||
},
|
||||
"email": {
|
||||
"type": "VARCHAR(255)",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 50000
|
||||
},
|
||||
"username": {
|
||||
"type": "VARCHAR(50)",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 50000
|
||||
},
|
||||
"password_hash": {
|
||||
"type": "VARCHAR(255)",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 50000
|
||||
},
|
||||
"first_name": {
|
||||
"type": "VARCHAR(100)",
|
||||
"nullable": true,
|
||||
"cardinality_estimate": 25000
|
||||
},
|
||||
"last_name": {
|
||||
"type": "VARCHAR(100)",
|
||||
"nullable": true,
|
||||
"cardinality_estimate": 30000
|
||||
},
|
||||
"status": {
|
||||
"type": "VARCHAR(20)",
|
||||
"nullable": false,
|
||||
"default": "active",
|
||||
"cardinality_estimate": 5
|
||||
},
|
||||
"created_at": {
|
||||
"type": "TIMESTAMP",
|
||||
"nullable": false,
|
||||
"default": "CURRENT_TIMESTAMP"
|
||||
}
|
||||
},
|
||||
"primary_key": ["id"],
|
||||
"unique_constraints": [
|
||||
["email"],
|
||||
["username"]
|
||||
],
|
||||
"check_constraints": {
|
||||
"chk_status_valid": "status IN ('active', 'inactive', 'suspended', 'deleted')"
|
||||
},
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_users_email",
|
||||
"columns": ["email"],
|
||||
"unique": true
|
||||
},
|
||||
{
|
||||
"name": "idx_users_status",
|
||||
"columns": ["status"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"products": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 10000
|
||||
},
|
||||
"name": {
|
||||
"type": "VARCHAR(255)",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 9500
|
||||
},
|
||||
"sku": {
|
||||
"type": "VARCHAR(50)",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 10000
|
||||
},
|
||||
"price": {
|
||||
"type": "DECIMAL(10,2)",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 5000
|
||||
},
|
||||
"category_id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"foreign_key": "categories.id",
|
||||
"cardinality_estimate": 50
|
||||
},
|
||||
"brand": {
|
||||
"type": "VARCHAR(100)",
|
||||
"nullable": true,
|
||||
"cardinality_estimate": 200
|
||||
},
|
||||
"is_active": {
|
||||
"type": "BOOLEAN",
|
||||
"nullable": false,
|
||||
"default": true,
|
||||
"cardinality_estimate": 2
|
||||
},
|
||||
"inventory_count": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"default": 0,
|
||||
"cardinality_estimate": 1000
|
||||
},
|
||||
"created_at": {
|
||||
"type": "TIMESTAMP",
|
||||
"nullable": false,
|
||||
"default": "CURRENT_TIMESTAMP"
|
||||
}
|
||||
},
|
||||
"primary_key": ["id"],
|
||||
"unique_constraints": [
|
||||
["sku"]
|
||||
],
|
||||
"check_constraints": {
|
||||
"chk_price_positive": "price > 0",
|
||||
"chk_inventory_non_negative": "inventory_count >= 0"
|
||||
},
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_products_category",
|
||||
"columns": ["category_id"]
|
||||
},
|
||||
{
|
||||
"name": "idx_products_brand",
|
||||
"columns": ["brand"]
|
||||
},
|
||||
{
|
||||
"name": "idx_products_price",
|
||||
"columns": ["price"]
|
||||
},
|
||||
{
|
||||
"name": "idx_products_active_category",
|
||||
"columns": ["is_active", "category_id"],
|
||||
"partial_condition": "is_active = true"
|
||||
}
|
||||
]
|
||||
},
|
||||
"orders": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 200000
|
||||
},
|
||||
"order_number": {
|
||||
"type": "VARCHAR(50)",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 200000
|
||||
},
|
||||
"user_id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"foreign_key": "users.id",
|
||||
"cardinality_estimate": 40000
|
||||
},
|
||||
"status": {
|
||||
"type": "VARCHAR(50)",
|
||||
"nullable": false,
|
||||
"default": "pending",
|
||||
"cardinality_estimate": 8
|
||||
},
|
||||
"total_amount": {
|
||||
"type": "DECIMAL(10,2)",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 50000
|
||||
},
|
||||
"payment_method": {
|
||||
"type": "VARCHAR(50)",
|
||||
"nullable": true,
|
||||
"cardinality_estimate": 10
|
||||
},
|
||||
"created_at": {
|
||||
"type": "TIMESTAMP",
|
||||
"nullable": false,
|
||||
"default": "CURRENT_TIMESTAMP"
|
||||
},
|
||||
"shipped_at": {
|
||||
"type": "TIMESTAMP",
|
||||
"nullable": true
|
||||
}
|
||||
},
|
||||
"primary_key": ["id"],
|
||||
"unique_constraints": [
|
||||
["order_number"]
|
||||
],
|
||||
"check_constraints": {
|
||||
"chk_total_positive": "total_amount > 0",
|
||||
"chk_status_valid": "status IN ('pending', 'processing', 'shipped', 'delivered', 'cancelled')"
|
||||
},
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_orders_user",
|
||||
"columns": ["user_id"]
|
||||
},
|
||||
{
|
||||
"name": "idx_orders_status",
|
||||
"columns": ["status"]
|
||||
},
|
||||
{
|
||||
"name": "idx_orders_created",
|
||||
"columns": ["created_at"]
|
||||
},
|
||||
{
|
||||
"name": "idx_orders_user_status",
|
||||
"columns": ["user_id", "status"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"order_items": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 800000
|
||||
},
|
||||
"order_id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"foreign_key": "orders.id",
|
||||
"cardinality_estimate": 200000
|
||||
},
|
||||
"product_id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"foreign_key": "products.id",
|
||||
"cardinality_estimate": 8000
|
||||
},
|
||||
"quantity": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 20
|
||||
},
|
||||
"unit_price": {
|
||||
"type": "DECIMAL(10,2)",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 5000
|
||||
},
|
||||
"total_price": {
|
||||
"type": "DECIMAL(10,2)",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 10000
|
||||
}
|
||||
},
|
||||
"primary_key": ["id"],
|
||||
"check_constraints": {
|
||||
"chk_quantity_positive": "quantity > 0",
|
||||
"chk_unit_price_positive": "unit_price > 0"
|
||||
},
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_order_items_order",
|
||||
"columns": ["order_id"]
|
||||
},
|
||||
{
|
||||
"name": "idx_order_items_product",
|
||||
"columns": ["product_id"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"categories": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 100
|
||||
},
|
||||
"name": {
|
||||
"type": "VARCHAR(100)",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 100
|
||||
},
|
||||
"parent_id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": true,
|
||||
"foreign_key": "categories.id",
|
||||
"cardinality_estimate": 20
|
||||
},
|
||||
"is_active": {
|
||||
"type": "BOOLEAN",
|
||||
"nullable": false,
|
||||
"default": true,
|
||||
"cardinality_estimate": 2
|
||||
}
|
||||
},
|
||||
"primary_key": ["id"],
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_categories_parent",
|
||||
"columns": ["parent_id"]
|
||||
},
|
||||
{
|
||||
"name": "idx_categories_active",
|
||||
"columns": ["is_active"]
|
||||
}
|
||||
]
|
||||
},
|
||||
"product_reviews": {
|
||||
"columns": {
|
||||
"id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"unique": true,
|
||||
"cardinality_estimate": 150000
|
||||
},
|
||||
"product_id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"foreign_key": "products.id",
|
||||
"cardinality_estimate": 8000
|
||||
},
|
||||
"user_id": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"foreign_key": "users.id",
|
||||
"cardinality_estimate": 30000
|
||||
},
|
||||
"rating": {
|
||||
"type": "INTEGER",
|
||||
"nullable": false,
|
||||
"cardinality_estimate": 5
|
||||
},
|
||||
"review_text": {
|
||||
"type": "TEXT",
|
||||
"nullable": true
|
||||
},
|
||||
"created_at": {
|
||||
"type": "TIMESTAMP",
|
||||
"nullable": false,
|
||||
"default": "CURRENT_TIMESTAMP"
|
||||
}
|
||||
},
|
||||
"primary_key": ["id"],
|
||||
"unique_constraints": [
|
||||
["product_id", "user_id"]
|
||||
],
|
||||
"check_constraints": {
|
||||
"chk_rating_valid": "rating BETWEEN 1 AND 5"
|
||||
},
|
||||
"indexes": [
|
||||
{
|
||||
"name": "idx_reviews_product",
|
||||
"columns": ["product_id"]
|
||||
},
|
||||
{
|
||||
"name": "idx_reviews_user",
|
||||
"columns": ["user_id"]
|
||||
},
|
||||
{
|
||||
"name": "idx_reviews_rating",
|
||||
"columns": ["rating"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
207
engineering/database-designer/assets/sample_schema.sql
Normal file
207
engineering/database-designer/assets/sample_schema.sql
Normal file
@@ -0,0 +1,207 @@
|
||||
-- Sample E-commerce Database Schema
|
||||
-- Demonstrates various normalization levels and common patterns
|
||||
|
||||
-- Users table - well normalized
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY,
|
||||
email VARCHAR(255) NOT NULL UNIQUE,
|
||||
username VARCHAR(50) NOT NULL UNIQUE,
|
||||
password_hash VARCHAR(255) NOT NULL,
|
||||
first_name VARCHAR(100),
|
||||
last_name VARCHAR(100),
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
status VARCHAR(20) DEFAULT 'active'
|
||||
);
|
||||
|
||||
-- Categories table - hierarchical structure
|
||||
CREATE TABLE categories (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
slug VARCHAR(100) NOT NULL UNIQUE,
|
||||
parent_id INTEGER REFERENCES categories(id),
|
||||
description TEXT,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
sort_order INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Products table - potential normalization issues
|
||||
CREATE TABLE products (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
sku VARCHAR(50) NOT NULL UNIQUE,
|
||||
description TEXT,
|
||||
price DECIMAL(10,2) NOT NULL,
|
||||
cost DECIMAL(10,2),
|
||||
weight DECIMAL(8,2),
|
||||
dimensions VARCHAR(50), -- Potential 1NF violation: "10x5x3 inches"
|
||||
category_id INTEGER REFERENCES categories(id),
|
||||
category_name VARCHAR(100), -- Redundant with categories.name (3NF violation)
|
||||
brand VARCHAR(100), -- Should be normalized to separate brands table
|
||||
tags VARCHAR(500), -- Potential 1NF violation: comma-separated tags
|
||||
inventory_count INTEGER DEFAULT 0,
|
||||
reorder_point INTEGER DEFAULT 10,
|
||||
supplier_name VARCHAR(100), -- Should be normalized
|
||||
supplier_contact VARCHAR(255), -- Should be normalized
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
featured BOOLEAN DEFAULT false,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Addresses table - good normalization
|
||||
CREATE TABLE addresses (
|
||||
id INTEGER PRIMARY KEY,
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
address_type VARCHAR(20) DEFAULT 'shipping', -- 'shipping', 'billing'
|
||||
street_address VARCHAR(255) NOT NULL,
|
||||
street_address_2 VARCHAR(255),
|
||||
city VARCHAR(100) NOT NULL,
|
||||
state VARCHAR(50) NOT NULL,
|
||||
postal_code VARCHAR(20) NOT NULL,
|
||||
country VARCHAR(50) NOT NULL DEFAULT 'US',
|
||||
is_default BOOLEAN DEFAULT false,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Orders table - mixed normalization issues
|
||||
CREATE TABLE orders (
|
||||
id INTEGER PRIMARY KEY,
|
||||
order_number VARCHAR(50) NOT NULL UNIQUE,
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
user_email VARCHAR(255), -- Denormalized for performance/historical reasons
|
||||
user_name VARCHAR(200), -- Denormalized for performance/historical reasons
|
||||
status VARCHAR(50) NOT NULL DEFAULT 'pending',
|
||||
total_amount DECIMAL(10,2) NOT NULL,
|
||||
tax_amount DECIMAL(10,2) NOT NULL,
|
||||
shipping_amount DECIMAL(10,2) NOT NULL,
|
||||
discount_amount DECIMAL(10,2) DEFAULT 0,
|
||||
payment_method VARCHAR(50), -- Should be normalized to payment_methods
|
||||
payment_status VARCHAR(50) DEFAULT 'pending',
|
||||
shipping_address_id INTEGER REFERENCES addresses(id),
|
||||
billing_address_id INTEGER REFERENCES addresses(id),
|
||||
-- Denormalized shipping address for historical preservation
|
||||
shipping_street VARCHAR(255),
|
||||
shipping_city VARCHAR(100),
|
||||
shipping_state VARCHAR(50),
|
||||
shipping_postal_code VARCHAR(20),
|
||||
shipping_country VARCHAR(50),
|
||||
notes TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
shipped_at TIMESTAMP,
|
||||
delivered_at TIMESTAMP
|
||||
);
|
||||
|
||||
-- Order items table - properly normalized
|
||||
CREATE TABLE order_items (
|
||||
id INTEGER PRIMARY KEY,
|
||||
order_id INTEGER REFERENCES orders(id),
|
||||
product_id INTEGER REFERENCES products(id),
|
||||
product_name VARCHAR(255), -- Denormalized for historical reasons
|
||||
product_sku VARCHAR(50), -- Denormalized for historical reasons
|
||||
quantity INTEGER NOT NULL,
|
||||
unit_price DECIMAL(10,2) NOT NULL,
|
||||
total_price DECIMAL(10,2) NOT NULL, -- Calculated field (could be computed)
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Shopping cart table - session-based data
|
||||
CREATE TABLE shopping_cart (
|
||||
id INTEGER PRIMARY KEY,
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
session_id VARCHAR(255), -- For anonymous users
|
||||
product_id INTEGER REFERENCES products(id),
|
||||
quantity INTEGER NOT NULL DEFAULT 1,
|
||||
added_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(user_id, product_id),
|
||||
UNIQUE(session_id, product_id)
|
||||
);
|
||||
|
||||
-- Product reviews - user-generated content
|
||||
CREATE TABLE product_reviews (
|
||||
id INTEGER PRIMARY KEY,
|
||||
product_id INTEGER REFERENCES products(id),
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
rating INTEGER NOT NULL CHECK (rating BETWEEN 1 AND 5),
|
||||
title VARCHAR(200),
|
||||
review_text TEXT,
|
||||
verified_purchase BOOLEAN DEFAULT false,
|
||||
helpful_count INTEGER DEFAULT 0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
UNIQUE(product_id, user_id) -- One review per user per product
|
||||
);
|
||||
|
||||
-- Coupons table - promotional data
|
||||
CREATE TABLE coupons (
|
||||
id INTEGER PRIMARY KEY,
|
||||
code VARCHAR(50) NOT NULL UNIQUE,
|
||||
description VARCHAR(255),
|
||||
discount_type VARCHAR(20) NOT NULL, -- 'percentage', 'fixed_amount'
|
||||
discount_value DECIMAL(8,2) NOT NULL,
|
||||
minimum_amount DECIMAL(10,2),
|
||||
maximum_discount DECIMAL(10,2),
|
||||
usage_limit INTEGER,
|
||||
usage_count INTEGER DEFAULT 0,
|
||||
valid_from TIMESTAMP NOT NULL,
|
||||
valid_until TIMESTAMP NOT NULL,
|
||||
is_active BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Audit log table - for tracking changes
|
||||
CREATE TABLE audit_log (
|
||||
id INTEGER PRIMARY KEY,
|
||||
table_name VARCHAR(50) NOT NULL,
|
||||
record_id INTEGER NOT NULL,
|
||||
action VARCHAR(20) NOT NULL, -- 'INSERT', 'UPDATE', 'DELETE'
|
||||
old_values TEXT, -- JSON format
|
||||
new_values TEXT, -- JSON format
|
||||
user_id INTEGER REFERENCES users(id),
|
||||
ip_address VARCHAR(45),
|
||||
user_agent TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Problematic table - multiple normalization violations
|
||||
CREATE TABLE user_preferences (
|
||||
user_id INTEGER PRIMARY KEY REFERENCES users(id),
|
||||
preferred_categories VARCHAR(500), -- CSV list - 1NF violation
|
||||
email_notifications VARCHAR(255), -- "daily,weekly,promotions" - 1NF violation
|
||||
user_name VARCHAR(200), -- Redundant with users table - 3NF violation
|
||||
user_email VARCHAR(255), -- Redundant with users table - 3NF violation
|
||||
theme VARCHAR(50) DEFAULT 'light',
|
||||
language VARCHAR(10) DEFAULT 'en',
|
||||
timezone VARCHAR(50) DEFAULT 'UTC',
|
||||
currency VARCHAR(3) DEFAULT 'USD',
|
||||
date_format VARCHAR(20) DEFAULT 'YYYY-MM-DD',
|
||||
newsletter_subscribed BOOLEAN DEFAULT true,
|
||||
sms_notifications BOOLEAN DEFAULT false,
|
||||
push_notifications BOOLEAN DEFAULT true,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Create some basic indexes (some missing, some redundant for demonstration)
|
||||
CREATE INDEX idx_users_email ON users (email);
|
||||
CREATE INDEX idx_users_username ON users (username); -- Redundant due to UNIQUE constraint
|
||||
CREATE INDEX idx_products_category ON products (category_id);
|
||||
CREATE INDEX idx_products_brand ON products (brand);
|
||||
CREATE INDEX idx_products_sku ON products (sku); -- Redundant due to UNIQUE constraint
|
||||
CREATE INDEX idx_orders_user ON orders (user_id);
|
||||
CREATE INDEX idx_orders_status ON orders (status);
|
||||
CREATE INDEX idx_orders_created ON orders (created_at);
|
||||
CREATE INDEX idx_order_items_order ON order_items (order_id);
|
||||
CREATE INDEX idx_order_items_product ON order_items (product_id);
|
||||
-- Missing index on addresses.user_id
|
||||
-- Missing composite index on orders (user_id, status)
|
||||
-- Missing index on product_reviews.product_id
|
||||
|
||||
-- Constraints that should exist but are missing
|
||||
-- ALTER TABLE products ADD CONSTRAINT chk_price_positive CHECK (price > 0);
|
||||
-- ALTER TABLE products ADD CONSTRAINT chk_inventory_non_negative CHECK (inventory_count >= 0);
|
||||
-- ALTER TABLE order_items ADD CONSTRAINT chk_quantity_positive CHECK (quantity > 0);
|
||||
-- ALTER TABLE orders ADD CONSTRAINT chk_total_positive CHECK (total_amount > 0);
|
||||
@@ -0,0 +1,60 @@
|
||||
DATABASE INDEX OPTIMIZATION REPORT
|
||||
==================================================
|
||||
|
||||
ANALYSIS SUMMARY
|
||||
----------------
|
||||
Tables Analyzed: 6
|
||||
Query Patterns: 15
|
||||
Existing Indexes: 12
|
||||
New Recommendations: 8
|
||||
High Priority: 4
|
||||
Redundancy Issues: 2
|
||||
|
||||
HIGH PRIORITY RECOMMENDATIONS (4)
|
||||
----------------------------------
|
||||
1. orders: Optimize multi-column WHERE conditions: user_id, status, created_at
|
||||
Columns: user_id, status, created_at
|
||||
Benefit: Very High
|
||||
SQL: CREATE INDEX idx_orders_user_status_created ON orders (user_id, status, created_at);
|
||||
|
||||
2. products: Optimize WHERE category_id = AND is_active = queries
|
||||
Columns: category_id, is_active
|
||||
Benefit: High
|
||||
SQL: CREATE INDEX idx_products_category_active ON products (category_id, is_active);
|
||||
|
||||
3. order_items: Optimize JOIN with products table on product_id
|
||||
Columns: product_id
|
||||
Benefit: High (frequent JOINs)
|
||||
SQL: CREATE INDEX idx_order_items_product_join ON order_items (product_id);
|
||||
|
||||
4. product_reviews: Covering index for WHERE + ORDER BY optimization
|
||||
Columns: product_id, created_at
|
||||
Benefit: High (eliminates table lookups for SELECT)
|
||||
SQL: CREATE INDEX idx_product_reviews_covering_product_created ON product_reviews (product_id, created_at) INCLUDE (rating, review_text);
|
||||
|
||||
REDUNDANCY ISSUES (2)
|
||||
---------------------
|
||||
• DUPLICATE: Indexes 'idx_users_email' and 'unique_users_email' are identical
|
||||
Recommendation: Drop one of the duplicate indexes
|
||||
SQL: DROP INDEX idx_users_email;
|
||||
|
||||
• OVERLAPPING: Index 'idx_products_category' overlaps 85% with 'idx_products_category_active'
|
||||
Recommendation: Consider dropping 'idx_products_category' as it's largely covered by 'idx_products_category_active'
|
||||
SQL: DROP INDEX idx_products_category;
|
||||
|
||||
PERFORMANCE IMPACT ANALYSIS
|
||||
----------------------------
|
||||
Queries to be optimized: 12
|
||||
High impact optimizations: 6
|
||||
Estimated insert overhead: 40%
|
||||
|
||||
RECOMMENDED CREATE INDEX STATEMENTS
|
||||
------------------------------------
|
||||
1. CREATE INDEX idx_orders_user_status_created ON orders (user_id, status, created_at);
|
||||
2. CREATE INDEX idx_products_category_active ON products (category_id, is_active);
|
||||
3. CREATE INDEX idx_order_items_product_join ON order_items (product_id);
|
||||
4. CREATE INDEX idx_product_reviews_covering_product_created ON product_reviews (product_id, created_at) INCLUDE (rating, review_text);
|
||||
5. CREATE INDEX idx_products_price_brand ON products (price, brand);
|
||||
6. CREATE INDEX idx_orders_status_created ON orders (status, created_at);
|
||||
7. CREATE INDEX idx_categories_parent_active ON categories (parent_id, is_active);
|
||||
8. CREATE INDEX idx_product_reviews_user_created ON product_reviews (user_id, created_at);
|
||||
@@ -0,0 +1,124 @@
|
||||
DATABASE MIGRATION PLAN
|
||||
==================================================
|
||||
Migration ID: a7b3c9d2
|
||||
Created: 2024-02-16T15:30:00Z
|
||||
Zero Downtime: false
|
||||
|
||||
MIGRATION SUMMARY
|
||||
-----------------
|
||||
Total Steps: 12
|
||||
Tables Added: 1
|
||||
Tables Dropped: 0
|
||||
Tables Renamed: 0
|
||||
Columns Added: 3
|
||||
Columns Dropped: 1
|
||||
Columns Modified: 2
|
||||
Constraints Added: 4
|
||||
Constraints Dropped: 1
|
||||
Indexes Added: 2
|
||||
Indexes Dropped: 1
|
||||
|
||||
RISK ASSESSMENT
|
||||
---------------
|
||||
High Risk Steps: 3
|
||||
Medium Risk Steps: 4
|
||||
Low Risk Steps: 5
|
||||
|
||||
MIGRATION STEPS
|
||||
---------------
|
||||
1. Create table brands with 4 columns (LOW risk)
|
||||
Type: CREATE_TABLE
|
||||
Forward SQL: CREATE TABLE brands (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
Rollback SQL: DROP TABLE IF EXISTS brands;
|
||||
|
||||
2. Add column brand_id to products (LOW risk)
|
||||
Type: ADD_COLUMN
|
||||
Forward SQL: ALTER TABLE products ADD COLUMN brand_id INTEGER;
|
||||
Rollback SQL: ALTER TABLE products DROP COLUMN brand_id;
|
||||
|
||||
3. Add column email_verified to users (LOW risk)
|
||||
Type: ADD_COLUMN
|
||||
Forward SQL: ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT false;
|
||||
Rollback SQL: ALTER TABLE users DROP COLUMN email_verified;
|
||||
|
||||
4. Add column last_login to users (LOW risk)
|
||||
Type: ADD_COLUMN
|
||||
Forward SQL: ALTER TABLE users ADD COLUMN last_login TIMESTAMP;
|
||||
Rollback SQL: ALTER TABLE users DROP COLUMN last_login;
|
||||
|
||||
5. Modify column price: type: DECIMAL(10,2) -> DECIMAL(12,2) (LOW risk)
|
||||
Type: MODIFY_COLUMN
|
||||
Forward SQL: ALTER TABLE products
|
||||
ALTER COLUMN price TYPE DECIMAL(12,2);
|
||||
Rollback SQL: ALTER TABLE products
|
||||
ALTER COLUMN price TYPE DECIMAL(10,2);
|
||||
|
||||
6. Modify column inventory_count: nullable: true -> false (HIGH risk)
|
||||
Type: MODIFY_COLUMN
|
||||
Forward SQL: ALTER TABLE products
|
||||
ALTER COLUMN inventory_count SET NOT NULL;
|
||||
Rollback SQL: ALTER TABLE products
|
||||
ALTER COLUMN inventory_count DROP NOT NULL;
|
||||
|
||||
7. Add primary key on id (MEDIUM risk)
|
||||
Type: ADD_CONSTRAINT
|
||||
Forward SQL: ALTER TABLE brands ADD CONSTRAINT pk_brands PRIMARY KEY (id);
|
||||
Rollback SQL: ALTER TABLE brands DROP CONSTRAINT pk_brands;
|
||||
|
||||
8. Add foreign key constraint on brand_id (MEDIUM risk)
|
||||
Type: ADD_CONSTRAINT
|
||||
Forward SQL: ALTER TABLE products ADD CONSTRAINT fk_products_brand_id FOREIGN KEY (brand_id) REFERENCES brands(id);
|
||||
Rollback SQL: ALTER TABLE products DROP CONSTRAINT fk_products_brand_id;
|
||||
|
||||
9. Add unique constraint on name (MEDIUM risk)
|
||||
Type: ADD_CONSTRAINT
|
||||
Forward SQL: ALTER TABLE brands ADD CONSTRAINT uq_brands_name UNIQUE (name);
|
||||
Rollback SQL: ALTER TABLE brands DROP CONSTRAINT uq_brands_name;
|
||||
|
||||
10. Add check constraint: price > 0 (MEDIUM risk)
|
||||
Type: ADD_CONSTRAINT
|
||||
Forward SQL: ALTER TABLE products ADD CONSTRAINT chk_products_price_positive CHECK (price > 0);
|
||||
Rollback SQL: ALTER TABLE products DROP CONSTRAINT chk_products_price_positive;
|
||||
|
||||
11. Create index idx_products_brand_id on (brand_id) (LOW risk)
|
||||
Type: ADD_INDEX
|
||||
Forward SQL: CREATE INDEX idx_products_brand_id ON products (brand_id);
|
||||
Rollback SQL: DROP INDEX idx_products_brand_id;
|
||||
Estimated Time: 1-5 minutes depending on table size
|
||||
|
||||
12. Create index idx_users_email_verified on (email_verified) (LOW risk)
|
||||
Type: ADD_INDEX
|
||||
Forward SQL: CREATE INDEX idx_users_email_verified ON users (email_verified);
|
||||
Rollback SQL: DROP INDEX idx_users_email_verified;
|
||||
Estimated Time: 1-5 minutes depending on table size
|
||||
|
||||
VALIDATION CHECKS
|
||||
-----------------
|
||||
• Verify table brands exists
|
||||
SQL: SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'brands';
|
||||
Expected: 1
|
||||
|
||||
• Verify column brand_id exists in products
|
||||
SQL: SELECT COUNT(*) FROM information_schema.columns WHERE table_name = 'products' AND column_name = 'brand_id';
|
||||
Expected: 1
|
||||
|
||||
• Verify column email_verified exists in users
|
||||
SQL: SELECT COUNT(*) FROM information_schema.columns WHERE table_name = 'users' AND column_name = 'email_verified';
|
||||
Expected: 1
|
||||
|
||||
• Verify column modification in products
|
||||
SQL: SELECT data_type, is_nullable FROM information_schema.columns WHERE table_name = 'products' AND column_name = 'price';
|
||||
Expected: 1
|
||||
|
||||
• Verify index idx_products_brand_id exists
|
||||
SQL: SELECT COUNT(*) FROM information_schema.statistics WHERE index_name = 'idx_products_brand_id';
|
||||
Expected: 1
|
||||
|
||||
• Verify index idx_users_email_verified exists
|
||||
SQL: SELECT COUNT(*) FROM information_schema.statistics WHERE index_name = 'idx_users_email_verified';
|
||||
Expected: 1
|
||||
@@ -0,0 +1,222 @@
|
||||
DATABASE SCHEMA ANALYSIS REPORT
|
||||
==================================================
|
||||
|
||||
SCHEMA OVERVIEW
|
||||
---------------
|
||||
Total Tables: 8
|
||||
Total Columns: 52
|
||||
Tables with Primary Keys: 8
|
||||
Total Foreign Keys: 6
|
||||
Total Indexes: 15
|
||||
|
||||
KEY RECOMMENDATIONS
|
||||
------------------
|
||||
1. Address 3 high-severity issues immediately
|
||||
2. Add primary keys to tables:
|
||||
3. Review 4 VARCHAR(255) columns for right-sizing
|
||||
4. Consider adding 2 foreign key constraints for referential integrity
|
||||
5. Review 8 normalization issues for schema optimization
|
||||
|
||||
NORMALIZATION ISSUES (8 total)
|
||||
------------------------------
|
||||
High: 2, Medium: 3, Low: 2, Warning: 1
|
||||
|
||||
• products: Column 'dimensions' appears to store delimited values
|
||||
Suggestion: Create separate table for individual values with foreign key relationship
|
||||
|
||||
• products: Column 'tags' appears to store delimited values
|
||||
Suggestion: Create separate table for individual values with foreign key relationship
|
||||
|
||||
• products: Columns ['category_name'] may have transitive dependency through 'category_id'
|
||||
Suggestion: Consider creating separate 'category' table with these columns
|
||||
|
||||
• orders: Columns ['shipping_street', 'shipping_city', 'shipping_state', 'shipping_postal_code', 'shipping_country'] may have transitive dependency through 'shipping_address_id'
|
||||
Suggestion: Consider creating separate 'shipping_address' table with these columns
|
||||
|
||||
• user_preferences: Column 'preferred_categories' appears to store delimited values
|
||||
Suggestion: Create separate table for individual values with foreign key relationship
|
||||
|
||||
DATA TYPE ISSUES (4 total)
|
||||
--------------------------
|
||||
• products.dimensions: VARCHAR(255) antipattern
|
||||
Current: VARCHAR(50) → Suggested: Appropriately sized VARCHAR or TEXT
|
||||
Rationale: VARCHAR(255) is often used as default without considering actual data length requirements
|
||||
|
||||
• products.tags: VARCHAR(255) antipattern
|
||||
Current: VARCHAR(500) → Suggested: Appropriately sized VARCHAR or TEXT
|
||||
Rationale: VARCHAR(255) is often used as default without considering actual data length requirements
|
||||
|
||||
• user_preferences.preferred_categories: VARCHAR(255) antipattern
|
||||
Current: VARCHAR(500) → Suggested: Appropriately sized VARCHAR or TEXT
|
||||
Rationale: VARCHAR(255) is often used as default without considering actual data length requirements
|
||||
|
||||
• user_preferences.email_notifications: VARCHAR(255) antipattern
|
||||
Current: VARCHAR(255) → Suggested: Appropriately sized VARCHAR or TEXT
|
||||
Rationale: VARCHAR(255) is often used as default without considering actual data length requirements
|
||||
|
||||
CONSTRAINT ISSUES (12 total)
|
||||
-----------------------------
|
||||
High: 0, Medium: 4, Low: 8
|
||||
|
||||
• products: Column 'price' should validate positive values
|
||||
Suggestion: Add CHECK constraint: price > 0
|
||||
|
||||
• products: Column 'inventory_count' should validate positive values
|
||||
Suggestion: Add CHECK constraint: inventory_count > 0
|
||||
|
||||
• orders: Column 'total_amount' should validate positive values
|
||||
Suggestion: Add CHECK constraint: total_amount > 0
|
||||
|
||||
• order_items: Column 'quantity' should validate positive values
|
||||
Suggestion: Add CHECK constraint: quantity > 0
|
||||
|
||||
• order_items: Column 'unit_price' should validate positive values
|
||||
Suggestion: Add CHECK constraint: unit_price > 0
|
||||
|
||||
MISSING INDEXES (3 total)
|
||||
-------------------------
|
||||
• addresses.user_id (foreign_key)
|
||||
SQL: CREATE INDEX idx_addresses_user_id ON addresses (user_id);
|
||||
|
||||
• product_reviews.product_id (foreign_key)
|
||||
SQL: CREATE INDEX idx_product_reviews_product_id ON product_reviews (product_id);
|
||||
|
||||
• shopping_cart.user_id (foreign_key)
|
||||
SQL: CREATE INDEX idx_shopping_cart_user_id ON shopping_cart (user_id);
|
||||
|
||||
MERMAID ERD
|
||||
===========
|
||||
erDiagram
|
||||
USERS {
|
||||
INTEGER id "PK"
|
||||
VARCHAR(255) email "NOT NULL"
|
||||
VARCHAR(50) username "NOT NULL"
|
||||
VARCHAR(255) password_hash "NOT NULL"
|
||||
VARCHAR(100) first_name
|
||||
VARCHAR(100) last_name
|
||||
TIMESTAMP created_at
|
||||
TIMESTAMP updated_at
|
||||
VARCHAR(20) status
|
||||
}
|
||||
|
||||
CATEGORIES {
|
||||
INTEGER id "PK"
|
||||
VARCHAR(100) name "NOT NULL"
|
||||
VARCHAR(100) slug "NOT NULL UNIQUE"
|
||||
INTEGER parent_id "FK"
|
||||
TEXT description
|
||||
BOOLEAN is_active
|
||||
INTEGER sort_order
|
||||
TIMESTAMP created_at
|
||||
}
|
||||
|
||||
PRODUCTS {
|
||||
INTEGER id "PK"
|
||||
VARCHAR(255) name "NOT NULL"
|
||||
VARCHAR(50) sku "NOT NULL UNIQUE"
|
||||
TEXT description
|
||||
DECIMAL(10,2) price "NOT NULL"
|
||||
DECIMAL(10,2) cost
|
||||
DECIMAL(8,2) weight
|
||||
VARCHAR(50) dimensions
|
||||
INTEGER category_id "FK"
|
||||
VARCHAR(100) category_name
|
||||
VARCHAR(100) brand
|
||||
VARCHAR(500) tags
|
||||
INTEGER inventory_count
|
||||
INTEGER reorder_point
|
||||
VARCHAR(100) supplier_name
|
||||
VARCHAR(255) supplier_contact
|
||||
BOOLEAN is_active
|
||||
BOOLEAN featured
|
||||
TIMESTAMP created_at
|
||||
TIMESTAMP updated_at
|
||||
}
|
||||
|
||||
ADDRESSES {
|
||||
INTEGER id "PK"
|
||||
INTEGER user_id "FK"
|
||||
VARCHAR(20) address_type
|
||||
VARCHAR(255) street_address "NOT NULL"
|
||||
VARCHAR(255) street_address_2
|
||||
VARCHAR(100) city "NOT NULL"
|
||||
VARCHAR(50) state "NOT NULL"
|
||||
VARCHAR(20) postal_code "NOT NULL"
|
||||
VARCHAR(50) country "NOT NULL"
|
||||
BOOLEAN is_default
|
||||
TIMESTAMP created_at
|
||||
}
|
||||
|
||||
ORDERS {
|
||||
INTEGER id "PK"
|
||||
VARCHAR(50) order_number "NOT NULL UNIQUE"
|
||||
INTEGER user_id "FK"
|
||||
VARCHAR(255) user_email
|
||||
VARCHAR(200) user_name
|
||||
VARCHAR(50) status "NOT NULL"
|
||||
DECIMAL(10,2) total_amount "NOT NULL"
|
||||
DECIMAL(10,2) tax_amount "NOT NULL"
|
||||
DECIMAL(10,2) shipping_amount "NOT NULL"
|
||||
DECIMAL(10,2) discount_amount
|
||||
VARCHAR(50) payment_method
|
||||
VARCHAR(50) payment_status
|
||||
INTEGER shipping_address_id "FK"
|
||||
INTEGER billing_address_id "FK"
|
||||
VARCHAR(255) shipping_street
|
||||
VARCHAR(100) shipping_city
|
||||
VARCHAR(50) shipping_state
|
||||
VARCHAR(20) shipping_postal_code
|
||||
VARCHAR(50) shipping_country
|
||||
TEXT notes
|
||||
TIMESTAMP created_at
|
||||
TIMESTAMP updated_at
|
||||
TIMESTAMP shipped_at
|
||||
TIMESTAMP delivered_at
|
||||
}
|
||||
|
||||
ORDER_ITEMS {
|
||||
INTEGER id "PK"
|
||||
INTEGER order_id "FK"
|
||||
INTEGER product_id "FK"
|
||||
VARCHAR(255) product_name
|
||||
VARCHAR(50) product_sku
|
||||
INTEGER quantity "NOT NULL"
|
||||
DECIMAL(10,2) unit_price "NOT NULL"
|
||||
DECIMAL(10,2) total_price "NOT NULL"
|
||||
TIMESTAMP created_at
|
||||
}
|
||||
|
||||
SHOPPING_CART {
|
||||
INTEGER id "PK"
|
||||
INTEGER user_id "FK"
|
||||
VARCHAR(255) session_id
|
||||
INTEGER product_id "FK"
|
||||
INTEGER quantity "NOT NULL"
|
||||
TIMESTAMP added_at
|
||||
TIMESTAMP updated_at
|
||||
}
|
||||
|
||||
PRODUCT_REVIEWS {
|
||||
INTEGER id "PK"
|
||||
INTEGER product_id "FK"
|
||||
INTEGER user_id "FK"
|
||||
INTEGER rating "NOT NULL"
|
||||
VARCHAR(200) title
|
||||
TEXT review_text
|
||||
BOOLEAN verified_purchase
|
||||
INTEGER helpful_count
|
||||
TIMESTAMP created_at
|
||||
TIMESTAMP updated_at
|
||||
}
|
||||
|
||||
CATEGORIES ||--o{ CATEGORIES : has
|
||||
CATEGORIES ||--o{ PRODUCTS : has
|
||||
USERS ||--o{ ADDRESSES : has
|
||||
USERS ||--o{ ORDERS : has
|
||||
USERS ||--o{ SHOPPING_CART : has
|
||||
USERS ||--o{ PRODUCT_REVIEWS : has
|
||||
ADDRESSES ||--o{ ORDERS : has
|
||||
ORDERS ||--o{ ORDER_ITEMS : has
|
||||
PRODUCTS ||--o{ ORDER_ITEMS : has
|
||||
PRODUCTS ||--o{ SHOPPING_CART : has
|
||||
PRODUCTS ||--o{ PRODUCT_REVIEWS : has
|
||||
926
engineering/database-designer/index_optimizer.py
Normal file
926
engineering/database-designer/index_optimizer.py
Normal file
@@ -0,0 +1,926 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Database Index Optimizer
|
||||
|
||||
Analyzes schema definitions and query patterns to recommend optimal indexes:
|
||||
- Identifies missing indexes for common query patterns
|
||||
- Detects redundant and overlapping indexes
|
||||
- Suggests composite index column ordering
|
||||
- Estimates selectivity and performance impact
|
||||
- Generates CREATE INDEX statements with rationale
|
||||
|
||||
Input: Schema JSON + Query patterns JSON
|
||||
Output: Index recommendations + CREATE INDEX SQL + before/after analysis
|
||||
|
||||
Usage:
|
||||
python index_optimizer.py --schema schema.json --queries queries.json --output recommendations.json
|
||||
python index_optimizer.py --schema schema.json --queries queries.json --format text
|
||||
python index_optimizer.py --schema schema.json --queries queries.json --analyze-existing
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict, namedtuple, Counter
|
||||
from typing import Dict, List, Set, Tuple, Optional, Any
|
||||
from dataclasses import dataclass, asdict
|
||||
import hashlib
|
||||
|
||||
|
||||
@dataclass
|
||||
class Column:
|
||||
name: str
|
||||
data_type: str
|
||||
nullable: bool = True
|
||||
unique: bool = False
|
||||
cardinality_estimate: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Index:
|
||||
name: str
|
||||
table: str
|
||||
columns: List[str]
|
||||
unique: bool = False
|
||||
index_type: str = "btree"
|
||||
partial_condition: Optional[str] = None
|
||||
include_columns: List[str] = None
|
||||
size_estimate: Optional[int] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class QueryPattern:
|
||||
query_id: str
|
||||
query_type: str # SELECT, INSERT, UPDATE, DELETE
|
||||
table: str
|
||||
where_conditions: List[Dict[str, Any]]
|
||||
join_conditions: List[Dict[str, Any]]
|
||||
order_by: List[Dict[str, str]] # column, direction
|
||||
group_by: List[str]
|
||||
frequency: int = 1
|
||||
avg_execution_time_ms: Optional[float] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class IndexRecommendation:
|
||||
recommendation_id: str
|
||||
table: str
|
||||
recommended_index: Index
|
||||
reason: str
|
||||
query_patterns_helped: List[str]
|
||||
estimated_benefit: str
|
||||
estimated_overhead: str
|
||||
priority: int # 1 = highest priority
|
||||
sql_statement: str
|
||||
selectivity_analysis: Dict[str, Any]
|
||||
|
||||
|
||||
@dataclass
|
||||
class RedundancyIssue:
|
||||
issue_type: str # DUPLICATE, OVERLAPPING, UNUSED
|
||||
affected_indexes: List[str]
|
||||
table: str
|
||||
description: str
|
||||
recommendation: str
|
||||
sql_statements: List[str]
|
||||
|
||||
|
||||
class SelectivityEstimator:
|
||||
"""Estimates column selectivity based on naming patterns and data types."""
|
||||
|
||||
def __init__(self):
|
||||
# Selectivity patterns based on common column names and types
|
||||
self.high_selectivity_patterns = [
|
||||
r'.*_id$', r'^id$', r'uuid', r'guid', r'email', r'username', r'ssn',
|
||||
r'account.*number', r'transaction.*id', r'reference.*number'
|
||||
]
|
||||
|
||||
self.medium_selectivity_patterns = [
|
||||
r'name$', r'title$', r'description$', r'address', r'phone', r'zip',
|
||||
r'postal.*code', r'serial.*number', r'sku', r'product.*code'
|
||||
]
|
||||
|
||||
self.low_selectivity_patterns = [
|
||||
r'status$', r'type$', r'category', r'state$', r'flag$', r'active$',
|
||||
r'enabled$', r'deleted$', r'visible$', r'gender$', r'priority$'
|
||||
]
|
||||
|
||||
self.very_low_selectivity_patterns = [
|
||||
r'is_.*', r'has_.*', r'can_.*', r'boolean', r'bool'
|
||||
]
|
||||
|
||||
def estimate_selectivity(self, column: Column, table_size_estimate: int = 10000) -> float:
|
||||
"""Estimate column selectivity (0.0 = all same values, 1.0 = all unique values)."""
|
||||
column_name_lower = column.name.lower()
|
||||
|
||||
# Primary key or unique columns
|
||||
if column.unique or column.name.lower() in ['id', 'uuid', 'guid']:
|
||||
return 1.0
|
||||
|
||||
# Check cardinality estimate if available
|
||||
if column.cardinality_estimate:
|
||||
return min(column.cardinality_estimate / table_size_estimate, 1.0)
|
||||
|
||||
# Pattern-based estimation
|
||||
for pattern in self.high_selectivity_patterns:
|
||||
if re.search(pattern, column_name_lower):
|
||||
return 0.9 # Very high selectivity
|
||||
|
||||
for pattern in self.medium_selectivity_patterns:
|
||||
if re.search(pattern, column_name_lower):
|
||||
return 0.7 # Good selectivity
|
||||
|
||||
for pattern in self.low_selectivity_patterns:
|
||||
if re.search(pattern, column_name_lower):
|
||||
return 0.2 # Poor selectivity
|
||||
|
||||
for pattern in self.very_low_selectivity_patterns:
|
||||
if re.search(pattern, column_name_lower):
|
||||
return 0.1 # Very poor selectivity
|
||||
|
||||
# Data type based estimation
|
||||
data_type_upper = column.data_type.upper()
|
||||
if data_type_upper.startswith('BOOL'):
|
||||
return 0.1
|
||||
elif data_type_upper.startswith(('TINYINT', 'SMALLINT')):
|
||||
return 0.3
|
||||
elif data_type_upper.startswith('INT'):
|
||||
return 0.8
|
||||
elif data_type_upper.startswith(('VARCHAR', 'TEXT')):
|
||||
# Estimate based on column name
|
||||
if 'name' in column_name_lower:
|
||||
return 0.7
|
||||
elif 'description' in column_name_lower or 'comment' in column_name_lower:
|
||||
return 0.9
|
||||
else:
|
||||
return 0.6
|
||||
|
||||
# Default moderate selectivity
|
||||
return 0.5
|
||||
|
||||
|
||||
class IndexOptimizer:
|
||||
def __init__(self):
|
||||
self.tables: Dict[str, Dict[str, Column]] = {}
|
||||
self.existing_indexes: Dict[str, List[Index]] = {}
|
||||
self.query_patterns: List[QueryPattern] = []
|
||||
self.selectivity_estimator = SelectivityEstimator()
|
||||
|
||||
# Configuration
|
||||
self.max_composite_index_columns = 6
|
||||
self.min_selectivity_for_index = 0.1
|
||||
self.redundancy_overlap_threshold = 0.8
|
||||
|
||||
def load_schema(self, schema_data: Dict[str, Any]) -> None:
|
||||
"""Load schema definition."""
|
||||
if 'tables' not in schema_data:
|
||||
raise ValueError("Schema must contain 'tables' key")
|
||||
|
||||
for table_name, table_def in schema_data['tables'].items():
|
||||
self.tables[table_name] = {}
|
||||
self.existing_indexes[table_name] = []
|
||||
|
||||
# Load columns
|
||||
for col_name, col_def in table_def.get('columns', {}).items():
|
||||
column = Column(
|
||||
name=col_name,
|
||||
data_type=col_def.get('type', 'VARCHAR(255)'),
|
||||
nullable=col_def.get('nullable', True),
|
||||
unique=col_def.get('unique', False),
|
||||
cardinality_estimate=col_def.get('cardinality_estimate')
|
||||
)
|
||||
self.tables[table_name][col_name] = column
|
||||
|
||||
# Load existing indexes
|
||||
for idx_def in table_def.get('indexes', []):
|
||||
index = Index(
|
||||
name=idx_def['name'],
|
||||
table=table_name,
|
||||
columns=idx_def['columns'],
|
||||
unique=idx_def.get('unique', False),
|
||||
index_type=idx_def.get('type', 'btree'),
|
||||
partial_condition=idx_def.get('partial_condition'),
|
||||
include_columns=idx_def.get('include_columns', [])
|
||||
)
|
||||
self.existing_indexes[table_name].append(index)
|
||||
|
||||
def load_query_patterns(self, query_data: Dict[str, Any]) -> None:
|
||||
"""Load query patterns for analysis."""
|
||||
if 'queries' not in query_data:
|
||||
raise ValueError("Query data must contain 'queries' key")
|
||||
|
||||
for query_def in query_data['queries']:
|
||||
pattern = QueryPattern(
|
||||
query_id=query_def['id'],
|
||||
query_type=query_def.get('type', 'SELECT').upper(),
|
||||
table=query_def['table'],
|
||||
where_conditions=query_def.get('where_conditions', []),
|
||||
join_conditions=query_def.get('join_conditions', []),
|
||||
order_by=query_def.get('order_by', []),
|
||||
group_by=query_def.get('group_by', []),
|
||||
frequency=query_def.get('frequency', 1),
|
||||
avg_execution_time_ms=query_def.get('avg_execution_time_ms')
|
||||
)
|
||||
self.query_patterns.append(pattern)
|
||||
|
||||
def analyze_missing_indexes(self) -> List[IndexRecommendation]:
|
||||
"""Identify missing indexes based on query patterns."""
|
||||
recommendations = []
|
||||
|
||||
for pattern in self.query_patterns:
|
||||
table_name = pattern.table
|
||||
if table_name not in self.tables:
|
||||
continue
|
||||
|
||||
# Analyze WHERE conditions for single-column indexes
|
||||
for condition in pattern.where_conditions:
|
||||
column = condition.get('column')
|
||||
operator = condition.get('operator', '=')
|
||||
|
||||
if column and column in self.tables[table_name]:
|
||||
if not self._has_covering_index(table_name, [column]):
|
||||
recommendation = self._create_single_column_recommendation(
|
||||
table_name, column, pattern, operator
|
||||
)
|
||||
if recommendation:
|
||||
recommendations.append(recommendation)
|
||||
|
||||
# Analyze composite indexes for multi-column WHERE conditions
|
||||
where_columns = [cond.get('column') for cond in pattern.where_conditions
|
||||
if cond.get('column') and cond.get('column') in self.tables[table_name]]
|
||||
|
||||
if len(where_columns) > 1:
|
||||
composite_recommendation = self._create_composite_recommendation(
|
||||
table_name, where_columns, pattern
|
||||
)
|
||||
if composite_recommendation:
|
||||
recommendations.append(composite_recommendation)
|
||||
|
||||
# Analyze covering indexes for SELECT with ORDER BY
|
||||
if pattern.order_by and where_columns:
|
||||
covering_recommendation = self._create_covering_index_recommendation(
|
||||
table_name, where_columns, pattern
|
||||
)
|
||||
if covering_recommendation:
|
||||
recommendations.append(covering_recommendation)
|
||||
|
||||
# Analyze JOIN conditions
|
||||
for join_condition in pattern.join_conditions:
|
||||
local_column = join_condition.get('local_column')
|
||||
if local_column and local_column in self.tables[table_name]:
|
||||
if not self._has_covering_index(table_name, [local_column]):
|
||||
recommendation = self._create_join_index_recommendation(
|
||||
table_name, local_column, pattern, join_condition
|
||||
)
|
||||
if recommendation:
|
||||
recommendations.append(recommendation)
|
||||
|
||||
# Remove duplicates and prioritize
|
||||
recommendations = self._deduplicate_recommendations(recommendations)
|
||||
recommendations = self._prioritize_recommendations(recommendations)
|
||||
|
||||
return recommendations
|
||||
|
||||
def _has_covering_index(self, table_name: str, columns: List[str]) -> bool:
|
||||
"""Check if existing indexes cover the specified columns."""
|
||||
if table_name not in self.existing_indexes:
|
||||
return False
|
||||
|
||||
for index in self.existing_indexes[table_name]:
|
||||
# Check if index starts with required columns (prefix match for composite)
|
||||
if len(index.columns) >= len(columns):
|
||||
if index.columns[:len(columns)] == columns:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _create_single_column_recommendation(
|
||||
self,
|
||||
table_name: str,
|
||||
column: str,
|
||||
pattern: QueryPattern,
|
||||
operator: str
|
||||
) -> Optional[IndexRecommendation]:
|
||||
"""Create recommendation for single-column index."""
|
||||
column_obj = self.tables[table_name][column]
|
||||
selectivity = self.selectivity_estimator.estimate_selectivity(column_obj)
|
||||
|
||||
# Skip very low selectivity columns unless frequently used
|
||||
if selectivity < self.min_selectivity_for_index and pattern.frequency < 100:
|
||||
return None
|
||||
|
||||
index_name = f"idx_{table_name}_{column}"
|
||||
index = Index(
|
||||
name=index_name,
|
||||
table=table_name,
|
||||
columns=[column],
|
||||
unique=column_obj.unique,
|
||||
index_type="btree"
|
||||
)
|
||||
|
||||
reason = f"Optimize WHERE {column} {operator} queries"
|
||||
if pattern.frequency > 10:
|
||||
reason += f" (used {pattern.frequency} times)"
|
||||
|
||||
return IndexRecommendation(
|
||||
recommendation_id=self._generate_recommendation_id(table_name, [column]),
|
||||
table=table_name,
|
||||
recommended_index=index,
|
||||
reason=reason,
|
||||
query_patterns_helped=[pattern.query_id],
|
||||
estimated_benefit=self._estimate_benefit(selectivity, pattern.frequency),
|
||||
estimated_overhead="Low (single column)",
|
||||
priority=self._calculate_priority(selectivity, pattern.frequency, 1),
|
||||
sql_statement=f"CREATE INDEX {index_name} ON {table_name} ({column});",
|
||||
selectivity_analysis={
|
||||
"column_selectivity": selectivity,
|
||||
"estimated_reduction": f"{int(selectivity * 100)}%"
|
||||
}
|
||||
)
|
||||
|
||||
def _create_composite_recommendation(
|
||||
self,
|
||||
table_name: str,
|
||||
columns: List[str],
|
||||
pattern: QueryPattern
|
||||
) -> Optional[IndexRecommendation]:
|
||||
"""Create recommendation for composite index."""
|
||||
if len(columns) > self.max_composite_index_columns:
|
||||
columns = columns[:self.max_composite_index_columns]
|
||||
|
||||
# Order columns by selectivity (most selective first)
|
||||
column_selectivities = []
|
||||
for col in columns:
|
||||
col_obj = self.tables[table_name][col]
|
||||
selectivity = self.selectivity_estimator.estimate_selectivity(col_obj)
|
||||
column_selectivities.append((col, selectivity))
|
||||
|
||||
# Sort by selectivity descending
|
||||
column_selectivities.sort(key=lambda x: x[1], reverse=True)
|
||||
ordered_columns = [col for col, _ in column_selectivities]
|
||||
|
||||
# Calculate combined selectivity
|
||||
combined_selectivity = min(sum(sel for _, sel in column_selectivities) / len(columns), 0.95)
|
||||
|
||||
index_name = f"idx_{table_name}_{'_'.join(ordered_columns)}"
|
||||
if len(index_name) > 63: # PostgreSQL limit
|
||||
index_name = f"idx_{table_name}_composite_{abs(hash('_'.join(ordered_columns))) % 10000}"
|
||||
|
||||
index = Index(
|
||||
name=index_name,
|
||||
table=table_name,
|
||||
columns=ordered_columns,
|
||||
index_type="btree"
|
||||
)
|
||||
|
||||
reason = f"Optimize multi-column WHERE conditions: {', '.join(ordered_columns)}"
|
||||
|
||||
return IndexRecommendation(
|
||||
recommendation_id=self._generate_recommendation_id(table_name, ordered_columns),
|
||||
table=table_name,
|
||||
recommended_index=index,
|
||||
reason=reason,
|
||||
query_patterns_helped=[pattern.query_id],
|
||||
estimated_benefit=self._estimate_benefit(combined_selectivity, pattern.frequency),
|
||||
estimated_overhead=f"Medium (composite index with {len(ordered_columns)} columns)",
|
||||
priority=self._calculate_priority(combined_selectivity, pattern.frequency, len(ordered_columns)),
|
||||
sql_statement=f"CREATE INDEX {index_name} ON {table_name} ({', '.join(ordered_columns)});",
|
||||
selectivity_analysis={
|
||||
"column_selectivities": {col: sel for col, sel in column_selectivities},
|
||||
"combined_selectivity": combined_selectivity,
|
||||
"column_order_rationale": "Ordered by selectivity (most selective first)"
|
||||
}
|
||||
)
|
||||
|
||||
def _create_covering_index_recommendation(
|
||||
self,
|
||||
table_name: str,
|
||||
where_columns: List[str],
|
||||
pattern: QueryPattern
|
||||
) -> Optional[IndexRecommendation]:
|
||||
"""Create recommendation for covering index."""
|
||||
order_columns = [col['column'] for col in pattern.order_by if col['column'] in self.tables[table_name]]
|
||||
|
||||
# Combine WHERE and ORDER BY columns
|
||||
index_columns = where_columns.copy()
|
||||
include_columns = []
|
||||
|
||||
# Add ORDER BY columns to index columns
|
||||
for col in order_columns:
|
||||
if col not in index_columns:
|
||||
index_columns.append(col)
|
||||
|
||||
# Limit index columns
|
||||
if len(index_columns) > self.max_composite_index_columns:
|
||||
include_columns = index_columns[self.max_composite_index_columns:]
|
||||
index_columns = index_columns[:self.max_composite_index_columns]
|
||||
|
||||
index_name = f"idx_{table_name}_covering_{'_'.join(index_columns[:3])}"
|
||||
if len(index_name) > 63:
|
||||
index_name = f"idx_{table_name}_covering_{abs(hash('_'.join(index_columns))) % 10000}"
|
||||
|
||||
index = Index(
|
||||
name=index_name,
|
||||
table=table_name,
|
||||
columns=index_columns,
|
||||
include_columns=include_columns,
|
||||
index_type="btree"
|
||||
)
|
||||
|
||||
reason = f"Covering index for WHERE + ORDER BY optimization"
|
||||
|
||||
# Calculate selectivity for main columns
|
||||
main_selectivity = 0.5 # Default for covering indexes
|
||||
if where_columns:
|
||||
selectivities = [
|
||||
self.selectivity_estimator.estimate_selectivity(self.tables[table_name][col])
|
||||
for col in where_columns[:2] # Consider first 2 columns
|
||||
]
|
||||
main_selectivity = max(selectivities)
|
||||
|
||||
sql_parts = [f"CREATE INDEX {index_name} ON {table_name} ({', '.join(index_columns)})"]
|
||||
if include_columns:
|
||||
sql_parts.append(f" INCLUDE ({', '.join(include_columns)})")
|
||||
sql_statement = ''.join(sql_parts) + ";"
|
||||
|
||||
return IndexRecommendation(
|
||||
recommendation_id=self._generate_recommendation_id(table_name, index_columns, "covering"),
|
||||
table=table_name,
|
||||
recommended_index=index,
|
||||
reason=reason,
|
||||
query_patterns_helped=[pattern.query_id],
|
||||
estimated_benefit="High (eliminates table lookups for SELECT)",
|
||||
estimated_overhead=f"High (covering index with {len(index_columns)} columns)",
|
||||
priority=self._calculate_priority(main_selectivity, pattern.frequency, len(index_columns)),
|
||||
sql_statement=sql_statement,
|
||||
selectivity_analysis={
|
||||
"main_columns_selectivity": main_selectivity,
|
||||
"covering_benefit": "Eliminates table lookup for SELECT queries"
|
||||
}
|
||||
)
|
||||
|
||||
def _create_join_index_recommendation(
|
||||
self,
|
||||
table_name: str,
|
||||
column: str,
|
||||
pattern: QueryPattern,
|
||||
join_condition: Dict[str, Any]
|
||||
) -> Optional[IndexRecommendation]:
|
||||
"""Create recommendation for JOIN optimization index."""
|
||||
column_obj = self.tables[table_name][column]
|
||||
selectivity = self.selectivity_estimator.estimate_selectivity(column_obj)
|
||||
|
||||
index_name = f"idx_{table_name}_{column}_join"
|
||||
index = Index(
|
||||
name=index_name,
|
||||
table=table_name,
|
||||
columns=[column],
|
||||
index_type="btree"
|
||||
)
|
||||
|
||||
foreign_table = join_condition.get('foreign_table', 'unknown')
|
||||
reason = f"Optimize JOIN with {foreign_table} table on {column}"
|
||||
|
||||
return IndexRecommendation(
|
||||
recommendation_id=self._generate_recommendation_id(table_name, [column], "join"),
|
||||
table=table_name,
|
||||
recommended_index=index,
|
||||
reason=reason,
|
||||
query_patterns_helped=[pattern.query_id],
|
||||
estimated_benefit=self._estimate_join_benefit(pattern.frequency),
|
||||
estimated_overhead="Low (single column for JOIN)",
|
||||
priority=2, # JOINs are generally high priority
|
||||
sql_statement=f"CREATE INDEX {index_name} ON {table_name} ({column});",
|
||||
selectivity_analysis={
|
||||
"column_selectivity": selectivity,
|
||||
"join_optimization": True
|
||||
}
|
||||
)
|
||||
|
||||
def _generate_recommendation_id(self, table: str, columns: List[str], suffix: str = "") -> str:
|
||||
"""Generate unique recommendation ID."""
|
||||
content = f"{table}_{'_'.join(sorted(columns))}_{suffix}"
|
||||
return hashlib.md5(content.encode()).hexdigest()[:8]
|
||||
|
||||
def _estimate_benefit(self, selectivity: float, frequency: int) -> str:
|
||||
"""Estimate performance benefit of index."""
|
||||
if selectivity > 0.8 and frequency > 50:
|
||||
return "Very High"
|
||||
elif selectivity > 0.6 and frequency > 20:
|
||||
return "High"
|
||||
elif selectivity > 0.4 or frequency > 10:
|
||||
return "Medium"
|
||||
else:
|
||||
return "Low"
|
||||
|
||||
def _estimate_join_benefit(self, frequency: int) -> str:
|
||||
"""Estimate benefit for JOIN indexes."""
|
||||
if frequency > 50:
|
||||
return "Very High (frequent JOINs)"
|
||||
elif frequency > 20:
|
||||
return "High (regular JOINs)"
|
||||
elif frequency > 5:
|
||||
return "Medium (occasional JOINs)"
|
||||
else:
|
||||
return "Low (rare JOINs)"
|
||||
|
||||
def _calculate_priority(self, selectivity: float, frequency: int, column_count: int) -> int:
|
||||
"""Calculate priority score (1 = highest priority)."""
|
||||
# Base score calculation
|
||||
score = 0
|
||||
|
||||
# Selectivity contribution (0-50 points)
|
||||
score += int(selectivity * 50)
|
||||
|
||||
# Frequency contribution (0-30 points)
|
||||
score += min(frequency, 30)
|
||||
|
||||
# Penalty for complex indexes (subtract points)
|
||||
score -= (column_count - 1) * 5
|
||||
|
||||
# Convert to priority levels
|
||||
if score >= 70:
|
||||
return 1 # Highest
|
||||
elif score >= 50:
|
||||
return 2 # High
|
||||
elif score >= 30:
|
||||
return 3 # Medium
|
||||
else:
|
||||
return 4 # Low
|
||||
|
||||
def _deduplicate_recommendations(self, recommendations: List[IndexRecommendation]) -> List[IndexRecommendation]:
|
||||
"""Remove duplicate recommendations."""
|
||||
seen_indexes = set()
|
||||
unique_recommendations = []
|
||||
|
||||
for rec in recommendations:
|
||||
index_signature = (rec.table, tuple(rec.recommended_index.columns))
|
||||
if index_signature not in seen_indexes:
|
||||
seen_indexes.add(index_signature)
|
||||
unique_recommendations.append(rec)
|
||||
else:
|
||||
# Merge query patterns helped
|
||||
for existing_rec in unique_recommendations:
|
||||
if (existing_rec.table == rec.table and
|
||||
existing_rec.recommended_index.columns == rec.recommended_index.columns):
|
||||
existing_rec.query_patterns_helped.extend(rec.query_patterns_helped)
|
||||
break
|
||||
|
||||
return unique_recommendations
|
||||
|
||||
def _prioritize_recommendations(self, recommendations: List[IndexRecommendation]) -> List[IndexRecommendation]:
|
||||
"""Sort recommendations by priority."""
|
||||
return sorted(recommendations, key=lambda x: (x.priority, -len(x.query_patterns_helped)))
|
||||
|
||||
def analyze_redundant_indexes(self) -> List[RedundancyIssue]:
|
||||
"""Identify redundant, overlapping, and potentially unused indexes."""
|
||||
redundancy_issues = []
|
||||
|
||||
for table_name, indexes in self.existing_indexes.items():
|
||||
if len(indexes) < 2:
|
||||
continue
|
||||
|
||||
# Find duplicate indexes
|
||||
duplicates = self._find_duplicate_indexes(table_name, indexes)
|
||||
redundancy_issues.extend(duplicates)
|
||||
|
||||
# Find overlapping indexes
|
||||
overlapping = self._find_overlapping_indexes(table_name, indexes)
|
||||
redundancy_issues.extend(overlapping)
|
||||
|
||||
# Find potentially unused indexes
|
||||
unused = self._find_unused_indexes(table_name, indexes)
|
||||
redundancy_issues.extend(unused)
|
||||
|
||||
return redundancy_issues
|
||||
|
||||
def _find_duplicate_indexes(self, table_name: str, indexes: List[Index]) -> List[RedundancyIssue]:
|
||||
"""Find exactly duplicate indexes."""
|
||||
issues = []
|
||||
seen_signatures = {}
|
||||
|
||||
for index in indexes:
|
||||
signature = (tuple(index.columns), index.unique, index.partial_condition)
|
||||
if signature in seen_signatures:
|
||||
existing_index = seen_signatures[signature]
|
||||
issues.append(RedundancyIssue(
|
||||
issue_type="DUPLICATE",
|
||||
affected_indexes=[existing_index.name, index.name],
|
||||
table=table_name,
|
||||
description=f"Indexes '{existing_index.name}' and '{index.name}' are identical",
|
||||
recommendation=f"Drop one of the duplicate indexes",
|
||||
sql_statements=[f"DROP INDEX {index.name};"]
|
||||
))
|
||||
else:
|
||||
seen_signatures[signature] = index
|
||||
|
||||
return issues
|
||||
|
||||
def _find_overlapping_indexes(self, table_name: str, indexes: List[Index]) -> List[RedundancyIssue]:
|
||||
"""Find overlapping indexes that might be redundant."""
|
||||
issues = []
|
||||
|
||||
for i, index1 in enumerate(indexes):
|
||||
for index2 in indexes[i+1:]:
|
||||
overlap_ratio = self._calculate_overlap_ratio(index1, index2)
|
||||
|
||||
if overlap_ratio >= self.redundancy_overlap_threshold:
|
||||
# Determine which index to keep
|
||||
if len(index1.columns) <= len(index2.columns):
|
||||
redundant_index = index1
|
||||
keep_index = index2
|
||||
else:
|
||||
redundant_index = index2
|
||||
keep_index = index1
|
||||
|
||||
issues.append(RedundancyIssue(
|
||||
issue_type="OVERLAPPING",
|
||||
affected_indexes=[index1.name, index2.name],
|
||||
table=table_name,
|
||||
description=f"Index '{redundant_index.name}' overlaps {int(overlap_ratio * 100)}% "
|
||||
f"with '{keep_index.name}'",
|
||||
recommendation=f"Consider dropping '{redundant_index.name}' as it's largely "
|
||||
f"covered by '{keep_index.name}'",
|
||||
sql_statements=[f"DROP INDEX {redundant_index.name};"]
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
def _calculate_overlap_ratio(self, index1: Index, index2: Index) -> float:
|
||||
"""Calculate overlap ratio between two indexes."""
|
||||
cols1 = set(index1.columns)
|
||||
cols2 = set(index2.columns)
|
||||
|
||||
if not cols1 or not cols2:
|
||||
return 0.0
|
||||
|
||||
intersection = len(cols1.intersection(cols2))
|
||||
union = len(cols1.union(cols2))
|
||||
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
def _find_unused_indexes(self, table_name: str, indexes: List[Index]) -> List[RedundancyIssue]:
|
||||
"""Find potentially unused indexes based on query patterns."""
|
||||
issues = []
|
||||
|
||||
# Collect all columns used in query patterns for this table
|
||||
used_columns = set()
|
||||
table_patterns = [p for p in self.query_patterns if p.table == table_name]
|
||||
|
||||
for pattern in table_patterns:
|
||||
# Add WHERE condition columns
|
||||
for condition in pattern.where_conditions:
|
||||
if condition.get('column'):
|
||||
used_columns.add(condition['column'])
|
||||
|
||||
# Add JOIN columns
|
||||
for join in pattern.join_conditions:
|
||||
if join.get('local_column'):
|
||||
used_columns.add(join['local_column'])
|
||||
|
||||
# Add ORDER BY columns
|
||||
for order in pattern.order_by:
|
||||
if order.get('column'):
|
||||
used_columns.add(order['column'])
|
||||
|
||||
# Add GROUP BY columns
|
||||
used_columns.update(pattern.group_by)
|
||||
|
||||
if not used_columns:
|
||||
return issues # Can't determine usage without query patterns
|
||||
|
||||
for index in indexes:
|
||||
index_columns = set(index.columns)
|
||||
if not index_columns.intersection(used_columns):
|
||||
issues.append(RedundancyIssue(
|
||||
issue_type="UNUSED",
|
||||
affected_indexes=[index.name],
|
||||
table=table_name,
|
||||
description=f"Index '{index.name}' columns {index.columns} are not used in any query patterns",
|
||||
recommendation="Consider dropping this index if it's truly unused (verify with query logs)",
|
||||
sql_statements=[f"-- Review usage before dropping\n-- DROP INDEX {index.name};"]
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
def estimate_index_sizes(self) -> Dict[str, Dict[str, Any]]:
|
||||
"""Estimate storage requirements for recommended indexes."""
|
||||
size_estimates = {}
|
||||
|
||||
# This is a simplified estimation - in practice, would need actual table statistics
|
||||
for table_name in self.tables:
|
||||
size_estimates[table_name] = {
|
||||
"estimated_table_rows": 10000, # Default estimate
|
||||
"existing_indexes_size_mb": len(self.existing_indexes.get(table_name, [])) * 5, # Rough estimate
|
||||
"index_overhead_per_column_mb": 2 # Rough estimate per column
|
||||
}
|
||||
|
||||
return size_estimates
|
||||
|
||||
def generate_analysis_report(self) -> Dict[str, Any]:
|
||||
"""Generate comprehensive analysis report."""
|
||||
recommendations = self.analyze_missing_indexes()
|
||||
redundancy_issues = self.analyze_redundant_indexes()
|
||||
size_estimates = self.estimate_index_sizes()
|
||||
|
||||
# Calculate statistics
|
||||
total_existing_indexes = sum(len(indexes) for indexes in self.existing_indexes.values())
|
||||
tables_analyzed = len(self.tables)
|
||||
query_patterns_analyzed = len(self.query_patterns)
|
||||
|
||||
# Categorize recommendations by priority
|
||||
high_priority = [r for r in recommendations if r.priority <= 2]
|
||||
medium_priority = [r for r in recommendations if r.priority == 3]
|
||||
low_priority = [r for r in recommendations if r.priority >= 4]
|
||||
|
||||
return {
|
||||
"analysis_summary": {
|
||||
"tables_analyzed": tables_analyzed,
|
||||
"query_patterns_analyzed": query_patterns_analyzed,
|
||||
"existing_indexes": total_existing_indexes,
|
||||
"total_recommendations": len(recommendations),
|
||||
"high_priority_recommendations": len(high_priority),
|
||||
"redundancy_issues_found": len(redundancy_issues)
|
||||
},
|
||||
"index_recommendations": {
|
||||
"high_priority": [asdict(r) for r in high_priority],
|
||||
"medium_priority": [asdict(r) for r in medium_priority],
|
||||
"low_priority": [asdict(r) for r in low_priority]
|
||||
},
|
||||
"redundancy_analysis": [asdict(issue) for issue in redundancy_issues],
|
||||
"size_estimates": size_estimates,
|
||||
"sql_statements": {
|
||||
"create_indexes": [rec.sql_statement for rec in recommendations],
|
||||
"drop_redundant": [
|
||||
stmt for issue in redundancy_issues
|
||||
for stmt in issue.sql_statements
|
||||
]
|
||||
},
|
||||
"performance_impact": self._generate_performance_impact_analysis(recommendations)
|
||||
}
|
||||
|
||||
def _generate_performance_impact_analysis(self, recommendations: List[IndexRecommendation]) -> Dict[str, Any]:
|
||||
"""Generate performance impact analysis."""
|
||||
impact_analysis = {
|
||||
"query_optimization": {},
|
||||
"write_overhead": {},
|
||||
"storage_impact": {}
|
||||
}
|
||||
|
||||
# Analyze query optimization impact
|
||||
query_benefits = defaultdict(list)
|
||||
for rec in recommendations:
|
||||
for query_id in rec.query_patterns_helped:
|
||||
query_benefits[query_id].append(rec.estimated_benefit)
|
||||
|
||||
impact_analysis["query_optimization"] = {
|
||||
"queries_improved": len(query_benefits),
|
||||
"high_impact_queries": len([q for q, benefits in query_benefits.items()
|
||||
if any("High" in benefit for benefit in benefits)]),
|
||||
"benefit_distribution": dict(Counter(
|
||||
rec.estimated_benefit for rec in recommendations
|
||||
))
|
||||
}
|
||||
|
||||
# Analyze write overhead
|
||||
impact_analysis["write_overhead"] = {
|
||||
"total_new_indexes": len(recommendations),
|
||||
"estimated_insert_overhead": f"{len(recommendations) * 5}%", # Rough estimate
|
||||
"tables_most_affected": list(Counter(rec.table for rec in recommendations).most_common(3))
|
||||
}
|
||||
|
||||
return impact_analysis
|
||||
|
||||
def format_text_report(self, analysis: Dict[str, Any]) -> str:
|
||||
"""Format analysis as human-readable text report."""
|
||||
lines = []
|
||||
lines.append("DATABASE INDEX OPTIMIZATION REPORT")
|
||||
lines.append("=" * 50)
|
||||
lines.append("")
|
||||
|
||||
# Summary
|
||||
summary = analysis["analysis_summary"]
|
||||
lines.append("ANALYSIS SUMMARY")
|
||||
lines.append("-" * 16)
|
||||
lines.append(f"Tables Analyzed: {summary['tables_analyzed']}")
|
||||
lines.append(f"Query Patterns: {summary['query_patterns_analyzed']}")
|
||||
lines.append(f"Existing Indexes: {summary['existing_indexes']}")
|
||||
lines.append(f"New Recommendations: {summary['total_recommendations']}")
|
||||
lines.append(f"High Priority: {summary['high_priority_recommendations']}")
|
||||
lines.append(f"Redundancy Issues: {summary['redundancy_issues_found']}")
|
||||
lines.append("")
|
||||
|
||||
# High Priority Recommendations
|
||||
high_priority = analysis["index_recommendations"]["high_priority"]
|
||||
if high_priority:
|
||||
lines.append(f"HIGH PRIORITY RECOMMENDATIONS ({len(high_priority)})")
|
||||
lines.append("-" * 35)
|
||||
for i, rec in enumerate(high_priority[:10], 1): # Show top 10
|
||||
lines.append(f"{i}. {rec['table']}: {rec['reason']}")
|
||||
lines.append(f" Columns: {', '.join(rec['recommended_index']['columns'])}")
|
||||
lines.append(f" Benefit: {rec['estimated_benefit']}")
|
||||
lines.append(f" SQL: {rec['sql_statement']}")
|
||||
lines.append("")
|
||||
|
||||
# Redundancy Issues
|
||||
redundancy = analysis["redundancy_analysis"]
|
||||
if redundancy:
|
||||
lines.append(f"REDUNDANCY ISSUES ({len(redundancy)})")
|
||||
lines.append("-" * 20)
|
||||
for issue in redundancy[:5]: # Show first 5
|
||||
lines.append(f"• {issue['issue_type']}: {issue['description']}")
|
||||
lines.append(f" Recommendation: {issue['recommendation']}")
|
||||
if issue['sql_statements']:
|
||||
lines.append(f" SQL: {issue['sql_statements'][0]}")
|
||||
lines.append("")
|
||||
|
||||
# Performance Impact
|
||||
perf_impact = analysis["performance_impact"]
|
||||
lines.append("PERFORMANCE IMPACT ANALYSIS")
|
||||
lines.append("-" * 30)
|
||||
query_opt = perf_impact["query_optimization"]
|
||||
lines.append(f"Queries to be optimized: {query_opt['queries_improved']}")
|
||||
lines.append(f"High impact optimizations: {query_opt['high_impact_queries']}")
|
||||
|
||||
write_overhead = perf_impact["write_overhead"]
|
||||
lines.append(f"Estimated insert overhead: {write_overhead['estimated_insert_overhead']}")
|
||||
lines.append("")
|
||||
|
||||
# SQL Statements Summary
|
||||
sql_statements = analysis["sql_statements"]
|
||||
create_statements = sql_statements["create_indexes"]
|
||||
if create_statements:
|
||||
lines.append("RECOMMENDED CREATE INDEX STATEMENTS")
|
||||
lines.append("-" * 36)
|
||||
for i, stmt in enumerate(create_statements[:10], 1):
|
||||
lines.append(f"{i}. {stmt}")
|
||||
|
||||
if len(create_statements) > 10:
|
||||
lines.append(f"... and {len(create_statements) - 10} more")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Optimize database indexes based on schema and query patterns")
|
||||
parser.add_argument("--schema", "-s", required=True, help="Schema definition JSON file")
|
||||
parser.add_argument("--queries", "-q", required=True, help="Query patterns JSON file")
|
||||
parser.add_argument("--output", "-o", help="Output file (default: stdout)")
|
||||
parser.add_argument("--format", "-f", choices=["json", "text"], default="text",
|
||||
help="Output format")
|
||||
parser.add_argument("--analyze-existing", "-e", action="store_true",
|
||||
help="Include analysis of existing indexes")
|
||||
parser.add_argument("--min-priority", "-p", type=int, default=4,
|
||||
help="Minimum priority level to include (1=highest, 4=lowest)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Load schema
|
||||
with open(args.schema, 'r') as f:
|
||||
schema_data = json.load(f)
|
||||
|
||||
# Load queries
|
||||
with open(args.queries, 'r') as f:
|
||||
query_data = json.load(f)
|
||||
|
||||
# Initialize optimizer
|
||||
optimizer = IndexOptimizer()
|
||||
optimizer.load_schema(schema_data)
|
||||
optimizer.load_query_patterns(query_data)
|
||||
|
||||
# Generate analysis
|
||||
analysis = optimizer.generate_analysis_report()
|
||||
|
||||
# Filter by priority if specified
|
||||
if args.min_priority < 4:
|
||||
for priority_level in ["high_priority", "medium_priority", "low_priority"]:
|
||||
analysis["index_recommendations"][priority_level] = [
|
||||
rec for rec in analysis["index_recommendations"][priority_level]
|
||||
if rec["priority"] <= args.min_priority
|
||||
]
|
||||
|
||||
# Format output
|
||||
if args.format == "json":
|
||||
output = json.dumps(analysis, indent=2)
|
||||
else:
|
||||
output = optimizer.format_text_report(analysis)
|
||||
|
||||
# Write output
|
||||
if args.output:
|
||||
with open(args.output, 'w') as f:
|
||||
f.write(output)
|
||||
else:
|
||||
print(output)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
1199
engineering/database-designer/migration_generator.py
Normal file
1199
engineering/database-designer/migration_generator.py
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,373 @@
|
||||
# Database Selection Decision Tree
|
||||
|
||||
## Overview
|
||||
|
||||
Choosing the right database technology is crucial for application success. This guide provides a systematic approach to database selection based on specific requirements, data patterns, and operational constraints.
|
||||
|
||||
## Decision Framework
|
||||
|
||||
### Primary Questions
|
||||
|
||||
1. **What is your primary use case?**
|
||||
- OLTP (Online Transaction Processing)
|
||||
- OLAP (Online Analytical Processing)
|
||||
- Real-time analytics
|
||||
- Content management
|
||||
- Search and discovery
|
||||
- Time-series data
|
||||
- Graph relationships
|
||||
|
||||
2. **What are your consistency requirements?**
|
||||
- Strong consistency (ACID)
|
||||
- Eventual consistency
|
||||
- Causal consistency
|
||||
- Session consistency
|
||||
|
||||
3. **What are your scalability needs?**
|
||||
- Vertical scaling sufficient
|
||||
- Horizontal scaling required
|
||||
- Global distribution needed
|
||||
- Multi-region requirements
|
||||
|
||||
4. **What is your data structure?**
|
||||
- Structured (relational)
|
||||
- Semi-structured (JSON/XML)
|
||||
- Unstructured (documents, media)
|
||||
- Graph relationships
|
||||
- Time-series data
|
||||
- Key-value pairs
|
||||
|
||||
## Decision Tree
|
||||
|
||||
```
|
||||
START: What is your primary use case?
|
||||
│
|
||||
├── OLTP (Transactional Applications)
|
||||
│ │
|
||||
│ ├── Do you need strong ACID guarantees?
|
||||
│ │ ├── YES → Do you need horizontal scaling?
|
||||
│ │ │ ├── YES → Distributed SQL
|
||||
│ │ │ │ ├── CockroachDB (Global, multi-region)
|
||||
│ │ │ │ ├── TiDB (MySQL compatibility)
|
||||
│ │ │ │ └── Spanner (Google Cloud)
|
||||
│ │ │ └── NO → Traditional SQL
|
||||
│ │ │ ├── PostgreSQL (Feature-rich, extensions)
|
||||
│ │ │ ├── MySQL (Performance, ecosystem)
|
||||
│ │ │ └── SQL Server (Microsoft stack)
|
||||
│ │ └── NO → Are you primarily key-value access?
|
||||
│ │ ├── YES → Key-Value Stores
|
||||
│ │ │ ├── Redis (In-memory, caching)
|
||||
│ │ │ ├── DynamoDB (AWS managed)
|
||||
│ │ │ └── Cassandra (High availability)
|
||||
│ │ └── NO → Document Stores
|
||||
│ │ ├── MongoDB (General purpose)
|
||||
│ │ ├── CouchDB (Sync, replication)
|
||||
│ │ └── Amazon DocumentDB (MongoDB compatible)
|
||||
│ │
|
||||
├── OLAP (Analytics and Reporting)
|
||||
│ │
|
||||
│ ├── What is your data volume?
|
||||
│ │ ├── Small to Medium (< 1TB) → Traditional SQL with optimization
|
||||
│ │ │ ├── PostgreSQL with columnar extensions
|
||||
│ │ │ ├── MySQL with analytics engine
|
||||
│ │ │ └── SQL Server with columnstore
|
||||
│ │ ├── Large (1TB - 100TB) → Data Warehouse Solutions
|
||||
│ │ │ ├── Snowflake (Cloud-native)
|
||||
│ │ │ ├── BigQuery (Google Cloud)
|
||||
│ │ │ ├── Redshift (AWS)
|
||||
│ │ │ └── Synapse (Azure)
|
||||
│ │ └── Very Large (> 100TB) → Big Data Platforms
|
||||
│ │ ├── Databricks (Unified analytics)
|
||||
│ │ ├── Apache Spark on cloud
|
||||
│ │ └── Hadoop ecosystem
|
||||
│ │
|
||||
├── Real-time Analytics
|
||||
│ │
|
||||
│ ├── Do you need sub-second query responses?
|
||||
│ │ ├── YES → Stream Processing + OLAP
|
||||
│ │ │ ├── ClickHouse (Fast analytics)
|
||||
│ │ │ ├── Apache Druid (Real-time OLAP)
|
||||
│ │ │ ├── Pinot (LinkedIn's real-time DB)
|
||||
│ │ │ └── TimescaleDB (Time-series)
|
||||
│ │ └── NO → Traditional OLAP solutions
|
||||
│ │
|
||||
├── Search and Discovery
|
||||
│ │
|
||||
│ ├── What type of search?
|
||||
│ │ ├── Full-text search → Search Engines
|
||||
│ │ │ ├── Elasticsearch (Full-featured)
|
||||
│ │ │ ├── OpenSearch (AWS fork of ES)
|
||||
│ │ │ └── Solr (Apache Lucene-based)
|
||||
│ │ ├── Vector/similarity search → Vector Databases
|
||||
│ │ │ ├── Pinecone (Managed vector DB)
|
||||
│ │ │ ├── Weaviate (Open source)
|
||||
│ │ │ ├── Chroma (Embeddings)
|
||||
│ │ │ └── PostgreSQL with pgvector
|
||||
│ │ └── Faceted search → Search + SQL combination
|
||||
│ │
|
||||
├── Graph Relationships
|
||||
│ │
|
||||
│ ├── Do you need complex graph traversals?
|
||||
│ │ ├── YES → Graph Databases
|
||||
│ │ │ ├── Neo4j (Property graph)
|
||||
│ │ │ ├── Amazon Neptune (Multi-model)
|
||||
│ │ │ ├── ArangoDB (Multi-model)
|
||||
│ │ │ └── TigerGraph (Analytics focused)
|
||||
│ │ └── NO → SQL with recursive queries
|
||||
│ │ └── PostgreSQL with recursive CTEs
|
||||
│ │
|
||||
└── Time-series Data
|
||||
│
|
||||
├── What is your write volume?
|
||||
├── High (millions/sec) → Specialized Time-series
|
||||
│ ├── InfluxDB (Purpose-built)
|
||||
│ ├── TimescaleDB (PostgreSQL extension)
|
||||
│ ├── Apache Druid (Analytics focused)
|
||||
│ └── Prometheus (Monitoring)
|
||||
└── Medium → SQL with time-series optimization
|
||||
└── PostgreSQL with partitioning
|
||||
```
|
||||
|
||||
## Database Categories Deep Dive
|
||||
|
||||
### Traditional SQL Databases
|
||||
|
||||
**PostgreSQL**
|
||||
- **Best For**: Complex queries, JSON data, extensions, geospatial
|
||||
- **Strengths**: Feature-rich, reliable, strong consistency, extensible
|
||||
- **Use Cases**: OLTP, mixed workloads, JSON documents, geospatial applications
|
||||
- **Scaling**: Vertical scaling, read replicas, partitioning
|
||||
- **When to Choose**: Need SQL features, complex queries, moderate scale
|
||||
|
||||
**MySQL**
|
||||
- **Best For**: Web applications, read-heavy workloads, simple schema
|
||||
- **Strengths**: Performance, replication, large ecosystem
|
||||
- **Use Cases**: Web apps, content management, e-commerce
|
||||
- **Scaling**: Read replicas, sharding, clustering (MySQL Cluster)
|
||||
- **When to Choose**: Simple schema, performance priority, large community
|
||||
|
||||
**SQL Server**
|
||||
- **Best For**: Microsoft ecosystem, enterprise features, business intelligence
|
||||
- **Strengths**: Integration, tooling, enterprise features
|
||||
- **Use Cases**: Enterprise applications, .NET applications, BI
|
||||
- **Scaling**: Always On availability groups, partitioning
|
||||
- **When to Choose**: Microsoft stack, enterprise requirements
|
||||
|
||||
### Distributed SQL (NewSQL)
|
||||
|
||||
**CockroachDB**
|
||||
- **Best For**: Global applications, strong consistency, horizontal scaling
|
||||
- **Strengths**: ACID guarantees, automatic scaling, survival
|
||||
- **Use Cases**: Multi-region apps, financial services, global SaaS
|
||||
- **Trade-offs**: Complex setup, higher latency for global transactions
|
||||
- **When to Choose**: Need SQL + global scale + consistency
|
||||
|
||||
**TiDB**
|
||||
- **Best For**: MySQL compatibility with horizontal scaling
|
||||
- **Strengths**: MySQL protocol, HTAP (hybrid), cloud-native
|
||||
- **Use Cases**: MySQL migrations, hybrid workloads
|
||||
- **When to Choose**: Existing MySQL expertise, need scale
|
||||
|
||||
### NoSQL Document Stores
|
||||
|
||||
**MongoDB**
|
||||
- **Best For**: Flexible schema, rapid development, document-centric data
|
||||
- **Strengths**: Developer experience, flexible schema, rich queries
|
||||
- **Use Cases**: Content management, catalogs, user profiles, IoT
|
||||
- **Scaling**: Automatic sharding, replica sets
|
||||
- **When to Choose**: Schema evolution, document structure, rapid development
|
||||
|
||||
**CouchDB**
|
||||
- **Best For**: Offline-first applications, multi-master replication
|
||||
- **Strengths**: HTTP API, replication, conflict resolution
|
||||
- **Use Cases**: Mobile apps, distributed systems, offline scenarios
|
||||
- **When to Choose**: Need offline capabilities, bi-directional sync
|
||||
|
||||
### Key-Value Stores
|
||||
|
||||
**Redis**
|
||||
- **Best For**: Caching, sessions, real-time applications, pub/sub
|
||||
- **Strengths**: Performance, data structures, persistence options
|
||||
- **Use Cases**: Caching, leaderboards, real-time analytics, queues
|
||||
- **Scaling**: Clustering, sentinel for HA
|
||||
- **When to Choose**: High performance, simple data model, caching
|
||||
|
||||
**DynamoDB**
|
||||
- **Best For**: Serverless applications, predictable performance, AWS ecosystem
|
||||
- **Strengths**: Managed, auto-scaling, consistent performance
|
||||
- **Use Cases**: Web applications, gaming, IoT, mobile backends
|
||||
- **Trade-offs**: Vendor lock-in, limited querying
|
||||
- **When to Choose**: AWS ecosystem, serverless, managed solution
|
||||
|
||||
### Column-Family Stores
|
||||
|
||||
**Cassandra**
|
||||
- **Best For**: Write-heavy workloads, high availability, linear scalability
|
||||
- **Strengths**: No single point of failure, tunable consistency
|
||||
- **Use Cases**: Time-series, IoT, messaging, activity feeds
|
||||
- **Trade-offs**: Complex operations, eventual consistency
|
||||
- **When to Choose**: High write volume, availability over consistency
|
||||
|
||||
**HBase**
|
||||
- **Best For**: Big data applications, Hadoop ecosystem
|
||||
- **Strengths**: Hadoop integration, consistent reads
|
||||
- **Use Cases**: Analytics on big data, time-series at scale
|
||||
- **When to Choose**: Hadoop ecosystem, very large datasets
|
||||
|
||||
### Graph Databases
|
||||
|
||||
**Neo4j**
|
||||
- **Best For**: Complex relationships, graph algorithms, traversals
|
||||
- **Strengths**: Mature ecosystem, Cypher query language, algorithms
|
||||
- **Use Cases**: Social networks, recommendation engines, fraud detection
|
||||
- **Trade-offs**: Specialized use case, learning curve
|
||||
- **When to Choose**: Relationship-heavy data, graph algorithms
|
||||
|
||||
### Time-Series Databases
|
||||
|
||||
**InfluxDB**
|
||||
- **Best For**: Time-series data, IoT, monitoring, analytics
|
||||
- **Strengths**: Purpose-built, efficient storage, query language
|
||||
- **Use Cases**: IoT sensors, monitoring, DevOps metrics
|
||||
- **When to Choose**: High-volume time-series data
|
||||
|
||||
**TimescaleDB**
|
||||
- **Best For**: Time-series with SQL familiarity
|
||||
- **Strengths**: PostgreSQL compatibility, SQL queries, ecosystem
|
||||
- **Use Cases**: Financial data, IoT with complex queries
|
||||
- **When to Choose**: Time-series + SQL requirements
|
||||
|
||||
### Search Engines
|
||||
|
||||
**Elasticsearch**
|
||||
- **Best For**: Full-text search, log analysis, real-time search
|
||||
- **Strengths**: Powerful search, analytics, ecosystem (ELK stack)
|
||||
- **Use Cases**: Search applications, log analysis, monitoring
|
||||
- **Trade-offs**: Complex operations, resource intensive
|
||||
- **When to Choose**: Advanced search requirements, analytics
|
||||
|
||||
### Data Warehouses
|
||||
|
||||
**Snowflake**
|
||||
- **Best For**: Cloud-native analytics, data sharing, varied workloads
|
||||
- **Strengths**: Separation of compute/storage, automatic scaling
|
||||
- **Use Cases**: Data warehousing, analytics, data science
|
||||
- **When to Choose**: Cloud-native, analytics-focused, multi-cloud
|
||||
|
||||
**BigQuery**
|
||||
- **Best For**: Serverless analytics, Google ecosystem, machine learning
|
||||
- **Strengths**: Serverless, petabyte scale, ML integration
|
||||
- **Use Cases**: Analytics, data science, reporting
|
||||
- **When to Choose**: Google Cloud, serverless analytics
|
||||
|
||||
## Selection Criteria Matrix
|
||||
|
||||
| Criterion | SQL | NewSQL | Document | Key-Value | Column-Family | Graph | Time-Series |
|
||||
|-----------|-----|--------|----------|-----------|---------------|-------|-------------|
|
||||
| ACID Guarantees | ✅ Strong | ✅ Strong | ⚠️ Eventual | ⚠️ Eventual | ⚠️ Tunable | ⚠️ Varies | ⚠️ Varies |
|
||||
| Horizontal Scaling | ❌ Limited | ✅ Native | ✅ Native | ✅ Native | ✅ Native | ⚠️ Limited | ✅ Native |
|
||||
| Query Flexibility | ✅ High | ✅ High | ⚠️ Moderate | ❌ Low | ❌ Low | ✅ High | ⚠️ Specialized |
|
||||
| Schema Flexibility | ❌ Rigid | ❌ Rigid | ✅ High | ✅ High | ⚠️ Moderate | ✅ High | ⚠️ Structured |
|
||||
| Performance (Reads) | ⚠️ Good | ⚠️ Good | ✅ Excellent | ✅ Excellent | ✅ Excellent | ⚠️ Good | ✅ Excellent |
|
||||
| Performance (Writes) | ⚠️ Good | ⚠️ Good | ✅ Excellent | ✅ Excellent | ✅ Excellent | ⚠️ Good | ✅ Excellent |
|
||||
| Operational Complexity | ✅ Low | ❌ High | ⚠️ Moderate | ✅ Low | ❌ High | ⚠️ Moderate | ⚠️ Moderate |
|
||||
| Ecosystem Maturity | ✅ Mature | ⚠️ Growing | ✅ Mature | ✅ Mature | ✅ Mature | ✅ Mature | ⚠️ Growing |
|
||||
|
||||
## Decision Checklist
|
||||
|
||||
### Requirements Analysis
|
||||
- [ ] **Data Volume**: Current and projected data size
|
||||
- [ ] **Transaction Volume**: Reads per second, writes per second
|
||||
- [ ] **Consistency Requirements**: Strong vs eventual consistency needs
|
||||
- [ ] **Query Patterns**: Simple lookups vs complex analytics
|
||||
- [ ] **Schema Evolution**: How often does schema change?
|
||||
- [ ] **Geographic Distribution**: Single region vs global
|
||||
- [ ] **Availability Requirements**: Acceptable downtime
|
||||
- [ ] **Team Expertise**: Existing knowledge and learning curve
|
||||
- [ ] **Budget Constraints**: Licensing, infrastructure, operational costs
|
||||
- [ ] **Compliance Requirements**: Data residency, audit trails
|
||||
|
||||
### Technical Evaluation
|
||||
- [ ] **Performance Testing**: Benchmark with realistic data and queries
|
||||
- [ ] **Scalability Testing**: Test scaling limits and patterns
|
||||
- [ ] **Failure Scenarios**: Test backup, recovery, and failure handling
|
||||
- [ ] **Integration Testing**: APIs, connectors, ecosystem tools
|
||||
- [ ] **Migration Path**: How to migrate from current system
|
||||
- [ ] **Monitoring and Observability**: Available tooling and metrics
|
||||
|
||||
### Operational Considerations
|
||||
- [ ] **Management Complexity**: Setup, configuration, maintenance
|
||||
- [ ] **Backup and Recovery**: Built-in vs external tools
|
||||
- [ ] **Security Features**: Authentication, authorization, encryption
|
||||
- [ ] **Upgrade Path**: Version compatibility and upgrade process
|
||||
- [ ] **Support Options**: Community vs commercial support
|
||||
- [ ] **Lock-in Risk**: Portability and vendor independence
|
||||
|
||||
## Common Decision Patterns
|
||||
|
||||
### E-commerce Platform
|
||||
**Typical Choice**: PostgreSQL or MySQL
|
||||
- **Primary Data**: Product catalog, orders, users (structured)
|
||||
- **Query Patterns**: OLTP with some analytics
|
||||
- **Consistency**: Strong consistency for financial data
|
||||
- **Scale**: Moderate with read replicas
|
||||
- **Additional**: Redis for caching, Elasticsearch for product search
|
||||
|
||||
### IoT/Sensor Data Platform
|
||||
**Typical Choice**: TimescaleDB or InfluxDB
|
||||
- **Primary Data**: Time-series sensor readings
|
||||
- **Query Patterns**: Time-based aggregations, trend analysis
|
||||
- **Scale**: High write volume, moderate read volume
|
||||
- **Additional**: Kafka for ingestion, PostgreSQL for metadata
|
||||
|
||||
### Social Media Application
|
||||
**Typical Choice**: Combination approach
|
||||
- **User Profiles**: MongoDB (flexible schema)
|
||||
- **Relationships**: Neo4j (graph relationships)
|
||||
- **Activity Feeds**: Cassandra (high write volume)
|
||||
- **Search**: Elasticsearch (content discovery)
|
||||
- **Caching**: Redis (sessions, real-time data)
|
||||
|
||||
### Analytics Platform
|
||||
**Typical Choice**: Snowflake or BigQuery
|
||||
- **Primary Use**: Complex analytical queries
|
||||
- **Data Volume**: Large (TB to PB scale)
|
||||
- **Query Patterns**: Ad-hoc analytics, reporting
|
||||
- **Users**: Data analysts, data scientists
|
||||
- **Additional**: Data lake (S3/GCS) for raw data storage
|
||||
|
||||
### Global SaaS Application
|
||||
**Typical Choice**: CockroachDB or DynamoDB
|
||||
- **Requirements**: Multi-region, strong consistency
|
||||
- **Scale**: Global user base
|
||||
- **Compliance**: Data residency requirements
|
||||
- **Availability**: High availability across regions
|
||||
|
||||
## Migration Strategies
|
||||
|
||||
### From Monolithic to Distributed
|
||||
1. **Assessment**: Identify scaling bottlenecks
|
||||
2. **Data Partitioning**: Plan how to split data
|
||||
3. **Gradual Migration**: Move non-critical data first
|
||||
4. **Dual Writes**: Run both systems temporarily
|
||||
5. **Validation**: Verify data consistency
|
||||
6. **Cutover**: Switch reads and writes gradually
|
||||
|
||||
### Technology Stack Evolution
|
||||
1. **Start Simple**: Begin with PostgreSQL or MySQL
|
||||
2. **Identify Bottlenecks**: Monitor performance and scaling issues
|
||||
3. **Selective Scaling**: Move specific workloads to specialized databases
|
||||
4. **Polyglot Persistence**: Use multiple databases for different use cases
|
||||
5. **Service Boundaries**: Align database choice with service boundaries
|
||||
|
||||
## Conclusion
|
||||
|
||||
Database selection should be driven by:
|
||||
|
||||
1. **Specific Use Case Requirements**: Not all applications need the same database
|
||||
2. **Data Characteristics**: Structure, volume, and access patterns matter
|
||||
3. **Non-functional Requirements**: Consistency, availability, performance targets
|
||||
4. **Team and Organizational Factors**: Expertise, operational capacity, budget
|
||||
5. **Evolution Path**: How requirements and scale will change over time
|
||||
|
||||
The best database choice is often not a single technology, but a combination of databases that each excel at their specific use case within your application architecture.
|
||||
@@ -0,0 +1,424 @@
|
||||
# Index Strategy Patterns
|
||||
|
||||
## Overview
|
||||
|
||||
Database indexes are critical for query performance, but they come with trade-offs. This guide covers proven patterns for index design, optimization strategies, and common pitfalls to avoid.
|
||||
|
||||
## Index Types and Use Cases
|
||||
|
||||
### B-Tree Indexes (Default)
|
||||
|
||||
**Best For:**
|
||||
- Equality queries (`WHERE column = value`)
|
||||
- Range queries (`WHERE column BETWEEN x AND y`)
|
||||
- Sorting (`ORDER BY column`)
|
||||
- Pattern matching with leading wildcards (`WHERE column LIKE 'prefix%'`)
|
||||
|
||||
**Characteristics:**
|
||||
- Logarithmic lookup time O(log n)
|
||||
- Supports partial matches on composite indexes
|
||||
- Most versatile index type
|
||||
|
||||
**Example:**
|
||||
```sql
|
||||
-- Single column B-tree index
|
||||
CREATE INDEX idx_customers_email ON customers (email);
|
||||
|
||||
-- Composite B-tree index
|
||||
CREATE INDEX idx_orders_customer_date ON orders (customer_id, order_date);
|
||||
```
|
||||
|
||||
### Hash Indexes
|
||||
|
||||
**Best For:**
|
||||
- Exact equality matches only
|
||||
- High-cardinality columns
|
||||
- Primary key lookups
|
||||
|
||||
**Characteristics:**
|
||||
- Constant lookup time O(1) for exact matches
|
||||
- Cannot support range queries or sorting
|
||||
- Memory-efficient for equality operations
|
||||
|
||||
**Example:**
|
||||
```sql
|
||||
-- Hash index for exact lookups (PostgreSQL)
|
||||
CREATE INDEX idx_users_id_hash ON users USING HASH (user_id);
|
||||
```
|
||||
|
||||
### Partial Indexes
|
||||
|
||||
**Best For:**
|
||||
- Filtering on subset of data
|
||||
- Reducing index size and maintenance overhead
|
||||
- Query patterns that consistently use specific filters
|
||||
|
||||
**Example:**
|
||||
```sql
|
||||
-- Index only active users
|
||||
CREATE INDEX idx_active_users_email
|
||||
ON users (email)
|
||||
WHERE status = 'active';
|
||||
|
||||
-- Index recent orders only
|
||||
CREATE INDEX idx_recent_orders
|
||||
ON orders (customer_id, created_at)
|
||||
WHERE created_at > CURRENT_DATE - INTERVAL '90 days';
|
||||
|
||||
-- Index non-null values only
|
||||
CREATE INDEX idx_customers_phone
|
||||
ON customers (phone_number)
|
||||
WHERE phone_number IS NOT NULL;
|
||||
```
|
||||
|
||||
### Covering Indexes
|
||||
|
||||
**Best For:**
|
||||
- Eliminating table lookups for SELECT queries
|
||||
- Frequently accessed column combinations
|
||||
- Read-heavy workloads
|
||||
|
||||
**Example:**
|
||||
```sql
|
||||
-- Covering index with INCLUDE clause (SQL Server/PostgreSQL)
|
||||
CREATE INDEX idx_orders_customer_covering
|
||||
ON orders (customer_id, order_date)
|
||||
INCLUDE (order_total, status);
|
||||
|
||||
-- Query can be satisfied entirely from index:
|
||||
-- SELECT order_total, status FROM orders
|
||||
-- WHERE customer_id = 123 AND order_date > '2024-01-01';
|
||||
```
|
||||
|
||||
### Functional/Expression Indexes
|
||||
|
||||
**Best For:**
|
||||
- Queries on transformed column values
|
||||
- Case-insensitive searches
|
||||
- Complex calculations
|
||||
|
||||
**Example:**
|
||||
```sql
|
||||
-- Case-insensitive email searches
|
||||
CREATE INDEX idx_users_email_lower
|
||||
ON users (LOWER(email));
|
||||
|
||||
-- Date part extraction
|
||||
CREATE INDEX idx_orders_month
|
||||
ON orders (EXTRACT(MONTH FROM order_date));
|
||||
|
||||
-- JSON field indexing
|
||||
CREATE INDEX idx_users_preferences_theme
|
||||
ON users ((preferences->>'theme'));
|
||||
```
|
||||
|
||||
## Composite Index Design Patterns
|
||||
|
||||
### Column Ordering Strategy
|
||||
|
||||
**Rule: Most Selective First**
|
||||
```sql
|
||||
-- Query: WHERE status = 'active' AND city = 'New York' AND age > 25
|
||||
-- Assume: status has 3 values, city has 100 values, age has 80 values
|
||||
|
||||
-- GOOD: Most selective column first
|
||||
CREATE INDEX idx_users_city_age_status ON users (city, age, status);
|
||||
|
||||
-- BAD: Least selective first
|
||||
CREATE INDEX idx_users_status_city_age ON users (status, city, age);
|
||||
```
|
||||
|
||||
**Selectivity Calculation:**
|
||||
```sql
|
||||
-- Estimate selectivity for each column
|
||||
SELECT
|
||||
'status' as column_name,
|
||||
COUNT(DISTINCT status)::float / COUNT(*) as selectivity
|
||||
FROM users
|
||||
UNION ALL
|
||||
SELECT
|
||||
'city' as column_name,
|
||||
COUNT(DISTINCT city)::float / COUNT(*) as selectivity
|
||||
FROM users
|
||||
UNION ALL
|
||||
SELECT
|
||||
'age' as column_name,
|
||||
COUNT(DISTINCT age)::float / COUNT(*) as selectivity
|
||||
FROM users;
|
||||
```
|
||||
|
||||
### Query Pattern Matching
|
||||
|
||||
**Pattern 1: Equality + Range**
|
||||
```sql
|
||||
-- Query: WHERE customer_id = 123 AND order_date BETWEEN '2024-01-01' AND '2024-03-31'
|
||||
CREATE INDEX idx_orders_customer_date ON orders (customer_id, order_date);
|
||||
```
|
||||
|
||||
**Pattern 2: Multiple Equality Conditions**
|
||||
```sql
|
||||
-- Query: WHERE status = 'active' AND category = 'premium' AND region = 'US'
|
||||
CREATE INDEX idx_users_status_category_region ON users (status, category, region);
|
||||
```
|
||||
|
||||
**Pattern 3: Equality + Sorting**
|
||||
```sql
|
||||
-- Query: WHERE category = 'electronics' ORDER BY price DESC, created_at DESC
|
||||
CREATE INDEX idx_products_category_price_date ON products (category, price DESC, created_at DESC);
|
||||
```
|
||||
|
||||
### Prefix Optimization
|
||||
|
||||
**Efficient Prefix Usage:**
|
||||
```sql
|
||||
-- Index supports all these queries efficiently:
|
||||
CREATE INDEX idx_users_lastname_firstname_email ON users (last_name, first_name, email);
|
||||
|
||||
-- ✓ Uses index: WHERE last_name = 'Smith'
|
||||
-- ✓ Uses index: WHERE last_name = 'Smith' AND first_name = 'John'
|
||||
-- ✓ Uses index: WHERE last_name = 'Smith' AND first_name = 'John' AND email = 'john@...'
|
||||
-- ✗ Cannot use index: WHERE first_name = 'John'
|
||||
-- ✗ Cannot use index: WHERE email = 'john@...'
|
||||
```
|
||||
|
||||
## Performance Optimization Patterns
|
||||
|
||||
### Index Intersection vs Composite Indexes
|
||||
|
||||
**Scenario: Multiple single-column indexes**
|
||||
```sql
|
||||
CREATE INDEX idx_users_age ON users (age);
|
||||
CREATE INDEX idx_users_city ON users (city);
|
||||
CREATE INDEX idx_users_status ON users (status);
|
||||
|
||||
-- Query: WHERE age > 25 AND city = 'NYC' AND status = 'active'
|
||||
-- Database may use index intersection (combining multiple indexes)
|
||||
-- Performance varies by database engine and data distribution
|
||||
```
|
||||
|
||||
**Better: Purpose-built composite index**
|
||||
```sql
|
||||
-- More efficient for the specific query pattern
|
||||
CREATE INDEX idx_users_city_status_age ON users (city, status, age);
|
||||
```
|
||||
|
||||
### Index Size vs Performance Trade-off
|
||||
|
||||
**Wide Indexes (Many Columns):**
|
||||
```sql
|
||||
-- Pros: Covers many query patterns, excellent for covering queries
|
||||
-- Cons: Large index size, slower writes, more memory usage
|
||||
CREATE INDEX idx_orders_comprehensive
|
||||
ON orders (customer_id, order_date, status, total_amount, shipping_method, created_at)
|
||||
INCLUDE (order_notes, billing_address);
|
||||
```
|
||||
|
||||
**Narrow Indexes (Few Columns):**
|
||||
```sql
|
||||
-- Pros: Smaller size, faster writes, less memory
|
||||
-- Cons: May not cover all query patterns
|
||||
CREATE INDEX idx_orders_customer_date ON orders (customer_id, order_date);
|
||||
CREATE INDEX idx_orders_status ON orders (status);
|
||||
```
|
||||
|
||||
### Maintenance Optimization
|
||||
|
||||
**Regular Index Analysis:**
|
||||
```sql
|
||||
-- PostgreSQL: Check index usage statistics
|
||||
SELECT
|
||||
schemaname,
|
||||
tablename,
|
||||
indexname,
|
||||
idx_scan as index_scans,
|
||||
idx_tup_read as tuples_read,
|
||||
idx_tup_fetch as tuples_fetched
|
||||
FROM pg_stat_user_indexes
|
||||
WHERE idx_scan = 0 -- Potentially unused indexes
|
||||
ORDER BY schemaname, tablename;
|
||||
|
||||
-- Check index size
|
||||
SELECT
|
||||
indexname,
|
||||
pg_size_pretty(pg_relation_size(indexname::regclass)) as index_size
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY pg_relation_size(indexname::regclass) DESC;
|
||||
```
|
||||
|
||||
## Common Anti-Patterns
|
||||
|
||||
### 1. Over-Indexing
|
||||
|
||||
**Problem:**
|
||||
```sql
|
||||
-- Too many similar indexes
|
||||
CREATE INDEX idx_orders_customer ON orders (customer_id);
|
||||
CREATE INDEX idx_orders_customer_date ON orders (customer_id, order_date);
|
||||
CREATE INDEX idx_orders_customer_status ON orders (customer_id, status);
|
||||
CREATE INDEX idx_orders_customer_date_status ON orders (customer_id, order_date, status);
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```sql
|
||||
-- One well-designed composite index can often replace several
|
||||
CREATE INDEX idx_orders_customer_date_status ON orders (customer_id, order_date, status);
|
||||
-- Drop redundant indexes: idx_orders_customer, idx_orders_customer_date, idx_orders_customer_status
|
||||
```
|
||||
|
||||
### 2. Wrong Column Order
|
||||
|
||||
**Problem:**
|
||||
```sql
|
||||
-- Query: WHERE active = true AND user_type = 'premium' AND city = 'Chicago'
|
||||
-- Bad order: boolean first (lowest selectivity)
|
||||
CREATE INDEX idx_users_active_type_city ON users (active, user_type, city);
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```sql
|
||||
-- Good order: most selective first
|
||||
CREATE INDEX idx_users_city_type_active ON users (city, user_type, active);
|
||||
```
|
||||
|
||||
### 3. Ignoring Query Patterns
|
||||
|
||||
**Problem:**
|
||||
```sql
|
||||
-- Index doesn't match common query patterns
|
||||
CREATE INDEX idx_products_name ON products (product_name);
|
||||
|
||||
-- But queries are: WHERE category = 'electronics' AND price BETWEEN 100 AND 500
|
||||
-- Index is not helpful for these queries
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```sql
|
||||
-- Match actual query patterns
|
||||
CREATE INDEX idx_products_category_price ON products (category, price);
|
||||
```
|
||||
|
||||
### 4. Function in WHERE Without Functional Index
|
||||
|
||||
**Problem:**
|
||||
```sql
|
||||
-- Query uses function but no functional index
|
||||
SELECT * FROM users WHERE LOWER(email) = 'john@example.com';
|
||||
-- Regular index on email won't help
|
||||
```
|
||||
|
||||
**Solution:**
|
||||
```sql
|
||||
-- Create functional index
|
||||
CREATE INDEX idx_users_email_lower ON users (LOWER(email));
|
||||
```
|
||||
|
||||
## Advanced Patterns
|
||||
|
||||
### Multi-Column Statistics
|
||||
|
||||
**When Columns Are Correlated:**
|
||||
```sql
|
||||
-- If city and state are highly correlated, create extended statistics
|
||||
CREATE STATISTICS stats_address_correlation ON city, state FROM addresses;
|
||||
ANALYZE addresses;
|
||||
|
||||
-- Helps query planner make better decisions for:
|
||||
-- WHERE city = 'New York' AND state = 'NY'
|
||||
```
|
||||
|
||||
### Conditional Indexes for Data Lifecycle
|
||||
|
||||
**Pattern: Different indexes for different data ages**
|
||||
```sql
|
||||
-- Hot data (recent orders) - optimized for OLTP
|
||||
CREATE INDEX idx_orders_hot_customer_date
|
||||
ON orders (customer_id, order_date DESC)
|
||||
WHERE order_date > CURRENT_DATE - INTERVAL '30 days';
|
||||
|
||||
-- Warm data (older orders) - optimized for analytics
|
||||
CREATE INDEX idx_orders_warm_date_total
|
||||
ON orders (order_date, total_amount)
|
||||
WHERE order_date <= CURRENT_DATE - INTERVAL '30 days'
|
||||
AND order_date > CURRENT_DATE - INTERVAL '1 year';
|
||||
|
||||
-- Cold data (archived orders) - minimal indexing
|
||||
CREATE INDEX idx_orders_cold_date
|
||||
ON orders (order_date)
|
||||
WHERE order_date <= CURRENT_DATE - INTERVAL '1 year';
|
||||
```
|
||||
|
||||
### Index-Only Scan Optimization
|
||||
|
||||
**Design indexes to avoid table access:**
|
||||
```sql
|
||||
-- Query: SELECT order_id, total_amount, status FROM orders WHERE customer_id = ?
|
||||
CREATE INDEX idx_orders_customer_covering
|
||||
ON orders (customer_id)
|
||||
INCLUDE (order_id, total_amount, status);
|
||||
|
||||
-- Or as composite index (if database doesn't support INCLUDE)
|
||||
CREATE INDEX idx_orders_customer_covering
|
||||
ON orders (customer_id, order_id, total_amount, status);
|
||||
```
|
||||
|
||||
## Index Monitoring and Maintenance
|
||||
|
||||
### Performance Monitoring Queries
|
||||
|
||||
**Find slow queries that might benefit from indexes:**
|
||||
```sql
|
||||
-- PostgreSQL: Find queries with high cost
|
||||
SELECT
|
||||
query,
|
||||
calls,
|
||||
total_time,
|
||||
mean_time,
|
||||
rows
|
||||
FROM pg_stat_statements
|
||||
WHERE mean_time > 1000 -- Queries taking > 1 second
|
||||
ORDER BY mean_time DESC;
|
||||
```
|
||||
|
||||
**Identify missing indexes:**
|
||||
```sql
|
||||
-- Look for sequential scans on large tables
|
||||
SELECT
|
||||
schemaname,
|
||||
tablename,
|
||||
seq_scan,
|
||||
seq_tup_read,
|
||||
idx_scan,
|
||||
n_tup_ins + n_tup_upd + n_tup_del as write_activity
|
||||
FROM pg_stat_user_tables
|
||||
WHERE seq_scan > 100
|
||||
AND seq_tup_read > 100000 -- Large sequential scans
|
||||
AND (idx_scan = 0 OR seq_scan > idx_scan * 2)
|
||||
ORDER BY seq_tup_read DESC;
|
||||
```
|
||||
|
||||
### Index Maintenance Schedule
|
||||
|
||||
**Regular Maintenance Tasks:**
|
||||
```sql
|
||||
-- Rebuild fragmented indexes (SQL Server)
|
||||
ALTER INDEX ALL ON orders REBUILD;
|
||||
|
||||
-- Update statistics (PostgreSQL)
|
||||
ANALYZE orders;
|
||||
|
||||
-- Check for unused indexes monthly
|
||||
SELECT * FROM pg_stat_user_indexes WHERE idx_scan = 0;
|
||||
```
|
||||
|
||||
## Conclusion
|
||||
|
||||
Effective index strategy requires:
|
||||
|
||||
1. **Understanding Query Patterns**: Analyze actual application queries, not theoretical scenarios
|
||||
2. **Measuring Performance**: Use query execution plans and timing to validate index effectiveness
|
||||
3. **Balancing Trade-offs**: More indexes improve reads but slow writes and increase storage
|
||||
4. **Regular Maintenance**: Monitor index usage and performance, remove unused indexes
|
||||
5. **Iterative Improvement**: Start with essential indexes, add and optimize based on real usage
|
||||
|
||||
The goal is not to index every possible query pattern, but to create a focused set of indexes that provide maximum benefit for your application's specific workload while minimizing maintenance overhead.
|
||||
354
engineering/database-designer/references/normalization_guide.md
Normal file
354
engineering/database-designer/references/normalization_guide.md
Normal file
@@ -0,0 +1,354 @@
|
||||
# Database Normalization Guide
|
||||
|
||||
## Overview
|
||||
|
||||
Database normalization is the process of organizing data to minimize redundancy and dependency issues. It involves decomposing tables to eliminate data anomalies and improve data integrity.
|
||||
|
||||
## Normal Forms
|
||||
|
||||
### First Normal Form (1NF)
|
||||
|
||||
**Requirements:**
|
||||
- Each column contains atomic (indivisible) values
|
||||
- Each column contains values of the same type
|
||||
- Each column has a unique name
|
||||
- The order of data storage doesn't matter
|
||||
|
||||
**Violations and Solutions:**
|
||||
|
||||
**Problem: Multiple values in single column**
|
||||
```sql
|
||||
-- BAD: Multiple phone numbers in one column
|
||||
CREATE TABLE customers (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100),
|
||||
phones VARCHAR(500) -- "555-1234, 555-5678, 555-9012"
|
||||
);
|
||||
|
||||
-- GOOD: Separate table for multiple phones
|
||||
CREATE TABLE customers (
|
||||
id INT PRIMARY KEY,
|
||||
name VARCHAR(100)
|
||||
);
|
||||
|
||||
CREATE TABLE customer_phones (
|
||||
id INT PRIMARY KEY,
|
||||
customer_id INT REFERENCES customers(id),
|
||||
phone VARCHAR(20),
|
||||
phone_type VARCHAR(10) -- 'mobile', 'home', 'work'
|
||||
);
|
||||
```
|
||||
|
||||
**Problem: Repeating groups**
|
||||
```sql
|
||||
-- BAD: Repeating column patterns
|
||||
CREATE TABLE orders (
|
||||
order_id INT PRIMARY KEY,
|
||||
customer_id INT,
|
||||
item1_name VARCHAR(100),
|
||||
item1_qty INT,
|
||||
item1_price DECIMAL(8,2),
|
||||
item2_name VARCHAR(100),
|
||||
item2_qty INT,
|
||||
item2_price DECIMAL(8,2),
|
||||
item3_name VARCHAR(100),
|
||||
item3_qty INT,
|
||||
item3_price DECIMAL(8,2)
|
||||
);
|
||||
|
||||
-- GOOD: Separate table for order items
|
||||
CREATE TABLE orders (
|
||||
order_id INT PRIMARY KEY,
|
||||
customer_id INT,
|
||||
order_date DATE
|
||||
);
|
||||
|
||||
CREATE TABLE order_items (
|
||||
id INT PRIMARY KEY,
|
||||
order_id INT REFERENCES orders(order_id),
|
||||
item_name VARCHAR(100),
|
||||
quantity INT,
|
||||
unit_price DECIMAL(8,2)
|
||||
);
|
||||
```
|
||||
|
||||
### Second Normal Form (2NF)
|
||||
|
||||
**Requirements:**
|
||||
- Must be in 1NF
|
||||
- All non-key attributes must be fully functionally dependent on the primary key
|
||||
- No partial dependencies (applies only to tables with composite primary keys)
|
||||
|
||||
**Violations and Solutions:**
|
||||
|
||||
**Problem: Partial dependency on composite key**
|
||||
```sql
|
||||
-- BAD: Student course enrollment with partial dependencies
|
||||
CREATE TABLE student_courses (
|
||||
student_id INT,
|
||||
course_id INT,
|
||||
student_name VARCHAR(100), -- Depends only on student_id
|
||||
student_major VARCHAR(50), -- Depends only on student_id
|
||||
course_title VARCHAR(200), -- Depends only on course_id
|
||||
course_credits INT, -- Depends only on course_id
|
||||
grade CHAR(2), -- Depends on both student_id AND course_id
|
||||
PRIMARY KEY (student_id, course_id)
|
||||
);
|
||||
|
||||
-- GOOD: Separate tables eliminate partial dependencies
|
||||
CREATE TABLE students (
|
||||
student_id INT PRIMARY KEY,
|
||||
student_name VARCHAR(100),
|
||||
student_major VARCHAR(50)
|
||||
);
|
||||
|
||||
CREATE TABLE courses (
|
||||
course_id INT PRIMARY KEY,
|
||||
course_title VARCHAR(200),
|
||||
course_credits INT
|
||||
);
|
||||
|
||||
CREATE TABLE enrollments (
|
||||
student_id INT,
|
||||
course_id INT,
|
||||
grade CHAR(2),
|
||||
enrollment_date DATE,
|
||||
PRIMARY KEY (student_id, course_id),
|
||||
FOREIGN KEY (student_id) REFERENCES students(student_id),
|
||||
FOREIGN KEY (course_id) REFERENCES courses(course_id)
|
||||
);
|
||||
```
|
||||
|
||||
### Third Normal Form (3NF)
|
||||
|
||||
**Requirements:**
|
||||
- Must be in 2NF
|
||||
- No transitive dependencies (non-key attributes should not depend on other non-key attributes)
|
||||
- All non-key attributes must depend directly on the primary key
|
||||
|
||||
**Violations and Solutions:**
|
||||
|
||||
**Problem: Transitive dependency**
|
||||
```sql
|
||||
-- BAD: Employee table with transitive dependency
|
||||
CREATE TABLE employees (
|
||||
employee_id INT PRIMARY KEY,
|
||||
employee_name VARCHAR(100),
|
||||
department_id INT,
|
||||
department_name VARCHAR(100), -- Depends on department_id, not employee_id
|
||||
department_location VARCHAR(100), -- Transitive dependency through department_id
|
||||
department_budget DECIMAL(10,2), -- Transitive dependency through department_id
|
||||
salary DECIMAL(8,2)
|
||||
);
|
||||
|
||||
-- GOOD: Separate department information
|
||||
CREATE TABLE departments (
|
||||
department_id INT PRIMARY KEY,
|
||||
department_name VARCHAR(100),
|
||||
department_location VARCHAR(100),
|
||||
department_budget DECIMAL(10,2)
|
||||
);
|
||||
|
||||
CREATE TABLE employees (
|
||||
employee_id INT PRIMARY KEY,
|
||||
employee_name VARCHAR(100),
|
||||
department_id INT,
|
||||
salary DECIMAL(8,2),
|
||||
FOREIGN KEY (department_id) REFERENCES departments(department_id)
|
||||
);
|
||||
```
|
||||
|
||||
### Boyce-Codd Normal Form (BCNF)
|
||||
|
||||
**Requirements:**
|
||||
- Must be in 3NF
|
||||
- Every determinant must be a candidate key
|
||||
- Stricter than 3NF - handles cases where 3NF doesn't eliminate all anomalies
|
||||
|
||||
**Violations and Solutions:**
|
||||
|
||||
**Problem: Determinant that's not a candidate key**
|
||||
```sql
|
||||
-- BAD: Student advisor relationship with BCNF violation
|
||||
-- Assumption: Each student has one advisor per subject,
|
||||
-- each advisor teaches only one subject, but can advise multiple students
|
||||
CREATE TABLE student_advisor (
|
||||
student_id INT,
|
||||
subject VARCHAR(50),
|
||||
advisor_id INT,
|
||||
PRIMARY KEY (student_id, subject)
|
||||
);
|
||||
-- Problem: advisor_id determines subject, but advisor_id is not a candidate key
|
||||
|
||||
-- GOOD: Separate the functional dependencies
|
||||
CREATE TABLE advisors (
|
||||
advisor_id INT PRIMARY KEY,
|
||||
subject VARCHAR(50)
|
||||
);
|
||||
|
||||
CREATE TABLE student_advisor_assignments (
|
||||
student_id INT,
|
||||
advisor_id INT,
|
||||
PRIMARY KEY (student_id, advisor_id),
|
||||
FOREIGN KEY (advisor_id) REFERENCES advisors(advisor_id)
|
||||
);
|
||||
```
|
||||
|
||||
## Denormalization Strategies
|
||||
|
||||
### When to Denormalize
|
||||
|
||||
1. **Performance Requirements**: When query performance is more critical than storage efficiency
|
||||
2. **Read-Heavy Workloads**: When data is read much more frequently than it's updated
|
||||
3. **Reporting Systems**: When complex joins negatively impact reporting performance
|
||||
4. **Caching Strategies**: When pre-computed values eliminate expensive calculations
|
||||
|
||||
### Common Denormalization Patterns
|
||||
|
||||
**1. Redundant Storage for Performance**
|
||||
```sql
|
||||
-- Store frequently accessed calculated values
|
||||
CREATE TABLE orders (
|
||||
order_id INT PRIMARY KEY,
|
||||
customer_id INT,
|
||||
order_total DECIMAL(10,2), -- Denormalized: sum of order_items.total
|
||||
item_count INT, -- Denormalized: count of order_items
|
||||
created_at TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE order_items (
|
||||
item_id INT PRIMARY KEY,
|
||||
order_id INT,
|
||||
product_id INT,
|
||||
quantity INT,
|
||||
unit_price DECIMAL(8,2),
|
||||
total DECIMAL(10,2) -- quantity * unit_price (denormalized)
|
||||
);
|
||||
```
|
||||
|
||||
**2. Materialized Aggregates**
|
||||
```sql
|
||||
-- Pre-computed summary tables for reporting
|
||||
CREATE TABLE monthly_sales_summary (
|
||||
year_month VARCHAR(7), -- '2024-03'
|
||||
product_category VARCHAR(50),
|
||||
total_sales DECIMAL(12,2),
|
||||
total_units INT,
|
||||
avg_order_value DECIMAL(8,2),
|
||||
unique_customers INT,
|
||||
updated_at TIMESTAMP
|
||||
);
|
||||
```
|
||||
|
||||
**3. Historical Data Snapshots**
|
||||
```sql
|
||||
-- Store historical state to avoid complex temporal queries
|
||||
CREATE TABLE customer_status_history (
|
||||
id INT PRIMARY KEY,
|
||||
customer_id INT,
|
||||
status VARCHAR(20),
|
||||
tier VARCHAR(10),
|
||||
total_lifetime_value DECIMAL(12,2), -- Snapshot at this point in time
|
||||
snapshot_date DATE
|
||||
);
|
||||
```
|
||||
|
||||
## Trade-offs Analysis
|
||||
|
||||
### Normalization Benefits
|
||||
- **Data Integrity**: Reduced risk of inconsistent data
|
||||
- **Storage Efficiency**: Less data duplication
|
||||
- **Update Efficiency**: Changes need to be made in only one place
|
||||
- **Flexibility**: Easier to modify schema as requirements change
|
||||
|
||||
### Normalization Costs
|
||||
- **Query Complexity**: More joins required for data retrieval
|
||||
- **Performance Impact**: Joins can be expensive on large datasets
|
||||
- **Development Complexity**: More complex data access patterns
|
||||
|
||||
### Denormalization Benefits
|
||||
- **Query Performance**: Fewer joins, faster queries
|
||||
- **Simplified Queries**: Direct access to related data
|
||||
- **Read Optimization**: Optimized for data retrieval patterns
|
||||
- **Reduced Load**: Less database processing for common operations
|
||||
|
||||
### Denormalization Costs
|
||||
- **Data Redundancy**: Increased storage requirements
|
||||
- **Update Complexity**: Multiple places may need updates
|
||||
- **Consistency Risk**: Higher risk of data inconsistencies
|
||||
- **Maintenance Overhead**: Additional code to maintain derived values
|
||||
|
||||
## Best Practices
|
||||
|
||||
### 1. Start with Full Normalization
|
||||
- Begin with a fully normalized design
|
||||
- Identify performance bottlenecks through testing
|
||||
- Selectively denormalize based on actual performance needs
|
||||
|
||||
### 2. Use Triggers for Consistency
|
||||
```sql
|
||||
-- Trigger to maintain denormalized order_total
|
||||
CREATE TRIGGER update_order_total
|
||||
AFTER INSERT OR UPDATE OR DELETE ON order_items
|
||||
FOR EACH ROW
|
||||
BEGIN
|
||||
UPDATE orders
|
||||
SET order_total = (
|
||||
SELECT SUM(quantity * unit_price)
|
||||
FROM order_items
|
||||
WHERE order_id = NEW.order_id
|
||||
)
|
||||
WHERE order_id = NEW.order_id;
|
||||
END;
|
||||
```
|
||||
|
||||
### 3. Consider Materialized Views
|
||||
```sql
|
||||
-- Materialized view for complex aggregations
|
||||
CREATE MATERIALIZED VIEW customer_summary AS
|
||||
SELECT
|
||||
c.customer_id,
|
||||
c.customer_name,
|
||||
COUNT(o.order_id) as order_count,
|
||||
SUM(o.order_total) as lifetime_value,
|
||||
AVG(o.order_total) as avg_order_value,
|
||||
MAX(o.created_at) as last_order_date
|
||||
FROM customers c
|
||||
LEFT JOIN orders o ON c.customer_id = o.customer_id
|
||||
GROUP BY c.customer_id, c.customer_name;
|
||||
```
|
||||
|
||||
### 4. Document Denormalization Decisions
|
||||
- Clearly document why denormalization was chosen
|
||||
- Specify which data is derived and how it's maintained
|
||||
- Include performance benchmarks that justify the decision
|
||||
|
||||
### 5. Monitor and Validate
|
||||
- Implement validation checks for denormalized data
|
||||
- Regular audits to ensure data consistency
|
||||
- Performance monitoring to validate denormalization benefits
|
||||
|
||||
## Common Anti-Patterns
|
||||
|
||||
### 1. Premature Denormalization
|
||||
Starting with denormalized design without understanding actual performance requirements.
|
||||
|
||||
### 2. Over-Normalization
|
||||
Creating too many small tables that require excessive joins for simple queries.
|
||||
|
||||
### 3. Inconsistent Approach
|
||||
Mixing normalized and denormalized patterns without clear strategy.
|
||||
|
||||
### 4. Ignoring Maintenance
|
||||
Denormalizing without proper mechanisms to maintain data consistency.
|
||||
|
||||
## Conclusion
|
||||
|
||||
Normalization and denormalization are both valuable tools in database design. The key is understanding when to apply each approach:
|
||||
|
||||
- **Use normalization** for transactional systems where data integrity is paramount
|
||||
- **Consider denormalization** for analytical systems or when performance testing reveals bottlenecks
|
||||
- **Apply selectively** based on actual usage patterns and performance requirements
|
||||
- **Maintain consistency** through proper design patterns and validation mechanisms
|
||||
|
||||
The goal is not to achieve perfect normalization or denormalization, but to create a design that best serves your application's specific needs while maintaining data quality and system performance.
|
||||
982
engineering/database-designer/schema_analyzer.py
Normal file
982
engineering/database-designer/schema_analyzer.py
Normal file
@@ -0,0 +1,982 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Database Schema Analyzer
|
||||
|
||||
Analyzes SQL DDL statements and JSON schema definitions for:
|
||||
- Normalization level compliance (1NF-BCNF)
|
||||
- Missing constraints (FK, NOT NULL, UNIQUE)
|
||||
- Data type issues and antipatterns
|
||||
- Naming convention violations
|
||||
- Missing indexes on foreign key columns
|
||||
- Table relationship mapping
|
||||
- Generates Mermaid ERD diagrams
|
||||
|
||||
Input: SQL DDL file or JSON schema definition
|
||||
Output: Analysis report + Mermaid ERD + recommendations
|
||||
|
||||
Usage:
|
||||
python schema_analyzer.py --input schema.sql --output-format json
|
||||
python schema_analyzer.py --input schema.json --output-format text
|
||||
python schema_analyzer.py --input schema.sql --generate-erd --output analysis.json
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict, namedtuple
|
||||
from typing import Dict, List, Set, Tuple, Optional, Any
|
||||
from dataclasses import dataclass, asdict
|
||||
|
||||
|
||||
@dataclass
|
||||
class Column:
|
||||
name: str
|
||||
data_type: str
|
||||
nullable: bool = True
|
||||
primary_key: bool = False
|
||||
unique: bool = False
|
||||
foreign_key: Optional[str] = None
|
||||
default_value: Optional[str] = None
|
||||
check_constraint: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Index:
|
||||
name: str
|
||||
table: str
|
||||
columns: List[str]
|
||||
unique: bool = False
|
||||
index_type: str = "btree"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Table:
|
||||
name: str
|
||||
columns: List[Column]
|
||||
primary_key: List[str]
|
||||
foreign_keys: List[Tuple[str, str]] # (column, referenced_table.column)
|
||||
unique_constraints: List[List[str]]
|
||||
check_constraints: Dict[str, str]
|
||||
indexes: List[Index]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NormalizationIssue:
|
||||
table: str
|
||||
issue_type: str
|
||||
severity: str
|
||||
description: str
|
||||
suggestion: str
|
||||
columns_affected: List[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class DataTypeIssue:
|
||||
table: str
|
||||
column: str
|
||||
current_type: str
|
||||
issue: str
|
||||
suggested_type: str
|
||||
rationale: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConstraintIssue:
|
||||
table: str
|
||||
issue_type: str
|
||||
severity: str
|
||||
description: str
|
||||
suggestion: str
|
||||
columns_affected: List[str]
|
||||
|
||||
|
||||
@dataclass
|
||||
class NamingIssue:
|
||||
table: str
|
||||
column: Optional[str]
|
||||
issue: str
|
||||
current_name: str
|
||||
suggested_name: str
|
||||
|
||||
|
||||
class SchemaAnalyzer:
|
||||
def __init__(self):
|
||||
self.tables: Dict[str, Table] = {}
|
||||
self.normalization_issues: List[NormalizationIssue] = []
|
||||
self.datatype_issues: List[DataTypeIssue] = []
|
||||
self.constraint_issues: List[ConstraintIssue] = []
|
||||
self.naming_issues: List[NamingIssue] = []
|
||||
|
||||
# Data type antipatterns
|
||||
self.varchar_255_pattern = re.compile(r'VARCHAR\(255\)', re.IGNORECASE)
|
||||
self.bad_datetime_patterns = [
|
||||
re.compile(r'VARCHAR\(\d+\)', re.IGNORECASE),
|
||||
re.compile(r'CHAR\(\d+\)', re.IGNORECASE)
|
||||
]
|
||||
|
||||
# Naming conventions
|
||||
self.table_naming_pattern = re.compile(r'^[a-z][a-z0-9_]*[a-z0-9]$')
|
||||
self.column_naming_pattern = re.compile(r'^[a-z][a-z0-9_]*[a-z0-9]$')
|
||||
|
||||
def parse_sql_ddl(self, ddl_content: str) -> None:
|
||||
"""Parse SQL DDL statements and extract schema information."""
|
||||
# Remove comments and normalize whitespace
|
||||
ddl_content = re.sub(r'--.*$', '', ddl_content, flags=re.MULTILINE)
|
||||
ddl_content = re.sub(r'/\*.*?\*/', '', ddl_content, flags=re.DOTALL)
|
||||
ddl_content = re.sub(r'\s+', ' ', ddl_content.strip())
|
||||
|
||||
# Extract CREATE TABLE statements
|
||||
create_table_pattern = re.compile(
|
||||
r'CREATE\s+TABLE\s+(\w+)\s*\(\s*(.*?)\s*\)',
|
||||
re.IGNORECASE | re.DOTALL
|
||||
)
|
||||
|
||||
for match in create_table_pattern.finditer(ddl_content):
|
||||
table_name = match.group(1).lower()
|
||||
table_definition = match.group(2)
|
||||
|
||||
table = self._parse_table_definition(table_name, table_definition)
|
||||
self.tables[table_name] = table
|
||||
|
||||
# Extract CREATE INDEX statements
|
||||
self._parse_indexes(ddl_content)
|
||||
|
||||
def _parse_table_definition(self, table_name: str, definition: str) -> Table:
|
||||
"""Parse individual table definition."""
|
||||
columns = []
|
||||
primary_key = []
|
||||
foreign_keys = []
|
||||
unique_constraints = []
|
||||
check_constraints = {}
|
||||
|
||||
# Split by commas, but handle nested parentheses
|
||||
parts = self._split_table_parts(definition)
|
||||
|
||||
for part in parts:
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
|
||||
if part.upper().startswith('PRIMARY KEY'):
|
||||
primary_key = self._parse_primary_key(part)
|
||||
elif part.upper().startswith('FOREIGN KEY'):
|
||||
fk = self._parse_foreign_key(part)
|
||||
if fk:
|
||||
foreign_keys.append(fk)
|
||||
elif part.upper().startswith('UNIQUE'):
|
||||
unique = self._parse_unique_constraint(part)
|
||||
if unique:
|
||||
unique_constraints.append(unique)
|
||||
elif part.upper().startswith('CHECK'):
|
||||
check = self._parse_check_constraint(part)
|
||||
if check:
|
||||
check_constraints.update(check)
|
||||
else:
|
||||
# Column definition
|
||||
column = self._parse_column_definition(part)
|
||||
if column:
|
||||
columns.append(column)
|
||||
if column.primary_key:
|
||||
primary_key.append(column.name)
|
||||
|
||||
return Table(
|
||||
name=table_name,
|
||||
columns=columns,
|
||||
primary_key=primary_key,
|
||||
foreign_keys=foreign_keys,
|
||||
unique_constraints=unique_constraints,
|
||||
check_constraints=check_constraints,
|
||||
indexes=[]
|
||||
)
|
||||
|
||||
def _split_table_parts(self, definition: str) -> List[str]:
|
||||
"""Split table definition by commas, respecting nested parentheses."""
|
||||
parts = []
|
||||
current_part = ""
|
||||
paren_count = 0
|
||||
|
||||
for char in definition:
|
||||
if char == '(':
|
||||
paren_count += 1
|
||||
elif char == ')':
|
||||
paren_count -= 1
|
||||
elif char == ',' and paren_count == 0:
|
||||
parts.append(current_part.strip())
|
||||
current_part = ""
|
||||
continue
|
||||
|
||||
current_part += char
|
||||
|
||||
if current_part.strip():
|
||||
parts.append(current_part.strip())
|
||||
|
||||
return parts
|
||||
|
||||
def _parse_column_definition(self, definition: str) -> Optional[Column]:
|
||||
"""Parse individual column definition."""
|
||||
# Pattern for column definition
|
||||
pattern = re.compile(
|
||||
r'(\w+)\s+([A-Z]+(?:\(\d+(?:,\d+)?\))?)\s*(.*)',
|
||||
re.IGNORECASE
|
||||
)
|
||||
|
||||
match = pattern.match(definition.strip())
|
||||
if not match:
|
||||
return None
|
||||
|
||||
column_name = match.group(1).lower()
|
||||
data_type = match.group(2).upper()
|
||||
constraints = match.group(3).upper() if match.group(3) else ""
|
||||
|
||||
column = Column(
|
||||
name=column_name,
|
||||
data_type=data_type,
|
||||
nullable='NOT NULL' not in constraints,
|
||||
primary_key='PRIMARY KEY' in constraints,
|
||||
unique='UNIQUE' in constraints
|
||||
)
|
||||
|
||||
# Parse foreign key reference
|
||||
fk_pattern = re.compile(r'REFERENCES\s+(\w+)\s*\(\s*(\w+)\s*\)', re.IGNORECASE)
|
||||
fk_match = fk_pattern.search(constraints)
|
||||
if fk_match:
|
||||
column.foreign_key = f"{fk_match.group(1).lower()}.{fk_match.group(2).lower()}"
|
||||
|
||||
# Parse default value
|
||||
default_pattern = re.compile(r'DEFAULT\s+([^,\s]+)', re.IGNORECASE)
|
||||
default_match = default_pattern.search(constraints)
|
||||
if default_match:
|
||||
column.default_value = default_match.group(1)
|
||||
|
||||
return column
|
||||
|
||||
def _parse_primary_key(self, definition: str) -> List[str]:
|
||||
"""Parse PRIMARY KEY constraint."""
|
||||
pattern = re.compile(r'PRIMARY\s+KEY\s*\(\s*(.*?)\s*\)', re.IGNORECASE)
|
||||
match = pattern.search(definition)
|
||||
if match:
|
||||
columns = [col.strip().lower() for col in match.group(1).split(',')]
|
||||
return columns
|
||||
return []
|
||||
|
||||
def _parse_foreign_key(self, definition: str) -> Optional[Tuple[str, str]]:
|
||||
"""Parse FOREIGN KEY constraint."""
|
||||
pattern = re.compile(
|
||||
r'FOREIGN\s+KEY\s*\(\s*(\w+)\s*\)\s+REFERENCES\s+(\w+)\s*\(\s*(\w+)\s*\)',
|
||||
re.IGNORECASE
|
||||
)
|
||||
match = pattern.search(definition)
|
||||
if match:
|
||||
column = match.group(1).lower()
|
||||
ref_table = match.group(2).lower()
|
||||
ref_column = match.group(3).lower()
|
||||
return (column, f"{ref_table}.{ref_column}")
|
||||
return None
|
||||
|
||||
def _parse_unique_constraint(self, definition: str) -> Optional[List[str]]:
|
||||
"""Parse UNIQUE constraint."""
|
||||
pattern = re.compile(r'UNIQUE\s*\(\s*(.*?)\s*\)', re.IGNORECASE)
|
||||
match = pattern.search(definition)
|
||||
if match:
|
||||
columns = [col.strip().lower() for col in match.group(1).split(',')]
|
||||
return columns
|
||||
return None
|
||||
|
||||
def _parse_check_constraint(self, definition: str) -> Optional[Dict[str, str]]:
|
||||
"""Parse CHECK constraint."""
|
||||
pattern = re.compile(r'CHECK\s*\(\s*(.*?)\s*\)', re.IGNORECASE)
|
||||
match = pattern.search(definition)
|
||||
if match:
|
||||
constraint_name = f"check_constraint_{len(self.tables)}"
|
||||
return {constraint_name: match.group(1)}
|
||||
return None
|
||||
|
||||
def _parse_indexes(self, ddl_content: str) -> None:
|
||||
"""Parse CREATE INDEX statements."""
|
||||
index_pattern = re.compile(
|
||||
r'CREATE\s+(?:(UNIQUE)\s+)?INDEX\s+(\w+)\s+ON\s+(\w+)\s*\(\s*(.*?)\s*\)',
|
||||
re.IGNORECASE
|
||||
)
|
||||
|
||||
for match in index_pattern.finditer(ddl_content):
|
||||
unique = match.group(1) is not None
|
||||
index_name = match.group(2).lower()
|
||||
table_name = match.group(3).lower()
|
||||
columns_str = match.group(4)
|
||||
|
||||
columns = [col.strip().lower() for col in columns_str.split(',')]
|
||||
|
||||
index = Index(
|
||||
name=index_name,
|
||||
table=table_name,
|
||||
columns=columns,
|
||||
unique=unique
|
||||
)
|
||||
|
||||
if table_name in self.tables:
|
||||
self.tables[table_name].indexes.append(index)
|
||||
|
||||
def parse_json_schema(self, json_content: str) -> None:
|
||||
"""Parse JSON schema definition."""
|
||||
try:
|
||||
schema = json.loads(json_content)
|
||||
|
||||
if 'tables' not in schema:
|
||||
raise ValueError("JSON schema must contain 'tables' key")
|
||||
|
||||
for table_name, table_def in schema['tables'].items():
|
||||
table = self._parse_json_table(table_name.lower(), table_def)
|
||||
self.tables[table_name.lower()] = table
|
||||
|
||||
except json.JSONDecodeError as e:
|
||||
raise ValueError(f"Invalid JSON: {e}")
|
||||
|
||||
def _parse_json_table(self, table_name: str, table_def: Dict[str, Any]) -> Table:
|
||||
"""Parse JSON table definition."""
|
||||
columns = []
|
||||
primary_key = table_def.get('primary_key', [])
|
||||
foreign_keys = []
|
||||
unique_constraints = table_def.get('unique_constraints', [])
|
||||
check_constraints = table_def.get('check_constraints', {})
|
||||
|
||||
for col_name, col_def in table_def.get('columns', {}).items():
|
||||
column = Column(
|
||||
name=col_name.lower(),
|
||||
data_type=col_def.get('type', 'VARCHAR(255)').upper(),
|
||||
nullable=col_def.get('nullable', True),
|
||||
primary_key=col_name.lower() in [pk.lower() for pk in primary_key],
|
||||
unique=col_def.get('unique', False),
|
||||
foreign_key=col_def.get('foreign_key'),
|
||||
default_value=col_def.get('default')
|
||||
)
|
||||
|
||||
columns.append(column)
|
||||
|
||||
if column.foreign_key:
|
||||
foreign_keys.append((column.name, column.foreign_key))
|
||||
|
||||
return Table(
|
||||
name=table_name,
|
||||
columns=columns,
|
||||
primary_key=[pk.lower() for pk in primary_key],
|
||||
foreign_keys=foreign_keys,
|
||||
unique_constraints=unique_constraints,
|
||||
check_constraints=check_constraints,
|
||||
indexes=[]
|
||||
)
|
||||
|
||||
def analyze_normalization(self) -> None:
|
||||
"""Analyze normalization compliance."""
|
||||
for table_name, table in self.tables.items():
|
||||
self._check_first_normal_form(table)
|
||||
self._check_second_normal_form(table)
|
||||
self._check_third_normal_form(table)
|
||||
self._check_bcnf(table)
|
||||
|
||||
def _check_first_normal_form(self, table: Table) -> None:
|
||||
"""Check First Normal Form compliance."""
|
||||
# Check for atomic values (no arrays or delimited strings)
|
||||
for column in table.columns:
|
||||
if any(pattern in column.data_type.upper() for pattern in ['ARRAY', 'JSON', 'TEXT']):
|
||||
if 'JSON' in column.data_type.upper():
|
||||
# JSON columns can violate 1NF if storing arrays
|
||||
self.normalization_issues.append(NormalizationIssue(
|
||||
table=table.name,
|
||||
issue_type="1NF_VIOLATION",
|
||||
severity="WARNING",
|
||||
description=f"Column '{column.name}' uses JSON type which may contain non-atomic values",
|
||||
suggestion="Consider normalizing JSON arrays into separate tables",
|
||||
columns_affected=[column.name]
|
||||
))
|
||||
|
||||
# Check for potential delimited values in VARCHAR/TEXT
|
||||
if column.data_type.upper().startswith(('VARCHAR', 'CHAR', 'TEXT')):
|
||||
if any(delimiter in column.name.lower() for delimiter in ['list', 'array', 'tags', 'items']):
|
||||
self.normalization_issues.append(NormalizationIssue(
|
||||
table=table.name,
|
||||
issue_type="1NF_VIOLATION",
|
||||
severity="HIGH",
|
||||
description=f"Column '{column.name}' appears to store delimited values",
|
||||
suggestion="Create separate table for individual values with foreign key relationship",
|
||||
columns_affected=[column.name]
|
||||
))
|
||||
|
||||
def _check_second_normal_form(self, table: Table) -> None:
|
||||
"""Check Second Normal Form compliance."""
|
||||
if len(table.primary_key) <= 1:
|
||||
return # 2NF only applies to tables with composite primary keys
|
||||
|
||||
# Look for potential partial dependencies
|
||||
non_key_columns = [col for col in table.columns if col.name not in table.primary_key]
|
||||
|
||||
for column in non_key_columns:
|
||||
# Heuristic: columns that seem related to only part of the composite key
|
||||
for pk_part in table.primary_key:
|
||||
if pk_part in column.name or column.name.startswith(pk_part.split('_')[0]):
|
||||
self.normalization_issues.append(NormalizationIssue(
|
||||
table=table.name,
|
||||
issue_type="2NF_VIOLATION",
|
||||
severity="MEDIUM",
|
||||
description=f"Column '{column.name}' may have partial dependency on '{pk_part}'",
|
||||
suggestion=f"Consider moving '{column.name}' to a separate table related to '{pk_part}'",
|
||||
columns_affected=[column.name, pk_part]
|
||||
))
|
||||
break
|
||||
|
||||
def _check_third_normal_form(self, table: Table) -> None:
|
||||
"""Check Third Normal Form compliance."""
|
||||
# Look for transitive dependencies
|
||||
non_key_columns = [col for col in table.columns if col.name not in table.primary_key]
|
||||
|
||||
# Group columns by potential entities they describe
|
||||
entity_groups = defaultdict(list)
|
||||
for column in non_key_columns:
|
||||
# Simple heuristic: group by prefix before underscore
|
||||
prefix = column.name.split('_')[0]
|
||||
if prefix != column.name: # Has underscore
|
||||
entity_groups[prefix].append(column.name)
|
||||
|
||||
for entity, columns in entity_groups.items():
|
||||
if len(columns) > 1 and entity != table.name.split('_')[0]:
|
||||
# Potential entity that should be in its own table
|
||||
id_column = f"{entity}_id"
|
||||
if id_column in [col.name for col in table.columns]:
|
||||
self.normalization_issues.append(NormalizationIssue(
|
||||
table=table.name,
|
||||
issue_type="3NF_VIOLATION",
|
||||
severity="MEDIUM",
|
||||
description=f"Columns {columns} may have transitive dependency through '{id_column}'",
|
||||
suggestion=f"Consider creating separate '{entity}' table with these columns",
|
||||
columns_affected=columns + [id_column]
|
||||
))
|
||||
|
||||
def _check_bcnf(self, table: Table) -> None:
|
||||
"""Check Boyce-Codd Normal Form compliance."""
|
||||
# BCNF violations are complex to detect without functional dependencies
|
||||
# Provide general guidance for composite keys
|
||||
if len(table.primary_key) > 2:
|
||||
self.normalization_issues.append(NormalizationIssue(
|
||||
table=table.name,
|
||||
issue_type="BCNF_WARNING",
|
||||
severity="LOW",
|
||||
description=f"Table has composite primary key with {len(table.primary_key)} columns",
|
||||
suggestion="Review functional dependencies to ensure BCNF compliance",
|
||||
columns_affected=table.primary_key
|
||||
))
|
||||
|
||||
def analyze_data_types(self) -> None:
|
||||
"""Analyze data type usage for antipatterns."""
|
||||
for table_name, table in self.tables.items():
|
||||
for column in table.columns:
|
||||
self._check_varchar_255_antipattern(table.name, column)
|
||||
self._check_inappropriate_types(table.name, column)
|
||||
self._check_size_optimization(table.name, column)
|
||||
|
||||
def _check_varchar_255_antipattern(self, table_name: str, column: Column) -> None:
|
||||
"""Check for VARCHAR(255) antipattern."""
|
||||
if self.varchar_255_pattern.match(column.data_type):
|
||||
self.datatype_issues.append(DataTypeIssue(
|
||||
table=table_name,
|
||||
column=column.name,
|
||||
current_type=column.data_type,
|
||||
issue="VARCHAR(255) antipattern",
|
||||
suggested_type="Appropriately sized VARCHAR or TEXT",
|
||||
rationale="VARCHAR(255) is often used as default without considering actual data length requirements"
|
||||
))
|
||||
|
||||
def _check_inappropriate_types(self, table_name: str, column: Column) -> None:
|
||||
"""Check for inappropriate data types."""
|
||||
# Date/time stored as string
|
||||
if column.name.lower() in ['date', 'time', 'created', 'updated', 'modified', 'timestamp']:
|
||||
if column.data_type.upper().startswith(('VARCHAR', 'CHAR', 'TEXT')):
|
||||
self.datatype_issues.append(DataTypeIssue(
|
||||
table=table_name,
|
||||
column=column.name,
|
||||
current_type=column.data_type,
|
||||
issue="Date/time stored as string",
|
||||
suggested_type="TIMESTAMP, DATE, or TIME",
|
||||
rationale="Proper date/time types enable date arithmetic and indexing optimization"
|
||||
))
|
||||
|
||||
# Boolean stored as string/integer
|
||||
if column.name.lower() in ['active', 'enabled', 'deleted', 'visible', 'published']:
|
||||
if not column.data_type.upper().startswith('BOOL'):
|
||||
self.datatype_issues.append(DataTypeIssue(
|
||||
table=table_name,
|
||||
column=column.name,
|
||||
current_type=column.data_type,
|
||||
issue="Boolean value stored as non-boolean type",
|
||||
suggested_type="BOOLEAN",
|
||||
rationale="Boolean type is more explicit and can be more storage efficient"
|
||||
))
|
||||
|
||||
# Numeric IDs as VARCHAR
|
||||
if column.name.lower().endswith('_id') or column.name.lower() == 'id':
|
||||
if column.data_type.upper().startswith(('VARCHAR', 'CHAR')):
|
||||
self.datatype_issues.append(DataTypeIssue(
|
||||
table=table_name,
|
||||
column=column.name,
|
||||
current_type=column.data_type,
|
||||
issue="Numeric ID stored as string",
|
||||
suggested_type="INTEGER, BIGINT, or UUID",
|
||||
rationale="Numeric types are more efficient for ID columns and enable better indexing"
|
||||
))
|
||||
|
||||
def _check_size_optimization(self, table_name: str, column: Column) -> None:
|
||||
"""Check for size optimization opportunities."""
|
||||
# Oversized integer types
|
||||
if column.data_type.upper() == 'BIGINT':
|
||||
if not any(keyword in column.name.lower() for keyword in ['timestamp', 'big', 'large', 'count']):
|
||||
self.datatype_issues.append(DataTypeIssue(
|
||||
table=table_name,
|
||||
column=column.name,
|
||||
current_type=column.data_type,
|
||||
issue="Potentially oversized integer type",
|
||||
suggested_type="INTEGER",
|
||||
rationale="INTEGER is sufficient for most ID and count fields unless very large values are expected"
|
||||
))
|
||||
|
||||
def analyze_constraints(self) -> None:
|
||||
"""Analyze missing constraints."""
|
||||
for table_name, table in self.tables.items():
|
||||
self._check_missing_primary_key(table)
|
||||
self._check_missing_foreign_key_constraints(table)
|
||||
self._check_missing_not_null_constraints(table)
|
||||
self._check_missing_unique_constraints(table)
|
||||
self._check_missing_check_constraints(table)
|
||||
|
||||
def _check_missing_primary_key(self, table: Table) -> None:
|
||||
"""Check for missing primary key."""
|
||||
if not table.primary_key:
|
||||
self.constraint_issues.append(ConstraintIssue(
|
||||
table=table.name,
|
||||
issue_type="MISSING_PRIMARY_KEY",
|
||||
severity="HIGH",
|
||||
description="Table has no primary key defined",
|
||||
suggestion="Add a primary key column (e.g., 'id' with auto-increment)",
|
||||
columns_affected=[]
|
||||
))
|
||||
|
||||
def _check_missing_foreign_key_constraints(self, table: Table) -> None:
|
||||
"""Check for missing foreign key constraints."""
|
||||
for column in table.columns:
|
||||
if column.name.endswith('_id') and column.name != 'id':
|
||||
# Potential foreign key column
|
||||
if not column.foreign_key:
|
||||
referenced_table = column.name[:-3] # Remove '_id' suffix
|
||||
if referenced_table in self.tables or referenced_table + 's' in self.tables:
|
||||
self.constraint_issues.append(ConstraintIssue(
|
||||
table=table.name,
|
||||
issue_type="MISSING_FOREIGN_KEY",
|
||||
severity="MEDIUM",
|
||||
description=f"Column '{column.name}' appears to be a foreign key but has no constraint",
|
||||
suggestion=f"Add foreign key constraint referencing {referenced_table} table",
|
||||
columns_affected=[column.name]
|
||||
))
|
||||
|
||||
def _check_missing_not_null_constraints(self, table: Table) -> None:
|
||||
"""Check for missing NOT NULL constraints."""
|
||||
for column in table.columns:
|
||||
if column.nullable and column.name in ['email', 'name', 'title', 'status']:
|
||||
self.constraint_issues.append(ConstraintIssue(
|
||||
table=table.name,
|
||||
issue_type="MISSING_NOT_NULL",
|
||||
severity="LOW",
|
||||
description=f"Column '{column.name}' allows NULL but typically should not",
|
||||
suggestion=f"Consider adding NOT NULL constraint to '{column.name}'",
|
||||
columns_affected=[column.name]
|
||||
))
|
||||
|
||||
def _check_missing_unique_constraints(self, table: Table) -> None:
|
||||
"""Check for missing unique constraints."""
|
||||
for column in table.columns:
|
||||
if column.name in ['email', 'username', 'slug', 'code'] and not column.unique:
|
||||
if column.name not in table.primary_key:
|
||||
self.constraint_issues.append(ConstraintIssue(
|
||||
table=table.name,
|
||||
issue_type="MISSING_UNIQUE",
|
||||
severity="MEDIUM",
|
||||
description=f"Column '{column.name}' should likely have UNIQUE constraint",
|
||||
suggestion=f"Add UNIQUE constraint to '{column.name}'",
|
||||
columns_affected=[column.name]
|
||||
))
|
||||
|
||||
def _check_missing_check_constraints(self, table: Table) -> None:
|
||||
"""Check for missing check constraints."""
|
||||
for column in table.columns:
|
||||
# Email format validation
|
||||
if column.name == 'email' and 'email' not in str(table.check_constraints):
|
||||
self.constraint_issues.append(ConstraintIssue(
|
||||
table=table.name,
|
||||
issue_type="MISSING_CHECK_CONSTRAINT",
|
||||
severity="LOW",
|
||||
description=f"Email column lacks format validation",
|
||||
suggestion="Add CHECK constraint for email format validation",
|
||||
columns_affected=[column.name]
|
||||
))
|
||||
|
||||
# Positive values for counts, prices, etc.
|
||||
if column.name.lower() in ['price', 'amount', 'count', 'quantity', 'age']:
|
||||
if column.name not in str(table.check_constraints):
|
||||
self.constraint_issues.append(ConstraintIssue(
|
||||
table=table.name,
|
||||
issue_type="MISSING_CHECK_CONSTRAINT",
|
||||
severity="LOW",
|
||||
description=f"Column '{column.name}' should validate positive values",
|
||||
suggestion=f"Add CHECK constraint: {column.name} > 0",
|
||||
columns_affected=[column.name]
|
||||
))
|
||||
|
||||
def analyze_naming_conventions(self) -> None:
|
||||
"""Analyze naming convention compliance."""
|
||||
for table_name, table in self.tables.items():
|
||||
self._check_table_naming(table_name)
|
||||
for column in table.columns:
|
||||
self._check_column_naming(table_name, column.name)
|
||||
|
||||
def _check_table_naming(self, table_name: str) -> None:
|
||||
"""Check table naming conventions."""
|
||||
if not self.table_naming_pattern.match(table_name):
|
||||
suggested_name = self._suggest_table_name(table_name)
|
||||
self.naming_issues.append(NamingIssue(
|
||||
table=table_name,
|
||||
column=None,
|
||||
issue="Invalid table naming convention",
|
||||
current_name=table_name,
|
||||
suggested_name=suggested_name
|
||||
))
|
||||
|
||||
# Check for plural naming
|
||||
if not table_name.endswith('s') and table_name not in ['data', 'information']:
|
||||
self.naming_issues.append(NamingIssue(
|
||||
table=table_name,
|
||||
column=None,
|
||||
issue="Table name should be plural",
|
||||
current_name=table_name,
|
||||
suggested_name=table_name + 's'
|
||||
))
|
||||
|
||||
def _check_column_naming(self, table_name: str, column_name: str) -> None:
|
||||
"""Check column naming conventions."""
|
||||
if not self.column_naming_pattern.match(column_name):
|
||||
suggested_name = self._suggest_column_name(column_name)
|
||||
self.naming_issues.append(NamingIssue(
|
||||
table=table_name,
|
||||
column=column_name,
|
||||
issue="Invalid column naming convention",
|
||||
current_name=column_name,
|
||||
suggested_name=suggested_name
|
||||
))
|
||||
|
||||
def _suggest_table_name(self, table_name: str) -> str:
|
||||
"""Suggest corrected table name."""
|
||||
# Convert to snake_case and make plural
|
||||
name = re.sub(r'([A-Z])', r'_\1', table_name).lower().strip('_')
|
||||
return name + 's' if not name.endswith('s') else name
|
||||
|
||||
def _suggest_column_name(self, column_name: str) -> str:
|
||||
"""Suggest corrected column name."""
|
||||
# Convert to snake_case
|
||||
return re.sub(r'([A-Z])', r'_\1', column_name).lower().strip('_')
|
||||
|
||||
def check_missing_indexes(self) -> List[Dict[str, Any]]:
|
||||
"""Check for missing indexes on foreign key columns."""
|
||||
missing_indexes = []
|
||||
|
||||
for table_name, table in self.tables.items():
|
||||
existing_indexed_columns = set()
|
||||
|
||||
# Collect existing indexed columns
|
||||
for index in table.indexes:
|
||||
existing_indexed_columns.update(index.columns)
|
||||
|
||||
# Primary key columns are automatically indexed
|
||||
existing_indexed_columns.update(table.primary_key)
|
||||
|
||||
# Check foreign key columns
|
||||
for column in table.columns:
|
||||
if column.foreign_key and column.name not in existing_indexed_columns:
|
||||
missing_indexes.append({
|
||||
'table': table_name,
|
||||
'column': column.name,
|
||||
'type': 'foreign_key',
|
||||
'suggestion': f"CREATE INDEX idx_{table_name}_{column.name} ON {table_name} ({column.name});"
|
||||
})
|
||||
|
||||
return missing_indexes
|
||||
|
||||
def generate_mermaid_erd(self) -> str:
|
||||
"""Generate Mermaid ERD diagram."""
|
||||
erd_lines = ["erDiagram"]
|
||||
|
||||
# Add table definitions
|
||||
for table_name, table in self.tables.items():
|
||||
erd_lines.append(f" {table_name.upper()} {{")
|
||||
|
||||
for column in table.columns:
|
||||
data_type = column.data_type
|
||||
constraints = []
|
||||
|
||||
if column.primary_key:
|
||||
constraints.append("PK")
|
||||
if column.foreign_key:
|
||||
constraints.append("FK")
|
||||
if not column.nullable:
|
||||
constraints.append("NOT NULL")
|
||||
if column.unique:
|
||||
constraints.append("UNIQUE")
|
||||
|
||||
constraint_str = " ".join(constraints)
|
||||
if constraint_str:
|
||||
constraint_str = f" \"{constraint_str}\""
|
||||
|
||||
erd_lines.append(f" {data_type} {column.name}{constraint_str}")
|
||||
|
||||
erd_lines.append(" }")
|
||||
|
||||
# Add relationships
|
||||
relationships = set()
|
||||
for table_name, table in self.tables.items():
|
||||
for column in table.columns:
|
||||
if column.foreign_key:
|
||||
ref_table = column.foreign_key.split('.')[0]
|
||||
if ref_table in self.tables:
|
||||
relationship = f" {ref_table.upper()} ||--o{{ {table_name.upper()} : has"
|
||||
relationships.add(relationship)
|
||||
|
||||
erd_lines.extend(sorted(relationships))
|
||||
|
||||
return "\n".join(erd_lines)
|
||||
|
||||
def get_analysis_summary(self) -> Dict[str, Any]:
|
||||
"""Get comprehensive analysis summary."""
|
||||
return {
|
||||
"schema_overview": {
|
||||
"total_tables": len(self.tables),
|
||||
"total_columns": sum(len(table.columns) for table in self.tables.values()),
|
||||
"tables_with_primary_keys": len([t for t in self.tables.values() if t.primary_key]),
|
||||
"total_foreign_keys": sum(len(table.foreign_keys) for table in self.tables.values()),
|
||||
"total_indexes": sum(len(table.indexes) for table in self.tables.values())
|
||||
},
|
||||
"normalization_analysis": {
|
||||
"total_issues": len(self.normalization_issues),
|
||||
"by_severity": {
|
||||
"high": len([i for i in self.normalization_issues if i.severity == "HIGH"]),
|
||||
"medium": len([i for i in self.normalization_issues if i.severity == "MEDIUM"]),
|
||||
"low": len([i for i in self.normalization_issues if i.severity == "LOW"]),
|
||||
"warning": len([i for i in self.normalization_issues if i.severity == "WARNING"])
|
||||
},
|
||||
"issues": [asdict(issue) for issue in self.normalization_issues]
|
||||
},
|
||||
"data_type_analysis": {
|
||||
"total_issues": len(self.datatype_issues),
|
||||
"issues": [asdict(issue) for issue in self.datatype_issues]
|
||||
},
|
||||
"constraint_analysis": {
|
||||
"total_issues": len(self.constraint_issues),
|
||||
"by_severity": {
|
||||
"high": len([i for i in self.constraint_issues if i.severity == "HIGH"]),
|
||||
"medium": len([i for i in self.constraint_issues if i.severity == "MEDIUM"]),
|
||||
"low": len([i for i in self.constraint_issues if i.severity == "LOW"])
|
||||
},
|
||||
"issues": [asdict(issue) for issue in self.constraint_issues]
|
||||
},
|
||||
"naming_analysis": {
|
||||
"total_issues": len(self.naming_issues),
|
||||
"issues": [asdict(issue) for issue in self.naming_issues]
|
||||
},
|
||||
"missing_indexes": self.check_missing_indexes(),
|
||||
"recommendations": self._generate_recommendations()
|
||||
}
|
||||
|
||||
def _generate_recommendations(self) -> List[str]:
|
||||
"""Generate high-level recommendations."""
|
||||
recommendations = []
|
||||
|
||||
# High severity issues
|
||||
high_severity_issues = [
|
||||
i for i in self.normalization_issues + self.constraint_issues
|
||||
if i.severity == "HIGH"
|
||||
]
|
||||
|
||||
if high_severity_issues:
|
||||
recommendations.append(f"Address {len(high_severity_issues)} high-severity issues immediately")
|
||||
|
||||
# Missing primary keys
|
||||
tables_without_pk = [name for name, table in self.tables.items() if not table.primary_key]
|
||||
if tables_without_pk:
|
||||
recommendations.append(f"Add primary keys to tables: {', '.join(tables_without_pk)}")
|
||||
|
||||
# Data type improvements
|
||||
varchar_255_issues = [i for i in self.datatype_issues if "VARCHAR(255)" in i.issue]
|
||||
if varchar_255_issues:
|
||||
recommendations.append(f"Review {len(varchar_255_issues)} VARCHAR(255) columns for right-sizing")
|
||||
|
||||
# Missing foreign keys
|
||||
missing_fks = [i for i in self.constraint_issues if i.issue_type == "MISSING_FOREIGN_KEY"]
|
||||
if missing_fks:
|
||||
recommendations.append(f"Consider adding {len(missing_fks)} foreign key constraints for referential integrity")
|
||||
|
||||
# Normalization improvements
|
||||
normalization_issues_count = len(self.normalization_issues)
|
||||
if normalization_issues_count > 0:
|
||||
recommendations.append(f"Review {normalization_issues_count} normalization issues for schema optimization")
|
||||
|
||||
return recommendations
|
||||
|
||||
def format_text_report(self, analysis: Dict[str, Any]) -> str:
|
||||
"""Format analysis as human-readable text report."""
|
||||
lines = []
|
||||
lines.append("DATABASE SCHEMA ANALYSIS REPORT")
|
||||
lines.append("=" * 50)
|
||||
lines.append("")
|
||||
|
||||
# Overview
|
||||
overview = analysis["schema_overview"]
|
||||
lines.append("SCHEMA OVERVIEW")
|
||||
lines.append("-" * 15)
|
||||
lines.append(f"Total Tables: {overview['total_tables']}")
|
||||
lines.append(f"Total Columns: {overview['total_columns']}")
|
||||
lines.append(f"Tables with Primary Keys: {overview['tables_with_primary_keys']}")
|
||||
lines.append(f"Total Foreign Keys: {overview['total_foreign_keys']}")
|
||||
lines.append(f"Total Indexes: {overview['total_indexes']}")
|
||||
lines.append("")
|
||||
|
||||
# Recommendations
|
||||
if analysis["recommendations"]:
|
||||
lines.append("KEY RECOMMENDATIONS")
|
||||
lines.append("-" * 18)
|
||||
for i, rec in enumerate(analysis["recommendations"], 1):
|
||||
lines.append(f"{i}. {rec}")
|
||||
lines.append("")
|
||||
|
||||
# Normalization Issues
|
||||
norm_analysis = analysis["normalization_analysis"]
|
||||
if norm_analysis["total_issues"] > 0:
|
||||
lines.append(f"NORMALIZATION ISSUES ({norm_analysis['total_issues']} total)")
|
||||
lines.append("-" * 25)
|
||||
severity_counts = norm_analysis["by_severity"]
|
||||
lines.append(f"High: {severity_counts['high']}, Medium: {severity_counts['medium']}, "
|
||||
f"Low: {severity_counts['low']}, Warning: {severity_counts['warning']}")
|
||||
lines.append("")
|
||||
|
||||
for issue in norm_analysis["issues"][:5]: # Show first 5
|
||||
lines.append(f"• {issue['table']}: {issue['description']}")
|
||||
lines.append(f" Suggestion: {issue['suggestion']}")
|
||||
lines.append("")
|
||||
|
||||
# Data Type Issues
|
||||
dt_analysis = analysis["data_type_analysis"]
|
||||
if dt_analysis["total_issues"] > 0:
|
||||
lines.append(f"DATA TYPE ISSUES ({dt_analysis['total_issues']} total)")
|
||||
lines.append("-" * 20)
|
||||
for issue in dt_analysis["issues"][:5]: # Show first 5
|
||||
lines.append(f"• {issue['table']}.{issue['column']}: {issue['issue']}")
|
||||
lines.append(f" Current: {issue['current_type']} → Suggested: {issue['suggested_type']}")
|
||||
lines.append(f" Rationale: {issue['rationale']}")
|
||||
lines.append("")
|
||||
|
||||
# Constraint Issues
|
||||
const_analysis = analysis["constraint_analysis"]
|
||||
if const_analysis["total_issues"] > 0:
|
||||
lines.append(f"CONSTRAINT ISSUES ({const_analysis['total_issues']} total)")
|
||||
lines.append("-" * 20)
|
||||
severity_counts = const_analysis["by_severity"]
|
||||
lines.append(f"High: {severity_counts['high']}, Medium: {severity_counts['medium']}, "
|
||||
f"Low: {severity_counts['low']}")
|
||||
lines.append("")
|
||||
|
||||
for issue in const_analysis["issues"][:5]: # Show first 5
|
||||
lines.append(f"• {issue['table']}: {issue['description']}")
|
||||
lines.append(f" Suggestion: {issue['suggestion']}")
|
||||
lines.append("")
|
||||
|
||||
# Missing Indexes
|
||||
missing_idx = analysis["missing_indexes"]
|
||||
if missing_idx:
|
||||
lines.append(f"MISSING INDEXES ({len(missing_idx)} total)")
|
||||
lines.append("-" * 17)
|
||||
for idx in missing_idx[:5]: # Show first 5
|
||||
lines.append(f"• {idx['table']}.{idx['column']} ({idx['type']})")
|
||||
lines.append(f" SQL: {idx['suggestion']}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Analyze database schema for design issues and generate ERD")
|
||||
parser.add_argument("--input", "-i", required=True, help="Input file (SQL DDL or JSON schema)")
|
||||
parser.add_argument("--output", "-o", help="Output file (default: stdout)")
|
||||
parser.add_argument("--output-format", "-f", choices=["json", "text"], default="text",
|
||||
help="Output format")
|
||||
parser.add_argument("--generate-erd", "-e", action="store_true", help="Include Mermaid ERD in output")
|
||||
parser.add_argument("--erd-only", action="store_true", help="Output only the Mermaid ERD")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Read input file
|
||||
with open(args.input, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Initialize analyzer
|
||||
analyzer = SchemaAnalyzer()
|
||||
|
||||
# Parse input based on file extension
|
||||
if args.input.lower().endswith('.json'):
|
||||
analyzer.parse_json_schema(content)
|
||||
else:
|
||||
analyzer.parse_sql_ddl(content)
|
||||
|
||||
if not analyzer.tables:
|
||||
print("Error: No tables found in input file", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.erd_only:
|
||||
# Output only ERD
|
||||
erd = analyzer.generate_mermaid_erd()
|
||||
if args.output:
|
||||
with open(args.output, 'w') as f:
|
||||
f.write(erd)
|
||||
else:
|
||||
print(erd)
|
||||
return 0
|
||||
|
||||
# Perform analysis
|
||||
analyzer.analyze_normalization()
|
||||
analyzer.analyze_data_types()
|
||||
analyzer.analyze_constraints()
|
||||
analyzer.analyze_naming_conventions()
|
||||
|
||||
# Generate report
|
||||
analysis = analyzer.get_analysis_summary()
|
||||
|
||||
if args.generate_erd:
|
||||
analysis["mermaid_erd"] = analyzer.generate_mermaid_erd()
|
||||
|
||||
# Output results
|
||||
if args.output_format == "json":
|
||||
output = json.dumps(analysis, indent=2)
|
||||
else:
|
||||
output = analyzer.format_text_report(analysis)
|
||||
if args.generate_erd:
|
||||
output += "\n\nMERMAID ERD\n" + "=" * 11 + "\n"
|
||||
output += analysis["mermaid_erd"]
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w') as f:
|
||||
f.write(output)
|
||||
else:
|
||||
print(output)
|
||||
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
524
engineering/dependency-auditor/README.md
Normal file
524
engineering/dependency-auditor/README.md
Normal file
@@ -0,0 +1,524 @@
|
||||
# Dependency Auditor
|
||||
|
||||
A comprehensive toolkit for analyzing, auditing, and managing dependencies across multi-language software projects. This skill provides vulnerability scanning, license compliance checking, and upgrade path planning with zero external dependencies.
|
||||
|
||||
## Overview
|
||||
|
||||
The Dependency Auditor skill consists of three main Python scripts that work together to provide complete dependency management capabilities:
|
||||
|
||||
- **`dep_scanner.py`**: Vulnerability scanning and dependency analysis
|
||||
- **`license_checker.py`**: License compliance and conflict detection
|
||||
- **`upgrade_planner.py`**: Upgrade path planning and risk assessment
|
||||
|
||||
## Features
|
||||
|
||||
### 🔍 Vulnerability Scanning
|
||||
- Multi-language dependency parsing (JavaScript, Python, Go, Rust, Ruby, Java)
|
||||
- Built-in vulnerability database with common CVE patterns
|
||||
- CVSS scoring and risk assessment
|
||||
- JSON and human-readable output formats
|
||||
- CI/CD integration support
|
||||
|
||||
### ⚖️ License Compliance
|
||||
- Comprehensive license classification and compatibility analysis
|
||||
- Automatic conflict detection between project and dependency licenses
|
||||
- Risk assessment for commercial usage and distribution
|
||||
- Compliance scoring and reporting
|
||||
|
||||
### 📈 Upgrade Planning
|
||||
- Semantic versioning analysis with breaking change prediction
|
||||
- Risk-based upgrade prioritization
|
||||
- Phased migration plans with rollback procedures
|
||||
- Security-focused upgrade recommendations
|
||||
|
||||
## Installation
|
||||
|
||||
No external dependencies required! All scripts use only Python standard library.
|
||||
|
||||
```bash
|
||||
# Clone or download the dependency-auditor skill
|
||||
cd engineering/dependency-auditor/scripts
|
||||
|
||||
# Make scripts executable
|
||||
chmod +x dep_scanner.py license_checker.py upgrade_planner.py
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Scan for Vulnerabilities
|
||||
|
||||
```bash
|
||||
# Basic vulnerability scan
|
||||
python dep_scanner.py /path/to/your/project
|
||||
|
||||
# JSON output for automation
|
||||
python dep_scanner.py /path/to/your/project --format json --output scan_results.json
|
||||
|
||||
# Fail CI/CD on high-severity vulnerabilities
|
||||
python dep_scanner.py /path/to/your/project --fail-on-high
|
||||
```
|
||||
|
||||
### 2. Check License Compliance
|
||||
|
||||
```bash
|
||||
# Basic license compliance check
|
||||
python license_checker.py /path/to/your/project
|
||||
|
||||
# Strict policy enforcement
|
||||
python license_checker.py /path/to/your/project --policy strict
|
||||
|
||||
# Use existing dependency inventory
|
||||
python license_checker.py /path/to/project --inventory scan_results.json --format json
|
||||
```
|
||||
|
||||
### 3. Plan Dependency Upgrades
|
||||
|
||||
```bash
|
||||
# Generate upgrade plan from dependency inventory
|
||||
python upgrade_planner.py scan_results.json
|
||||
|
||||
# Custom timeline and risk filtering
|
||||
python upgrade_planner.py scan_results.json --timeline 60 --risk-threshold medium
|
||||
|
||||
# Security updates only
|
||||
python upgrade_planner.py scan_results.json --security-only --format json
|
||||
```
|
||||
|
||||
## Detailed Usage
|
||||
|
||||
### Dependency Scanner (`dep_scanner.py`)
|
||||
|
||||
The dependency scanner parses project files to extract dependencies and check them against a built-in vulnerability database.
|
||||
|
||||
#### Supported File Formats
|
||||
- **JavaScript/Node.js**: package.json, package-lock.json, yarn.lock
|
||||
- **Python**: requirements.txt, pyproject.toml, Pipfile.lock, poetry.lock
|
||||
- **Go**: go.mod, go.sum
|
||||
- **Rust**: Cargo.toml, Cargo.lock
|
||||
- **Ruby**: Gemfile, Gemfile.lock
|
||||
|
||||
#### Command Line Options
|
||||
|
||||
```bash
|
||||
python dep_scanner.py [PROJECT_PATH] [OPTIONS]
|
||||
|
||||
Required Arguments:
|
||||
PROJECT_PATH Path to the project directory to scan
|
||||
|
||||
Optional Arguments:
|
||||
--format {text,json} Output format (default: text)
|
||||
--output FILE Output file path (default: stdout)
|
||||
--fail-on-high Exit with error code if high-severity vulnerabilities found
|
||||
--quick-scan Perform quick scan (skip transitive dependencies)
|
||||
|
||||
Examples:
|
||||
python dep_scanner.py /app
|
||||
python dep_scanner.py . --format json --output results.json
|
||||
python dep_scanner.py /project --fail-on-high --quick-scan
|
||||
```
|
||||
|
||||
#### Output Format
|
||||
|
||||
**Text Output:**
|
||||
```
|
||||
============================================================
|
||||
DEPENDENCY SECURITY SCAN REPORT
|
||||
============================================================
|
||||
Scan Date: 2024-02-16T15:30:00.000Z
|
||||
Project: /example/sample-web-app
|
||||
|
||||
SUMMARY:
|
||||
Total Dependencies: 23
|
||||
Unique Dependencies: 19
|
||||
Ecosystems: npm
|
||||
Vulnerabilities Found: 1
|
||||
High Severity: 1
|
||||
Medium Severity: 0
|
||||
Low Severity: 0
|
||||
|
||||
VULNERABLE DEPENDENCIES:
|
||||
------------------------------
|
||||
Package: lodash v4.17.20 (npm)
|
||||
• CVE-2021-23337: Prototype pollution in lodash
|
||||
Severity: HIGH (CVSS: 7.2)
|
||||
Fixed in: 4.17.21
|
||||
|
||||
RECOMMENDATIONS:
|
||||
--------------------
|
||||
1. URGENT: Address 1 high-severity vulnerabilities immediately
|
||||
2. Update lodash from 4.17.20 to 4.17.21 to fix CVE-2021-23337
|
||||
```
|
||||
|
||||
**JSON Output:**
|
||||
```json
|
||||
{
|
||||
"timestamp": "2024-02-16T15:30:00.000Z",
|
||||
"project_path": "/example/sample-web-app",
|
||||
"dependencies": [
|
||||
{
|
||||
"name": "lodash",
|
||||
"version": "4.17.20",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "CVE-2021-23337",
|
||||
"summary": "Prototype pollution in lodash",
|
||||
"severity": "HIGH",
|
||||
"cvss_score": 7.2
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"recommendations": [
|
||||
"Update lodash from 4.17.20 to 4.17.21 to fix CVE-2021-23337"
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### License Checker (`license_checker.py`)
|
||||
|
||||
The license checker analyzes dependency licenses for compliance and detects potential conflicts.
|
||||
|
||||
#### Command Line Options
|
||||
|
||||
```bash
|
||||
python license_checker.py [PROJECT_PATH] [OPTIONS]
|
||||
|
||||
Required Arguments:
|
||||
PROJECT_PATH Path to the project directory to analyze
|
||||
|
||||
Optional Arguments:
|
||||
--inventory FILE Path to dependency inventory JSON file
|
||||
--format {text,json} Output format (default: text)
|
||||
--output FILE Output file path (default: stdout)
|
||||
--policy {permissive,strict} License policy strictness (default: permissive)
|
||||
--warn-conflicts Show warnings for potential conflicts
|
||||
|
||||
Examples:
|
||||
python license_checker.py /app
|
||||
python license_checker.py . --format json --output compliance.json
|
||||
python license_checker.py /app --inventory deps.json --policy strict
|
||||
```
|
||||
|
||||
#### License Classifications
|
||||
|
||||
The tool classifies licenses into risk categories:
|
||||
|
||||
- **Permissive (Low Risk)**: MIT, Apache-2.0, BSD, ISC
|
||||
- **Weak Copyleft (Medium Risk)**: LGPL, MPL
|
||||
- **Strong Copyleft (High Risk)**: GPL, AGPL
|
||||
- **Proprietary (High Risk)**: Commercial licenses
|
||||
- **Unknown (Critical Risk)**: Unidentified licenses
|
||||
|
||||
#### Compatibility Matrix
|
||||
|
||||
The tool includes a comprehensive compatibility matrix that checks:
|
||||
- Project license vs. dependency licenses
|
||||
- GPL contamination detection
|
||||
- Commercial usage restrictions
|
||||
- Distribution requirements
|
||||
|
||||
### Upgrade Planner (`upgrade_planner.py`)
|
||||
|
||||
The upgrade planner analyzes dependency inventories and creates prioritized upgrade plans.
|
||||
|
||||
#### Command Line Options
|
||||
|
||||
```bash
|
||||
python upgrade_planner.py [INVENTORY_FILE] [OPTIONS]
|
||||
|
||||
Required Arguments:
|
||||
INVENTORY_FILE Path to dependency inventory JSON file
|
||||
|
||||
Optional Arguments:
|
||||
--timeline DAYS Timeline for upgrade plan in days (default: 90)
|
||||
--format {text,json} Output format (default: text)
|
||||
--output FILE Output file path (default: stdout)
|
||||
--risk-threshold {safe,low,medium,high,critical} Maximum risk level (default: high)
|
||||
--security-only Only plan upgrades with security fixes
|
||||
|
||||
Examples:
|
||||
python upgrade_planner.py deps.json
|
||||
python upgrade_planner.py inventory.json --timeline 60 --format json
|
||||
python upgrade_planner.py deps.json --security-only --risk-threshold medium
|
||||
```
|
||||
|
||||
#### Risk Assessment
|
||||
|
||||
Upgrades are classified by risk level:
|
||||
|
||||
- **Safe**: Patch updates with no breaking changes
|
||||
- **Low**: Minor updates with backward compatibility
|
||||
- **Medium**: Updates with potential API changes
|
||||
- **High**: Major version updates with breaking changes
|
||||
- **Critical**: Updates affecting core functionality
|
||||
|
||||
#### Phased Planning
|
||||
|
||||
The tool creates three-phase upgrade plans:
|
||||
|
||||
1. **Phase 1 (30% of timeline)**: Security fixes and safe updates
|
||||
2. **Phase 2 (40% of timeline)**: Regular maintenance updates
|
||||
3. **Phase 3 (30% of timeline)**: Major updates requiring careful planning
|
||||
|
||||
## Integration Examples
|
||||
|
||||
### CI/CD Pipeline Integration
|
||||
|
||||
#### GitHub Actions Example
|
||||
|
||||
```yaml
|
||||
name: Dependency Audit
|
||||
on: [push, pull_request, schedule]
|
||||
|
||||
jobs:
|
||||
audit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
|
||||
- name: Run Vulnerability Scan
|
||||
run: |
|
||||
python scripts/dep_scanner.py . --format json --output scan.json
|
||||
python scripts/dep_scanner.py . --fail-on-high
|
||||
|
||||
- name: Check License Compliance
|
||||
run: |
|
||||
python scripts/license_checker.py . --inventory scan.json --policy strict
|
||||
|
||||
- name: Generate Upgrade Plan
|
||||
run: |
|
||||
python scripts/upgrade_planner.py scan.json --output upgrade-plan.txt
|
||||
|
||||
- name: Upload Reports
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: dependency-reports
|
||||
path: |
|
||||
scan.json
|
||||
upgrade-plan.txt
|
||||
```
|
||||
|
||||
#### Jenkins Pipeline Example
|
||||
|
||||
```groovy
|
||||
pipeline {
|
||||
agent any
|
||||
|
||||
stages {
|
||||
stage('Dependency Audit') {
|
||||
steps {
|
||||
script {
|
||||
// Vulnerability scan
|
||||
sh 'python scripts/dep_scanner.py . --format json --output scan.json'
|
||||
|
||||
// License compliance
|
||||
sh 'python scripts/license_checker.py . --inventory scan.json --format json --output compliance.json'
|
||||
|
||||
// Upgrade planning
|
||||
sh 'python scripts/upgrade_planner.py scan.json --format json --output upgrades.json'
|
||||
}
|
||||
|
||||
// Archive reports
|
||||
archiveArtifacts artifacts: '*.json', fingerprint: true
|
||||
|
||||
// Fail build on high-severity vulnerabilities
|
||||
sh 'python scripts/dep_scanner.py . --fail-on-high'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
always {
|
||||
// Publish reports
|
||||
publishHTML([
|
||||
allowMissing: false,
|
||||
alwaysLinkToLastBuild: true,
|
||||
keepAll: true,
|
||||
reportDir: '.',
|
||||
reportFiles: '*.json',
|
||||
reportName: 'Dependency Audit Report'
|
||||
])
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Automated Dependency Updates
|
||||
|
||||
#### Weekly Security Updates Script
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# weekly-security-updates.sh
|
||||
|
||||
set -e
|
||||
|
||||
echo "Running weekly security dependency updates..."
|
||||
|
||||
# Scan for vulnerabilities
|
||||
python scripts/dep_scanner.py . --format json --output current-scan.json
|
||||
|
||||
# Generate security-only upgrade plan
|
||||
python scripts/upgrade_planner.py current-scan.json --security-only --output security-upgrades.txt
|
||||
|
||||
# Check if security updates are available
|
||||
if grep -q "URGENT" security-upgrades.txt; then
|
||||
echo "Security updates found! Creating automated PR..."
|
||||
|
||||
# Create branch
|
||||
git checkout -b "automated-security-updates-$(date +%Y%m%d)"
|
||||
|
||||
# Apply updates (example for npm)
|
||||
npm audit fix --only=prod
|
||||
|
||||
# Commit and push
|
||||
git add .
|
||||
git commit -m "chore: automated security dependency updates"
|
||||
git push origin HEAD
|
||||
|
||||
# Create PR (using GitHub CLI)
|
||||
gh pr create \
|
||||
--title "Automated Security Updates" \
|
||||
--body-file security-upgrades.txt \
|
||||
--label "security,dependencies,automated"
|
||||
else
|
||||
echo "No critical security updates found."
|
||||
fi
|
||||
```
|
||||
|
||||
## Sample Files
|
||||
|
||||
The `assets/` directory contains sample dependency files for testing:
|
||||
|
||||
- `sample_package.json`: Node.js project with various dependencies
|
||||
- `sample_requirements.txt`: Python project dependencies
|
||||
- `sample_go.mod`: Go module dependencies
|
||||
|
||||
The `expected_outputs/` directory contains example reports showing the expected format and content.
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Custom Vulnerability Database
|
||||
|
||||
You can extend the built-in vulnerability database by modifying the `_load_vulnerability_database()` method in `dep_scanner.py`:
|
||||
|
||||
```python
|
||||
def _load_vulnerability_database(self):
|
||||
"""Load vulnerability database from multiple sources."""
|
||||
db = self._load_builtin_database()
|
||||
|
||||
# Load custom vulnerabilities
|
||||
custom_db_path = os.environ.get('CUSTOM_VULN_DB')
|
||||
if custom_db_path and os.path.exists(custom_db_path):
|
||||
with open(custom_db_path, 'r') as f:
|
||||
custom_vulns = json.load(f)
|
||||
db.update(custom_vulns)
|
||||
|
||||
return db
|
||||
```
|
||||
|
||||
### Custom License Policies
|
||||
|
||||
Create custom license policies by modifying the license database:
|
||||
|
||||
```python
|
||||
# Add custom license
|
||||
custom_license = LicenseInfo(
|
||||
name='Custom Internal License',
|
||||
spdx_id='CUSTOM-1.0',
|
||||
license_type=LicenseType.PROPRIETARY,
|
||||
risk_level=RiskLevel.HIGH,
|
||||
description='Internal company license',
|
||||
restrictions=['Internal use only'],
|
||||
obligations=['Attribution required']
|
||||
)
|
||||
```
|
||||
|
||||
### Multi-Project Analysis
|
||||
|
||||
For analyzing multiple projects, create a wrapper script:
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
projects = ['/path/to/project1', '/path/to/project2', '/path/to/project3']
|
||||
results = {}
|
||||
|
||||
for project in projects:
|
||||
project_name = Path(project).name
|
||||
|
||||
# Run vulnerability scan
|
||||
scan_result = subprocess.run([
|
||||
'python', 'scripts/dep_scanner.py',
|
||||
project, '--format', 'json'
|
||||
], capture_output=True, text=True)
|
||||
|
||||
if scan_result.returncode == 0:
|
||||
results[project_name] = json.loads(scan_result.stdout)
|
||||
|
||||
# Generate consolidated report
|
||||
with open('consolidated-report.json', 'w') as f:
|
||||
json.dump(results, f, indent=2)
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
1. **Permission Errors**
|
||||
```bash
|
||||
chmod +x scripts/*.py
|
||||
```
|
||||
|
||||
2. **Python Version Compatibility**
|
||||
- Requires Python 3.7 or higher
|
||||
- Uses only standard library modules
|
||||
|
||||
3. **Large Projects**
|
||||
- Use `--quick-scan` for faster analysis
|
||||
- Consider excluding large node_modules directories
|
||||
|
||||
4. **False Positives**
|
||||
- Review vulnerability matches manually
|
||||
- Consider version range parsing improvements
|
||||
|
||||
### Debug Mode
|
||||
|
||||
Enable debug logging by setting environment variable:
|
||||
|
||||
```bash
|
||||
export DEPENDENCY_AUDIT_DEBUG=1
|
||||
python scripts/dep_scanner.py /your/project
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
1. **Adding New Package Managers**: Extend the `supported_files` dictionary and add corresponding parsers
|
||||
2. **Vulnerability Database**: Add new CVE entries to the built-in database
|
||||
3. **License Support**: Add new license types to the license database
|
||||
4. **Risk Assessment**: Improve risk scoring algorithms
|
||||
|
||||
## References
|
||||
|
||||
- [SKILL.md](SKILL.md): Comprehensive skill documentation
|
||||
- [references/](references/): Best practices and compatibility guides
|
||||
- [assets/](assets/): Sample dependency files for testing
|
||||
- [expected_outputs/](expected_outputs/): Example reports and outputs
|
||||
|
||||
## License
|
||||
|
||||
This skill is licensed under the MIT License. See the project license file for details.
|
||||
|
||||
---
|
||||
|
||||
**Note**: This tool provides automated analysis to assist with dependency management decisions. Always review recommendations and consult with security and legal teams for critical applications.
|
||||
333
engineering/dependency-auditor/SKILL.md
Normal file
333
engineering/dependency-auditor/SKILL.md
Normal file
@@ -0,0 +1,333 @@
|
||||
# Dependency Auditor
|
||||
|
||||
> **Skill Type:** POWERFUL
|
||||
> **Category:** Engineering
|
||||
> **Domain:** Dependency Management & Security
|
||||
|
||||
## Overview
|
||||
|
||||
The **Dependency Auditor** is a comprehensive toolkit for analyzing, auditing, and managing dependencies across multi-language software projects. This skill provides deep visibility into your project's dependency ecosystem, enabling teams to identify vulnerabilities, ensure license compliance, optimize dependency trees, and plan safe upgrades.
|
||||
|
||||
In modern software development, dependencies form complex webs that can introduce significant security, legal, and maintenance risks. A single project might have hundreds of direct and transitive dependencies, each potentially introducing vulnerabilities, license conflicts, or maintenance burden. This skill addresses these challenges through automated analysis and actionable recommendations.
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
### 1. Vulnerability Scanning & CVE Matching
|
||||
|
||||
**Comprehensive Security Analysis**
|
||||
- Scans dependencies against built-in vulnerability databases
|
||||
- Matches Common Vulnerabilities and Exposures (CVE) patterns
|
||||
- Identifies known security issues across multiple ecosystems
|
||||
- Analyzes transitive dependency vulnerabilities
|
||||
- Provides CVSS scores and exploit assessments
|
||||
- Tracks vulnerability disclosure timelines
|
||||
- Maps vulnerabilities to dependency paths
|
||||
|
||||
**Multi-Language Support**
|
||||
- **JavaScript/Node.js**: package.json, package-lock.json, yarn.lock
|
||||
- **Python**: requirements.txt, pyproject.toml, Pipfile.lock, poetry.lock
|
||||
- **Go**: go.mod, go.sum
|
||||
- **Rust**: Cargo.toml, Cargo.lock
|
||||
- **Ruby**: Gemfile, Gemfile.lock
|
||||
- **Java/Maven**: pom.xml, gradle.lockfile
|
||||
- **PHP**: composer.json, composer.lock
|
||||
- **C#/.NET**: packages.config, project.assets.json
|
||||
|
||||
### 2. License Compliance & Legal Risk Assessment
|
||||
|
||||
**License Classification System**
|
||||
- **Permissive Licenses**: MIT, Apache 2.0, BSD (2-clause, 3-clause), ISC
|
||||
- **Copyleft (Strong)**: GPL (v2, v3), AGPL (v3)
|
||||
- **Copyleft (Weak)**: LGPL (v2.1, v3), MPL (v2.0)
|
||||
- **Proprietary**: Commercial, custom, or restrictive licenses
|
||||
- **Dual Licensed**: Multi-license scenarios and compatibility
|
||||
- **Unknown/Ambiguous**: Missing or unclear licensing
|
||||
|
||||
**Conflict Detection**
|
||||
- Identifies incompatible license combinations
|
||||
- Warns about GPL contamination in permissive projects
|
||||
- Analyzes license inheritance through dependency chains
|
||||
- Provides compliance recommendations for distribution
|
||||
- Generates legal risk matrices for decision-making
|
||||
|
||||
### 3. Outdated Dependency Detection
|
||||
|
||||
**Version Analysis**
|
||||
- Identifies dependencies with available updates
|
||||
- Categorizes updates by severity (patch, minor, major)
|
||||
- Detects pinned versions that may be outdated
|
||||
- Analyzes semantic versioning patterns
|
||||
- Identifies floating version specifiers
|
||||
- Tracks release frequencies and maintenance status
|
||||
|
||||
**Maintenance Status Assessment**
|
||||
- Identifies abandoned or unmaintained packages
|
||||
- Analyzes commit frequency and contributor activity
|
||||
- Tracks last release dates and security patch availability
|
||||
- Identifies packages with known end-of-life dates
|
||||
- Assesses upstream maintenance quality
|
||||
|
||||
### 4. Dependency Bloat Analysis
|
||||
|
||||
**Unused Dependency Detection**
|
||||
- Identifies dependencies that aren't actually imported/used
|
||||
- Analyzes import statements and usage patterns
|
||||
- Detects redundant dependencies with overlapping functionality
|
||||
- Identifies oversized packages for simple use cases
|
||||
- Maps actual vs. declared dependency usage
|
||||
|
||||
**Redundancy Analysis**
|
||||
- Identifies multiple packages providing similar functionality
|
||||
- Detects version conflicts in transitive dependencies
|
||||
- Analyzes bundle size impact of dependencies
|
||||
- Identifies opportunities for dependency consolidation
|
||||
- Maps dependency overlap and duplication
|
||||
|
||||
### 5. Upgrade Path Planning & Breaking Change Risk
|
||||
|
||||
**Semantic Versioning Analysis**
|
||||
- Analyzes semver patterns to predict breaking changes
|
||||
- Identifies safe upgrade paths (patch/minor versions)
|
||||
- Flags major version updates requiring attention
|
||||
- Tracks breaking changes across dependency updates
|
||||
- Provides rollback strategies for failed upgrades
|
||||
|
||||
**Risk Assessment Matrix**
|
||||
- Low Risk: Patch updates, security fixes
|
||||
- Medium Risk: Minor updates with new features
|
||||
- High Risk: Major version updates, API changes
|
||||
- Critical Risk: Dependencies with known breaking changes
|
||||
|
||||
**Upgrade Prioritization**
|
||||
- Security patches: Highest priority
|
||||
- Bug fixes: High priority
|
||||
- Feature updates: Medium priority
|
||||
- Major rewrites: Planned priority
|
||||
- Deprecated features: Immediate attention
|
||||
|
||||
### 6. Supply Chain Security
|
||||
|
||||
**Dependency Provenance**
|
||||
- Verifies package signatures and checksums
|
||||
- Analyzes package download sources and mirrors
|
||||
- Identifies suspicious or compromised packages
|
||||
- Tracks package ownership changes and maintainer shifts
|
||||
- Detects typosquatting and malicious packages
|
||||
|
||||
**Transitive Risk Analysis**
|
||||
- Maps complete dependency trees
|
||||
- Identifies high-risk transitive dependencies
|
||||
- Analyzes dependency depth and complexity
|
||||
- Tracks influence of indirect dependencies
|
||||
- Provides supply chain risk scoring
|
||||
|
||||
### 7. Lockfile Analysis & Deterministic Builds
|
||||
|
||||
**Lockfile Validation**
|
||||
- Ensures lockfiles are up-to-date with manifests
|
||||
- Validates integrity hashes and version consistency
|
||||
- Identifies drift between environments
|
||||
- Analyzes lockfile conflicts and resolution strategies
|
||||
- Ensures deterministic, reproducible builds
|
||||
|
||||
**Environment Consistency**
|
||||
- Compares dependencies across environments (dev/staging/prod)
|
||||
- Identifies version mismatches between team members
|
||||
- Validates CI/CD environment consistency
|
||||
- Tracks dependency resolution differences
|
||||
|
||||
## Technical Architecture
|
||||
|
||||
### Scanner Engine (`dep_scanner.py`)
|
||||
- Multi-format parser supporting 8+ package ecosystems
|
||||
- Built-in vulnerability database with 500+ CVE patterns
|
||||
- Transitive dependency resolution from lockfiles
|
||||
- JSON and human-readable output formats
|
||||
- Configurable scanning depth and exclusion patterns
|
||||
|
||||
### License Analyzer (`license_checker.py`)
|
||||
- License detection from package metadata and files
|
||||
- Compatibility matrix with 20+ license types
|
||||
- Conflict detection engine with remediation suggestions
|
||||
- Risk scoring based on distribution and usage context
|
||||
- Export capabilities for legal review
|
||||
|
||||
### Upgrade Planner (`upgrade_planner.py`)
|
||||
- Semantic version analysis with breaking change prediction
|
||||
- Dependency ordering based on risk and interdependence
|
||||
- Migration checklists with testing recommendations
|
||||
- Rollback procedures for failed upgrades
|
||||
- Timeline estimation for upgrade cycles
|
||||
|
||||
## Use Cases & Applications
|
||||
|
||||
### Security Teams
|
||||
- **Vulnerability Management**: Continuous scanning for security issues
|
||||
- **Incident Response**: Rapid assessment of vulnerable dependencies
|
||||
- **Supply Chain Monitoring**: Tracking third-party security posture
|
||||
- **Compliance Reporting**: Automated security compliance documentation
|
||||
|
||||
### Legal & Compliance Teams
|
||||
- **License Auditing**: Comprehensive license compliance verification
|
||||
- **Risk Assessment**: Legal risk analysis for software distribution
|
||||
- **Due Diligence**: Dependency licensing for M&A activities
|
||||
- **Policy Enforcement**: Automated license policy compliance
|
||||
|
||||
### Development Teams
|
||||
- **Dependency Hygiene**: Regular cleanup of unused dependencies
|
||||
- **Upgrade Planning**: Strategic dependency update scheduling
|
||||
- **Performance Optimization**: Bundle size optimization through dep analysis
|
||||
- **Technical Debt**: Identifying and prioritizing dependency technical debt
|
||||
|
||||
### DevOps & Platform Teams
|
||||
- **Build Optimization**: Faster builds through dependency optimization
|
||||
- **Security Automation**: Automated vulnerability scanning in CI/CD
|
||||
- **Environment Consistency**: Ensuring consistent dependencies across environments
|
||||
- **Release Management**: Dependency-aware release planning
|
||||
|
||||
## Integration Patterns
|
||||
|
||||
### CI/CD Pipeline Integration
|
||||
```bash
|
||||
# Security gate in CI
|
||||
python dep_scanner.py /project --format json --fail-on-high
|
||||
python license_checker.py /project --policy strict --format json
|
||||
```
|
||||
|
||||
### Scheduled Audits
|
||||
```bash
|
||||
# Weekly dependency audit
|
||||
./audit_dependencies.sh > weekly_report.html
|
||||
python upgrade_planner.py deps.json --timeline 30days
|
||||
```
|
||||
|
||||
### Development Workflow
|
||||
```bash
|
||||
# Pre-commit dependency check
|
||||
python dep_scanner.py . --quick-scan
|
||||
python license_checker.py . --warn-conflicts
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### Custom Vulnerability Databases
|
||||
- Support for internal/proprietary vulnerability feeds
|
||||
- Custom CVE pattern definitions
|
||||
- Organization-specific risk scoring
|
||||
- Integration with enterprise security tools
|
||||
|
||||
### Policy-Based Scanning
|
||||
- Configurable license policies by project type
|
||||
- Custom risk thresholds and escalation rules
|
||||
- Automated policy enforcement and notifications
|
||||
- Exception management for approved violations
|
||||
|
||||
### Reporting & Dashboards
|
||||
- Executive summaries for management
|
||||
- Technical reports for development teams
|
||||
- Trend analysis and dependency health metrics
|
||||
- Integration with project management tools
|
||||
|
||||
### Multi-Project Analysis
|
||||
- Portfolio-level dependency analysis
|
||||
- Shared dependency impact analysis
|
||||
- Organization-wide license compliance
|
||||
- Cross-project vulnerability propagation
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Scanning Frequency
|
||||
- **Security Scans**: Daily or on every commit
|
||||
- **License Audits**: Weekly or monthly
|
||||
- **Upgrade Planning**: Monthly or quarterly
|
||||
- **Full Dependency Audit**: Quarterly
|
||||
|
||||
### Risk Management
|
||||
1. **Prioritize Security**: Address high/critical CVEs immediately
|
||||
2. **License First**: Ensure compliance before functionality
|
||||
3. **Gradual Updates**: Incremental dependency updates
|
||||
4. **Test Thoroughly**: Comprehensive testing after updates
|
||||
5. **Monitor Continuously**: Automated monitoring and alerting
|
||||
|
||||
### Team Workflows
|
||||
1. **Security Champions**: Designate dependency security owners
|
||||
2. **Review Process**: Mandatory review for new dependencies
|
||||
3. **Update Cycles**: Regular, scheduled dependency updates
|
||||
4. **Documentation**: Maintain dependency rationale and decisions
|
||||
5. **Training**: Regular team education on dependency security
|
||||
|
||||
## Metrics & KPIs
|
||||
|
||||
### Security Metrics
|
||||
- Mean Time to Patch (MTTP) for vulnerabilities
|
||||
- Number of high/critical vulnerabilities
|
||||
- Percentage of dependencies with known vulnerabilities
|
||||
- Security debt accumulation rate
|
||||
|
||||
### Compliance Metrics
|
||||
- License compliance percentage
|
||||
- Number of license conflicts
|
||||
- Time to resolve compliance issues
|
||||
- Policy violation frequency
|
||||
|
||||
### Maintenance Metrics
|
||||
- Percentage of up-to-date dependencies
|
||||
- Average dependency age
|
||||
- Number of abandoned dependencies
|
||||
- Upgrade success rate
|
||||
|
||||
### Efficiency Metrics
|
||||
- Bundle size reduction percentage
|
||||
- Unused dependency elimination rate
|
||||
- Build time improvement
|
||||
- Developer productivity impact
|
||||
|
||||
## Troubleshooting Guide
|
||||
|
||||
### Common Issues
|
||||
1. **False Positives**: Tuning vulnerability detection sensitivity
|
||||
2. **License Ambiguity**: Resolving unclear or multiple licenses
|
||||
3. **Breaking Changes**: Managing major version upgrades
|
||||
4. **Performance Impact**: Optimizing scanning for large codebases
|
||||
|
||||
### Resolution Strategies
|
||||
- Whitelist false positives with documentation
|
||||
- Contact maintainers for license clarification
|
||||
- Implement feature flags for risky upgrades
|
||||
- Use incremental scanning for large projects
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### Planned Features
|
||||
- Machine learning for vulnerability prediction
|
||||
- Automated dependency update pull requests
|
||||
- Integration with container image scanning
|
||||
- Real-time dependency monitoring dashboards
|
||||
- Natural language policy definition
|
||||
|
||||
### Ecosystem Expansion
|
||||
- Additional language support (Swift, Kotlin, Dart)
|
||||
- Container and infrastructure dependencies
|
||||
- Development tool and build system dependencies
|
||||
- Cloud service and SaaS dependency tracking
|
||||
|
||||
---
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Scan project for vulnerabilities and licenses
|
||||
python scripts/dep_scanner.py /path/to/project
|
||||
|
||||
# Check license compliance
|
||||
python scripts/license_checker.py /path/to/project --policy strict
|
||||
|
||||
# Plan dependency upgrades
|
||||
python scripts/upgrade_planner.py deps.json --risk-threshold medium
|
||||
```
|
||||
|
||||
For detailed usage instructions, see [README.md](README.md).
|
||||
|
||||
---
|
||||
|
||||
*This skill provides comprehensive dependency management capabilities essential for maintaining secure, compliant, and efficient software projects. Regular use helps teams stay ahead of security threats, maintain legal compliance, and optimize their dependency ecosystems.*
|
||||
53
engineering/dependency-auditor/assets/sample_go.mod
Normal file
53
engineering/dependency-auditor/assets/sample_go.mod
Normal file
@@ -0,0 +1,53 @@
|
||||
module github.com/example/sample-go-service
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/gin-gonic/gin v1.9.1
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/gorilla/websocket v1.5.0
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/stretchr/testify v1.8.2
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.9.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
gorm.io/driver/postgres v1.5.0
|
||||
gorm.io/gorm v1.25.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/bytedance/sonic v1.8.8 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.13.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/pgx/v5 v5.3.1 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/mattn/go-isatty v0.0.18 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.7 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.11 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/arch v0.3.0 // indirect
|
||||
golang.org/x/net v0.10.0 // indirect
|
||||
golang.org/x/sys v0.8.0 // indirect
|
||||
golang.org/x/text v0.9.0 // indirect
|
||||
)
|
||||
72
engineering/dependency-auditor/assets/sample_package.json
Normal file
72
engineering/dependency-auditor/assets/sample_package.json
Normal file
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"name": "sample-web-app",
|
||||
"version": "1.2.3",
|
||||
"description": "A sample web application with various dependencies for testing dependency auditing",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"dev": "nodemon index.js",
|
||||
"build": "webpack --mode production",
|
||||
"test": "jest",
|
||||
"lint": "eslint src/",
|
||||
"audit": "npm audit"
|
||||
},
|
||||
"keywords": ["web", "app", "sample", "dependency", "audit"],
|
||||
"author": "Claude Skills Team",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"express": "4.18.1",
|
||||
"lodash": "4.17.20",
|
||||
"axios": "1.5.0",
|
||||
"jsonwebtoken": "8.5.1",
|
||||
"bcrypt": "5.1.0",
|
||||
"mongoose": "6.10.0",
|
||||
"cors": "2.8.5",
|
||||
"helmet": "6.1.5",
|
||||
"winston": "3.8.2",
|
||||
"dotenv": "16.0.3",
|
||||
"express-rate-limit": "6.7.0",
|
||||
"multer": "1.4.5-lts.1",
|
||||
"sharp": "0.32.1",
|
||||
"nodemailer": "6.9.1",
|
||||
"socket.io": "4.6.1",
|
||||
"redis": "4.6.5",
|
||||
"moment": "2.29.4",
|
||||
"chalk": "4.1.2",
|
||||
"commander": "9.4.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "2.0.22",
|
||||
"jest": "29.5.0",
|
||||
"supertest": "6.3.3",
|
||||
"eslint": "8.40.0",
|
||||
"eslint-config-airbnb-base": "15.0.0",
|
||||
"eslint-plugin-import": "2.27.5",
|
||||
"webpack": "5.82.1",
|
||||
"webpack-cli": "5.1.1",
|
||||
"babel-loader": "9.1.2",
|
||||
"@babel/core": "7.22.1",
|
||||
"@babel/preset-env": "7.22.2",
|
||||
"css-loader": "6.7.4",
|
||||
"style-loader": "3.3.3",
|
||||
"html-webpack-plugin": "5.5.1",
|
||||
"mini-css-extract-plugin": "2.7.6",
|
||||
"postcss": "8.4.23",
|
||||
"postcss-loader": "7.3.0",
|
||||
"autoprefixer": "10.4.14",
|
||||
"cross-env": "7.0.3",
|
||||
"rimraf": "5.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16.0.0",
|
||||
"npm": ">=8.0.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/example/sample-web-app.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/example/sample-web-app/issues"
|
||||
},
|
||||
"homepage": "https://github.com/example/sample-web-app#readme"
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
# Core web framework
|
||||
Django==4.1.7
|
||||
djangorestframework==3.14.0
|
||||
django-cors-headers==3.14.0
|
||||
django-environ==0.10.0
|
||||
django-extensions==3.2.1
|
||||
|
||||
# Database and ORM
|
||||
psycopg2-binary==2.9.6
|
||||
redis==4.5.4
|
||||
celery==5.2.7
|
||||
|
||||
# Authentication and Security
|
||||
django-allauth==0.54.0
|
||||
djangorestframework-simplejwt==5.2.2
|
||||
cryptography==40.0.1
|
||||
bcrypt==4.0.1
|
||||
|
||||
# HTTP and API clients
|
||||
requests==2.28.2
|
||||
httpx==0.24.1
|
||||
urllib3==1.26.15
|
||||
|
||||
# Data processing and analysis
|
||||
pandas==2.0.1
|
||||
numpy==1.24.3
|
||||
Pillow==9.5.0
|
||||
openpyxl==3.1.2
|
||||
|
||||
# Monitoring and logging
|
||||
sentry-sdk==1.21.1
|
||||
structlog==23.1.0
|
||||
|
||||
# Testing
|
||||
pytest==7.3.1
|
||||
pytest-django==4.5.2
|
||||
pytest-cov==4.0.0
|
||||
factory-boy==3.2.1
|
||||
freezegun==1.2.2
|
||||
|
||||
# Development tools
|
||||
black==23.3.0
|
||||
flake8==6.0.0
|
||||
isort==5.12.0
|
||||
pre-commit==3.3.2
|
||||
django-debug-toolbar==4.0.0
|
||||
|
||||
# Documentation
|
||||
Sphinx==6.2.1
|
||||
sphinx-rtd-theme==1.2.0
|
||||
|
||||
# Deployment and server
|
||||
gunicorn==20.1.0
|
||||
whitenoise==6.4.0
|
||||
|
||||
# Environment and configuration
|
||||
python-decouple==3.8
|
||||
pyyaml==6.0
|
||||
|
||||
# Utilities
|
||||
click==8.1.3
|
||||
python-dateutil==2.8.2
|
||||
pytz==2023.3
|
||||
six==1.16.0
|
||||
|
||||
# AWS integration
|
||||
boto3==1.26.137
|
||||
botocore==1.29.137
|
||||
|
||||
# Email
|
||||
django-anymail==10.0
|
||||
@@ -0,0 +1,37 @@
|
||||
============================================================
|
||||
LICENSE COMPLIANCE REPORT
|
||||
============================================================
|
||||
Analysis Date: 2024-02-16T15:30:00.000Z
|
||||
Project: /example/sample-web-app
|
||||
Project License: MIT
|
||||
|
||||
SUMMARY:
|
||||
Total Dependencies: 23
|
||||
Compliance Score: 92.5/100
|
||||
Overall Risk: LOW
|
||||
License Conflicts: 0
|
||||
|
||||
LICENSE DISTRIBUTION:
|
||||
Permissive: 21
|
||||
Copyleft_weak: 1
|
||||
Copyleft_strong: 0
|
||||
Proprietary: 0
|
||||
Unknown: 1
|
||||
|
||||
RISK BREAKDOWN:
|
||||
Low: 21
|
||||
Medium: 1
|
||||
High: 0
|
||||
Critical: 1
|
||||
|
||||
HIGH-RISK DEPENDENCIES:
|
||||
------------------------------
|
||||
moment v2.29.4: Unknown (CRITICAL)
|
||||
|
||||
RECOMMENDATIONS:
|
||||
--------------------
|
||||
1. Investigate and clarify licenses for 1 dependencies with unknown licensing
|
||||
2. Overall compliance score is high - maintain current practices
|
||||
3. Consider updating moment.js which has been deprecated by maintainers
|
||||
|
||||
============================================================
|
||||
@@ -0,0 +1,59 @@
|
||||
============================================================
|
||||
DEPENDENCY UPGRADE PLAN
|
||||
============================================================
|
||||
Generated: 2024-02-16T15:30:00.000Z
|
||||
Timeline: 90 days
|
||||
|
||||
UPGRADE SUMMARY:
|
||||
Total Upgrades Available: 12
|
||||
Security Updates: 2
|
||||
Major Version Updates: 3
|
||||
High Risk Updates: 2
|
||||
|
||||
RISK ASSESSMENT:
|
||||
Overall Risk Level: MEDIUM
|
||||
Key Risk Factors:
|
||||
• 2 critical risk upgrades requiring careful planning
|
||||
• Core framework upgrades: ['express', 'webpack', 'eslint']
|
||||
• 1 major version upgrades with potential breaking changes
|
||||
|
||||
TOP PRIORITY UPGRADES:
|
||||
------------------------------
|
||||
🔒 lodash: 4.17.20 → 4.17.21 🔒
|
||||
Type: Patch | Risk: Low | Priority: 95.0
|
||||
Security: CVE-2021-23337: Prototype pollution vulnerability
|
||||
|
||||
🟡 express: 4.18.1 → 4.18.2
|
||||
Type: Patch | Risk: Low | Priority: 85.0
|
||||
|
||||
🟡 webpack: 5.82.1 → 5.88.0
|
||||
Type: Minor | Risk: Medium | Priority: 75.0
|
||||
|
||||
🔴 eslint: 8.40.0 → 9.0.0
|
||||
Type: Major | Risk: High | Priority: 65.0
|
||||
|
||||
🟢 cors: 2.8.5 → 2.8.7
|
||||
Type: Patch | Risk: Safe | Priority: 80.0
|
||||
|
||||
PHASED UPGRADE PLANS:
|
||||
------------------------------
|
||||
Phase 1: Security & Safe Updates (30 days)
|
||||
Dependencies: lodash, cors, helmet, dotenv, bcrypt
|
||||
Key Steps: Create feature branch; Update dependency versions in manifest files; Run dependency install/update commands
|
||||
|
||||
Phase 2: Regular Updates (36 days)
|
||||
Dependencies: express, axios, winston, multer
|
||||
Key Steps: Create feature branch; Update dependency versions in manifest files; Run dependency install/update commands
|
||||
|
||||
Phase 3: Major Updates (30 days)
|
||||
Dependencies: webpack, eslint, jest
|
||||
... and 2 more
|
||||
Key Steps: Create feature branch; Update dependency versions in manifest files; Run dependency install/update commands
|
||||
|
||||
RECOMMENDATIONS:
|
||||
--------------------
|
||||
1. URGENT: 2 security updates available - prioritize immediately
|
||||
2. Quick wins: 6 safe updates can be applied with minimal risk
|
||||
3. Plan carefully: 2 high-risk upgrades need thorough testing
|
||||
|
||||
============================================================
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"timestamp": "2024-02-16T15:30:00.000Z",
|
||||
"project_path": "/example/sample-web-app",
|
||||
"dependencies": [
|
||||
{
|
||||
"name": "lodash",
|
||||
"version": "4.17.20",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": "MIT",
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "CVE-2021-23337",
|
||||
"summary": "Prototype pollution in lodash",
|
||||
"severity": "HIGH",
|
||||
"cvss_score": 7.2,
|
||||
"affected_versions": "<4.17.21",
|
||||
"fixed_version": "4.17.21",
|
||||
"published_date": "2021-02-15",
|
||||
"references": [
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2021-23337"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "axios",
|
||||
"version": "1.5.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": "MIT",
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "express",
|
||||
"version": "4.18.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": "MIT",
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "jsonwebtoken",
|
||||
"version": "8.5.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": "MIT",
|
||||
"vulnerabilities": []
|
||||
}
|
||||
],
|
||||
"vulnerabilities_found": 1,
|
||||
"high_severity_count": 1,
|
||||
"medium_severity_count": 0,
|
||||
"low_severity_count": 0,
|
||||
"ecosystems": ["npm"],
|
||||
"scan_summary": {
|
||||
"total_dependencies": 4,
|
||||
"unique_dependencies": 4,
|
||||
"ecosystems_found": 1,
|
||||
"vulnerable_dependencies": 1,
|
||||
"vulnerability_breakdown": {
|
||||
"high": 1,
|
||||
"medium": 0,
|
||||
"low": 0
|
||||
}
|
||||
},
|
||||
"recommendations": [
|
||||
"URGENT: Address 1 high-severity vulnerabilities immediately",
|
||||
"Update lodash from 4.17.20 to 4.17.21 to fix CVE-2021-23337"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,643 @@
|
||||
# Dependency Management Best Practices
|
||||
|
||||
A comprehensive guide to effective dependency management across the software development lifecycle, covering strategy, governance, security, and operational practices.
|
||||
|
||||
## Strategic Foundation
|
||||
|
||||
### Dependency Strategy
|
||||
|
||||
#### Philosophy and Principles
|
||||
1. **Minimize Dependencies**: Every dependency is a liability
|
||||
- Prefer standard library solutions when possible
|
||||
- Evaluate alternatives before adding new dependencies
|
||||
- Regularly audit and remove unused dependencies
|
||||
|
||||
2. **Quality Over Convenience**: Choose well-maintained, secure dependencies
|
||||
- Active maintenance and community
|
||||
- Strong security track record
|
||||
- Comprehensive documentation and testing
|
||||
|
||||
3. **Stability Over Novelty**: Prefer proven, stable solutions
|
||||
- Avoid dependencies with frequent breaking changes
|
||||
- Consider long-term support and backwards compatibility
|
||||
- Evaluate dependency maturity and adoption
|
||||
|
||||
4. **Transparency and Control**: Understand what you're depending on
|
||||
- Review dependency source code when possible
|
||||
- Understand licensing implications
|
||||
- Monitor dependency behavior and updates
|
||||
|
||||
#### Decision Framework
|
||||
|
||||
##### Evaluation Criteria
|
||||
```
|
||||
Dependency Evaluation Scorecard:
|
||||
│
|
||||
├── Necessity (25 points)
|
||||
│ ├── Problem complexity (10)
|
||||
│ ├── Standard library alternatives (8)
|
||||
│ └── Internal implementation effort (7)
|
||||
│
|
||||
├── Quality (30 points)
|
||||
│ ├── Code quality and architecture (10)
|
||||
│ ├── Test coverage and reliability (10)
|
||||
│ └── Documentation completeness (10)
|
||||
│
|
||||
├── Maintenance (25 points)
|
||||
│ ├── Active development and releases (10)
|
||||
│ ├── Issue response time (8)
|
||||
│ └── Community size and engagement (7)
|
||||
│
|
||||
└── Compatibility (20 points)
|
||||
├── License compatibility (10)
|
||||
├── Version stability (5)
|
||||
└── Platform/runtime compatibility (5)
|
||||
|
||||
Scoring:
|
||||
- 80-100: Excellent choice
|
||||
- 60-79: Good choice with monitoring
|
||||
- 40-59: Acceptable with caution
|
||||
- Below 40: Avoid or find alternatives
|
||||
```
|
||||
|
||||
### Governance Framework
|
||||
|
||||
#### Dependency Approval Process
|
||||
|
||||
##### New Dependency Approval
|
||||
```
|
||||
New Dependency Workflow:
|
||||
│
|
||||
1. Developer identifies need
|
||||
├── Documents use case and requirements
|
||||
├── Researches available options
|
||||
└── Proposes recommendation
|
||||
↓
|
||||
2. Technical review
|
||||
├── Architecture team evaluates fit
|
||||
├── Security team assesses risks
|
||||
└── Legal team reviews licensing
|
||||
↓
|
||||
3. Management approval
|
||||
├── Low risk: Tech lead approval
|
||||
├── Medium risk: Architecture board
|
||||
└── High risk: CTO approval
|
||||
↓
|
||||
4. Implementation
|
||||
├── Add to approved dependencies list
|
||||
├── Document usage guidelines
|
||||
└── Configure monitoring and alerts
|
||||
```
|
||||
|
||||
##### Risk Classification
|
||||
- **Low Risk**: Well-known libraries, permissive licenses, stable APIs
|
||||
- **Medium Risk**: Less common libraries, weak copyleft licenses, evolving APIs
|
||||
- **High Risk**: New/experimental libraries, strong copyleft licenses, breaking changes
|
||||
|
||||
#### Dependency Policies
|
||||
|
||||
##### Licensing Policy
|
||||
```yaml
|
||||
licensing_policy:
|
||||
allowed_licenses:
|
||||
- MIT
|
||||
- Apache-2.0
|
||||
- BSD-3-Clause
|
||||
- BSD-2-Clause
|
||||
- ISC
|
||||
|
||||
conditional_licenses:
|
||||
- LGPL-2.1 # Library linking only
|
||||
- LGPL-3.0 # With legal review
|
||||
- MPL-2.0 # File-level copyleft acceptable
|
||||
|
||||
prohibited_licenses:
|
||||
- GPL-2.0 # Strong copyleft
|
||||
- GPL-3.0 # Strong copyleft
|
||||
- AGPL-3.0 # Network copyleft
|
||||
- SSPL # Server-side public license
|
||||
- Custom # Unknown/proprietary licenses
|
||||
|
||||
exceptions:
|
||||
process: "Legal and executive approval required"
|
||||
documentation: "Risk assessment and mitigation plan"
|
||||
```
|
||||
|
||||
##### Security Policy
|
||||
```yaml
|
||||
security_policy:
|
||||
vulnerability_response:
|
||||
critical: "24 hours"
|
||||
high: "1 week"
|
||||
medium: "1 month"
|
||||
low: "Next release cycle"
|
||||
|
||||
scanning_requirements:
|
||||
frequency: "Daily automated scans"
|
||||
tools: ["Snyk", "OWASP Dependency Check"]
|
||||
ci_cd_integration: "Mandatory security gates"
|
||||
|
||||
approval_thresholds:
|
||||
known_vulnerabilities: "Zero tolerance for high/critical"
|
||||
maintenance_status: "Must be actively maintained"
|
||||
community_size: "Minimum 10 contributors or enterprise backing"
|
||||
```
|
||||
|
||||
## Operational Practices
|
||||
|
||||
### Dependency Lifecycle Management
|
||||
|
||||
#### Addition Process
|
||||
1. **Research and Evaluation**
|
||||
```bash
|
||||
# Example evaluation script
|
||||
#!/bin/bash
|
||||
PACKAGE=$1
|
||||
|
||||
echo "=== Package Analysis: $PACKAGE ==="
|
||||
|
||||
# Check package stats
|
||||
npm view $PACKAGE
|
||||
|
||||
# Security audit
|
||||
npm audit $PACKAGE
|
||||
|
||||
# License check
|
||||
npm view $PACKAGE license
|
||||
|
||||
# Dependency tree
|
||||
npm ls $PACKAGE
|
||||
|
||||
# Recent activity
|
||||
npm view $PACKAGE --json | jq '.time'
|
||||
```
|
||||
|
||||
2. **Documentation Requirements**
|
||||
- **Purpose**: Why this dependency is needed
|
||||
- **Alternatives**: Other options considered and why rejected
|
||||
- **Risk Assessment**: Security, licensing, maintenance risks
|
||||
- **Usage Guidelines**: How to use safely within the project
|
||||
- **Exit Strategy**: How to remove/replace if needed
|
||||
|
||||
3. **Integration Standards**
|
||||
- Pin to specific versions (avoid wildcards)
|
||||
- Document version constraints and reasoning
|
||||
- Configure automated update policies
|
||||
- Add monitoring and alerting
|
||||
|
||||
#### Update Management
|
||||
|
||||
##### Update Strategy
|
||||
```
|
||||
Update Prioritization:
|
||||
│
|
||||
├── Security Updates (P0)
|
||||
│ ├── Critical vulnerabilities: Immediate
|
||||
│ ├── High vulnerabilities: Within 1 week
|
||||
│ └── Medium vulnerabilities: Within 1 month
|
||||
│
|
||||
├── Maintenance Updates (P1)
|
||||
│ ├── Bug fixes: Next minor release
|
||||
│ ├── Performance improvements: Next minor release
|
||||
│ └── Deprecation warnings: Plan for major release
|
||||
│
|
||||
└── Feature Updates (P2)
|
||||
├── Minor versions: Quarterly review
|
||||
├── Major versions: Annual planning cycle
|
||||
└── Breaking changes: Dedicated migration projects
|
||||
```
|
||||
|
||||
##### Update Process
|
||||
```yaml
|
||||
update_workflow:
|
||||
automated:
|
||||
patch_updates:
|
||||
enabled: true
|
||||
auto_merge: true
|
||||
conditions:
|
||||
- tests_pass: true
|
||||
- security_scan_clean: true
|
||||
- no_breaking_changes: true
|
||||
|
||||
minor_updates:
|
||||
enabled: true
|
||||
auto_merge: false
|
||||
requires: "Manual review and testing"
|
||||
|
||||
major_updates:
|
||||
enabled: false
|
||||
requires: "Full impact assessment and planning"
|
||||
|
||||
testing_requirements:
|
||||
unit_tests: "100% pass rate"
|
||||
integration_tests: "Full test suite"
|
||||
security_tests: "Vulnerability scan clean"
|
||||
performance_tests: "No regression"
|
||||
|
||||
rollback_plan:
|
||||
automated: "Failed CI/CD triggers automatic rollback"
|
||||
manual: "Documented rollback procedure"
|
||||
monitoring: "Real-time health checks post-deployment"
|
||||
```
|
||||
|
||||
#### Removal Process
|
||||
1. **Deprecation Planning**
|
||||
- Identify deprecated/unused dependencies
|
||||
- Assess removal impact and effort
|
||||
- Plan migration timeline and strategy
|
||||
- Communicate to stakeholders
|
||||
|
||||
2. **Safe Removal**
|
||||
```bash
|
||||
# Example removal checklist
|
||||
echo "Dependency Removal Checklist:"
|
||||
echo "1. [ ] Grep codebase for all imports/usage"
|
||||
echo "2. [ ] Check if any other dependencies require it"
|
||||
echo "3. [ ] Remove from package files"
|
||||
echo "4. [ ] Run full test suite"
|
||||
echo "5. [ ] Update documentation"
|
||||
echo "6. [ ] Deploy with monitoring"
|
||||
```
|
||||
|
||||
### Version Management
|
||||
|
||||
#### Semantic Versioning Strategy
|
||||
|
||||
##### Version Pinning Policies
|
||||
```yaml
|
||||
version_pinning:
|
||||
production_dependencies:
|
||||
strategy: "Exact pinning"
|
||||
example: "react: 18.2.0"
|
||||
rationale: "Predictable builds, security control"
|
||||
|
||||
development_dependencies:
|
||||
strategy: "Compatible range"
|
||||
example: "eslint: ^8.0.0"
|
||||
rationale: "Allow bug fixes and improvements"
|
||||
|
||||
internal_libraries:
|
||||
strategy: "Compatible range"
|
||||
example: "^1.2.0"
|
||||
rationale: "Internal control, faster iteration"
|
||||
```
|
||||
|
||||
##### Update Windows
|
||||
- **Patch Updates (x.y.Z)**: Allow automatically with testing
|
||||
- **Minor Updates (x.Y.z)**: Review monthly, apply quarterly
|
||||
- **Major Updates (X.y.z)**: Annual review cycle, planned migrations
|
||||
|
||||
#### Lockfile Management
|
||||
|
||||
##### Best Practices
|
||||
1. **Always Commit Lockfiles**
|
||||
- package-lock.json (npm)
|
||||
- yarn.lock (Yarn)
|
||||
- Pipfile.lock (Python)
|
||||
- Cargo.lock (Rust)
|
||||
- go.sum (Go)
|
||||
|
||||
2. **Lockfile Validation**
|
||||
```bash
|
||||
# Example CI validation
|
||||
- name: Validate lockfile
|
||||
run: |
|
||||
npm ci --audit
|
||||
npm audit --audit-level moderate
|
||||
# Verify lockfile is up to date
|
||||
npm install --package-lock-only
|
||||
git diff --exit-code package-lock.json
|
||||
```
|
||||
|
||||
3. **Regeneration Policy**
|
||||
- Regenerate monthly or after significant updates
|
||||
- Always regenerate after security updates
|
||||
- Document regeneration in change logs
|
||||
|
||||
## Security Management
|
||||
|
||||
### Vulnerability Management
|
||||
|
||||
#### Continuous Monitoring
|
||||
```yaml
|
||||
monitoring_stack:
|
||||
scanning_tools:
|
||||
- name: "Snyk"
|
||||
scope: "All ecosystems"
|
||||
frequency: "Daily"
|
||||
integration: "CI/CD + IDE"
|
||||
|
||||
- name: "GitHub Dependabot"
|
||||
scope: "GitHub repositories"
|
||||
frequency: "Real-time"
|
||||
integration: "Pull requests"
|
||||
|
||||
- name: "OWASP Dependency Check"
|
||||
scope: "Java/.NET focus"
|
||||
frequency: "Build pipeline"
|
||||
integration: "CI/CD gates"
|
||||
|
||||
alerting:
|
||||
channels: ["Slack", "Email", "PagerDuty"]
|
||||
escalation:
|
||||
critical: "Immediate notification"
|
||||
high: "Within 1 hour"
|
||||
medium: "Daily digest"
|
||||
```
|
||||
|
||||
#### Response Procedures
|
||||
|
||||
##### Critical Vulnerability Response
|
||||
```
|
||||
Critical Vulnerability (CVSS 9.0+) Response:
|
||||
│
|
||||
0-2 hours: Detection & Assessment
|
||||
├── Automated scan identifies vulnerability
|
||||
├── Security team notified immediately
|
||||
└── Initial impact assessment started
|
||||
│
|
||||
2-6 hours: Planning & Communication
|
||||
├── Detailed impact analysis completed
|
||||
├── Fix strategy determined
|
||||
├── Stakeholder communication initiated
|
||||
└── Emergency change approval obtained
|
||||
│
|
||||
6-24 hours: Implementation & Testing
|
||||
├── Fix implemented in development
|
||||
├── Security testing performed
|
||||
├── Limited rollout to staging
|
||||
└── Production deployment prepared
|
||||
│
|
||||
24-48 hours: Deployment & Validation
|
||||
├── Production deployment executed
|
||||
├── Monitoring and validation performed
|
||||
├── Post-deployment testing completed
|
||||
└── Incident documentation finalized
|
||||
```
|
||||
|
||||
### Supply Chain Security
|
||||
|
||||
#### Source Verification
|
||||
1. **Package Authenticity**
|
||||
- Verify package signatures when available
|
||||
- Use official package registries
|
||||
- Check package maintainer reputation
|
||||
- Validate download checksums
|
||||
|
||||
2. **Build Reproducibility**
|
||||
- Use deterministic builds where possible
|
||||
- Pin dependency versions exactly
|
||||
- Document build environment requirements
|
||||
- Maintain build artifact checksums
|
||||
|
||||
#### Dependency Provenance
|
||||
```yaml
|
||||
provenance_tracking:
|
||||
metadata_collection:
|
||||
- package_name: "Library identification"
|
||||
- version: "Exact version used"
|
||||
- source_url: "Official repository"
|
||||
- maintainer: "Package maintainer info"
|
||||
- license: "License verification"
|
||||
- checksum: "Content verification"
|
||||
|
||||
verification_process:
|
||||
- signature_check: "GPG signature validation"
|
||||
- reputation_check: "Maintainer history review"
|
||||
- content_analysis: "Static code analysis"
|
||||
- behavior_monitoring: "Runtime behavior analysis"
|
||||
```
|
||||
|
||||
## Multi-Language Considerations
|
||||
|
||||
### Ecosystem-Specific Practices
|
||||
|
||||
#### JavaScript/Node.js
|
||||
```json
|
||||
{
|
||||
"npm_practices": {
|
||||
"package_json": {
|
||||
"engines": "Specify Node.js version requirements",
|
||||
"dependencies": "Production dependencies only",
|
||||
"devDependencies": "Development tools and testing",
|
||||
"optionalDependencies": "Use sparingly, document why"
|
||||
},
|
||||
"security": {
|
||||
"npm_audit": "Run in CI/CD pipeline",
|
||||
"package_lock": "Always commit to repository",
|
||||
"registry": "Use official npm registry or approved mirrors"
|
||||
},
|
||||
"performance": {
|
||||
"bundle_analysis": "Regular bundle size monitoring",
|
||||
"tree_shaking": "Ensure unused code is eliminated",
|
||||
"code_splitting": "Lazy load dependencies when possible"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
#### Python
|
||||
```yaml
|
||||
python_practices:
|
||||
dependency_files:
|
||||
requirements.txt: "Pin exact versions for production"
|
||||
requirements-dev.txt: "Development dependencies"
|
||||
setup.py: "Package distribution metadata"
|
||||
pyproject.toml: "Modern Python packaging"
|
||||
|
||||
virtual_environments:
|
||||
purpose: "Isolate project dependencies"
|
||||
tools: ["venv", "virtualenv", "conda", "poetry"]
|
||||
best_practice: "One environment per project"
|
||||
|
||||
security:
|
||||
tools: ["safety", "pip-audit", "bandit"]
|
||||
practices: ["Pin versions", "Use private PyPI if needed"]
|
||||
```
|
||||
|
||||
#### Java/Maven
|
||||
```xml
|
||||
<!-- Maven best practices -->
|
||||
<properties>
|
||||
<!-- Define version properties -->
|
||||
<spring.version>5.3.21</spring.version>
|
||||
<junit.version>5.8.2</junit.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
<!-- Centralize version management -->
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
<artifactId>spring-bom</artifactId>
|
||||
<version>${spring.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
```
|
||||
|
||||
### Cross-Language Integration
|
||||
|
||||
#### API Boundaries
|
||||
- Define clear service interfaces
|
||||
- Use standard protocols (HTTP, gRPC)
|
||||
- Document API contracts
|
||||
- Version APIs independently
|
||||
|
||||
#### Shared Dependencies
|
||||
- Minimize shared dependencies across services
|
||||
- Use containerization for isolation
|
||||
- Document shared dependency policies
|
||||
- Monitor for version conflicts
|
||||
|
||||
## Performance and Optimization
|
||||
|
||||
### Bundle Size Management
|
||||
|
||||
#### Analysis Tools
|
||||
```bash
|
||||
# JavaScript bundle analysis
|
||||
npm install -g webpack-bundle-analyzer
|
||||
webpack-bundle-analyzer dist/main.js
|
||||
|
||||
# Python package size analysis
|
||||
pip install pip-audit
|
||||
pip-audit --format json | jq '.dependencies[].package_size'
|
||||
|
||||
# General dependency tree analysis
|
||||
dep-tree analyze --format json --output deps.json
|
||||
```
|
||||
|
||||
#### Optimization Strategies
|
||||
1. **Tree Shaking**: Remove unused code
|
||||
2. **Code Splitting**: Load dependencies on demand
|
||||
3. **Polyfill Optimization**: Only include needed polyfills
|
||||
4. **Alternative Packages**: Choose smaller alternatives when possible
|
||||
|
||||
### Build Performance
|
||||
|
||||
#### Dependency Caching
|
||||
```yaml
|
||||
# Example CI/CD caching
|
||||
cache_strategy:
|
||||
node_modules:
|
||||
key: "npm-{{ checksum 'package-lock.json' }}"
|
||||
paths: ["~/.npm", "node_modules"]
|
||||
|
||||
pip_cache:
|
||||
key: "pip-{{ checksum 'requirements.txt' }}"
|
||||
paths: ["~/.cache/pip"]
|
||||
|
||||
maven_cache:
|
||||
key: "maven-{{ checksum 'pom.xml' }}"
|
||||
paths: ["~/.m2/repository"]
|
||||
```
|
||||
|
||||
#### Parallel Installation
|
||||
- Configure package managers for parallel downloads
|
||||
- Use local package caches
|
||||
- Consider dependency proxies for enterprise environments
|
||||
|
||||
## Monitoring and Metrics
|
||||
|
||||
### Key Performance Indicators
|
||||
|
||||
#### Security Metrics
|
||||
```yaml
|
||||
security_kpis:
|
||||
vulnerability_metrics:
|
||||
- mean_time_to_detection: "Average time to identify vulnerabilities"
|
||||
- mean_time_to_patch: "Average time to fix vulnerabilities"
|
||||
- vulnerability_density: "Vulnerabilities per 1000 dependencies"
|
||||
- false_positive_rate: "Percentage of false vulnerability reports"
|
||||
|
||||
compliance_metrics:
|
||||
- license_compliance_rate: "Percentage of compliant dependencies"
|
||||
- policy_violation_rate: "Rate of policy violations"
|
||||
- security_gate_success_rate: "CI/CD security gate pass rate"
|
||||
```
|
||||
|
||||
#### Operational Metrics
|
||||
```yaml
|
||||
operational_kpis:
|
||||
maintenance_metrics:
|
||||
- dependency_freshness: "Average age of dependencies"
|
||||
- update_frequency: "Rate of dependency updates"
|
||||
- technical_debt: "Number of outdated dependencies"
|
||||
|
||||
performance_metrics:
|
||||
- build_time: "Time to install/build dependencies"
|
||||
- bundle_size: "Final application size"
|
||||
- dependency_count: "Total number of dependencies"
|
||||
```
|
||||
|
||||
### Dashboard and Reporting
|
||||
|
||||
#### Executive Dashboard
|
||||
- Overall risk score and trend
|
||||
- Security compliance status
|
||||
- Cost of dependency management
|
||||
- Policy violation summary
|
||||
|
||||
#### Technical Dashboard
|
||||
- Vulnerability count by severity
|
||||
- Outdated dependency count
|
||||
- Build performance metrics
|
||||
- License compliance details
|
||||
|
||||
#### Automated Reports
|
||||
- Weekly security summary
|
||||
- Monthly compliance report
|
||||
- Quarterly dependency review
|
||||
- Annual strategy assessment
|
||||
|
||||
## Team Organization and Training
|
||||
|
||||
### Roles and Responsibilities
|
||||
|
||||
#### Security Champions
|
||||
- Monitor security advisories
|
||||
- Review dependency security scans
|
||||
- Coordinate vulnerability responses
|
||||
- Maintain security policies
|
||||
|
||||
#### Platform Engineers
|
||||
- Maintain dependency management infrastructure
|
||||
- Configure automated scanning and updates
|
||||
- Manage package registries and mirrors
|
||||
- Support development teams
|
||||
|
||||
#### Development Teams
|
||||
- Follow dependency policies
|
||||
- Perform regular security updates
|
||||
- Document dependency decisions
|
||||
- Participate in security training
|
||||
|
||||
### Training Programs
|
||||
|
||||
#### Security Training
|
||||
- Dependency security fundamentals
|
||||
- Vulnerability assessment and response
|
||||
- Secure coding practices
|
||||
- Supply chain attack awareness
|
||||
|
||||
#### Tool Training
|
||||
- Package manager best practices
|
||||
- Security scanning tool usage
|
||||
- CI/CD security integration
|
||||
- Incident response procedures
|
||||
|
||||
## Conclusion
|
||||
|
||||
Effective dependency management requires a holistic approach combining technical practices, organizational policies, and cultural awareness. Key success factors:
|
||||
|
||||
1. **Proactive Strategy**: Plan dependency management from project inception
|
||||
2. **Clear Governance**: Establish and enforce dependency policies
|
||||
3. **Automated Processes**: Use tools to scale security and maintenance
|
||||
4. **Continuous Monitoring**: Stay informed about dependency risks and updates
|
||||
5. **Team Training**: Ensure all team members understand security implications
|
||||
6. **Regular Review**: Periodically assess and improve dependency practices
|
||||
|
||||
Remember that dependency management is an investment in long-term project health, security, and maintainability. The upfront effort to establish good practices pays dividends in reduced security risks, easier maintenance, and more stable software systems.
|
||||
@@ -0,0 +1,238 @@
|
||||
# License Compatibility Matrix
|
||||
|
||||
This document provides a comprehensive reference for understanding license compatibility when combining open source software dependencies in your projects.
|
||||
|
||||
## Understanding License Types
|
||||
|
||||
### Permissive Licenses
|
||||
- **MIT License**: Very permissive, allows commercial use, modification, and distribution
|
||||
- **Apache 2.0**: Permissive with patent grant and trademark restrictions
|
||||
- **BSD 3-Clause**: Permissive with non-endorsement clause
|
||||
- **BSD 2-Clause**: Simple permissive license
|
||||
- **ISC License**: Functionally equivalent to MIT
|
||||
|
||||
### Weak Copyleft Licenses
|
||||
- **LGPL 2.1/3.0**: Library-level copyleft, allows linking but requires modifications to be shared
|
||||
- **MPL 2.0**: File-level copyleft, compatible with many licenses
|
||||
|
||||
### Strong Copyleft Licenses
|
||||
- **GPL 2.0/3.0**: Requires entire derivative work to be GPL-licensed
|
||||
- **AGPL 3.0**: Extends GPL to network services (SaaS applications)
|
||||
|
||||
## Compatibility Matrix
|
||||
|
||||
| Project License | MIT | Apache-2.0 | BSD-3 | LGPL-2.1 | LGPL-3.0 | MPL-2.0 | GPL-2.0 | GPL-3.0 | AGPL-3.0 |
|
||||
|----------------|-----|------------|-------|----------|----------|---------|---------|---------|----------|
|
||||
| **MIT** | ✅ | ✅ | ✅ | ⚠️ | ⚠️ | ⚠️ | ❌ | ❌ | ❌ |
|
||||
| **Apache-2.0** | ✅ | ✅ | ✅ | ❌ | ⚠️ | ✅ | ❌ | ⚠️ | ⚠️ |
|
||||
| **BSD-3** | ✅ | ✅ | ✅ | ⚠️ | ⚠️ | ⚠️ | ❌ | ❌ | ❌ |
|
||||
| **LGPL-2.1** | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ |
|
||||
| **LGPL-3.0** | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ |
|
||||
| **MPL-2.0** | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ |
|
||||
| **GPL-2.0** | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ |
|
||||
| **GPL-3.0** | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ |
|
||||
| **AGPL-3.0** | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ✅ | ✅ |
|
||||
|
||||
**Legend:**
|
||||
- ✅ Generally Compatible
|
||||
- ⚠️ Compatible with conditions/restrictions
|
||||
- ❌ Incompatible
|
||||
|
||||
## Detailed Compatibility Rules
|
||||
|
||||
### MIT Project with Other Licenses
|
||||
|
||||
**Compatible:**
|
||||
- MIT, Apache-2.0, BSD (all variants), ISC: Full compatibility
|
||||
- LGPL 2.1/3.0: Can use LGPL libraries via dynamic linking
|
||||
- MPL 2.0: Can use MPL modules, must keep MPL files under MPL
|
||||
|
||||
**Incompatible:**
|
||||
- GPL 2.0/3.0: GPL requires entire project to be GPL
|
||||
- AGPL 3.0: AGPL extends to network services
|
||||
|
||||
### Apache 2.0 Project with Other Licenses
|
||||
|
||||
**Compatible:**
|
||||
- MIT, BSD, ISC: Full compatibility
|
||||
- LGPL 3.0: Compatible (LGPL 3.0 has Apache compatibility clause)
|
||||
- MPL 2.0: Compatible
|
||||
- GPL 3.0: Compatible (GPL 3.0 has Apache compatibility clause)
|
||||
|
||||
**Incompatible:**
|
||||
- LGPL 2.1: License incompatibility
|
||||
- GPL 2.0: License incompatibility (no Apache clause)
|
||||
|
||||
### GPL Projects
|
||||
|
||||
**GPL 2.0 Compatible:**
|
||||
- MIT, BSD, ISC: Can incorporate permissive code
|
||||
- LGPL 2.1: Compatible
|
||||
- Other GPL 2.0: Compatible
|
||||
|
||||
**GPL 2.0 Incompatible:**
|
||||
- Apache 2.0: Different patent clauses
|
||||
- LGPL 3.0: Version incompatibility
|
||||
- GPL 3.0: Version incompatibility
|
||||
|
||||
**GPL 3.0 Compatible:**
|
||||
- All permissive licenses (MIT, Apache, BSD, ISC)
|
||||
- LGPL 3.0: Version compatibility
|
||||
- MPL 2.0: Explicit compatibility
|
||||
|
||||
## Common Compatibility Scenarios
|
||||
|
||||
### Scenario 1: Permissive Project with GPL Dependency
|
||||
**Problem:** MIT-licensed project wants to use GPL library
|
||||
**Impact:** Entire project must become GPL-licensed
|
||||
**Solutions:**
|
||||
1. Find alternative non-GPL library
|
||||
2. Use dynamic linking (if possible)
|
||||
3. Change project license to GPL
|
||||
4. Remove the dependency
|
||||
|
||||
### Scenario 2: Apache Project with GPL 2.0 Dependency
|
||||
**Problem:** Apache 2.0 project with GPL 2.0 dependency
|
||||
**Impact:** License incompatibility due to patent clauses
|
||||
**Solutions:**
|
||||
1. Upgrade to GPL 3.0 if available
|
||||
2. Find alternative library
|
||||
3. Use via separate service (API boundary)
|
||||
|
||||
### Scenario 3: Commercial Product with AGPL Dependency
|
||||
**Problem:** Proprietary software using AGPL library
|
||||
**Impact:** AGPL copyleft extends to network services
|
||||
**Solutions:**
|
||||
1. Obtain commercial license
|
||||
2. Replace with permissive alternative
|
||||
3. Use via separate service with API boundary
|
||||
4. Make entire application AGPL
|
||||
|
||||
## License Combination Rules
|
||||
|
||||
### Safe Combinations
|
||||
1. **Permissive + Permissive**: Always safe
|
||||
2. **Permissive + Weak Copyleft**: Usually safe with proper attribution
|
||||
3. **GPL + Compatible Permissive**: Safe, result is GPL
|
||||
|
||||
### Risky Combinations
|
||||
1. **Apache 2.0 + GPL 2.0**: Incompatible patent terms
|
||||
2. **Different GPL versions**: Version compatibility issues
|
||||
3. **Permissive + Strong Copyleft**: Changes project licensing
|
||||
|
||||
### Forbidden Combinations
|
||||
1. **MIT + GPL** (without relicensing)
|
||||
2. **Proprietary + Any Copyleft**
|
||||
3. **LGPL 2.1 + Apache 2.0**
|
||||
|
||||
## Distribution Considerations
|
||||
|
||||
### Binary Distribution
|
||||
- Must include all required license texts
|
||||
- Must preserve copyright notices
|
||||
- Must include source code for copyleft licenses
|
||||
- Must provide installation instructions for LGPL
|
||||
|
||||
### Source Distribution
|
||||
- Must include original license files
|
||||
- Must preserve copyright headers
|
||||
- Must document any modifications
|
||||
- Must provide clear licensing information
|
||||
|
||||
### SaaS/Network Services
|
||||
- AGPL extends copyleft to network services
|
||||
- GPL/LGPL generally don't apply to network services
|
||||
- Consider service boundaries carefully
|
||||
|
||||
## Compliance Best Practices
|
||||
|
||||
### 1. License Inventory
|
||||
- Maintain complete list of all dependencies
|
||||
- Track license changes in updates
|
||||
- Document license obligations
|
||||
|
||||
### 2. Compatibility Checking
|
||||
- Use automated tools for license scanning
|
||||
- Implement CI/CD license gates
|
||||
- Regular compliance audits
|
||||
|
||||
### 3. Documentation
|
||||
- Clear project license declaration
|
||||
- Complete attribution files
|
||||
- License change history
|
||||
|
||||
### 4. Legal Review
|
||||
- Consult legal counsel for complex scenarios
|
||||
- Review before major releases
|
||||
- Consider business model implications
|
||||
|
||||
## Risk Mitigation Strategies
|
||||
|
||||
### High-Risk Licenses
|
||||
- **AGPL**: Avoid in commercial/proprietary projects
|
||||
- **GPL in permissive projects**: Plan migration strategy
|
||||
- **Unknown licenses**: Investigate immediately
|
||||
|
||||
### Medium-Risk Scenarios
|
||||
- **Version incompatibilities**: Upgrade when possible
|
||||
- **Patent clause conflicts**: Seek legal advice
|
||||
- **Multiple copyleft licenses**: Verify compatibility
|
||||
|
||||
### Risk Assessment Framework
|
||||
1. **Identify** all dependencies and their licenses
|
||||
2. **Classify** by license type and risk level
|
||||
3. **Analyze** compatibility with project license
|
||||
4. **Document** decisions and rationale
|
||||
5. **Monitor** for license changes
|
||||
|
||||
## Common Misconceptions
|
||||
|
||||
### ❌ Wrong Assumptions
|
||||
- "MIT allows everything" (still requires attribution)
|
||||
- "Linking doesn't create derivatives" (depends on license)
|
||||
- "GPL only affects distribution" (AGPL affects network use)
|
||||
- "Commercial use is always forbidden" (most FOSS allows it)
|
||||
|
||||
### ✅ Correct Understanding
|
||||
- Each license has specific requirements
|
||||
- Combination creates most restrictive terms
|
||||
- Network use may trigger copyleft (AGPL)
|
||||
- Commercial licensing options often available
|
||||
|
||||
## Quick Reference Decision Tree
|
||||
|
||||
```
|
||||
Is the dependency GPL/AGPL?
|
||||
├─ YES → Is your project commercial/proprietary?
|
||||
│ ├─ YES → ❌ Incompatible (find alternative)
|
||||
│ └─ NO → ✅ Compatible (if same GPL version)
|
||||
└─ NO → Is it permissive (MIT/Apache/BSD)?
|
||||
├─ YES → ✅ Generally compatible
|
||||
└─ NO → Check specific compatibility matrix
|
||||
```
|
||||
|
||||
## Tools and Resources
|
||||
|
||||
### Automated Tools
|
||||
- **FOSSA**: Commercial license scanning
|
||||
- **WhiteSource**: Enterprise license management
|
||||
- **ORT**: Open source license scanning
|
||||
- **License Finder**: Ruby-based license detection
|
||||
|
||||
### Manual Review Resources
|
||||
- **choosealicense.com**: License picker and comparison
|
||||
- **SPDX License List**: Standardized license identifiers
|
||||
- **FSF License List**: Free Software Foundation compatibility
|
||||
- **OSI Approved Licenses**: Open Source Initiative approved licenses
|
||||
|
||||
## Conclusion
|
||||
|
||||
License compatibility is crucial for legal compliance and risk management. When in doubt:
|
||||
|
||||
1. **Choose permissive licenses** for maximum compatibility
|
||||
2. **Avoid strong copyleft** in proprietary projects
|
||||
3. **Document all license decisions** thoroughly
|
||||
4. **Consult legal experts** for complex scenarios
|
||||
5. **Use automated tools** for continuous monitoring
|
||||
|
||||
Remember: This matrix provides general guidance but legal requirements may vary by jurisdiction and specific use cases. Always consult with legal counsel for important licensing decisions.
|
||||
@@ -0,0 +1,461 @@
|
||||
# Vulnerability Assessment Guide
|
||||
|
||||
A comprehensive guide to assessing, prioritizing, and managing security vulnerabilities in software dependencies.
|
||||
|
||||
## Overview
|
||||
|
||||
Dependency vulnerabilities represent one of the most significant attack vectors in modern software systems. This guide provides a structured approach to vulnerability assessment, risk scoring, and remediation planning.
|
||||
|
||||
## Vulnerability Classification System
|
||||
|
||||
### Severity Levels (CVSS 3.1)
|
||||
|
||||
#### Critical (9.0 - 10.0)
|
||||
- **Impact**: Complete system compromise possible
|
||||
- **Examples**: Remote code execution, privilege escalation to admin
|
||||
- **Response Time**: Immediate (within 24 hours)
|
||||
- **Business Risk**: System shutdown, data breach, regulatory violations
|
||||
|
||||
#### High (7.0 - 8.9)
|
||||
- **Impact**: Significant security impact
|
||||
- **Examples**: SQL injection, authentication bypass, sensitive data exposure
|
||||
- **Response Time**: 7 days maximum
|
||||
- **Business Risk**: Data compromise, service disruption
|
||||
|
||||
#### Medium (4.0 - 6.9)
|
||||
- **Impact**: Moderate security impact
|
||||
- **Examples**: Cross-site scripting (XSS), information disclosure
|
||||
- **Response Time**: 30 days
|
||||
- **Business Risk**: Limited data exposure, minor service impact
|
||||
|
||||
#### Low (0.1 - 3.9)
|
||||
- **Impact**: Limited security impact
|
||||
- **Examples**: Denial of service (limited), minor information leakage
|
||||
- **Response Time**: Next planned release cycle
|
||||
- **Business Risk**: Minimal impact on operations
|
||||
|
||||
## Vulnerability Types and Patterns
|
||||
|
||||
### Code Injection Vulnerabilities
|
||||
|
||||
#### SQL Injection
|
||||
- **CWE-89**: Improper neutralization of SQL commands
|
||||
- **Common in**: Database interaction libraries, ORM frameworks
|
||||
- **Detection**: Parameter handling analysis, query construction review
|
||||
- **Mitigation**: Parameterized queries, input validation, least privilege DB access
|
||||
|
||||
#### Command Injection
|
||||
- **CWE-78**: OS command injection
|
||||
- **Common in**: System utilities, file processing libraries
|
||||
- **Detection**: System call analysis, user input handling
|
||||
- **Mitigation**: Input sanitization, avoid system calls, sandboxing
|
||||
|
||||
#### Code Injection
|
||||
- **CWE-94**: Code injection
|
||||
- **Common in**: Template engines, dynamic code evaluation
|
||||
- **Detection**: eval() usage, dynamic code generation
|
||||
- **Mitigation**: Avoid dynamic code execution, input validation, sandboxing
|
||||
|
||||
### Authentication and Authorization
|
||||
|
||||
#### Authentication Bypass
|
||||
- **CWE-287**: Improper authentication
|
||||
- **Common in**: Authentication libraries, session management
|
||||
- **Detection**: Authentication flow analysis, session handling review
|
||||
- **Mitigation**: Multi-factor authentication, secure session management
|
||||
|
||||
#### Privilege Escalation
|
||||
- **CWE-269**: Improper privilege management
|
||||
- **Common in**: Authorization frameworks, access control libraries
|
||||
- **Detection**: Permission checking analysis, role validation
|
||||
- **Mitigation**: Principle of least privilege, proper access controls
|
||||
|
||||
### Data Exposure
|
||||
|
||||
#### Sensitive Data Exposure
|
||||
- **CWE-200**: Information exposure
|
||||
- **Common in**: Logging libraries, error handling, API responses
|
||||
- **Detection**: Log output analysis, error message review
|
||||
- **Mitigation**: Data classification, sanitized logging, proper error handling
|
||||
|
||||
#### Cryptographic Failures
|
||||
- **CWE-327**: Broken cryptography
|
||||
- **Common in**: Cryptographic libraries, hash functions
|
||||
- **Detection**: Algorithm analysis, key management review
|
||||
- **Mitigation**: Modern cryptographic standards, proper key management
|
||||
|
||||
### Input Validation Issues
|
||||
|
||||
#### Cross-Site Scripting (XSS)
|
||||
- **CWE-79**: Improper neutralization of input
|
||||
- **Common in**: Web frameworks, template engines
|
||||
- **Detection**: Input handling analysis, output encoding review
|
||||
- **Mitigation**: Input validation, output encoding, Content Security Policy
|
||||
|
||||
#### Deserialization Vulnerabilities
|
||||
- **CWE-502**: Deserialization of untrusted data
|
||||
- **Common in**: Serialization libraries, data processing
|
||||
- **Detection**: Deserialization usage analysis
|
||||
- **Mitigation**: Avoid untrusted deserialization, input validation
|
||||
|
||||
## Risk Assessment Framework
|
||||
|
||||
### CVSS Scoring Components
|
||||
|
||||
#### Base Metrics
|
||||
1. **Attack Vector (AV)**
|
||||
- Network (N): 0.85
|
||||
- Adjacent (A): 0.62
|
||||
- Local (L): 0.55
|
||||
- Physical (P): 0.2
|
||||
|
||||
2. **Attack Complexity (AC)**
|
||||
- Low (L): 0.77
|
||||
- High (H): 0.44
|
||||
|
||||
3. **Privileges Required (PR)**
|
||||
- None (N): 0.85
|
||||
- Low (L): 0.62/0.68
|
||||
- High (H): 0.27/0.50
|
||||
|
||||
4. **User Interaction (UI)**
|
||||
- None (N): 0.85
|
||||
- Required (R): 0.62
|
||||
|
||||
5. **Impact Metrics (C/I/A)**
|
||||
- High (H): 0.56
|
||||
- Low (L): 0.22
|
||||
- None (N): 0
|
||||
|
||||
#### Temporal Metrics
|
||||
- **Exploit Code Maturity**: Proof of concept availability
|
||||
- **Remediation Level**: Official fix availability
|
||||
- **Report Confidence**: Vulnerability confirmation level
|
||||
|
||||
#### Environmental Metrics
|
||||
- **Confidentiality/Integrity/Availability Requirements**: Business impact
|
||||
- **Modified Base Metrics**: Environment-specific adjustments
|
||||
|
||||
### Custom Risk Factors
|
||||
|
||||
#### Business Context
|
||||
1. **Data Sensitivity**
|
||||
- Public data: Low risk multiplier (1.0x)
|
||||
- Internal data: Medium risk multiplier (1.2x)
|
||||
- Customer data: High risk multiplier (1.5x)
|
||||
- Regulated data: Critical risk multiplier (2.0x)
|
||||
|
||||
2. **System Criticality**
|
||||
- Development: Low impact (1.0x)
|
||||
- Staging: Medium impact (1.3x)
|
||||
- Production: High impact (1.8x)
|
||||
- Core infrastructure: Critical impact (2.5x)
|
||||
|
||||
3. **Exposure Level**
|
||||
- Internal systems: Base risk
|
||||
- Partner access: +1 risk level
|
||||
- Public internet: +2 risk levels
|
||||
- High-value target: +3 risk levels
|
||||
|
||||
#### Technical Factors
|
||||
|
||||
1. **Dependency Type**
|
||||
- Direct dependencies: Higher priority
|
||||
- Transitive dependencies: Lower priority (unless critical path)
|
||||
- Development dependencies: Lowest priority
|
||||
|
||||
2. **Usage Pattern**
|
||||
- Core functionality: Highest priority
|
||||
- Optional features: Medium priority
|
||||
- Unused code paths: Lowest priority
|
||||
|
||||
3. **Fix Availability**
|
||||
- Official patch available: Standard timeline
|
||||
- Workaround available: Extended timeline acceptable
|
||||
- No fix available: Risk acceptance or replacement needed
|
||||
|
||||
## Vulnerability Discovery and Monitoring
|
||||
|
||||
### Automated Scanning
|
||||
|
||||
#### Dependency Scanners
|
||||
- **npm audit**: Node.js ecosystem
|
||||
- **pip-audit**: Python ecosystem
|
||||
- **bundler-audit**: Ruby ecosystem
|
||||
- **OWASP Dependency Check**: Multi-language support
|
||||
|
||||
#### Continuous Monitoring
|
||||
```bash
|
||||
# Example CI/CD integration
|
||||
name: Security Scan
|
||||
on: [push, pull_request, schedule]
|
||||
jobs:
|
||||
security-scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Run dependency audit
|
||||
run: |
|
||||
npm audit --audit-level high
|
||||
python -m pip_audit
|
||||
bundle audit
|
||||
```
|
||||
|
||||
#### Commercial Tools
|
||||
- **Snyk**: Developer-first security platform
|
||||
- **WhiteSource**: Enterprise dependency management
|
||||
- **Veracode**: Application security platform
|
||||
- **Checkmarx**: Static application security testing
|
||||
|
||||
### Manual Assessment
|
||||
|
||||
#### Code Review Checklist
|
||||
1. **Input Validation**
|
||||
- [ ] All user inputs validated
|
||||
- [ ] Proper sanitization applied
|
||||
- [ ] Length and format restrictions
|
||||
|
||||
2. **Authentication/Authorization**
|
||||
- [ ] Proper authentication checks
|
||||
- [ ] Authorization at every access point
|
||||
- [ ] Session management secure
|
||||
|
||||
3. **Data Handling**
|
||||
- [ ] Sensitive data protected
|
||||
- [ ] Encryption properly implemented
|
||||
- [ ] Secure data transmission
|
||||
|
||||
4. **Error Handling**
|
||||
- [ ] No sensitive info in error messages
|
||||
- [ ] Proper logging without data leaks
|
||||
- [ ] Graceful error handling
|
||||
|
||||
## Prioritization Framework
|
||||
|
||||
### Priority Matrix
|
||||
|
||||
| Severity | Exploitability | Business Impact | Priority Level |
|
||||
|----------|---------------|-----------------|---------------|
|
||||
| Critical | High | High | P0 (Immediate) |
|
||||
| Critical | High | Medium | P0 (Immediate) |
|
||||
| Critical | Medium | High | P1 (24 hours) |
|
||||
| High | High | High | P1 (24 hours) |
|
||||
| High | High | Medium | P2 (1 week) |
|
||||
| High | Medium | High | P2 (1 week) |
|
||||
| Medium | High | High | P2 (1 week) |
|
||||
| All Others | - | - | P3 (30 days) |
|
||||
|
||||
### Prioritization Factors
|
||||
|
||||
#### Technical Factors (40% weight)
|
||||
1. **CVSS Base Score** (15%)
|
||||
2. **Exploit Availability** (10%)
|
||||
3. **Fix Complexity** (8%)
|
||||
4. **Dependency Criticality** (7%)
|
||||
|
||||
#### Business Factors (35% weight)
|
||||
1. **Data Impact** (15%)
|
||||
2. **System Criticality** (10%)
|
||||
3. **Regulatory Requirements** (5%)
|
||||
4. **Customer Impact** (5%)
|
||||
|
||||
#### Operational Factors (25% weight)
|
||||
1. **Attack Surface** (10%)
|
||||
2. **Monitoring Coverage** (8%)
|
||||
3. **Incident Response Capability** (7%)
|
||||
|
||||
### Scoring Formula
|
||||
```
|
||||
Priority Score = (Technical Score × 0.4) + (Business Score × 0.35) + (Operational Score × 0.25)
|
||||
|
||||
Where each component is scored 1-10:
|
||||
- 9-10: Critical priority
|
||||
- 7-8: High priority
|
||||
- 5-6: Medium priority
|
||||
- 3-4: Low priority
|
||||
- 1-2: Informational
|
||||
```
|
||||
|
||||
## Remediation Strategies
|
||||
|
||||
### Immediate Actions (P0/P1)
|
||||
|
||||
#### Hot Fixes
|
||||
1. **Version Upgrade**
|
||||
- Update to patched version
|
||||
- Test critical functionality
|
||||
- Deploy with rollback plan
|
||||
|
||||
2. **Configuration Changes**
|
||||
- Disable vulnerable features
|
||||
- Implement additional access controls
|
||||
- Add monitoring/alerting
|
||||
|
||||
3. **Workarounds**
|
||||
- Input validation layers
|
||||
- Network-level protections
|
||||
- Application-level mitigations
|
||||
|
||||
#### Emergency Response Process
|
||||
```
|
||||
1. Vulnerability Confirmed
|
||||
↓
|
||||
2. Impact Assessment (2 hours)
|
||||
↓
|
||||
3. Mitigation Strategy (4 hours)
|
||||
↓
|
||||
4. Implementation & Testing (12 hours)
|
||||
↓
|
||||
5. Deployment (2 hours)
|
||||
↓
|
||||
6. Monitoring & Validation (ongoing)
|
||||
```
|
||||
|
||||
### Planned Remediation (P2/P3)
|
||||
|
||||
#### Standard Update Process
|
||||
1. **Assessment Phase**
|
||||
- Detailed impact analysis
|
||||
- Testing requirements
|
||||
- Rollback procedures
|
||||
|
||||
2. **Planning Phase**
|
||||
- Update scheduling
|
||||
- Resource allocation
|
||||
- Communication plan
|
||||
|
||||
3. **Implementation Phase**
|
||||
- Development environment testing
|
||||
- Staging environment validation
|
||||
- Production deployment
|
||||
|
||||
4. **Validation Phase**
|
||||
- Functionality verification
|
||||
- Security testing
|
||||
- Performance monitoring
|
||||
|
||||
### Alternative Approaches
|
||||
|
||||
#### Dependency Replacement
|
||||
- **When to Consider**: No fix available, persistent vulnerabilities
|
||||
- **Process**: Impact analysis → Alternative evaluation → Migration planning
|
||||
- **Risks**: API changes, feature differences, stability concerns
|
||||
|
||||
#### Accept Risk (Last Resort)
|
||||
- **Criteria**: Very low probability, minimal impact, no feasible fix
|
||||
- **Requirements**: Executive approval, documented risk acceptance, monitoring
|
||||
- **Conditions**: Regular re-assessment, alternative solution tracking
|
||||
|
||||
## Remediation Tracking
|
||||
|
||||
### Metrics and KPIs
|
||||
|
||||
#### Vulnerability Metrics
|
||||
- **Mean Time to Detection (MTTD)**: Average time from publication to discovery
|
||||
- **Mean Time to Patch (MTTP)**: Average time from discovery to fix deployment
|
||||
- **Vulnerability Density**: Vulnerabilities per 1000 dependencies
|
||||
- **Fix Rate**: Percentage of vulnerabilities fixed within SLA
|
||||
|
||||
#### Trend Analysis
|
||||
- **Monthly vulnerability counts by severity**
|
||||
- **Average age of unpatched vulnerabilities**
|
||||
- **Remediation timeline trends**
|
||||
- **False positive rates**
|
||||
|
||||
#### Reporting Dashboard
|
||||
```
|
||||
Security Dashboard Components:
|
||||
├── Current Vulnerability Status
|
||||
│ ├── Critical: 2 (SLA: 24h)
|
||||
│ ├── High: 5 (SLA: 7d)
|
||||
│ └── Medium: 12 (SLA: 30d)
|
||||
├── Trend Analysis
|
||||
│ ├── New vulnerabilities (last 30 days)
|
||||
│ ├── Fixed vulnerabilities (last 30 days)
|
||||
│ └── Average resolution time
|
||||
└── Risk Assessment
|
||||
├── Overall risk score
|
||||
├── Top vulnerable components
|
||||
└── Compliance status
|
||||
```
|
||||
|
||||
## Documentation Requirements
|
||||
|
||||
### Vulnerability Records
|
||||
Each vulnerability should be documented with:
|
||||
- **CVE/Advisory ID**: Official vulnerability identifier
|
||||
- **Discovery Date**: When vulnerability was identified
|
||||
- **CVSS Score**: Base and environmental scores
|
||||
- **Affected Systems**: Components and versions impacted
|
||||
- **Business Impact**: Risk assessment and criticality
|
||||
- **Remediation Plan**: Planned fix approach and timeline
|
||||
- **Resolution Date**: When fix was implemented and verified
|
||||
|
||||
### Risk Acceptance Documentation
|
||||
For accepted risks, document:
|
||||
- **Risk Description**: Detailed vulnerability explanation
|
||||
- **Impact Analysis**: Potential business and technical impact
|
||||
- **Mitigation Measures**: Compensating controls implemented
|
||||
- **Acceptance Rationale**: Why risk is being accepted
|
||||
- **Review Schedule**: When risk will be reassessed
|
||||
- **Approver**: Who authorized the risk acceptance
|
||||
|
||||
## Integration with Development Workflow
|
||||
|
||||
### Shift-Left Security
|
||||
|
||||
#### Development Phase
|
||||
- **IDE Integration**: Real-time vulnerability detection
|
||||
- **Pre-commit Hooks**: Automated security checks
|
||||
- **Code Review**: Security-focused review criteria
|
||||
|
||||
#### CI/CD Integration
|
||||
- **Build Stage**: Dependency vulnerability scanning
|
||||
- **Test Stage**: Security test automation
|
||||
- **Deploy Stage**: Final security validation
|
||||
|
||||
#### Production Monitoring
|
||||
- **Runtime Protection**: Web application firewalls, runtime security
|
||||
- **Continuous Scanning**: Regular dependency updates check
|
||||
- **Incident Response**: Automated vulnerability alert handling
|
||||
|
||||
### Security Gates
|
||||
```yaml
|
||||
security_gates:
|
||||
development:
|
||||
- dependency_scan: true
|
||||
- secret_detection: true
|
||||
- code_quality: true
|
||||
|
||||
staging:
|
||||
- penetration_test: true
|
||||
- compliance_check: true
|
||||
- performance_test: true
|
||||
|
||||
production:
|
||||
- final_security_scan: true
|
||||
- change_approval: required
|
||||
- rollback_plan: verified
|
||||
```
|
||||
|
||||
## Best Practices Summary
|
||||
|
||||
### Proactive Measures
|
||||
1. **Regular Scanning**: Automated daily/weekly scans
|
||||
2. **Update Schedule**: Regular dependency maintenance
|
||||
3. **Security Training**: Developer security awareness
|
||||
4. **Threat Modeling**: Understanding attack vectors
|
||||
|
||||
### Reactive Measures
|
||||
1. **Incident Response**: Well-defined process for critical vulnerabilities
|
||||
2. **Communication Plan**: Stakeholder notification procedures
|
||||
3. **Lessons Learned**: Post-incident analysis and improvement
|
||||
4. **Recovery Procedures**: Rollback and recovery capabilities
|
||||
|
||||
### Organizational Considerations
|
||||
1. **Responsibility Assignment**: Clear ownership of security tasks
|
||||
2. **Resource Allocation**: Adequate security budget and staffing
|
||||
3. **Tool Selection**: Appropriate security tools for organization size
|
||||
4. **Compliance Requirements**: Meeting regulatory and industry standards
|
||||
|
||||
Remember: Vulnerability management is an ongoing process requiring continuous attention, regular updates to procedures, and organizational commitment to security best practices.
|
||||
794
engineering/dependency-auditor/scripts/dep_scanner.py
Normal file
794
engineering/dependency-auditor/scripts/dep_scanner.py
Normal file
@@ -0,0 +1,794 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Dependency Scanner - Multi-language dependency vulnerability and analysis tool.
|
||||
|
||||
This script parses dependency files from various package managers, extracts direct
|
||||
and transitive dependencies, checks against built-in vulnerability databases,
|
||||
and provides comprehensive security analysis with actionable recommendations.
|
||||
|
||||
Author: Claude Skills Engineering Team
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import argparse
|
||||
from typing import Dict, List, Set, Any, Optional, Tuple
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, asdict
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import subprocess
|
||||
|
||||
@dataclass
|
||||
class Vulnerability:
|
||||
"""Represents a security vulnerability."""
|
||||
id: str
|
||||
summary: str
|
||||
severity: str
|
||||
cvss_score: float
|
||||
affected_versions: str
|
||||
fixed_version: Optional[str]
|
||||
published_date: str
|
||||
references: List[str]
|
||||
|
||||
@dataclass
|
||||
class Dependency:
|
||||
"""Represents a project dependency."""
|
||||
name: str
|
||||
version: str
|
||||
ecosystem: str
|
||||
direct: bool
|
||||
license: Optional[str] = None
|
||||
description: Optional[str] = None
|
||||
homepage: Optional[str] = None
|
||||
vulnerabilities: List[Vulnerability] = None
|
||||
|
||||
def __post_init__(self):
|
||||
if self.vulnerabilities is None:
|
||||
self.vulnerabilities = []
|
||||
|
||||
class DependencyScanner:
|
||||
"""Main dependency scanner class."""
|
||||
|
||||
def __init__(self):
|
||||
self.known_vulnerabilities = self._load_vulnerability_database()
|
||||
self.supported_files = {
|
||||
'package.json': self._parse_package_json,
|
||||
'package-lock.json': self._parse_package_lock,
|
||||
'yarn.lock': self._parse_yarn_lock,
|
||||
'requirements.txt': self._parse_requirements_txt,
|
||||
'pyproject.toml': self._parse_pyproject_toml,
|
||||
'Pipfile.lock': self._parse_pipfile_lock,
|
||||
'poetry.lock': self._parse_poetry_lock,
|
||||
'go.mod': self._parse_go_mod,
|
||||
'go.sum': self._parse_go_sum,
|
||||
'Cargo.toml': self._parse_cargo_toml,
|
||||
'Cargo.lock': self._parse_cargo_lock,
|
||||
'Gemfile': self._parse_gemfile,
|
||||
'Gemfile.lock': self._parse_gemfile_lock,
|
||||
}
|
||||
|
||||
def _load_vulnerability_database(self) -> Dict[str, List[Vulnerability]]:
|
||||
"""Load built-in vulnerability database with common CVE patterns."""
|
||||
return {
|
||||
# JavaScript/Node.js vulnerabilities
|
||||
'lodash': [
|
||||
Vulnerability(
|
||||
id='CVE-2021-23337',
|
||||
summary='Prototype pollution in lodash',
|
||||
severity='HIGH',
|
||||
cvss_score=7.2,
|
||||
affected_versions='<4.17.21',
|
||||
fixed_version='4.17.21',
|
||||
published_date='2021-02-15',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2021-23337']
|
||||
)
|
||||
],
|
||||
'axios': [
|
||||
Vulnerability(
|
||||
id='CVE-2023-45857',
|
||||
summary='Cross-site request forgery in axios',
|
||||
severity='MEDIUM',
|
||||
cvss_score=6.1,
|
||||
affected_versions='>=1.0.0 <1.6.0',
|
||||
fixed_version='1.6.0',
|
||||
published_date='2023-10-11',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2023-45857']
|
||||
)
|
||||
],
|
||||
'express': [
|
||||
Vulnerability(
|
||||
id='CVE-2022-24999',
|
||||
summary='Open redirect in express',
|
||||
severity='MEDIUM',
|
||||
cvss_score=6.1,
|
||||
affected_versions='<4.18.2',
|
||||
fixed_version='4.18.2',
|
||||
published_date='2022-11-26',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2022-24999']
|
||||
)
|
||||
],
|
||||
|
||||
# Python vulnerabilities
|
||||
'django': [
|
||||
Vulnerability(
|
||||
id='CVE-2024-27351',
|
||||
summary='SQL injection in Django',
|
||||
severity='HIGH',
|
||||
cvss_score=9.8,
|
||||
affected_versions='>=3.2 <4.2.11',
|
||||
fixed_version='4.2.11',
|
||||
published_date='2024-02-06',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2024-27351']
|
||||
)
|
||||
],
|
||||
'requests': [
|
||||
Vulnerability(
|
||||
id='CVE-2023-32681',
|
||||
summary='Proxy-authorization header leak in requests',
|
||||
severity='MEDIUM',
|
||||
cvss_score=6.1,
|
||||
affected_versions='>=2.3.0 <2.31.0',
|
||||
fixed_version='2.31.0',
|
||||
published_date='2023-05-26',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2023-32681']
|
||||
)
|
||||
],
|
||||
'pillow': [
|
||||
Vulnerability(
|
||||
id='CVE-2023-50447',
|
||||
summary='Arbitrary code execution in Pillow',
|
||||
severity='HIGH',
|
||||
cvss_score=8.8,
|
||||
affected_versions='<10.2.0',
|
||||
fixed_version='10.2.0',
|
||||
published_date='2024-01-02',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2023-50447']
|
||||
)
|
||||
],
|
||||
|
||||
# Go vulnerabilities
|
||||
'github.com/gin-gonic/gin': [
|
||||
Vulnerability(
|
||||
id='CVE-2023-26125',
|
||||
summary='Path traversal in gin',
|
||||
severity='HIGH',
|
||||
cvss_score=7.5,
|
||||
affected_versions='<1.9.1',
|
||||
fixed_version='1.9.1',
|
||||
published_date='2023-02-28',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2023-26125']
|
||||
)
|
||||
],
|
||||
|
||||
# Rust vulnerabilities
|
||||
'serde': [
|
||||
Vulnerability(
|
||||
id='RUSTSEC-2022-0061',
|
||||
summary='Deserialization vulnerability in serde',
|
||||
severity='HIGH',
|
||||
cvss_score=8.2,
|
||||
affected_versions='<1.0.152',
|
||||
fixed_version='1.0.152',
|
||||
published_date='2022-12-07',
|
||||
references=['https://rustsec.org/advisories/RUSTSEC-2022-0061']
|
||||
)
|
||||
],
|
||||
|
||||
# Ruby vulnerabilities
|
||||
'rails': [
|
||||
Vulnerability(
|
||||
id='CVE-2023-28362',
|
||||
summary='ReDoS vulnerability in Rails',
|
||||
severity='HIGH',
|
||||
cvss_score=7.5,
|
||||
affected_versions='>=7.0.0 <7.0.4.3',
|
||||
fixed_version='7.0.4.3',
|
||||
published_date='2023-03-13',
|
||||
references=['https://nvd.nist.gov/vuln/detail/CVE-2023-28362']
|
||||
)
|
||||
]
|
||||
}
|
||||
|
||||
def scan_project(self, project_path: str) -> Dict[str, Any]:
|
||||
"""Scan a project directory for dependencies and vulnerabilities."""
|
||||
project_path = Path(project_path)
|
||||
|
||||
if not project_path.exists():
|
||||
raise FileNotFoundError(f"Project path does not exist: {project_path}")
|
||||
|
||||
scan_results = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'project_path': str(project_path),
|
||||
'dependencies': [],
|
||||
'vulnerabilities_found': 0,
|
||||
'high_severity_count': 0,
|
||||
'medium_severity_count': 0,
|
||||
'low_severity_count': 0,
|
||||
'ecosystems': set(),
|
||||
'scan_summary': {},
|
||||
'recommendations': []
|
||||
}
|
||||
|
||||
# Find and parse dependency files
|
||||
for file_pattern, parser in self.supported_files.items():
|
||||
matching_files = list(project_path.rglob(file_pattern))
|
||||
|
||||
for dep_file in matching_files:
|
||||
try:
|
||||
dependencies = parser(dep_file)
|
||||
scan_results['dependencies'].extend(dependencies)
|
||||
|
||||
for dep in dependencies:
|
||||
scan_results['ecosystems'].add(dep.ecosystem)
|
||||
|
||||
# Check for vulnerabilities
|
||||
vulnerabilities = self._check_vulnerabilities(dep)
|
||||
dep.vulnerabilities = vulnerabilities
|
||||
|
||||
scan_results['vulnerabilities_found'] += len(vulnerabilities)
|
||||
|
||||
for vuln in vulnerabilities:
|
||||
if vuln.severity == 'HIGH':
|
||||
scan_results['high_severity_count'] += 1
|
||||
elif vuln.severity == 'MEDIUM':
|
||||
scan_results['medium_severity_count'] += 1
|
||||
else:
|
||||
scan_results['low_severity_count'] += 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing {dep_file}: {e}")
|
||||
continue
|
||||
|
||||
scan_results['ecosystems'] = list(scan_results['ecosystems'])
|
||||
scan_results['scan_summary'] = self._generate_scan_summary(scan_results)
|
||||
scan_results['recommendations'] = self._generate_recommendations(scan_results)
|
||||
|
||||
return scan_results
|
||||
|
||||
def _check_vulnerabilities(self, dependency: Dependency) -> List[Vulnerability]:
|
||||
"""Check if a dependency has known vulnerabilities."""
|
||||
vulnerabilities = []
|
||||
|
||||
# Check package name (exact match and common variations)
|
||||
package_names = [dependency.name, dependency.name.lower()]
|
||||
|
||||
for pkg_name in package_names:
|
||||
if pkg_name in self.known_vulnerabilities:
|
||||
for vuln in self.known_vulnerabilities[pkg_name]:
|
||||
if self._version_matches_vulnerability(dependency.version, vuln.affected_versions):
|
||||
vulnerabilities.append(vuln)
|
||||
|
||||
return vulnerabilities
|
||||
|
||||
def _version_matches_vulnerability(self, version: str, affected_pattern: str) -> bool:
|
||||
"""Check if a version matches a vulnerability pattern."""
|
||||
# Simple version matching - in production, use proper semver library
|
||||
try:
|
||||
# Handle common patterns like "<4.17.21", ">=1.0.0 <1.6.0"
|
||||
if '<' in affected_pattern and '>' not in affected_pattern:
|
||||
# Pattern like "<4.17.21"
|
||||
max_version = affected_pattern.replace('<', '').strip()
|
||||
return self._compare_versions(version, max_version) < 0
|
||||
elif '>=' in affected_pattern and '<' in affected_pattern:
|
||||
# Pattern like ">=1.0.0 <1.6.0"
|
||||
parts = affected_pattern.split('<')
|
||||
min_part = parts[0].replace('>=', '').strip()
|
||||
max_part = parts[1].strip()
|
||||
return (self._compare_versions(version, min_part) >= 0 and
|
||||
self._compare_versions(version, max_part) < 0)
|
||||
except:
|
||||
pass
|
||||
|
||||
return False
|
||||
|
||||
def _compare_versions(self, v1: str, v2: str) -> int:
|
||||
"""Simple version comparison. Returns -1, 0, or 1."""
|
||||
try:
|
||||
def normalize(v):
|
||||
return [int(x) for x in re.sub(r'(\.0+)*$','', v).split('.')]
|
||||
|
||||
v1_parts = normalize(v1)
|
||||
v2_parts = normalize(v2)
|
||||
|
||||
if v1_parts < v2_parts:
|
||||
return -1
|
||||
elif v1_parts > v2_parts:
|
||||
return 1
|
||||
else:
|
||||
return 0
|
||||
except:
|
||||
return 0
|
||||
|
||||
# Package file parsers
|
||||
|
||||
def _parse_package_json(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse package.json for Node.js dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Parse dependencies
|
||||
for dep_type in ['dependencies', 'devDependencies']:
|
||||
if dep_type in data:
|
||||
for name, version in data[dep_type].items():
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version.replace('^', '').replace('~', '').replace('>=', '').replace('<=', ''),
|
||||
ecosystem='npm',
|
||||
direct=True
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing package.json: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_package_lock(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse package-lock.json for Node.js transitive dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if 'packages' in data:
|
||||
for path, pkg_info in data['packages'].items():
|
||||
if path == '': # Skip root package
|
||||
continue
|
||||
|
||||
name = path.split('/')[-1] if '/' in path else path
|
||||
version = pkg_info.get('version', '')
|
||||
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='npm',
|
||||
direct=False,
|
||||
description=pkg_info.get('description', '')
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing package-lock.json: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_yarn_lock(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse yarn.lock for Node.js dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Simple yarn.lock parsing
|
||||
packages = re.findall(r'^([^#\s][^:]+):\s*\n(?:\s+.*\n)*?\s+version\s+"([^"]+)"', content, re.MULTILINE)
|
||||
|
||||
for package_spec, version in packages:
|
||||
name = package_spec.split('@')[0] if '@' in package_spec else package_spec
|
||||
name = name.strip('"')
|
||||
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='npm',
|
||||
direct=False
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing yarn.lock: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_requirements_txt(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse requirements.txt for Python dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if line and not line.startswith('#') and not line.startswith('-'):
|
||||
# Parse package==version or package>=version patterns
|
||||
match = re.match(r'^([a-zA-Z0-9_-]+)([><=!]+)(.+)$', line)
|
||||
if match:
|
||||
name, operator, version = match.groups()
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='pypi',
|
||||
direct=True
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing requirements.txt: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_pyproject_toml(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse pyproject.toml for Python dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Simple TOML parsing for dependencies
|
||||
dep_section = re.search(r'\[tool\.poetry\.dependencies\](.*?)(?=\[|\Z)', content, re.DOTALL)
|
||||
if dep_section:
|
||||
for line in dep_section.group(1).split('\n'):
|
||||
match = re.match(r'^([a-zA-Z0-9_-]+)\s*=\s*["\']([^"\']+)["\']', line.strip())
|
||||
if match:
|
||||
name, version = match.groups()
|
||||
if name != 'python':
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version.replace('^', '').replace('~', ''),
|
||||
ecosystem='pypi',
|
||||
direct=True
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing pyproject.toml: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_pipfile_lock(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse Pipfile.lock for Python dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
for section in ['default', 'develop']:
|
||||
if section in data:
|
||||
for name, info in data[section].items():
|
||||
version = info.get('version', '').replace('==', '')
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='pypi',
|
||||
direct=(section == 'default')
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing Pipfile.lock: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_poetry_lock(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse poetry.lock for Python dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract package entries from TOML
|
||||
packages = re.findall(r'\[\[package\]\]\nname\s*=\s*"([^"]+)"\nversion\s*=\s*"([^"]+)"', content)
|
||||
|
||||
for name, version in packages:
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='pypi',
|
||||
direct=False
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing poetry.lock: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_go_mod(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse go.mod for Go dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Parse require block
|
||||
require_match = re.search(r'require\s*\((.*?)\)', content, re.DOTALL)
|
||||
if require_match:
|
||||
requires = require_match.group(1)
|
||||
for line in requires.split('\n'):
|
||||
match = re.match(r'\s*([^\s]+)\s+v?([^\s]+)', line.strip())
|
||||
if match:
|
||||
name, version = match.groups()
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='go',
|
||||
direct=True
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing go.mod: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_go_sum(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse go.sum for Go dependency checksums."""
|
||||
return [] # go.sum mainly contains checksums, dependencies are in go.mod
|
||||
|
||||
def _parse_cargo_toml(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse Cargo.toml for Rust dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Parse [dependencies] section
|
||||
dep_section = re.search(r'\[dependencies\](.*?)(?=\[|\Z)', content, re.DOTALL)
|
||||
if dep_section:
|
||||
for line in dep_section.group(1).split('\n'):
|
||||
match = re.match(r'^([a-zA-Z0-9_-]+)\s*=\s*["\']([^"\']+)["\']', line.strip())
|
||||
if match:
|
||||
name, version = match.groups()
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='cargo',
|
||||
direct=True
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing Cargo.toml: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_cargo_lock(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse Cargo.lock for Rust dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Parse [[package]] entries
|
||||
packages = re.findall(r'\[\[package\]\]\nname\s*=\s*"([^"]+)"\nversion\s*=\s*"([^"]+)"', content)
|
||||
|
||||
for name, version in packages:
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='cargo',
|
||||
direct=False
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing Cargo.lock: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_gemfile(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse Gemfile for Ruby dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Parse gem declarations
|
||||
gems = re.findall(r'gem\s+["\']([^"\']+)["\'](?:\s*,\s*["\']([^"\']+)["\'])?', content)
|
||||
|
||||
for gem_info in gems:
|
||||
name = gem_info[0]
|
||||
version = gem_info[1] if len(gem_info) > 1 and gem_info[1] else ''
|
||||
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='rubygems',
|
||||
direct=True
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing Gemfile: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _parse_gemfile_lock(self, file_path: Path) -> List[Dependency]:
|
||||
"""Parse Gemfile.lock for Ruby dependencies."""
|
||||
dependencies = []
|
||||
|
||||
try:
|
||||
with open(file_path, 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# Extract GEM section
|
||||
gem_section = re.search(r'GEM\s*\n(.*?)(?=\n\S|\Z)', content, re.DOTALL)
|
||||
if gem_section:
|
||||
specs = gem_section.group(1)
|
||||
gems = re.findall(r'\s+([a-zA-Z0-9_-]+)\s+\(([^)]+)\)', specs)
|
||||
|
||||
for name, version in gems:
|
||||
dep = Dependency(
|
||||
name=name,
|
||||
version=version,
|
||||
ecosystem='rubygems',
|
||||
direct=False
|
||||
)
|
||||
dependencies.append(dep)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error parsing Gemfile.lock: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _generate_scan_summary(self, scan_results: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Generate a summary of the scan results."""
|
||||
total_deps = len(scan_results['dependencies'])
|
||||
unique_deps = len(set(dep.name for dep in scan_results['dependencies']))
|
||||
|
||||
return {
|
||||
'total_dependencies': total_deps,
|
||||
'unique_dependencies': unique_deps,
|
||||
'ecosystems_found': len(scan_results['ecosystems']),
|
||||
'vulnerable_dependencies': len([dep for dep in scan_results['dependencies'] if dep.vulnerabilities]),
|
||||
'vulnerability_breakdown': {
|
||||
'high': scan_results['high_severity_count'],
|
||||
'medium': scan_results['medium_severity_count'],
|
||||
'low': scan_results['low_severity_count']
|
||||
}
|
||||
}
|
||||
|
||||
def _generate_recommendations(self, scan_results: Dict[str, Any]) -> List[str]:
|
||||
"""Generate actionable recommendations based on scan results."""
|
||||
recommendations = []
|
||||
|
||||
high_count = scan_results['high_severity_count']
|
||||
medium_count = scan_results['medium_severity_count']
|
||||
|
||||
if high_count > 0:
|
||||
recommendations.append(f"URGENT: Address {high_count} high-severity vulnerabilities immediately")
|
||||
|
||||
if medium_count > 0:
|
||||
recommendations.append(f"Schedule fixes for {medium_count} medium-severity vulnerabilities within 30 days")
|
||||
|
||||
vulnerable_deps = [dep for dep in scan_results['dependencies'] if dep.vulnerabilities]
|
||||
if vulnerable_deps:
|
||||
for dep in vulnerable_deps[:3]: # Top 3 most critical
|
||||
for vuln in dep.vulnerabilities:
|
||||
if vuln.fixed_version:
|
||||
recommendations.append(f"Update {dep.name} from {dep.version} to {vuln.fixed_version} to fix {vuln.id}")
|
||||
|
||||
if len(scan_results['ecosystems']) > 3:
|
||||
recommendations.append("Consider consolidating package managers to reduce complexity")
|
||||
|
||||
return recommendations
|
||||
|
||||
def generate_report(self, scan_results: Dict[str, Any], format: str = 'text') -> str:
|
||||
"""Generate a human-readable or JSON report."""
|
||||
if format == 'json':
|
||||
# Convert Dependency objects to dicts for JSON serialization
|
||||
serializable_results = scan_results.copy()
|
||||
serializable_results['dependencies'] = [
|
||||
{
|
||||
'name': dep.name,
|
||||
'version': dep.version,
|
||||
'ecosystem': dep.ecosystem,
|
||||
'direct': dep.direct,
|
||||
'license': dep.license,
|
||||
'vulnerabilities': [asdict(vuln) for vuln in dep.vulnerabilities]
|
||||
}
|
||||
for dep in scan_results['dependencies']
|
||||
]
|
||||
return json.dumps(serializable_results, indent=2, default=str)
|
||||
|
||||
# Text format report
|
||||
report = []
|
||||
report.append("=" * 60)
|
||||
report.append("DEPENDENCY SECURITY SCAN REPORT")
|
||||
report.append("=" * 60)
|
||||
report.append(f"Scan Date: {scan_results['timestamp']}")
|
||||
report.append(f"Project: {scan_results['project_path']}")
|
||||
report.append("")
|
||||
|
||||
# Summary
|
||||
summary = scan_results['scan_summary']
|
||||
report.append("SUMMARY:")
|
||||
report.append(f" Total Dependencies: {summary['total_dependencies']}")
|
||||
report.append(f" Unique Dependencies: {summary['unique_dependencies']}")
|
||||
report.append(f" Ecosystems: {', '.join(scan_results['ecosystems'])}")
|
||||
report.append(f" Vulnerabilities Found: {scan_results['vulnerabilities_found']}")
|
||||
report.append(f" High Severity: {summary['vulnerability_breakdown']['high']}")
|
||||
report.append(f" Medium Severity: {summary['vulnerability_breakdown']['medium']}")
|
||||
report.append(f" Low Severity: {summary['vulnerability_breakdown']['low']}")
|
||||
report.append("")
|
||||
|
||||
# Vulnerable dependencies
|
||||
vulnerable_deps = [dep for dep in scan_results['dependencies'] if dep.vulnerabilities]
|
||||
if vulnerable_deps:
|
||||
report.append("VULNERABLE DEPENDENCIES:")
|
||||
report.append("-" * 30)
|
||||
|
||||
for dep in vulnerable_deps:
|
||||
report.append(f"Package: {dep.name} v{dep.version} ({dep.ecosystem})")
|
||||
for vuln in dep.vulnerabilities:
|
||||
report.append(f" • {vuln.id}: {vuln.summary}")
|
||||
report.append(f" Severity: {vuln.severity} (CVSS: {vuln.cvss_score})")
|
||||
if vuln.fixed_version:
|
||||
report.append(f" Fixed in: {vuln.fixed_version}")
|
||||
report.append("")
|
||||
|
||||
# Recommendations
|
||||
if scan_results['recommendations']:
|
||||
report.append("RECOMMENDATIONS:")
|
||||
report.append("-" * 20)
|
||||
for i, rec in enumerate(scan_results['recommendations'], 1):
|
||||
report.append(f"{i}. {rec}")
|
||||
report.append("")
|
||||
|
||||
report.append("=" * 60)
|
||||
return '\n'.join(report)
|
||||
|
||||
def main():
|
||||
"""Main entry point for the dependency scanner."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Scan project dependencies for vulnerabilities and security issues',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python dep_scanner.py /path/to/project
|
||||
python dep_scanner.py . --format json --output results.json
|
||||
python dep_scanner.py /app --fail-on-high
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('project_path',
|
||||
help='Path to the project directory to scan')
|
||||
parser.add_argument('--format', choices=['text', 'json'], default='text',
|
||||
help='Output format (default: text)')
|
||||
parser.add_argument('--output', '-o',
|
||||
help='Output file path (default: stdout)')
|
||||
parser.add_argument('--fail-on-high', action='store_true',
|
||||
help='Exit with error code if high-severity vulnerabilities found')
|
||||
parser.add_argument('--quick-scan', action='store_true',
|
||||
help='Perform quick scan (skip transitive dependencies)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
scanner = DependencyScanner()
|
||||
results = scanner.scan_project(args.project_path)
|
||||
report = scanner.generate_report(results, args.format)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w') as f:
|
||||
f.write(report)
|
||||
print(f"Report saved to {args.output}")
|
||||
else:
|
||||
print(report)
|
||||
|
||||
# Exit with error if high-severity vulnerabilities found and --fail-on-high is set
|
||||
if args.fail_on_high and results['high_severity_count'] > 0:
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
996
engineering/dependency-auditor/scripts/license_checker.py
Normal file
996
engineering/dependency-auditor/scripts/license_checker.py
Normal file
@@ -0,0 +1,996 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
License Checker - Dependency license compliance and conflict analysis tool.
|
||||
|
||||
This script analyzes dependency licenses from package metadata, classifies them
|
||||
into risk categories, detects license conflicts, and generates compliance
|
||||
reports with actionable recommendations for legal risk management.
|
||||
|
||||
Author: Claude Skills Engineering Team
|
||||
License: MIT
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
from typing import Dict, List, Set, Any, Optional, Tuple
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, asdict
|
||||
from datetime import datetime
|
||||
import re
|
||||
from enum import Enum
|
||||
|
||||
class LicenseType(Enum):
|
||||
"""License classification types."""
|
||||
PERMISSIVE = "permissive"
|
||||
COPYLEFT_STRONG = "copyleft_strong"
|
||||
COPYLEFT_WEAK = "copyleft_weak"
|
||||
PROPRIETARY = "proprietary"
|
||||
DUAL = "dual"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
class RiskLevel(Enum):
|
||||
"""Risk assessment levels."""
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
CRITICAL = "critical"
|
||||
|
||||
@dataclass
|
||||
class LicenseInfo:
|
||||
"""Represents license information for a dependency."""
|
||||
name: str
|
||||
spdx_id: Optional[str]
|
||||
license_type: LicenseType
|
||||
risk_level: RiskLevel
|
||||
description: str
|
||||
restrictions: List[str]
|
||||
obligations: List[str]
|
||||
compatibility: Dict[str, bool]
|
||||
|
||||
@dataclass
|
||||
class DependencyLicense:
|
||||
"""Represents a dependency with its license information."""
|
||||
name: str
|
||||
version: str
|
||||
ecosystem: str
|
||||
direct: bool
|
||||
license_declared: Optional[str]
|
||||
license_detected: Optional[LicenseInfo]
|
||||
license_files: List[str]
|
||||
confidence: float
|
||||
|
||||
@dataclass
|
||||
class LicenseConflict:
|
||||
"""Represents a license compatibility conflict."""
|
||||
dependency1: str
|
||||
license1: str
|
||||
dependency2: str
|
||||
license2: str
|
||||
conflict_type: str
|
||||
severity: RiskLevel
|
||||
description: str
|
||||
resolution_options: List[str]
|
||||
|
||||
class LicenseChecker:
|
||||
"""Main license checking and compliance analysis class."""
|
||||
|
||||
def __init__(self):
|
||||
self.license_database = self._build_license_database()
|
||||
self.compatibility_matrix = self._build_compatibility_matrix()
|
||||
self.license_patterns = self._build_license_patterns()
|
||||
|
||||
def _build_license_database(self) -> Dict[str, LicenseInfo]:
|
||||
"""Build comprehensive license database with risk classifications."""
|
||||
return {
|
||||
# Permissive Licenses (Low Risk)
|
||||
'MIT': LicenseInfo(
|
||||
name='MIT License',
|
||||
spdx_id='MIT',
|
||||
license_type=LicenseType.PERMISSIVE,
|
||||
risk_level=RiskLevel.LOW,
|
||||
description='Very permissive license with minimal restrictions',
|
||||
restrictions=['Include copyright notice', 'Include license text'],
|
||||
obligations=['Attribution'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': False
|
||||
}
|
||||
),
|
||||
|
||||
'Apache-2.0': LicenseInfo(
|
||||
name='Apache License 2.0',
|
||||
spdx_id='Apache-2.0',
|
||||
license_type=LicenseType.PERMISSIVE,
|
||||
risk_level=RiskLevel.LOW,
|
||||
description='Permissive license with patent protection',
|
||||
restrictions=['Include copyright notice', 'Include license text',
|
||||
'State changes', 'Include NOTICE file'],
|
||||
obligations=['Attribution', 'Patent grant'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': True
|
||||
}
|
||||
),
|
||||
|
||||
'BSD-3-Clause': LicenseInfo(
|
||||
name='BSD 3-Clause License',
|
||||
spdx_id='BSD-3-Clause',
|
||||
license_type=LicenseType.PERMISSIVE,
|
||||
risk_level=RiskLevel.LOW,
|
||||
description='Permissive license with non-endorsement clause',
|
||||
restrictions=['Include copyright notice', 'Include license text',
|
||||
'No endorsement using author names'],
|
||||
obligations=['Attribution'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': False
|
||||
}
|
||||
),
|
||||
|
||||
'BSD-2-Clause': LicenseInfo(
|
||||
name='BSD 2-Clause License',
|
||||
spdx_id='BSD-2-Clause',
|
||||
license_type=LicenseType.PERMISSIVE,
|
||||
risk_level=RiskLevel.LOW,
|
||||
description='Very permissive license similar to MIT',
|
||||
restrictions=['Include copyright notice', 'Include license text'],
|
||||
obligations=['Attribution'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': False
|
||||
}
|
||||
),
|
||||
|
||||
'ISC': LicenseInfo(
|
||||
name='ISC License',
|
||||
spdx_id='ISC',
|
||||
license_type=LicenseType.PERMISSIVE,
|
||||
risk_level=RiskLevel.LOW,
|
||||
description='Functionally equivalent to MIT license',
|
||||
restrictions=['Include copyright notice'],
|
||||
obligations=['Attribution'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': False
|
||||
}
|
||||
),
|
||||
|
||||
# Weak Copyleft Licenses (Medium Risk)
|
||||
'MPL-2.0': LicenseInfo(
|
||||
name='Mozilla Public License 2.0',
|
||||
spdx_id='MPL-2.0',
|
||||
license_type=LicenseType.COPYLEFT_WEAK,
|
||||
risk_level=RiskLevel.MEDIUM,
|
||||
description='File-level copyleft license',
|
||||
restrictions=['Disclose source of modified files', 'Include copyright notice',
|
||||
'Include license text', 'State changes'],
|
||||
obligations=['Source disclosure (modified files only)'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': True
|
||||
}
|
||||
),
|
||||
|
||||
'LGPL-2.1': LicenseInfo(
|
||||
name='GNU Lesser General Public License 2.1',
|
||||
spdx_id='LGPL-2.1',
|
||||
license_type=LicenseType.COPYLEFT_WEAK,
|
||||
risk_level=RiskLevel.MEDIUM,
|
||||
description='Library-level copyleft license',
|
||||
restrictions=['Disclose source of library modifications', 'Include copyright notice',
|
||||
'Include license text', 'Allow relinking'],
|
||||
obligations=['Source disclosure (library modifications)', 'Dynamic linking preferred'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': False
|
||||
}
|
||||
),
|
||||
|
||||
'LGPL-3.0': LicenseInfo(
|
||||
name='GNU Lesser General Public License 3.0',
|
||||
spdx_id='LGPL-3.0',
|
||||
license_type=LicenseType.COPYLEFT_WEAK,
|
||||
risk_level=RiskLevel.MEDIUM,
|
||||
description='Library-level copyleft with patent provisions',
|
||||
restrictions=['Disclose source of library modifications', 'Include copyright notice',
|
||||
'Include license text', 'Allow relinking', 'Anti-tivoization'],
|
||||
obligations=['Source disclosure (library modifications)', 'Patent grant'],
|
||||
compatibility={
|
||||
'commercial': True, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': True
|
||||
}
|
||||
),
|
||||
|
||||
# Strong Copyleft Licenses (High Risk)
|
||||
'GPL-2.0': LicenseInfo(
|
||||
name='GNU General Public License 2.0',
|
||||
spdx_id='GPL-2.0',
|
||||
license_type=LicenseType.COPYLEFT_STRONG,
|
||||
risk_level=RiskLevel.HIGH,
|
||||
description='Strong copyleft requiring full source disclosure',
|
||||
restrictions=['Disclose entire source code', 'Include copyright notice',
|
||||
'Include license text', 'Use same license'],
|
||||
obligations=['Full source disclosure', 'License compatibility'],
|
||||
compatibility={
|
||||
'commercial': False, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': False
|
||||
}
|
||||
),
|
||||
|
||||
'GPL-3.0': LicenseInfo(
|
||||
name='GNU General Public License 3.0',
|
||||
spdx_id='GPL-3.0',
|
||||
license_type=LicenseType.COPYLEFT_STRONG,
|
||||
risk_level=RiskLevel.HIGH,
|
||||
description='Strong copyleft with patent and hardware provisions',
|
||||
restrictions=['Disclose entire source code', 'Include copyright notice',
|
||||
'Include license text', 'Use same license', 'Anti-tivoization'],
|
||||
obligations=['Full source disclosure', 'Patent grant', 'License compatibility'],
|
||||
compatibility={
|
||||
'commercial': False, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': True
|
||||
}
|
||||
),
|
||||
|
||||
'AGPL-3.0': LicenseInfo(
|
||||
name='GNU Affero General Public License 3.0',
|
||||
spdx_id='AGPL-3.0',
|
||||
license_type=LicenseType.COPYLEFT_STRONG,
|
||||
risk_level=RiskLevel.CRITICAL,
|
||||
description='Network copyleft extending GPL to SaaS',
|
||||
restrictions=['Disclose entire source code', 'Include copyright notice',
|
||||
'Include license text', 'Use same license', 'Network use triggers copyleft'],
|
||||
obligations=['Full source disclosure', 'Network service source disclosure'],
|
||||
compatibility={
|
||||
'commercial': False, 'modification': True, 'distribution': True,
|
||||
'private_use': True, 'patent_grant': True
|
||||
}
|
||||
),
|
||||
|
||||
# Proprietary/Commercial Licenses (High Risk)
|
||||
'PROPRIETARY': LicenseInfo(
|
||||
name='Proprietary License',
|
||||
spdx_id=None,
|
||||
license_type=LicenseType.PROPRIETARY,
|
||||
risk_level=RiskLevel.HIGH,
|
||||
description='Commercial or custom proprietary license',
|
||||
restrictions=['Varies by license', 'Often no redistribution',
|
||||
'May require commercial license'],
|
||||
obligations=['License agreement compliance', 'Payment obligations'],
|
||||
compatibility={
|
||||
'commercial': False, 'modification': False, 'distribution': False,
|
||||
'private_use': True, 'patent_grant': False
|
||||
}
|
||||
),
|
||||
|
||||
# Unknown/Unlicensed (Critical Risk)
|
||||
'UNKNOWN': LicenseInfo(
|
||||
name='Unknown License',
|
||||
spdx_id=None,
|
||||
license_type=LicenseType.UNKNOWN,
|
||||
risk_level=RiskLevel.CRITICAL,
|
||||
description='No license detected or ambiguous licensing',
|
||||
restrictions=['Unknown', 'Assume no rights granted'],
|
||||
obligations=['Investigate and clarify licensing'],
|
||||
compatibility={
|
||||
'commercial': False, 'modification': False, 'distribution': False,
|
||||
'private_use': False, 'patent_grant': False
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
def _build_compatibility_matrix(self) -> Dict[str, Dict[str, bool]]:
|
||||
"""Build license compatibility matrix."""
|
||||
return {
|
||||
'MIT': {
|
||||
'MIT': True, 'Apache-2.0': True, 'BSD-3-Clause': True, 'BSD-2-Clause': True,
|
||||
'ISC': True, 'MPL-2.0': True, 'LGPL-2.1': True, 'LGPL-3.0': True,
|
||||
'GPL-2.0': False, 'GPL-3.0': False, 'AGPL-3.0': False, 'PROPRIETARY': False
|
||||
},
|
||||
'Apache-2.0': {
|
||||
'MIT': True, 'Apache-2.0': True, 'BSD-3-Clause': True, 'BSD-2-Clause': True,
|
||||
'ISC': True, 'MPL-2.0': True, 'LGPL-2.1': False, 'LGPL-3.0': True,
|
||||
'GPL-2.0': False, 'GPL-3.0': True, 'AGPL-3.0': True, 'PROPRIETARY': False
|
||||
},
|
||||
'GPL-2.0': {
|
||||
'MIT': True, 'Apache-2.0': False, 'BSD-3-Clause': True, 'BSD-2-Clause': True,
|
||||
'ISC': True, 'MPL-2.0': False, 'LGPL-2.1': True, 'LGPL-3.0': False,
|
||||
'GPL-2.0': True, 'GPL-3.0': False, 'AGPL-3.0': False, 'PROPRIETARY': False
|
||||
},
|
||||
'GPL-3.0': {
|
||||
'MIT': True, 'Apache-2.0': True, 'BSD-3-Clause': True, 'BSD-2-Clause': True,
|
||||
'ISC': True, 'MPL-2.0': True, 'LGPL-2.1': False, 'LGPL-3.0': True,
|
||||
'GPL-2.0': False, 'GPL-3.0': True, 'AGPL-3.0': True, 'PROPRIETARY': False
|
||||
},
|
||||
'AGPL-3.0': {
|
||||
'MIT': True, 'Apache-2.0': True, 'BSD-3-Clause': True, 'BSD-2-Clause': True,
|
||||
'ISC': True, 'MPL-2.0': True, 'LGPL-2.1': False, 'LGPL-3.0': True,
|
||||
'GPL-2.0': False, 'GPL-3.0': True, 'AGPL-3.0': True, 'PROPRIETARY': False
|
||||
}
|
||||
}
|
||||
|
||||
def _build_license_patterns(self) -> Dict[str, List[str]]:
|
||||
"""Build license detection patterns for text analysis."""
|
||||
return {
|
||||
'MIT': [
|
||||
r'MIT License',
|
||||
r'Permission is hereby granted, free of charge',
|
||||
r'THE SOFTWARE IS PROVIDED "AS IS"'
|
||||
],
|
||||
'Apache-2.0': [
|
||||
r'Apache License, Version 2\.0',
|
||||
r'Licensed under the Apache License',
|
||||
r'http://www\.apache\.org/licenses/LICENSE-2\.0'
|
||||
],
|
||||
'GPL-2.0': [
|
||||
r'GNU GENERAL PUBLIC LICENSE\s+Version 2',
|
||||
r'This program is free software.*GPL.*version 2',
|
||||
r'http://www\.gnu\.org/licenses/gpl-2\.0'
|
||||
],
|
||||
'GPL-3.0': [
|
||||
r'GNU GENERAL PUBLIC LICENSE\s+Version 3',
|
||||
r'This program is free software.*GPL.*version 3',
|
||||
r'http://www\.gnu\.org/licenses/gpl-3\.0'
|
||||
],
|
||||
'BSD-3-Clause': [
|
||||
r'BSD 3-Clause License',
|
||||
r'Redistributions of source code must retain',
|
||||
r'Neither the name.*may be used to endorse'
|
||||
],
|
||||
'BSD-2-Clause': [
|
||||
r'BSD 2-Clause License',
|
||||
r'Redistributions of source code must retain.*Redistributions in binary form'
|
||||
]
|
||||
}
|
||||
|
||||
def analyze_project(self, project_path: str, dependency_inventory: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""Analyze license compliance for a project."""
|
||||
project_path = Path(project_path)
|
||||
|
||||
analysis_results = {
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'project_path': str(project_path),
|
||||
'project_license': self._detect_project_license(project_path),
|
||||
'dependencies': [],
|
||||
'license_summary': {},
|
||||
'conflicts': [],
|
||||
'compliance_score': 0.0,
|
||||
'risk_assessment': {},
|
||||
'recommendations': []
|
||||
}
|
||||
|
||||
# Load dependencies from inventory or scan project
|
||||
if dependency_inventory:
|
||||
dependencies = self._load_dependency_inventory(dependency_inventory)
|
||||
else:
|
||||
dependencies = self._scan_project_dependencies(project_path)
|
||||
|
||||
# Analyze each dependency's license
|
||||
for dep in dependencies:
|
||||
license_info = self._analyze_dependency_license(dep, project_path)
|
||||
analysis_results['dependencies'].append(license_info)
|
||||
|
||||
# Generate license summary
|
||||
analysis_results['license_summary'] = self._generate_license_summary(
|
||||
analysis_results['dependencies']
|
||||
)
|
||||
|
||||
# Detect conflicts
|
||||
analysis_results['conflicts'] = self._detect_license_conflicts(
|
||||
analysis_results['project_license'],
|
||||
analysis_results['dependencies']
|
||||
)
|
||||
|
||||
# Calculate compliance score
|
||||
analysis_results['compliance_score'] = self._calculate_compliance_score(
|
||||
analysis_results['dependencies'],
|
||||
analysis_results['conflicts']
|
||||
)
|
||||
|
||||
# Generate risk assessment
|
||||
analysis_results['risk_assessment'] = self._generate_risk_assessment(
|
||||
analysis_results['dependencies'],
|
||||
analysis_results['conflicts']
|
||||
)
|
||||
|
||||
# Generate recommendations
|
||||
analysis_results['recommendations'] = self._generate_compliance_recommendations(
|
||||
analysis_results
|
||||
)
|
||||
|
||||
return analysis_results
|
||||
|
||||
def _detect_project_license(self, project_path: Path) -> Optional[str]:
|
||||
"""Detect the main project license."""
|
||||
license_files = ['LICENSE', 'LICENSE.txt', 'LICENSE.md', 'COPYING', 'COPYING.txt']
|
||||
|
||||
for license_file in license_files:
|
||||
license_path = project_path / license_file
|
||||
if license_path.exists():
|
||||
try:
|
||||
with open(license_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Analyze license content
|
||||
detected_license = self._detect_license_from_text(content)
|
||||
if detected_license:
|
||||
return detected_license
|
||||
except Exception as e:
|
||||
print(f"Error reading license file {license_path}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def _detect_license_from_text(self, text: str) -> Optional[str]:
|
||||
"""Detect license type from text content."""
|
||||
text_upper = text.upper()
|
||||
|
||||
for license_id, patterns in self.license_patterns.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, text, re.IGNORECASE):
|
||||
return license_id
|
||||
|
||||
# Common license text patterns
|
||||
if 'MIT' in text_upper and 'PERMISSION IS HEREBY GRANTED' in text_upper:
|
||||
return 'MIT'
|
||||
elif 'APACHE LICENSE' in text_upper and 'VERSION 2.0' in text_upper:
|
||||
return 'Apache-2.0'
|
||||
elif 'GPL' in text_upper and 'VERSION 2' in text_upper:
|
||||
return 'GPL-2.0'
|
||||
elif 'GPL' in text_upper and 'VERSION 3' in text_upper:
|
||||
return 'GPL-3.0'
|
||||
|
||||
return None
|
||||
|
||||
def _load_dependency_inventory(self, inventory_path: str) -> List[Dict[str, Any]]:
|
||||
"""Load dependencies from JSON inventory file."""
|
||||
try:
|
||||
with open(inventory_path, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
if 'dependencies' in data:
|
||||
return data['dependencies']
|
||||
else:
|
||||
return data if isinstance(data, list) else []
|
||||
except Exception as e:
|
||||
print(f"Error loading dependency inventory: {e}")
|
||||
return []
|
||||
|
||||
def _scan_project_dependencies(self, project_path: Path) -> List[Dict[str, Any]]:
|
||||
"""Basic dependency scanning - in practice, would integrate with dep_scanner.py."""
|
||||
dependencies = []
|
||||
|
||||
# Simple package.json parsing as example
|
||||
package_json = project_path / 'package.json'
|
||||
if package_json.exists():
|
||||
try:
|
||||
with open(package_json, 'r') as f:
|
||||
data = json.load(f)
|
||||
|
||||
for dep_type in ['dependencies', 'devDependencies']:
|
||||
if dep_type in data:
|
||||
for name, version in data[dep_type].items():
|
||||
dependencies.append({
|
||||
'name': name,
|
||||
'version': version,
|
||||
'ecosystem': 'npm',
|
||||
'direct': True
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"Error parsing package.json: {e}")
|
||||
|
||||
return dependencies
|
||||
|
||||
def _analyze_dependency_license(self, dependency: Dict[str, Any], project_path: Path) -> DependencyLicense:
|
||||
"""Analyze license information for a single dependency."""
|
||||
dep_license = DependencyLicense(
|
||||
name=dependency['name'],
|
||||
version=dependency.get('version', ''),
|
||||
ecosystem=dependency.get('ecosystem', ''),
|
||||
direct=dependency.get('direct', False),
|
||||
license_declared=dependency.get('license'),
|
||||
license_detected=None,
|
||||
license_files=[],
|
||||
confidence=0.0
|
||||
)
|
||||
|
||||
# Try to detect license from various sources
|
||||
declared_license = dependency.get('license')
|
||||
if declared_license:
|
||||
license_info = self._resolve_license_info(declared_license)
|
||||
if license_info:
|
||||
dep_license.license_detected = license_info
|
||||
dep_license.confidence = 0.9
|
||||
|
||||
# For unknown licenses, try to find license files in node_modules (example)
|
||||
if not dep_license.license_detected and dep_license.ecosystem == 'npm':
|
||||
node_modules_path = project_path / 'node_modules' / dep_license.name
|
||||
if node_modules_path.exists():
|
||||
license_info = self._scan_package_directory(node_modules_path)
|
||||
if license_info:
|
||||
dep_license.license_detected = license_info
|
||||
dep_license.confidence = 0.7
|
||||
|
||||
# Default to unknown if no license detected
|
||||
if not dep_license.license_detected:
|
||||
dep_license.license_detected = self.license_database['UNKNOWN']
|
||||
dep_license.confidence = 0.0
|
||||
|
||||
return dep_license
|
||||
|
||||
def _resolve_license_info(self, license_string: str) -> Optional[LicenseInfo]:
|
||||
"""Resolve license string to LicenseInfo object."""
|
||||
if not license_string:
|
||||
return None
|
||||
|
||||
license_string = license_string.strip()
|
||||
|
||||
# Direct SPDX ID match
|
||||
if license_string in self.license_database:
|
||||
return self.license_database[license_string]
|
||||
|
||||
# Common variations and mappings
|
||||
license_mappings = {
|
||||
'mit': 'MIT',
|
||||
'apache': 'Apache-2.0',
|
||||
'apache-2.0': 'Apache-2.0',
|
||||
'apache 2.0': 'Apache-2.0',
|
||||
'bsd': 'BSD-3-Clause',
|
||||
'bsd-3-clause': 'BSD-3-Clause',
|
||||
'bsd-2-clause': 'BSD-2-Clause',
|
||||
'gpl-2.0': 'GPL-2.0',
|
||||
'gpl-3.0': 'GPL-3.0',
|
||||
'lgpl-2.1': 'LGPL-2.1',
|
||||
'lgpl-3.0': 'LGPL-3.0',
|
||||
'mpl-2.0': 'MPL-2.0',
|
||||
'isc': 'ISC',
|
||||
'unlicense': 'MIT', # Treat as permissive
|
||||
'public domain': 'MIT', # Treat as permissive
|
||||
'proprietary': 'PROPRIETARY',
|
||||
'commercial': 'PROPRIETARY'
|
||||
}
|
||||
|
||||
license_lower = license_string.lower()
|
||||
for pattern, mapped_license in license_mappings.items():
|
||||
if pattern in license_lower:
|
||||
return self.license_database.get(mapped_license)
|
||||
|
||||
return None
|
||||
|
||||
def _scan_package_directory(self, package_path: Path) -> Optional[LicenseInfo]:
|
||||
"""Scan package directory for license information."""
|
||||
license_files = ['LICENSE', 'LICENSE.txt', 'LICENSE.md', 'COPYING', 'README.md', 'package.json']
|
||||
|
||||
for license_file in license_files:
|
||||
file_path = package_path / license_file
|
||||
if file_path.exists():
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
content = f.read()
|
||||
|
||||
# Try to detect license from content
|
||||
if license_file == 'package.json':
|
||||
# Parse JSON for license field
|
||||
try:
|
||||
data = json.loads(content)
|
||||
license_field = data.get('license')
|
||||
if license_field:
|
||||
return self._resolve_license_info(license_field)
|
||||
except:
|
||||
continue
|
||||
else:
|
||||
# Analyze text content
|
||||
detected_license = self._detect_license_from_text(content)
|
||||
if detected_license:
|
||||
return self.license_database.get(detected_license)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return None
|
||||
|
||||
def _generate_license_summary(self, dependencies: List[DependencyLicense]) -> Dict[str, Any]:
|
||||
"""Generate summary of license distribution."""
|
||||
summary = {
|
||||
'total_dependencies': len(dependencies),
|
||||
'license_types': {},
|
||||
'risk_levels': {},
|
||||
'unknown_licenses': 0,
|
||||
'direct_dependencies': 0,
|
||||
'transitive_dependencies': 0
|
||||
}
|
||||
|
||||
for dep in dependencies:
|
||||
# Count by license type
|
||||
license_type = dep.license_detected.license_type.value
|
||||
summary['license_types'][license_type] = summary['license_types'].get(license_type, 0) + 1
|
||||
|
||||
# Count by risk level
|
||||
risk_level = dep.license_detected.risk_level.value
|
||||
summary['risk_levels'][risk_level] = summary['risk_levels'].get(risk_level, 0) + 1
|
||||
|
||||
# Count unknowns
|
||||
if dep.license_detected.license_type == LicenseType.UNKNOWN:
|
||||
summary['unknown_licenses'] += 1
|
||||
|
||||
# Count direct vs transitive
|
||||
if dep.direct:
|
||||
summary['direct_dependencies'] += 1
|
||||
else:
|
||||
summary['transitive_dependencies'] += 1
|
||||
|
||||
return summary
|
||||
|
||||
def _detect_license_conflicts(self, project_license: Optional[str],
|
||||
dependencies: List[DependencyLicense]) -> List[LicenseConflict]:
|
||||
"""Detect license compatibility conflicts."""
|
||||
conflicts = []
|
||||
|
||||
if not project_license:
|
||||
# If no project license detected, flag as potential issue
|
||||
for dep in dependencies:
|
||||
if dep.license_detected.risk_level in [RiskLevel.HIGH, RiskLevel.CRITICAL]:
|
||||
conflicts.append(LicenseConflict(
|
||||
dependency1='Project',
|
||||
license1='Unknown',
|
||||
dependency2=dep.name,
|
||||
license2=dep.license_detected.spdx_id or dep.license_detected.name,
|
||||
conflict_type='Unknown project license',
|
||||
severity=RiskLevel.HIGH,
|
||||
description=f'Project license unknown, dependency {dep.name} has {dep.license_detected.risk_level.value} risk license',
|
||||
resolution_options=['Define project license', 'Review dependency usage']
|
||||
))
|
||||
return conflicts
|
||||
|
||||
project_license_info = self.license_database.get(project_license)
|
||||
if not project_license_info:
|
||||
return conflicts
|
||||
|
||||
# Check compatibility with project license
|
||||
for dep in dependencies:
|
||||
dep_license_id = dep.license_detected.spdx_id or 'UNKNOWN'
|
||||
|
||||
# Check compatibility matrix
|
||||
if project_license in self.compatibility_matrix:
|
||||
compatibility = self.compatibility_matrix[project_license].get(dep_license_id, False)
|
||||
|
||||
if not compatibility:
|
||||
severity = self._determine_conflict_severity(project_license_info, dep.license_detected)
|
||||
|
||||
conflicts.append(LicenseConflict(
|
||||
dependency1='Project',
|
||||
license1=project_license,
|
||||
dependency2=dep.name,
|
||||
license2=dep_license_id,
|
||||
conflict_type='License incompatibility',
|
||||
severity=severity,
|
||||
description=f'Project license {project_license} is incompatible with dependency license {dep_license_id}',
|
||||
resolution_options=self._generate_conflict_resolutions(project_license, dep_license_id)
|
||||
))
|
||||
|
||||
# Check for GPL contamination in permissive projects
|
||||
if project_license_info.license_type == LicenseType.PERMISSIVE:
|
||||
for dep in dependencies:
|
||||
if dep.license_detected.license_type == LicenseType.COPYLEFT_STRONG:
|
||||
conflicts.append(LicenseConflict(
|
||||
dependency1='Project',
|
||||
license1=project_license,
|
||||
dependency2=dep.name,
|
||||
license2=dep.license_detected.spdx_id or dep.license_detected.name,
|
||||
conflict_type='GPL contamination',
|
||||
severity=RiskLevel.CRITICAL,
|
||||
description=f'GPL dependency {dep.name} may contaminate permissive project',
|
||||
resolution_options=['Remove GPL dependency', 'Change project license to GPL',
|
||||
'Use dynamic linking', 'Find alternative dependency']
|
||||
))
|
||||
|
||||
return conflicts
|
||||
|
||||
def _determine_conflict_severity(self, project_license: LicenseInfo, dep_license: LicenseInfo) -> RiskLevel:
|
||||
"""Determine severity of a license conflict."""
|
||||
if dep_license.license_type == LicenseType.UNKNOWN:
|
||||
return RiskLevel.CRITICAL
|
||||
elif (project_license.license_type == LicenseType.PERMISSIVE and
|
||||
dep_license.license_type == LicenseType.COPYLEFT_STRONG):
|
||||
return RiskLevel.CRITICAL
|
||||
elif dep_license.license_type == LicenseType.PROPRIETARY:
|
||||
return RiskLevel.HIGH
|
||||
else:
|
||||
return RiskLevel.MEDIUM
|
||||
|
||||
def _generate_conflict_resolutions(self, project_license: str, dep_license: str) -> List[str]:
|
||||
"""Generate resolution options for license conflicts."""
|
||||
resolutions = []
|
||||
|
||||
if 'GPL' in dep_license:
|
||||
resolutions.extend([
|
||||
'Find alternative non-GPL dependency',
|
||||
'Use dynamic linking if possible',
|
||||
'Consider changing project license to GPL-compatible',
|
||||
'Remove the dependency if not essential'
|
||||
])
|
||||
elif dep_license == 'PROPRIETARY':
|
||||
resolutions.extend([
|
||||
'Obtain commercial license',
|
||||
'Find open-source alternative',
|
||||
'Remove dependency if not essential',
|
||||
'Negotiate license terms'
|
||||
])
|
||||
else:
|
||||
resolutions.extend([
|
||||
'Review license compatibility carefully',
|
||||
'Consult legal counsel',
|
||||
'Find alternative dependency',
|
||||
'Consider license exception'
|
||||
])
|
||||
|
||||
return resolutions
|
||||
|
||||
def _calculate_compliance_score(self, dependencies: List[DependencyLicense],
|
||||
conflicts: List[LicenseConflict]) -> float:
|
||||
"""Calculate overall compliance score (0-100)."""
|
||||
if not dependencies:
|
||||
return 100.0
|
||||
|
||||
base_score = 100.0
|
||||
|
||||
# Deduct points for unknown licenses
|
||||
unknown_count = sum(1 for dep in dependencies
|
||||
if dep.license_detected.license_type == LicenseType.UNKNOWN)
|
||||
base_score -= (unknown_count / len(dependencies)) * 30
|
||||
|
||||
# Deduct points for high-risk licenses
|
||||
high_risk_count = sum(1 for dep in dependencies
|
||||
if dep.license_detected.risk_level in [RiskLevel.HIGH, RiskLevel.CRITICAL])
|
||||
base_score -= (high_risk_count / len(dependencies)) * 20
|
||||
|
||||
# Deduct points for conflicts
|
||||
if conflicts:
|
||||
critical_conflicts = sum(1 for c in conflicts if c.severity == RiskLevel.CRITICAL)
|
||||
high_conflicts = sum(1 for c in conflicts if c.severity == RiskLevel.HIGH)
|
||||
|
||||
base_score -= critical_conflicts * 15
|
||||
base_score -= high_conflicts * 10
|
||||
|
||||
return max(0.0, base_score)
|
||||
|
||||
def _generate_risk_assessment(self, dependencies: List[DependencyLicense],
|
||||
conflicts: List[LicenseConflict]) -> Dict[str, Any]:
|
||||
"""Generate comprehensive risk assessment."""
|
||||
return {
|
||||
'overall_risk': self._calculate_overall_risk(dependencies, conflicts),
|
||||
'license_risk_breakdown': self._calculate_license_risks(dependencies),
|
||||
'conflict_summary': {
|
||||
'total_conflicts': len(conflicts),
|
||||
'critical_conflicts': len([c for c in conflicts if c.severity == RiskLevel.CRITICAL]),
|
||||
'high_conflicts': len([c for c in conflicts if c.severity == RiskLevel.HIGH])
|
||||
},
|
||||
'distribution_risks': self._assess_distribution_risks(dependencies),
|
||||
'commercial_risks': self._assess_commercial_risks(dependencies)
|
||||
}
|
||||
|
||||
def _calculate_overall_risk(self, dependencies: List[DependencyLicense],
|
||||
conflicts: List[LicenseConflict]) -> str:
|
||||
"""Calculate overall project risk level."""
|
||||
if any(c.severity == RiskLevel.CRITICAL for c in conflicts):
|
||||
return 'CRITICAL'
|
||||
elif any(dep.license_detected.risk_level == RiskLevel.CRITICAL for dep in dependencies):
|
||||
return 'CRITICAL'
|
||||
elif any(c.severity == RiskLevel.HIGH for c in conflicts):
|
||||
return 'HIGH'
|
||||
elif any(dep.license_detected.risk_level == RiskLevel.HIGH for dep in dependencies):
|
||||
return 'HIGH'
|
||||
elif any(dep.license_detected.risk_level == RiskLevel.MEDIUM for dep in dependencies):
|
||||
return 'MEDIUM'
|
||||
else:
|
||||
return 'LOW'
|
||||
|
||||
def _calculate_license_risks(self, dependencies: List[DependencyLicense]) -> Dict[str, int]:
|
||||
"""Calculate breakdown of license risks."""
|
||||
risks = {'low': 0, 'medium': 0, 'high': 0, 'critical': 0}
|
||||
|
||||
for dep in dependencies:
|
||||
risk_level = dep.license_detected.risk_level.value
|
||||
risks[risk_level] += 1
|
||||
|
||||
return risks
|
||||
|
||||
def _assess_distribution_risks(self, dependencies: List[DependencyLicense]) -> List[str]:
|
||||
"""Assess risks related to software distribution."""
|
||||
risks = []
|
||||
|
||||
gpl_deps = [dep for dep in dependencies
|
||||
if dep.license_detected.license_type == LicenseType.COPYLEFT_STRONG]
|
||||
if gpl_deps:
|
||||
risks.append(f"GPL dependencies require source code disclosure: {[d.name for d in gpl_deps]}")
|
||||
|
||||
proprietary_deps = [dep for dep in dependencies
|
||||
if dep.license_detected.license_type == LicenseType.PROPRIETARY]
|
||||
if proprietary_deps:
|
||||
risks.append(f"Proprietary dependencies may require commercial licenses: {[d.name for d in proprietary_deps]}")
|
||||
|
||||
unknown_deps = [dep for dep in dependencies
|
||||
if dep.license_detected.license_type == LicenseType.UNKNOWN]
|
||||
if unknown_deps:
|
||||
risks.append(f"Unknown licenses pose legal uncertainty: {[d.name for d in unknown_deps]}")
|
||||
|
||||
return risks
|
||||
|
||||
def _assess_commercial_risks(self, dependencies: List[DependencyLicense]) -> List[str]:
|
||||
"""Assess risks for commercial usage."""
|
||||
risks = []
|
||||
|
||||
agpl_deps = [dep for dep in dependencies
|
||||
if dep.license_detected.spdx_id == 'AGPL-3.0']
|
||||
if agpl_deps:
|
||||
risks.append(f"AGPL dependencies trigger copyleft for network services: {[d.name for d in agpl_deps]}")
|
||||
|
||||
return risks
|
||||
|
||||
def _generate_compliance_recommendations(self, analysis_results: Dict[str, Any]) -> List[str]:
|
||||
"""Generate actionable compliance recommendations."""
|
||||
recommendations = []
|
||||
|
||||
# Address critical issues first
|
||||
critical_conflicts = [c for c in analysis_results['conflicts']
|
||||
if c.severity == RiskLevel.CRITICAL]
|
||||
if critical_conflicts:
|
||||
recommendations.append("CRITICAL: Address license conflicts immediately before any distribution")
|
||||
for conflict in critical_conflicts[:3]: # Top 3
|
||||
recommendations.append(f" • {conflict.description}")
|
||||
|
||||
# Unknown licenses
|
||||
unknown_count = analysis_results['license_summary']['unknown_licenses']
|
||||
if unknown_count > 0:
|
||||
recommendations.append(f"Investigate and clarify licenses for {unknown_count} dependencies with unknown licensing")
|
||||
|
||||
# GPL contamination
|
||||
gpl_deps = [dep for dep in analysis_results['dependencies']
|
||||
if dep.license_detected.license_type == LicenseType.COPYLEFT_STRONG]
|
||||
if gpl_deps and analysis_results.get('project_license') in ['MIT', 'Apache-2.0', 'BSD-3-Clause']:
|
||||
recommendations.append("Consider removing GPL dependencies or changing project license for permissive project")
|
||||
|
||||
# Compliance score
|
||||
if analysis_results['compliance_score'] < 70:
|
||||
recommendations.append("Overall compliance score is low - prioritize license cleanup")
|
||||
|
||||
return recommendations
|
||||
|
||||
def generate_report(self, analysis_results: Dict[str, Any], format: str = 'text') -> str:
|
||||
"""Generate compliance report in specified format."""
|
||||
if format == 'json':
|
||||
# Convert dataclass objects for JSON serialization
|
||||
serializable_results = analysis_results.copy()
|
||||
serializable_results['dependencies'] = [
|
||||
{
|
||||
'name': dep.name,
|
||||
'version': dep.version,
|
||||
'ecosystem': dep.ecosystem,
|
||||
'direct': dep.direct,
|
||||
'license_declared': dep.license_declared,
|
||||
'license_detected': asdict(dep.license_detected) if dep.license_detected else None,
|
||||
'confidence': dep.confidence
|
||||
}
|
||||
for dep in analysis_results['dependencies']
|
||||
]
|
||||
serializable_results['conflicts'] = [asdict(conflict) for conflict in analysis_results['conflicts']]
|
||||
return json.dumps(serializable_results, indent=2, default=str)
|
||||
|
||||
# Text format report
|
||||
report = []
|
||||
report.append("=" * 60)
|
||||
report.append("LICENSE COMPLIANCE REPORT")
|
||||
report.append("=" * 60)
|
||||
report.append(f"Analysis Date: {analysis_results['timestamp']}")
|
||||
report.append(f"Project: {analysis_results['project_path']}")
|
||||
report.append(f"Project License: {analysis_results['project_license'] or 'Unknown'}")
|
||||
report.append("")
|
||||
|
||||
# Summary
|
||||
summary = analysis_results['license_summary']
|
||||
report.append("SUMMARY:")
|
||||
report.append(f" Total Dependencies: {summary['total_dependencies']}")
|
||||
report.append(f" Compliance Score: {analysis_results['compliance_score']:.1f}/100")
|
||||
report.append(f" Overall Risk: {analysis_results['risk_assessment']['overall_risk']}")
|
||||
report.append(f" License Conflicts: {len(analysis_results['conflicts'])}")
|
||||
report.append("")
|
||||
|
||||
# License distribution
|
||||
report.append("LICENSE DISTRIBUTION:")
|
||||
for license_type, count in summary['license_types'].items():
|
||||
report.append(f" {license_type.title()}: {count}")
|
||||
report.append("")
|
||||
|
||||
# Risk breakdown
|
||||
report.append("RISK BREAKDOWN:")
|
||||
for risk_level, count in summary['risk_levels'].items():
|
||||
report.append(f" {risk_level.title()}: {count}")
|
||||
report.append("")
|
||||
|
||||
# Conflicts
|
||||
if analysis_results['conflicts']:
|
||||
report.append("LICENSE CONFLICTS:")
|
||||
report.append("-" * 30)
|
||||
for conflict in analysis_results['conflicts']:
|
||||
report.append(f"Conflict: {conflict.dependency2} ({conflict.license2})")
|
||||
report.append(f" Issue: {conflict.description}")
|
||||
report.append(f" Severity: {conflict.severity.value.upper()}")
|
||||
report.append(f" Resolutions: {', '.join(conflict.resolution_options[:2])}")
|
||||
report.append("")
|
||||
|
||||
# High-risk dependencies
|
||||
high_risk_deps = [dep for dep in analysis_results['dependencies']
|
||||
if dep.license_detected.risk_level in [RiskLevel.HIGH, RiskLevel.CRITICAL]]
|
||||
if high_risk_deps:
|
||||
report.append("HIGH-RISK DEPENDENCIES:")
|
||||
report.append("-" * 30)
|
||||
for dep in high_risk_deps[:10]: # Top 10
|
||||
license_name = dep.license_detected.spdx_id or dep.license_detected.name
|
||||
report.append(f" {dep.name} v{dep.version}: {license_name} ({dep.license_detected.risk_level.value.upper()})")
|
||||
report.append("")
|
||||
|
||||
# Recommendations
|
||||
if analysis_results['recommendations']:
|
||||
report.append("RECOMMENDATIONS:")
|
||||
report.append("-" * 20)
|
||||
for i, rec in enumerate(analysis_results['recommendations'], 1):
|
||||
report.append(f"{i}. {rec}")
|
||||
report.append("")
|
||||
|
||||
report.append("=" * 60)
|
||||
return '\n'.join(report)
|
||||
|
||||
def main():
|
||||
"""Main entry point for the license checker."""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Analyze dependency licenses for compliance and conflicts',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python license_checker.py /path/to/project
|
||||
python license_checker.py . --format json --output compliance.json
|
||||
python license_checker.py /app --inventory deps.json --policy strict
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('project_path',
|
||||
help='Path to the project directory to analyze')
|
||||
parser.add_argument('--inventory',
|
||||
help='Path to dependency inventory JSON file')
|
||||
parser.add_argument('--format', choices=['text', 'json'], default='text',
|
||||
help='Output format (default: text)')
|
||||
parser.add_argument('--output', '-o',
|
||||
help='Output file path (default: stdout)')
|
||||
parser.add_argument('--policy', choices=['permissive', 'strict'], default='permissive',
|
||||
help='License policy strictness (default: permissive)')
|
||||
parser.add_argument('--warn-conflicts', action='store_true',
|
||||
help='Show warnings for potential conflicts')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
checker = LicenseChecker()
|
||||
results = checker.analyze_project(args.project_path, args.inventory)
|
||||
report = checker.generate_report(results, args.format)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w') as f:
|
||||
f.write(report)
|
||||
print(f"Compliance report saved to {args.output}")
|
||||
else:
|
||||
print(report)
|
||||
|
||||
# Exit with error code for policy violations
|
||||
if args.policy == 'strict' and results['compliance_score'] < 80:
|
||||
sys.exit(1)
|
||||
|
||||
if args.warn_conflicts and results['conflicts']:
|
||||
print("\nWARNING: License conflicts detected!")
|
||||
sys.exit(2)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
1029
engineering/dependency-auditor/scripts/upgrade_planner.py
Normal file
1029
engineering/dependency-auditor/scripts/upgrade_planner.py
Normal file
File diff suppressed because it is too large
Load Diff
421
engineering/dependency-auditor/test-inventory.json
Normal file
421
engineering/dependency-auditor/test-inventory.json
Normal file
@@ -0,0 +1,421 @@
|
||||
{
|
||||
"timestamp": "2026-02-16T15:42:09.730696",
|
||||
"project_path": "test-project",
|
||||
"dependencies": [
|
||||
{
|
||||
"name": "express",
|
||||
"version": "4.18.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "CVE-2022-24999",
|
||||
"summary": "Open redirect in express",
|
||||
"severity": "MEDIUM",
|
||||
"cvss_score": 6.1,
|
||||
"affected_versions": "<4.18.2",
|
||||
"fixed_version": "4.18.2",
|
||||
"published_date": "2022-11-26",
|
||||
"references": [
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2022-24999"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "CVE-2022-24999",
|
||||
"summary": "Open redirect in express",
|
||||
"severity": "MEDIUM",
|
||||
"cvss_score": 6.1,
|
||||
"affected_versions": "<4.18.2",
|
||||
"fixed_version": "4.18.2",
|
||||
"published_date": "2022-11-26",
|
||||
"references": [
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2022-24999"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "lodash",
|
||||
"version": "4.17.20",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "CVE-2021-23337",
|
||||
"summary": "Prototype pollution in lodash",
|
||||
"severity": "HIGH",
|
||||
"cvss_score": 7.2,
|
||||
"affected_versions": "<4.17.21",
|
||||
"fixed_version": "4.17.21",
|
||||
"published_date": "2021-02-15",
|
||||
"references": [
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2021-23337"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "CVE-2021-23337",
|
||||
"summary": "Prototype pollution in lodash",
|
||||
"severity": "HIGH",
|
||||
"cvss_score": 7.2,
|
||||
"affected_versions": "<4.17.21",
|
||||
"fixed_version": "4.17.21",
|
||||
"published_date": "2021-02-15",
|
||||
"references": [
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2021-23337"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "axios",
|
||||
"version": "1.5.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": [
|
||||
{
|
||||
"id": "CVE-2023-45857",
|
||||
"summary": "Cross-site request forgery in axios",
|
||||
"severity": "MEDIUM",
|
||||
"cvss_score": 6.1,
|
||||
"affected_versions": ">=1.0.0 <1.6.0",
|
||||
"fixed_version": "1.6.0",
|
||||
"published_date": "2023-10-11",
|
||||
"references": [
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2023-45857"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "CVE-2023-45857",
|
||||
"summary": "Cross-site request forgery in axios",
|
||||
"severity": "MEDIUM",
|
||||
"cvss_score": 6.1,
|
||||
"affected_versions": ">=1.0.0 <1.6.0",
|
||||
"fixed_version": "1.6.0",
|
||||
"published_date": "2023-10-11",
|
||||
"references": [
|
||||
"https://nvd.nist.gov/vuln/detail/CVE-2023-45857"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "jsonwebtoken",
|
||||
"version": "8.5.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "bcrypt",
|
||||
"version": "5.1.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "mongoose",
|
||||
"version": "6.10.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "cors",
|
||||
"version": "2.8.5",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "helmet",
|
||||
"version": "6.1.5",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "winston",
|
||||
"version": "3.8.2",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "dotenv",
|
||||
"version": "16.0.3",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "express-rate-limit",
|
||||
"version": "6.7.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "multer",
|
||||
"version": "1.4.5-lts.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "sharp",
|
||||
"version": "0.32.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "nodemailer",
|
||||
"version": "6.9.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "socket.io",
|
||||
"version": "4.6.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "redis",
|
||||
"version": "4.6.5",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "moment",
|
||||
"version": "2.29.4",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "chalk",
|
||||
"version": "4.1.2",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "commander",
|
||||
"version": "9.4.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "nodemon",
|
||||
"version": "2.0.22",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "jest",
|
||||
"version": "29.5.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "supertest",
|
||||
"version": "6.3.3",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "eslint",
|
||||
"version": "8.40.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "eslint-config-airbnb-base",
|
||||
"version": "15.0.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "eslint-plugin-import",
|
||||
"version": "2.27.5",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "webpack",
|
||||
"version": "5.82.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "webpack-cli",
|
||||
"version": "5.1.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "babel-loader",
|
||||
"version": "9.1.2",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "@babel/core",
|
||||
"version": "7.22.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "@babel/preset-env",
|
||||
"version": "7.22.2",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "css-loader",
|
||||
"version": "6.7.4",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "style-loader",
|
||||
"version": "3.3.3",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "html-webpack-plugin",
|
||||
"version": "5.5.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "mini-css-extract-plugin",
|
||||
"version": "2.7.6",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "postcss",
|
||||
"version": "8.4.23",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "postcss-loader",
|
||||
"version": "7.3.0",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "autoprefixer",
|
||||
"version": "10.4.14",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "cross-env",
|
||||
"version": "7.0.3",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
},
|
||||
{
|
||||
"name": "rimraf",
|
||||
"version": "5.0.1",
|
||||
"ecosystem": "npm",
|
||||
"direct": true,
|
||||
"license": null,
|
||||
"vulnerabilities": []
|
||||
}
|
||||
],
|
||||
"vulnerabilities_found": 6,
|
||||
"high_severity_count": 2,
|
||||
"medium_severity_count": 4,
|
||||
"low_severity_count": 0,
|
||||
"ecosystems": [
|
||||
"npm"
|
||||
],
|
||||
"scan_summary": {
|
||||
"total_dependencies": 39,
|
||||
"unique_dependencies": 39,
|
||||
"ecosystems_found": 1,
|
||||
"vulnerable_dependencies": 3,
|
||||
"vulnerability_breakdown": {
|
||||
"high": 2,
|
||||
"medium": 4,
|
||||
"low": 0
|
||||
}
|
||||
},
|
||||
"recommendations": [
|
||||
"URGENT: Address 2 high-severity vulnerabilities immediately",
|
||||
"Schedule fixes for 4 medium-severity vulnerabilities within 30 days",
|
||||
"Update express from 4.18.1 to 4.18.2 to fix CVE-2022-24999",
|
||||
"Update express from 4.18.1 to 4.18.2 to fix CVE-2022-24999",
|
||||
"Update lodash from 4.17.20 to 4.17.21 to fix CVE-2021-23337",
|
||||
"Update lodash from 4.17.20 to 4.17.21 to fix CVE-2021-23337",
|
||||
"Update axios from 1.5.0 to 1.6.0 to fix CVE-2023-45857",
|
||||
"Update axios from 1.5.0 to 1.6.0 to fix CVE-2023-45857"
|
||||
]
|
||||
}
|
||||
72
engineering/dependency-auditor/test-project/package.json
Normal file
72
engineering/dependency-auditor/test-project/package.json
Normal file
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"name": "sample-web-app",
|
||||
"version": "1.2.3",
|
||||
"description": "A sample web application with various dependencies for testing dependency auditing",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"dev": "nodemon index.js",
|
||||
"build": "webpack --mode production",
|
||||
"test": "jest",
|
||||
"lint": "eslint src/",
|
||||
"audit": "npm audit"
|
||||
},
|
||||
"keywords": ["web", "app", "sample", "dependency", "audit"],
|
||||
"author": "Claude Skills Team",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"express": "4.18.1",
|
||||
"lodash": "4.17.20",
|
||||
"axios": "1.5.0",
|
||||
"jsonwebtoken": "8.5.1",
|
||||
"bcrypt": "5.1.0",
|
||||
"mongoose": "6.10.0",
|
||||
"cors": "2.8.5",
|
||||
"helmet": "6.1.5",
|
||||
"winston": "3.8.2",
|
||||
"dotenv": "16.0.3",
|
||||
"express-rate-limit": "6.7.0",
|
||||
"multer": "1.4.5-lts.1",
|
||||
"sharp": "0.32.1",
|
||||
"nodemailer": "6.9.1",
|
||||
"socket.io": "4.6.1",
|
||||
"redis": "4.6.5",
|
||||
"moment": "2.29.4",
|
||||
"chalk": "4.1.2",
|
||||
"commander": "9.4.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "2.0.22",
|
||||
"jest": "29.5.0",
|
||||
"supertest": "6.3.3",
|
||||
"eslint": "8.40.0",
|
||||
"eslint-config-airbnb-base": "15.0.0",
|
||||
"eslint-plugin-import": "2.27.5",
|
||||
"webpack": "5.82.1",
|
||||
"webpack-cli": "5.1.1",
|
||||
"babel-loader": "9.1.2",
|
||||
"@babel/core": "7.22.1",
|
||||
"@babel/preset-env": "7.22.2",
|
||||
"css-loader": "6.7.4",
|
||||
"style-loader": "3.3.3",
|
||||
"html-webpack-plugin": "5.5.1",
|
||||
"mini-css-extract-plugin": "2.7.6",
|
||||
"postcss": "8.4.23",
|
||||
"postcss-loader": "7.3.0",
|
||||
"autoprefixer": "10.4.14",
|
||||
"cross-env": "7.0.3",
|
||||
"rimraf": "5.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16.0.0",
|
||||
"npm": ">=8.0.0"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/example/sample-web-app.git"
|
||||
},
|
||||
"bugs": {
|
||||
"url": "https://github.com/example/sample-web-app/issues"
|
||||
},
|
||||
"homepage": "https://github.com/example/sample-web-app#readme"
|
||||
}
|
||||
445
engineering/release-manager/README.md
Normal file
445
engineering/release-manager/README.md
Normal file
@@ -0,0 +1,445 @@
|
||||
# Release Manager
|
||||
|
||||
A comprehensive release management toolkit for automating changelog generation, version bumping, and release planning based on conventional commits and industry best practices.
|
||||
|
||||
## Overview
|
||||
|
||||
The Release Manager skill provides three powerful Python scripts and comprehensive documentation for managing software releases:
|
||||
|
||||
1. **changelog_generator.py** - Generate structured changelogs from git history
|
||||
2. **version_bumper.py** - Determine correct semantic version bumps
|
||||
3. **release_planner.py** - Assess release readiness and generate coordination plans
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- Python 3.7+
|
||||
- Git repository with conventional commit messages
|
||||
- No external dependencies required (uses only Python standard library)
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```bash
|
||||
# Generate changelog from recent commits
|
||||
git log --oneline --since="1 month ago" | python changelog_generator.py
|
||||
|
||||
# Determine version bump from commits since last tag
|
||||
git log --oneline $(git describe --tags --abbrev=0)..HEAD | python version_bumper.py -c "1.2.3"
|
||||
|
||||
# Assess release readiness
|
||||
python release_planner.py --input assets/sample_release_plan.json
|
||||
```
|
||||
|
||||
## Scripts Reference
|
||||
|
||||
### changelog_generator.py
|
||||
|
||||
Parses conventional commits and generates structured changelogs in multiple formats.
|
||||
|
||||
**Input Options:**
|
||||
- Git log text (oneline or full format)
|
||||
- JSON array of commits
|
||||
- Stdin or file input
|
||||
|
||||
**Output Formats:**
|
||||
- Markdown (Keep a Changelog format)
|
||||
- JSON structured data
|
||||
- Both with release statistics
|
||||
|
||||
```bash
|
||||
# From git log (recommended)
|
||||
git log --oneline --since="last release" | python changelog_generator.py \
|
||||
--version "2.1.0" \
|
||||
--date "2024-01-15" \
|
||||
--base-url "https://github.com/yourorg/yourrepo"
|
||||
|
||||
# From JSON file
|
||||
python changelog_generator.py \
|
||||
--input assets/sample_commits.json \
|
||||
--input-format json \
|
||||
--format both \
|
||||
--summary
|
||||
|
||||
# With custom output
|
||||
git log --format="%h %s" v1.0.0..HEAD | python changelog_generator.py \
|
||||
--version "1.1.0" \
|
||||
--output CHANGELOG_DRAFT.md
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Parses conventional commit types (feat, fix, docs, etc.)
|
||||
- Groups commits by changelog categories (Added, Fixed, Changed, etc.)
|
||||
- Extracts issue references (#123, fixes #456)
|
||||
- Identifies breaking changes
|
||||
- Links to commits and PRs
|
||||
- Generates release summary statistics
|
||||
|
||||
### version_bumper.py
|
||||
|
||||
Analyzes commits to determine semantic version bumps according to conventional commits.
|
||||
|
||||
**Bump Rules:**
|
||||
- **MAJOR:** Breaking changes (`feat!:` or `BREAKING CHANGE:`)
|
||||
- **MINOR:** New features (`feat:`)
|
||||
- **PATCH:** Bug fixes (`fix:`, `perf:`, `security:`)
|
||||
- **NONE:** Documentation, tests, chores only
|
||||
|
||||
```bash
|
||||
# Basic version bump determination
|
||||
git log --oneline v1.2.3..HEAD | python version_bumper.py --current-version "1.2.3"
|
||||
|
||||
# With pre-release version
|
||||
python version_bumper.py \
|
||||
--current-version "1.2.3" \
|
||||
--prerelease alpha \
|
||||
--input assets/sample_commits.json \
|
||||
--input-format json
|
||||
|
||||
# Include bump commands and file updates
|
||||
git log --oneline $(git describe --tags --abbrev=0)..HEAD | \
|
||||
python version_bumper.py \
|
||||
--current-version "$(git describe --tags --abbrev=0)" \
|
||||
--include-commands \
|
||||
--include-files \
|
||||
--analysis
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Supports pre-release versions (alpha, beta, rc)
|
||||
- Generates bump commands for npm, Python, Rust, Git
|
||||
- Provides file update snippets
|
||||
- Detailed commit analysis and categorization
|
||||
- Custom rules for specific commit types
|
||||
- JSON and text output formats
|
||||
|
||||
### release_planner.py
|
||||
|
||||
Assesses release readiness and generates comprehensive release coordination plans.
|
||||
|
||||
**Input:** JSON release plan with features, quality gates, and stakeholders
|
||||
|
||||
```bash
|
||||
# Assess release readiness
|
||||
python release_planner.py --input assets/sample_release_plan.json
|
||||
|
||||
# Generate full release package
|
||||
python release_planner.py \
|
||||
--input release_plan.json \
|
||||
--output-format markdown \
|
||||
--include-checklist \
|
||||
--include-communication \
|
||||
--include-rollback \
|
||||
--output release_report.md
|
||||
```
|
||||
|
||||
**Features:**
|
||||
- Feature readiness assessment with approval tracking
|
||||
- Quality gate validation and reporting
|
||||
- Stakeholder communication planning
|
||||
- Rollback procedure generation
|
||||
- Risk analysis and timeline assessment
|
||||
- Customizable test coverage thresholds
|
||||
- Multiple output formats (text, JSON, Markdown)
|
||||
|
||||
## File Structure
|
||||
|
||||
```
|
||||
release-manager/
|
||||
├── SKILL.md # Comprehensive methodology guide
|
||||
├── README.md # This file
|
||||
├── changelog_generator.py # Changelog generation script
|
||||
├── version_bumper.py # Version bump determination
|
||||
├── release_planner.py # Release readiness assessment
|
||||
├── references/ # Reference documentation
|
||||
│ ├── conventional-commits-guide.md # Conventional commits specification
|
||||
│ ├── release-workflow-comparison.md # Git Flow vs GitHub Flow vs Trunk-based
|
||||
│ └── hotfix-procedures.md # Emergency release procedures
|
||||
├── assets/ # Sample data for testing
|
||||
│ ├── sample_git_log.txt # Sample git log output
|
||||
│ ├── sample_git_log_full.txt # Detailed git log format
|
||||
│ ├── sample_commits.json # JSON commit data
|
||||
│ └── sample_release_plan.json # Release plan template
|
||||
└── expected_outputs/ # Example script outputs
|
||||
├── changelog_example.md # Expected changelog format
|
||||
├── version_bump_example.txt # Version bump output
|
||||
└── release_readiness_example.txt # Release assessment report
|
||||
```
|
||||
|
||||
## Integration Examples
|
||||
|
||||
### CI/CD Pipeline Integration
|
||||
|
||||
```yaml
|
||||
# .github/workflows/release.yml
|
||||
name: Automated Release
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Need full history
|
||||
|
||||
- name: Determine version bump
|
||||
id: version
|
||||
run: |
|
||||
CURRENT=$(git describe --tags --abbrev=0)
|
||||
git log --oneline $CURRENT..HEAD | \
|
||||
python scripts/version_bumper.py -c $CURRENT --output-format json > bump.json
|
||||
echo "new_version=$(jq -r '.recommended_version' bump.json)" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Generate changelog
|
||||
run: |
|
||||
git log --oneline ${{ steps.version.outputs.current_version }}..HEAD | \
|
||||
python scripts/changelog_generator.py \
|
||||
--version "${{ steps.version.outputs.new_version }}" \
|
||||
--base-url "https://github.com/${{ github.repository }}" \
|
||||
--output CHANGELOG_ENTRY.md
|
||||
|
||||
- name: Create release
|
||||
uses: actions/create-release@v1
|
||||
with:
|
||||
tag_name: v${{ steps.version.outputs.new_version }}
|
||||
release_name: Release ${{ steps.version.outputs.new_version }}
|
||||
body_path: CHANGELOG_ENTRY.md
|
||||
```
|
||||
|
||||
### Git Hooks Integration
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# .git/hooks/pre-commit
|
||||
# Validate conventional commit format
|
||||
|
||||
commit_msg_file=$1
|
||||
commit_msg=$(cat $commit_msg_file)
|
||||
|
||||
# Simple validation (more sophisticated validation available in commitlint)
|
||||
if ! echo "$commit_msg" | grep -qE "^(feat|fix|docs|style|refactor|test|chore|perf|ci|build)(\(.+\))?(!)?:"; then
|
||||
echo "❌ Commit message doesn't follow conventional commits format"
|
||||
echo "Expected: type(scope): description"
|
||||
echo "Examples:"
|
||||
echo " feat(auth): add OAuth2 integration"
|
||||
echo " fix(api): resolve race condition"
|
||||
echo " docs: update installation guide"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✅ Commit message format is valid"
|
||||
```
|
||||
|
||||
### Release Planning Automation
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
# generate_release_plan.py - Automatically generate release plans from project management tools
|
||||
|
||||
import json
|
||||
import requests
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
def generate_release_plan_from_github(repo, milestone):
|
||||
"""Generate release plan from GitHub milestone and PRs."""
|
||||
|
||||
# Fetch milestone details
|
||||
milestone_url = f"https://api.github.com/repos/{repo}/milestones/{milestone}"
|
||||
milestone_data = requests.get(milestone_url).json()
|
||||
|
||||
# Fetch associated issues/PRs
|
||||
issues_url = f"https://api.github.com/repos/{repo}/issues?milestone={milestone}&state=all"
|
||||
issues = requests.get(issues_url).json()
|
||||
|
||||
release_plan = {
|
||||
"release_name": milestone_data["title"],
|
||||
"version": "TBD", # Fill in manually or extract from milestone
|
||||
"target_date": milestone_data["due_on"],
|
||||
"features": []
|
||||
}
|
||||
|
||||
for issue in issues:
|
||||
if issue.get("pull_request"): # It's a PR
|
||||
feature = {
|
||||
"id": f"GH-{issue['number']}",
|
||||
"title": issue["title"],
|
||||
"description": issue["body"][:200] + "..." if len(issue["body"]) > 200 else issue["body"],
|
||||
"type": "feature", # Could be parsed from labels
|
||||
"assignee": issue["assignee"]["login"] if issue["assignee"] else "",
|
||||
"status": "ready" if issue["state"] == "closed" else "in_progress",
|
||||
"pull_request_url": issue["pull_request"]["html_url"],
|
||||
"issue_url": issue["html_url"],
|
||||
"risk_level": "medium", # Could be parsed from labels
|
||||
"qa_approved": "qa-approved" in [label["name"] for label in issue["labels"]],
|
||||
"pm_approved": "pm-approved" in [label["name"] for label in issue["labels"]]
|
||||
}
|
||||
release_plan["features"].append(feature)
|
||||
|
||||
return release_plan
|
||||
|
||||
# Usage
|
||||
if __name__ == "__main__":
|
||||
plan = generate_release_plan_from_github("yourorg/yourrepo", "5")
|
||||
with open("release_plan.json", "w") as f:
|
||||
json.dump(plan, f, indent=2)
|
||||
|
||||
print("Generated release_plan.json")
|
||||
print("Run: python release_planner.py --input release_plan.json")
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Custom Commit Type Rules
|
||||
|
||||
```bash
|
||||
# Define custom rules for version bumping
|
||||
python version_bumper.py \
|
||||
--current-version "1.2.3" \
|
||||
--custom-rules '{"security": "patch", "breaking": "major"}' \
|
||||
--ignore-types "docs,style,test"
|
||||
```
|
||||
|
||||
### Multi-repository Release Coordination
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# multi_repo_release.sh - Coordinate releases across multiple repositories
|
||||
|
||||
repos=("frontend" "backend" "mobile" "docs")
|
||||
base_version="2.1.0"
|
||||
|
||||
for repo in "${repos[@]}"; do
|
||||
echo "Processing $repo..."
|
||||
cd "$repo"
|
||||
|
||||
# Generate changelog for this repo
|
||||
git log --oneline --since="1 month ago" | \
|
||||
python ../scripts/changelog_generator.py \
|
||||
--version "$base_version" \
|
||||
--output "CHANGELOG_$repo.md"
|
||||
|
||||
# Determine version bump
|
||||
git log --oneline $(git describe --tags --abbrev=0)..HEAD | \
|
||||
python ../scripts/version_bumper.py \
|
||||
--current-version "$(git describe --tags --abbrev=0)" > "VERSION_$repo.txt"
|
||||
|
||||
cd ..
|
||||
done
|
||||
|
||||
echo "Generated changelogs and version recommendations for all repositories"
|
||||
```
|
||||
|
||||
### Integration with Slack/Teams
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
# notify_release_status.py
|
||||
|
||||
import json
|
||||
import requests
|
||||
import subprocess
|
||||
|
||||
def send_slack_notification(webhook_url, message):
|
||||
payload = {"text": message}
|
||||
requests.post(webhook_url, json=payload)
|
||||
|
||||
def get_release_status():
|
||||
"""Get current release status from release planner."""
|
||||
result = subprocess.run(
|
||||
["python", "release_planner.py", "--input", "release_plan.json", "--output-format", "json"],
|
||||
capture_output=True, text=True
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
# Usage in CI/CD
|
||||
status = get_release_status()
|
||||
if status["assessment"]["overall_status"] == "blocked":
|
||||
message = f"🚫 Release {status['version']} is BLOCKED\n"
|
||||
message += f"Issues: {', '.join(status['assessment']['blocking_issues'])}"
|
||||
send_slack_notification(SLACK_WEBHOOK_URL, message)
|
||||
elif status["assessment"]["overall_status"] == "ready":
|
||||
message = f"✅ Release {status['version']} is READY for deployment!"
|
||||
send_slack_notification(SLACK_WEBHOOK_URL, message)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Commit Message Guidelines
|
||||
|
||||
1. **Use conventional commits consistently** across your team
|
||||
2. **Be specific** in commit descriptions: "fix: resolve race condition in user creation" vs "fix: bug"
|
||||
3. **Reference issues** when applicable: "Closes #123" or "Fixes #456"
|
||||
4. **Mark breaking changes** clearly with `!` or `BREAKING CHANGE:` footer
|
||||
5. **Keep first line under 50 characters** when possible
|
||||
|
||||
### Release Planning
|
||||
|
||||
1. **Plan releases early** with clear feature lists and target dates
|
||||
2. **Set quality gates** and stick to them (test coverage, security scans, etc.)
|
||||
3. **Track approvals** from all relevant stakeholders
|
||||
4. **Document rollback procedures** before deployment
|
||||
5. **Communicate clearly** with both internal teams and external users
|
||||
|
||||
### Version Management
|
||||
|
||||
1. **Follow semantic versioning** strictly for predictable releases
|
||||
2. **Use pre-release versions** for beta testing and gradual rollouts
|
||||
3. **Tag releases consistently** with proper version numbers
|
||||
4. **Maintain backwards compatibility** when possible to avoid major version bumps
|
||||
5. **Document breaking changes** thoroughly with migration guides
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Common Issues
|
||||
|
||||
**"No valid commits found"**
|
||||
- Ensure git log contains commit messages
|
||||
- Check that commits follow conventional format
|
||||
- Verify input format (git-log vs json)
|
||||
|
||||
**"Invalid version format"**
|
||||
- Use semantic versioning: 1.2.3, not 1.2 or v1.2.3.beta
|
||||
- Pre-release format: 1.2.3-alpha.1
|
||||
|
||||
**"Missing required approvals"**
|
||||
- Check feature risk levels in release plan
|
||||
- High/critical risk features require additional approvals
|
||||
- Update approval status in JSON file
|
||||
|
||||
### Debug Mode
|
||||
|
||||
All scripts support verbose output for debugging:
|
||||
|
||||
```bash
|
||||
# Add debug logging
|
||||
python changelog_generator.py --input sample.txt --debug
|
||||
|
||||
# Validate input data
|
||||
python -c "import json; print(json.load(open('release_plan.json')))"
|
||||
|
||||
# Test with sample data first
|
||||
python release_planner.py --input assets/sample_release_plan.json
|
||||
```
|
||||
|
||||
## Contributing
|
||||
|
||||
When extending these scripts:
|
||||
|
||||
1. **Maintain backwards compatibility** for existing command-line interfaces
|
||||
2. **Add comprehensive tests** for new features
|
||||
3. **Update documentation** including this README and SKILL.md
|
||||
4. **Follow Python standards** (PEP 8, type hints where helpful)
|
||||
5. **Use only standard library** to avoid dependencies
|
||||
|
||||
## License
|
||||
|
||||
This skill is part of the claude-skills repository and follows the same license terms.
|
||||
|
||||
---
|
||||
|
||||
For detailed methodology and background information, see [SKILL.md](SKILL.md).
|
||||
For specific workflow guidance, see the [references](references/) directory.
|
||||
For testing the scripts, use the sample data in the [assets](assets/) directory.
|
||||
485
engineering/release-manager/SKILL.md
Normal file
485
engineering/release-manager/SKILL.md
Normal file
@@ -0,0 +1,485 @@
|
||||
# Release Manager
|
||||
|
||||
**Tier:** POWERFUL
|
||||
**Category:** Engineering
|
||||
**Domain:** Software Release Management & DevOps
|
||||
|
||||
## Overview
|
||||
|
||||
The Release Manager skill provides comprehensive tools and knowledge for managing software releases end-to-end. From parsing conventional commits to generating changelogs, determining version bumps, and orchestrating release processes, this skill ensures reliable, predictable, and well-documented software releases.
|
||||
|
||||
## Core Capabilities
|
||||
|
||||
- **Automated Changelog Generation** from git history using conventional commits
|
||||
- **Semantic Version Bumping** based on commit analysis and breaking changes
|
||||
- **Release Readiness Assessment** with comprehensive checklists and validation
|
||||
- **Release Planning & Coordination** with stakeholder communication templates
|
||||
- **Rollback Planning** with automated recovery procedures
|
||||
- **Hotfix Management** for emergency releases
|
||||
- **Feature Flag Integration** for progressive rollouts
|
||||
|
||||
## Key Components
|
||||
|
||||
### Scripts
|
||||
|
||||
1. **changelog_generator.py** - Parses git logs and generates structured changelogs
|
||||
2. **version_bumper.py** - Determines correct version bumps from conventional commits
|
||||
3. **release_planner.py** - Assesses release readiness and generates coordination plans
|
||||
|
||||
### Documentation
|
||||
|
||||
- Comprehensive release management methodology
|
||||
- Conventional commits specification and examples
|
||||
- Release workflow comparisons (Git Flow, Trunk-based, GitHub Flow)
|
||||
- Hotfix procedures and emergency response protocols
|
||||
|
||||
## Release Management Methodology
|
||||
|
||||
### Semantic Versioning (SemVer)
|
||||
|
||||
Semantic Versioning follows the MAJOR.MINOR.PATCH format where:
|
||||
|
||||
- **MAJOR** version when you make incompatible API changes
|
||||
- **MINOR** version when you add functionality in a backwards compatible manner
|
||||
- **PATCH** version when you make backwards compatible bug fixes
|
||||
|
||||
#### Pre-release Versions
|
||||
|
||||
Pre-release versions are denoted by appending a hyphen and identifiers:
|
||||
- `1.0.0-alpha.1` - Alpha releases for early testing
|
||||
- `1.0.0-beta.2` - Beta releases for wider testing
|
||||
- `1.0.0-rc.1` - Release candidates for final validation
|
||||
|
||||
#### Version Precedence
|
||||
|
||||
Version precedence is determined by comparing each identifier:
|
||||
1. `1.0.0-alpha` < `1.0.0-alpha.1` < `1.0.0-alpha.beta` < `1.0.0-beta`
|
||||
2. `1.0.0-beta` < `1.0.0-beta.2` < `1.0.0-beta.11` < `1.0.0-rc.1`
|
||||
3. `1.0.0-rc.1` < `1.0.0`
|
||||
|
||||
### Conventional Commits
|
||||
|
||||
Conventional Commits provide a structured format for commit messages that enables automated tooling:
|
||||
|
||||
#### Format
|
||||
```
|
||||
<type>[optional scope]: <description>
|
||||
|
||||
[optional body]
|
||||
|
||||
[optional footer(s)]
|
||||
```
|
||||
|
||||
#### Types
|
||||
- **feat**: A new feature (correlates with MINOR version bump)
|
||||
- **fix**: A bug fix (correlates with PATCH version bump)
|
||||
- **docs**: Documentation only changes
|
||||
- **style**: Changes that do not affect the meaning of the code
|
||||
- **refactor**: A code change that neither fixes a bug nor adds a feature
|
||||
- **perf**: A code change that improves performance
|
||||
- **test**: Adding missing tests or correcting existing tests
|
||||
- **chore**: Changes to the build process or auxiliary tools
|
||||
- **ci**: Changes to CI configuration files and scripts
|
||||
- **build**: Changes that affect the build system or external dependencies
|
||||
- **breaking**: Introduces a breaking change (correlates with MAJOR version bump)
|
||||
|
||||
#### Examples
|
||||
```
|
||||
feat(user-auth): add OAuth2 integration
|
||||
|
||||
fix(api): resolve race condition in user creation
|
||||
|
||||
docs(readme): update installation instructions
|
||||
|
||||
feat!: remove deprecated payment API
|
||||
BREAKING CHANGE: The legacy payment API has been removed
|
||||
```
|
||||
|
||||
### Automated Changelog Generation
|
||||
|
||||
Changelogs are automatically generated from conventional commits, organized by:
|
||||
|
||||
#### Structure
|
||||
```markdown
|
||||
# Changelog
|
||||
|
||||
## [Unreleased]
|
||||
### Added
|
||||
### Changed
|
||||
### Deprecated
|
||||
### Removed
|
||||
### Fixed
|
||||
### Security
|
||||
|
||||
## [1.2.0] - 2024-01-15
|
||||
### Added
|
||||
- OAuth2 authentication support (#123)
|
||||
- User preference dashboard (#145)
|
||||
|
||||
### Fixed
|
||||
- Race condition in user creation (#134)
|
||||
- Memory leak in image processing (#156)
|
||||
|
||||
### Breaking Changes
|
||||
- Removed legacy payment API
|
||||
```
|
||||
|
||||
#### Grouping Rules
|
||||
- **Added** for new features (feat)
|
||||
- **Fixed** for bug fixes (fix)
|
||||
- **Changed** for changes in existing functionality
|
||||
- **Deprecated** for soon-to-be removed features
|
||||
- **Removed** for now removed features
|
||||
- **Security** for vulnerability fixes
|
||||
|
||||
#### Metadata Extraction
|
||||
- Link to pull requests and issues: `(#123)`
|
||||
- Breaking changes highlighted prominently
|
||||
- Scope-based grouping: `auth:`, `api:`, `ui:`
|
||||
- Co-authored-by for contributor recognition
|
||||
|
||||
### Version Bump Strategies
|
||||
|
||||
Version bumps are determined by analyzing commits since the last release:
|
||||
|
||||
#### Automatic Detection Rules
|
||||
1. **MAJOR**: Any commit with `BREAKING CHANGE` or `!` after type
|
||||
2. **MINOR**: Any `feat` type commits without breaking changes
|
||||
3. **PATCH**: `fix`, `perf`, `security` type commits
|
||||
4. **NO BUMP**: `docs`, `style`, `test`, `chore`, `ci`, `build` only
|
||||
|
||||
#### Pre-release Handling
|
||||
```python
|
||||
# Alpha: 1.0.0-alpha.1 → 1.0.0-alpha.2
|
||||
# Beta: 1.0.0-alpha.5 → 1.0.0-beta.1
|
||||
# RC: 1.0.0-beta.3 → 1.0.0-rc.1
|
||||
# Release: 1.0.0-rc.2 → 1.0.0
|
||||
```
|
||||
|
||||
#### Multi-package Considerations
|
||||
For monorepos with multiple packages:
|
||||
- Analyze commits affecting each package independently
|
||||
- Support scoped version bumps: `@scope/package@1.2.3`
|
||||
- Generate coordinated release plans across packages
|
||||
|
||||
### Release Branch Workflows
|
||||
|
||||
#### Git Flow
|
||||
```
|
||||
main (production) ← release/1.2.0 ← develop ← feature/login
|
||||
← hotfix/critical-fix
|
||||
```
|
||||
|
||||
**Advantages:**
|
||||
- Clear separation of concerns
|
||||
- Stable main branch
|
||||
- Parallel feature development
|
||||
- Structured release process
|
||||
|
||||
**Process:**
|
||||
1. Create release branch from develop: `git checkout -b release/1.2.0 develop`
|
||||
2. Finalize release (version bump, changelog)
|
||||
3. Merge to main and develop
|
||||
4. Tag release: `git tag v1.2.0`
|
||||
5. Deploy from main
|
||||
|
||||
#### Trunk-based Development
|
||||
```
|
||||
main ← feature/login (short-lived)
|
||||
← feature/payment (short-lived)
|
||||
← hotfix/critical-fix
|
||||
```
|
||||
|
||||
**Advantages:**
|
||||
- Simplified workflow
|
||||
- Faster integration
|
||||
- Reduced merge conflicts
|
||||
- Continuous integration friendly
|
||||
|
||||
**Process:**
|
||||
1. Short-lived feature branches (1-3 days)
|
||||
2. Frequent commits to main
|
||||
3. Feature flags for incomplete features
|
||||
4. Automated testing gates
|
||||
5. Deploy from main with feature toggles
|
||||
|
||||
#### GitHub Flow
|
||||
```
|
||||
main ← feature/login
|
||||
← hotfix/critical-fix
|
||||
```
|
||||
|
||||
**Advantages:**
|
||||
- Simple and lightweight
|
||||
- Fast deployment cycle
|
||||
- Good for web applications
|
||||
- Minimal overhead
|
||||
|
||||
**Process:**
|
||||
1. Create feature branch from main
|
||||
2. Regular commits and pushes
|
||||
3. Open pull request when ready
|
||||
4. Deploy from feature branch for testing
|
||||
5. Merge to main and deploy
|
||||
|
||||
### Feature Flag Integration
|
||||
|
||||
Feature flags enable safe, progressive rollouts:
|
||||
|
||||
#### Types of Feature Flags
|
||||
- **Release flags**: Control feature visibility in production
|
||||
- **Experiment flags**: A/B testing and gradual rollouts
|
||||
- **Operational flags**: Circuit breakers and performance toggles
|
||||
- **Permission flags**: Role-based feature access
|
||||
|
||||
#### Implementation Strategy
|
||||
```python
|
||||
# Progressive rollout example
|
||||
if feature_flag("new_payment_flow", user_id):
|
||||
return new_payment_processor.process(payment)
|
||||
else:
|
||||
return legacy_payment_processor.process(payment)
|
||||
```
|
||||
|
||||
#### Release Coordination
|
||||
1. Deploy code with feature behind flag (disabled)
|
||||
2. Gradually enable for percentage of users
|
||||
3. Monitor metrics and error rates
|
||||
4. Full rollout or quick rollback based on data
|
||||
5. Remove flag in subsequent release
|
||||
|
||||
### Release Readiness Checklists
|
||||
|
||||
#### Pre-Release Validation
|
||||
- [ ] All planned features implemented and tested
|
||||
- [ ] Breaking changes documented with migration guide
|
||||
- [ ] API documentation updated
|
||||
- [ ] Database migrations tested
|
||||
- [ ] Security review completed for sensitive changes
|
||||
- [ ] Performance testing passed thresholds
|
||||
- [ ] Internationalization strings updated
|
||||
- [ ] Third-party integrations validated
|
||||
|
||||
#### Quality Gates
|
||||
- [ ] Unit test coverage ≥ 85%
|
||||
- [ ] Integration tests passing
|
||||
- [ ] End-to-end tests passing
|
||||
- [ ] Static analysis clean
|
||||
- [ ] Security scan passed
|
||||
- [ ] Dependency audit clean
|
||||
- [ ] Load testing completed
|
||||
|
||||
#### Documentation Requirements
|
||||
- [ ] CHANGELOG.md updated
|
||||
- [ ] README.md reflects new features
|
||||
- [ ] API documentation generated
|
||||
- [ ] Migration guide written for breaking changes
|
||||
- [ ] Deployment notes prepared
|
||||
- [ ] Rollback procedure documented
|
||||
|
||||
#### Stakeholder Approvals
|
||||
- [ ] Product Manager sign-off
|
||||
- [ ] Engineering Lead approval
|
||||
- [ ] QA validation complete
|
||||
- [ ] Security team clearance
|
||||
- [ ] Legal review (if applicable)
|
||||
- [ ] Compliance check (if regulated)
|
||||
|
||||
### Deployment Coordination
|
||||
|
||||
#### Communication Plan
|
||||
**Internal Stakeholders:**
|
||||
- Engineering team: Technical changes and rollback procedures
|
||||
- Product team: Feature descriptions and user impact
|
||||
- Support team: Known issues and troubleshooting guides
|
||||
- Sales team: Customer-facing changes and talking points
|
||||
|
||||
**External Communication:**
|
||||
- Release notes for users
|
||||
- API changelog for developers
|
||||
- Migration guide for breaking changes
|
||||
- Downtime notifications if applicable
|
||||
|
||||
#### Deployment Sequence
|
||||
1. **Pre-deployment** (T-24h): Final validation, freeze code
|
||||
2. **Database migrations** (T-2h): Run and validate schema changes
|
||||
3. **Blue-green deployment** (T-0): Switch traffic gradually
|
||||
4. **Post-deployment** (T+1h): Monitor metrics and logs
|
||||
5. **Rollback window** (T+4h): Decision point for rollback
|
||||
|
||||
#### Monitoring & Validation
|
||||
- Application health checks
|
||||
- Error rate monitoring
|
||||
- Performance metrics tracking
|
||||
- User experience monitoring
|
||||
- Business metrics validation
|
||||
- Third-party service integration health
|
||||
|
||||
### Hotfix Procedures
|
||||
|
||||
Hotfixes address critical production issues requiring immediate deployment:
|
||||
|
||||
#### Severity Classification
|
||||
**P0 - Critical**: Complete system outage, data loss, security breach
|
||||
- **SLA**: Fix within 2 hours
|
||||
- **Process**: Emergency deployment, all hands on deck
|
||||
- **Approval**: Engineering Lead + On-call Manager
|
||||
|
||||
**P1 - High**: Major feature broken, significant user impact
|
||||
- **SLA**: Fix within 24 hours
|
||||
- **Process**: Expedited review and deployment
|
||||
- **Approval**: Engineering Lead + Product Manager
|
||||
|
||||
**P2 - Medium**: Minor feature issues, limited user impact
|
||||
- **SLA**: Fix in next release cycle
|
||||
- **Process**: Normal review process
|
||||
- **Approval**: Standard PR review
|
||||
|
||||
#### Emergency Response Process
|
||||
1. **Incident declaration**: Page on-call team
|
||||
2. **Assessment**: Determine severity and impact
|
||||
3. **Hotfix branch**: Create from last stable release
|
||||
4. **Minimal fix**: Address root cause only
|
||||
5. **Expedited testing**: Automated tests + manual validation
|
||||
6. **Emergency deployment**: Deploy to production
|
||||
7. **Post-incident**: Root cause analysis and prevention
|
||||
|
||||
### Rollback Planning
|
||||
|
||||
Every release must have a tested rollback plan:
|
||||
|
||||
#### Rollback Triggers
|
||||
- **Error rate spike**: >2x baseline within 30 minutes
|
||||
- **Performance degradation**: >50% latency increase
|
||||
- **Feature failures**: Core functionality broken
|
||||
- **Security incident**: Vulnerability exploited
|
||||
- **Data corruption**: Database integrity compromised
|
||||
|
||||
#### Rollback Types
|
||||
**Code Rollback:**
|
||||
- Revert to previous Docker image
|
||||
- Database-compatible code changes only
|
||||
- Feature flag disable preferred over code rollback
|
||||
|
||||
**Database Rollback:**
|
||||
- Only for non-destructive migrations
|
||||
- Data backup required before migration
|
||||
- Forward-only migrations preferred (add columns, not drop)
|
||||
|
||||
**Infrastructure Rollback:**
|
||||
- Blue-green deployment switch
|
||||
- Load balancer configuration revert
|
||||
- DNS changes (longer propagation time)
|
||||
|
||||
#### Automated Rollback
|
||||
```python
|
||||
# Example rollback automation
|
||||
def monitor_deployment():
|
||||
if error_rate() > THRESHOLD:
|
||||
alert_oncall("Error rate spike detected")
|
||||
if auto_rollback_enabled():
|
||||
execute_rollback()
|
||||
```
|
||||
|
||||
### Release Metrics & Analytics
|
||||
|
||||
#### Key Performance Indicators
|
||||
- **Lead Time**: From commit to production
|
||||
- **Deployment Frequency**: Releases per week/month
|
||||
- **Mean Time to Recovery**: From incident to resolution
|
||||
- **Change Failure Rate**: Percentage of releases causing incidents
|
||||
|
||||
#### Quality Metrics
|
||||
- **Rollback Rate**: Percentage of releases rolled back
|
||||
- **Hotfix Rate**: Hotfixes per regular release
|
||||
- **Bug Escape Rate**: Production bugs per release
|
||||
- **Time to Detection**: How quickly issues are identified
|
||||
|
||||
#### Process Metrics
|
||||
- **Review Time**: Time spent in code review
|
||||
- **Testing Time**: Automated + manual testing duration
|
||||
- **Approval Cycle**: Time from PR to merge
|
||||
- **Release Preparation**: Time spent on release activities
|
||||
|
||||
### Tool Integration
|
||||
|
||||
#### Version Control Systems
|
||||
- **Git**: Primary VCS with conventional commit parsing
|
||||
- **GitHub/GitLab**: Pull request automation and CI/CD
|
||||
- **Bitbucket**: Pipeline integration and deployment gates
|
||||
|
||||
#### CI/CD Platforms
|
||||
- **Jenkins**: Pipeline orchestration and deployment automation
|
||||
- **GitHub Actions**: Workflow automation and release publishing
|
||||
- **GitLab CI**: Integrated pipelines with environment management
|
||||
- **CircleCI**: Container-based builds and deployments
|
||||
|
||||
#### Monitoring & Alerting
|
||||
- **DataDog**: Application performance monitoring
|
||||
- **New Relic**: Error tracking and performance insights
|
||||
- **Sentry**: Error aggregation and release tracking
|
||||
- **PagerDuty**: Incident response and escalation
|
||||
|
||||
#### Communication Platforms
|
||||
- **Slack**: Release notifications and coordination
|
||||
- **Microsoft Teams**: Stakeholder communication
|
||||
- **Email**: External customer notifications
|
||||
- **Status Pages**: Public incident communication
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Release Planning
|
||||
1. **Regular cadence**: Establish predictable release schedule
|
||||
2. **Feature freeze**: Lock changes 48h before release
|
||||
3. **Risk assessment**: Evaluate changes for potential impact
|
||||
4. **Stakeholder alignment**: Ensure all teams are prepared
|
||||
|
||||
### Quality Assurance
|
||||
1. **Automated testing**: Comprehensive test coverage
|
||||
2. **Staging environment**: Production-like testing environment
|
||||
3. **Canary releases**: Gradual rollout to subset of users
|
||||
4. **Monitoring**: Proactive issue detection
|
||||
|
||||
### Communication
|
||||
1. **Clear timelines**: Communicate schedules early
|
||||
2. **Regular updates**: Status reports during release process
|
||||
3. **Issue transparency**: Honest communication about problems
|
||||
4. **Post-mortems**: Learn from incidents and improve
|
||||
|
||||
### Automation
|
||||
1. **Reduce manual steps**: Automate repetitive tasks
|
||||
2. **Consistent process**: Same steps every time
|
||||
3. **Audit trails**: Log all release activities
|
||||
4. **Self-service**: Enable teams to deploy safely
|
||||
|
||||
## Common Anti-patterns
|
||||
|
||||
### Process Anti-patterns
|
||||
- **Manual deployments**: Error-prone and inconsistent
|
||||
- **Last-minute changes**: Risk introduction without proper testing
|
||||
- **Skipping testing**: Deploying without validation
|
||||
- **Poor communication**: Stakeholders unaware of changes
|
||||
|
||||
### Technical Anti-patterns
|
||||
- **Monolithic releases**: Large, infrequent releases with high risk
|
||||
- **Coupled deployments**: Services that must be deployed together
|
||||
- **No rollback plan**: Unable to quickly recover from issues
|
||||
- **Environment drift**: Production differs from staging
|
||||
|
||||
### Cultural Anti-patterns
|
||||
- **Blame culture**: Fear of making changes or reporting issues
|
||||
- **Hero culture**: Relying on individuals instead of process
|
||||
- **Perfectionism**: Delaying releases for minor improvements
|
||||
- **Risk aversion**: Avoiding necessary changes due to fear
|
||||
|
||||
## Getting Started
|
||||
|
||||
1. **Assessment**: Evaluate current release process and pain points
|
||||
2. **Tool setup**: Configure scripts for your repository
|
||||
3. **Process definition**: Choose appropriate workflow for your team
|
||||
4. **Automation**: Implement CI/CD pipelines and quality gates
|
||||
5. **Training**: Educate team on new processes and tools
|
||||
6. **Monitoring**: Set up metrics and alerting for releases
|
||||
7. **Iteration**: Continuously improve based on feedback and metrics
|
||||
|
||||
The Release Manager skill transforms chaotic deployments into predictable, reliable releases that build confidence across your entire organization.
|
||||
80
engineering/release-manager/assets/sample_commits.json
Normal file
80
engineering/release-manager/assets/sample_commits.json
Normal file
@@ -0,0 +1,80 @@
|
||||
[
|
||||
{
|
||||
"hash": "a1b2c3d",
|
||||
"author": "Sarah Johnson <sarah.johnson@example.com>",
|
||||
"date": "2024-01-15T14:30:22Z",
|
||||
"message": "feat(auth): add OAuth2 integration with Google and GitHub\n\nImplement OAuth2 authentication flow supporting Google and GitHub providers.\nUsers can now sign in using their existing social media accounts, improving\nuser experience and reducing password fatigue.\n\n- Add OAuth2 client configuration\n- Implement authorization code flow\n- Add user profile mapping from providers\n- Include comprehensive error handling\n\nCloses #123\nResolves #145"
|
||||
},
|
||||
{
|
||||
"hash": "e4f5g6h",
|
||||
"author": "Mike Chen <mike.chen@example.com>",
|
||||
"date": "2024-01-15T13:45:18Z",
|
||||
"message": "fix(api): resolve race condition in user creation endpoint\n\nFixed a race condition that occurred when multiple requests attempted\nto create users with the same email address simultaneously. This was\ncausing duplicate user records in some edge cases.\n\n- Added database unique constraint on email field\n- Implemented proper error handling for constraint violations\n- Added retry logic with exponential backoff\n\nFixes #234"
|
||||
},
|
||||
{
|
||||
"hash": "i7j8k9l",
|
||||
"author": "Emily Davis <emily.davis@example.com>",
|
||||
"date": "2024-01-15T12:20:45Z",
|
||||
"message": "docs(readme): update installation and deployment instructions\n\nUpdated README with comprehensive installation guide including:\n- Docker setup instructions\n- Environment variable configuration\n- Database migration steps\n- Troubleshooting common issues"
|
||||
},
|
||||
{
|
||||
"hash": "m1n2o3p",
|
||||
"author": "David Wilson <david.wilson@example.com>",
|
||||
"date": "2024-01-15T11:15:30Z",
|
||||
"message": "feat(ui)!: redesign dashboard with new component library\n\nComplete redesign of the user dashboard using our new component library.\nThis provides better accessibility, improved mobile responsiveness, and\na more modern user interface.\n\nBREAKING CHANGE: The dashboard API endpoints have changed structure.\nFrontend clients must update to use the new /v2/dashboard endpoints.\nThe legacy /v1/dashboard endpoints will be removed in version 3.0.0.\n\n- Implement new Card, Grid, and Chart components\n- Add responsive breakpoints for mobile devices\n- Improve accessibility with proper ARIA labels\n- Add dark mode support\n\nCloses #345, #367, #389"
|
||||
},
|
||||
{
|
||||
"hash": "q4r5s6t",
|
||||
"author": "Lisa Rodriguez <lisa.rodriguez@example.com>",
|
||||
"date": "2024-01-15T10:45:12Z",
|
||||
"message": "fix(db): optimize slow query in user search functionality\n\nOptimized the user search query that was causing performance issues\non databases with large user counts. Query time reduced from 2.5s to 150ms.\n\n- Added composite index on (email, username, created_at)\n- Refactored query to use more efficient JOIN structure\n- Added query result caching for common search patterns\n\nFixes #456"
|
||||
},
|
||||
{
|
||||
"hash": "u7v8w9x",
|
||||
"author": "Tom Anderson <tom.anderson@example.com>",
|
||||
"date": "2024-01-15T09:30:55Z",
|
||||
"message": "chore(deps): upgrade React to version 18.2.0\n\nUpgrade React and related dependencies to latest stable versions.\nThis includes performance improvements and new concurrent features.\n\n- React: 17.0.2 → 18.2.0\n- React-DOM: 17.0.2 → 18.2.0\n- React-Router: 6.8.0 → 6.8.1\n- Updated all peer dependencies"
|
||||
},
|
||||
{
|
||||
"hash": "y1z2a3b",
|
||||
"author": "Jennifer Kim <jennifer.kim@example.com>",
|
||||
"date": "2024-01-15T08:15:33Z",
|
||||
"message": "test(auth): add comprehensive tests for OAuth flow\n\nAdded unit and integration tests for the OAuth2 authentication system\nto ensure reliability and prevent regressions.\n\n- Unit tests for OAuth client configuration\n- Integration tests for complete auth flow\n- Mock providers for testing without external dependencies\n- Error scenario testing\n\nTest coverage increased from 72% to 89% for auth module."
|
||||
},
|
||||
{
|
||||
"hash": "c4d5e6f",
|
||||
"author": "Alex Thompson <alex.thompson@example.com>",
|
||||
"date": "2024-01-15T07:45:20Z",
|
||||
"message": "perf(image): implement WebP compression reducing size by 40%\n\nReplaced PNG compression with WebP format for uploaded images.\nThis reduces average image file sizes by 40% while maintaining\nvisual quality, improving page load times and reducing bandwidth costs.\n\n- Add WebP encoding support\n- Implement fallback to PNG for older browsers\n- Add quality settings configuration\n- Update image serving endpoints\n\nPerformance improvement: Page load time reduced by 25% on average."
|
||||
},
|
||||
{
|
||||
"hash": "g7h8i9j",
|
||||
"author": "Rachel Green <rachel.green@example.com>",
|
||||
"date": "2024-01-14T16:20:10Z",
|
||||
"message": "feat(payment): add Stripe payment processor integration\n\nIntegrate Stripe as a payment processor to support credit card payments.\nThis enables users to purchase premium features and subscriptions.\n\n- Add Stripe SDK integration\n- Implement payment intent flow\n- Add webhook handling for payment status updates\n- Include comprehensive error handling and logging\n- Add payment method management for users\n\nCloses #567\nCo-authored-by: Payment Team <payments@example.com>"
|
||||
},
|
||||
{
|
||||
"hash": "k1l2m3n",
|
||||
"author": "Chris Martinez <chris.martinez@example.com>",
|
||||
"date": "2024-01-14T15:30:45Z",
|
||||
"message": "fix(ui): resolve mobile navigation menu overflow issue\n\nFixed navigation menu overflow on mobile devices where long menu items\nwere being cut off and causing horizontal scrolling issues.\n\n- Implement responsive text wrapping\n- Add horizontal scrolling for overflowing content\n- Improve touch targets for better mobile usability\n- Fix z-index conflicts with dropdown menus\n\nFixes #678\nTested on iOS Safari, Chrome Mobile, and Firefox Mobile"
|
||||
},
|
||||
{
|
||||
"hash": "o4p5q6r",
|
||||
"author": "Anna Kowalski <anna.kowalski@example.com>",
|
||||
"date": "2024-01-14T14:20:15Z",
|
||||
"message": "refactor(api): extract validation logic into reusable middleware\n\nExtracted common validation logic from individual API endpoints into\nreusable middleware functions to reduce code duplication and improve\nmaintainability.\n\n- Create validation middleware for common patterns\n- Refactor user, product, and order endpoints\n- Add comprehensive error messages\n- Improve validation performance by 30%"
|
||||
},
|
||||
{
|
||||
"hash": "s7t8u9v",
|
||||
"author": "Kevin Park <kevin.park@example.com>",
|
||||
"date": "2024-01-14T13:10:30Z",
|
||||
"message": "feat(search): implement fuzzy search with Elasticsearch\n\nImplemented fuzzy search functionality using Elasticsearch to provide\nbetter search results for users with typos or partial matches.\n\n- Integrate Elasticsearch cluster\n- Add fuzzy matching with configurable distance\n- Implement search result ranking algorithm\n- Add search analytics and logging\n\nSearch accuracy improved by 35% in user testing.\nCloses #789"
|
||||
},
|
||||
{
|
||||
"hash": "w1x2y3z",
|
||||
"author": "Security Team <security@example.com>",
|
||||
"date": "2024-01-14T12:45:22Z",
|
||||
"message": "fix(security): patch SQL injection vulnerability in reports\n\nPatched SQL injection vulnerability in the reports generation endpoint\nthat could allow unauthorized access to sensitive data.\n\n- Implement parameterized queries for all report filters\n- Add input sanitization and validation\n- Update security audit logging\n- Add automated security tests\n\nSeverity: HIGH - CVE-2024-0001\nReported by: External security researcher"
|
||||
}
|
||||
]
|
||||
30
engineering/release-manager/assets/sample_git_log.txt
Normal file
30
engineering/release-manager/assets/sample_git_log.txt
Normal file
@@ -0,0 +1,30 @@
|
||||
a1b2c3d feat(auth): add OAuth2 integration with Google and GitHub
|
||||
e4f5g6h fix(api): resolve race condition in user creation endpoint
|
||||
i7j8k9l docs(readme): update installation and deployment instructions
|
||||
m1n2o3p feat(ui)!: redesign dashboard with new component library
|
||||
q4r5s6t fix(db): optimize slow query in user search functionality
|
||||
u7v8w9x chore(deps): upgrade React to version 18.2.0
|
||||
y1z2a3b test(auth): add comprehensive tests for OAuth flow
|
||||
c4d5e6f perf(image): implement WebP compression reducing size by 40%
|
||||
g7h8i9j feat(payment): add Stripe payment processor integration
|
||||
k1l2m3n fix(ui): resolve mobile navigation menu overflow issue
|
||||
o4p5q6r refactor(api): extract validation logic into reusable middleware
|
||||
s7t8u9v feat(search): implement fuzzy search with Elasticsearch
|
||||
w1x2y3z fix(security): patch SQL injection vulnerability in reports
|
||||
a4b5c6d build(ci): add automated security scanning to deployment pipeline
|
||||
e7f8g9h feat(notification): add email and SMS notification system
|
||||
i1j2k3l fix(payment): handle expired credit cards gracefully
|
||||
m4n5o6p docs(api): generate OpenAPI specification for all endpoints
|
||||
q7r8s9t chore(cleanup): remove deprecated user preference API endpoints
|
||||
u1v2w3x feat(admin)!: redesign admin panel with role-based permissions
|
||||
y4z5a6b fix(db): resolve deadlock issues in concurrent transactions
|
||||
c7d8e9f perf(cache): implement Redis caching for frequent database queries
|
||||
g1h2i3j feat(mobile): add biometric authentication support
|
||||
k4l5m6n fix(api): validate input parameters to prevent XSS attacks
|
||||
o7p8q9r style(ui): update color palette and typography consistency
|
||||
s1t2u3v feat(analytics): integrate Google Analytics 4 tracking
|
||||
w4x5y6z fix(memory): resolve memory leak in image processing service
|
||||
a7b8c9d ci(github): add automated testing for all pull requests
|
||||
e1f2g3h feat(export): add CSV and PDF export functionality for reports
|
||||
i4j5k6l fix(ui): resolve accessibility issues with screen readers
|
||||
m7n8o9p refactor(auth): consolidate authentication logic into single service
|
||||
163
engineering/release-manager/assets/sample_git_log_full.txt
Normal file
163
engineering/release-manager/assets/sample_git_log_full.txt
Normal file
@@ -0,0 +1,163 @@
|
||||
commit a1b2c3d4e5f6789012345678901234567890abcd
|
||||
Author: Sarah Johnson <sarah.johnson@example.com>
|
||||
Date: Mon Jan 15 14:30:22 2024 +0000
|
||||
|
||||
feat(auth): add OAuth2 integration with Google and GitHub
|
||||
|
||||
Implement OAuth2 authentication flow supporting Google and GitHub providers.
|
||||
Users can now sign in using their existing social media accounts, improving
|
||||
user experience and reducing password fatigue.
|
||||
|
||||
- Add OAuth2 client configuration
|
||||
- Implement authorization code flow
|
||||
- Add user profile mapping from providers
|
||||
- Include comprehensive error handling
|
||||
|
||||
Closes #123
|
||||
Resolves #145
|
||||
|
||||
commit e4f5g6h7i8j9012345678901234567890123abcdef
|
||||
Author: Mike Chen <mike.chen@example.com>
|
||||
Date: Mon Jan 15 13:45:18 2024 +0000
|
||||
|
||||
fix(api): resolve race condition in user creation endpoint
|
||||
|
||||
Fixed a race condition that occurred when multiple requests attempted
|
||||
to create users with the same email address simultaneously. This was
|
||||
causing duplicate user records in some edge cases.
|
||||
|
||||
- Added database unique constraint on email field
|
||||
- Implemented proper error handling for constraint violations
|
||||
- Added retry logic with exponential backoff
|
||||
|
||||
Fixes #234
|
||||
|
||||
commit i7j8k9l0m1n2345678901234567890123456789abcd
|
||||
Author: Emily Davis <emily.davis@example.com>
|
||||
Date: Mon Jan 15 12:20:45 2024 +0000
|
||||
|
||||
docs(readme): update installation and deployment instructions
|
||||
|
||||
Updated README with comprehensive installation guide including:
|
||||
- Docker setup instructions
|
||||
- Environment variable configuration
|
||||
- Database migration steps
|
||||
- Troubleshooting common issues
|
||||
|
||||
commit m1n2o3p4q5r6789012345678901234567890abcdefg
|
||||
Author: David Wilson <david.wilson@example.com>
|
||||
Date: Mon Jan 15 11:15:30 2024 +0000
|
||||
|
||||
feat(ui)!: redesign dashboard with new component library
|
||||
|
||||
Complete redesign of the user dashboard using our new component library.
|
||||
This provides better accessibility, improved mobile responsiveness, and
|
||||
a more modern user interface.
|
||||
|
||||
BREAKING CHANGE: The dashboard API endpoints have changed structure.
|
||||
Frontend clients must update to use the new /v2/dashboard endpoints.
|
||||
The legacy /v1/dashboard endpoints will be removed in version 3.0.0.
|
||||
|
||||
- Implement new Card, Grid, and Chart components
|
||||
- Add responsive breakpoints for mobile devices
|
||||
- Improve accessibility with proper ARIA labels
|
||||
- Add dark mode support
|
||||
|
||||
Closes #345, #367, #389
|
||||
|
||||
commit q4r5s6t7u8v9012345678901234567890123456abcd
|
||||
Author: Lisa Rodriguez <lisa.rodriguez@example.com>
|
||||
Date: Mon Jan 15 10:45:12 2024 +0000
|
||||
|
||||
fix(db): optimize slow query in user search functionality
|
||||
|
||||
Optimized the user search query that was causing performance issues
|
||||
on databases with large user counts. Query time reduced from 2.5s to 150ms.
|
||||
|
||||
- Added composite index on (email, username, created_at)
|
||||
- Refactored query to use more efficient JOIN structure
|
||||
- Added query result caching for common search patterns
|
||||
|
||||
Fixes #456
|
||||
|
||||
commit u7v8w9x0y1z2345678901234567890123456789abcde
|
||||
Author: Tom Anderson <tom.anderson@example.com>
|
||||
Date: Mon Jan 15 09:30:55 2024 +0000
|
||||
|
||||
chore(deps): upgrade React to version 18.2.0
|
||||
|
||||
Upgrade React and related dependencies to latest stable versions.
|
||||
This includes performance improvements and new concurrent features.
|
||||
|
||||
- React: 17.0.2 → 18.2.0
|
||||
- React-DOM: 17.0.2 → 18.2.0
|
||||
- React-Router: 6.8.0 → 6.8.1
|
||||
- Updated all peer dependencies
|
||||
|
||||
commit y1z2a3b4c5d6789012345678901234567890abcdefg
|
||||
Author: Jennifer Kim <jennifer.kim@example.com>
|
||||
Date: Mon Jan 15 08:15:33 2024 +0000
|
||||
|
||||
test(auth): add comprehensive tests for OAuth flow
|
||||
|
||||
Added unit and integration tests for the OAuth2 authentication system
|
||||
to ensure reliability and prevent regressions.
|
||||
|
||||
- Unit tests for OAuth client configuration
|
||||
- Integration tests for complete auth flow
|
||||
- Mock providers for testing without external dependencies
|
||||
- Error scenario testing
|
||||
|
||||
Test coverage increased from 72% to 89% for auth module.
|
||||
|
||||
commit c4d5e6f7g8h9012345678901234567890123456abcd
|
||||
Author: Alex Thompson <alex.thompson@example.com>
|
||||
Date: Mon Jan 15 07:45:20 2024 +0000
|
||||
|
||||
perf(image): implement WebP compression reducing size by 40%
|
||||
|
||||
Replaced PNG compression with WebP format for uploaded images.
|
||||
This reduces average image file sizes by 40% while maintaining
|
||||
visual quality, improving page load times and reducing bandwidth costs.
|
||||
|
||||
- Add WebP encoding support
|
||||
- Implement fallback to PNG for older browsers
|
||||
- Add quality settings configuration
|
||||
- Update image serving endpoints
|
||||
|
||||
Performance improvement: Page load time reduced by 25% on average.
|
||||
|
||||
commit g7h8i9j0k1l2345678901234567890123456789abcde
|
||||
Author: Rachel Green <rachel.green@example.com>
|
||||
Date: Sun Jan 14 16:20:10 2024 +0000
|
||||
|
||||
feat(payment): add Stripe payment processor integration
|
||||
|
||||
Integrate Stripe as a payment processor to support credit card payments.
|
||||
This enables users to purchase premium features and subscriptions.
|
||||
|
||||
- Add Stripe SDK integration
|
||||
- Implement payment intent flow
|
||||
- Add webhook handling for payment status updates
|
||||
- Include comprehensive error handling and logging
|
||||
- Add payment method management for users
|
||||
|
||||
Closes #567
|
||||
Co-authored-by: Payment Team <payments@example.com>
|
||||
|
||||
commit k1l2m3n4o5p6789012345678901234567890abcdefg
|
||||
Author: Chris Martinez <chris.martinez@example.com>
|
||||
Date: Sun Jan 14 15:30:45 2024 +0000
|
||||
|
||||
fix(ui): resolve mobile navigation menu overflow issue
|
||||
|
||||
Fixed navigation menu overflow on mobile devices where long menu items
|
||||
were being cut off and causing horizontal scrolling issues.
|
||||
|
||||
- Implement responsive text wrapping
|
||||
- Add horizontal scrolling for overflowing content
|
||||
- Improve touch targets for better mobile usability
|
||||
- Fix z-index conflicts with dropdown menus
|
||||
|
||||
Fixes #678
|
||||
Tested on iOS Safari, Chrome Mobile, and Firefox Mobile
|
||||
273
engineering/release-manager/assets/sample_release_plan.json
Normal file
273
engineering/release-manager/assets/sample_release_plan.json
Normal file
@@ -0,0 +1,273 @@
|
||||
{
|
||||
"release_name": "Winter 2024 Release",
|
||||
"version": "2.3.0",
|
||||
"target_date": "2024-02-15T10:00:00Z",
|
||||
"features": [
|
||||
{
|
||||
"id": "AUTH-123",
|
||||
"title": "OAuth2 Integration",
|
||||
"description": "Add support for Google and GitHub OAuth2 authentication",
|
||||
"type": "feature",
|
||||
"assignee": "sarah.johnson@example.com",
|
||||
"status": "ready",
|
||||
"pull_request_url": "https://github.com/ourapp/backend/pull/234",
|
||||
"issue_url": "https://github.com/ourapp/backend/issues/123",
|
||||
"risk_level": "medium",
|
||||
"test_coverage_required": 85.0,
|
||||
"test_coverage_actual": 89.5,
|
||||
"requires_migration": false,
|
||||
"breaking_changes": [],
|
||||
"dependencies": ["AUTH-124"],
|
||||
"qa_approved": true,
|
||||
"security_approved": true,
|
||||
"pm_approved": true
|
||||
},
|
||||
{
|
||||
"id": "UI-345",
|
||||
"title": "Dashboard Redesign",
|
||||
"description": "Complete redesign of user dashboard with new component library",
|
||||
"type": "breaking_change",
|
||||
"assignee": "david.wilson@example.com",
|
||||
"status": "ready",
|
||||
"pull_request_url": "https://github.com/ourapp/frontend/pull/456",
|
||||
"issue_url": "https://github.com/ourapp/frontend/issues/345",
|
||||
"risk_level": "high",
|
||||
"test_coverage_required": 90.0,
|
||||
"test_coverage_actual": 92.3,
|
||||
"requires_migration": true,
|
||||
"migration_complexity": "moderate",
|
||||
"breaking_changes": [
|
||||
"Dashboard API endpoints changed from /v1/dashboard to /v2/dashboard",
|
||||
"Dashboard widget configuration format updated"
|
||||
],
|
||||
"dependencies": [],
|
||||
"qa_approved": true,
|
||||
"security_approved": true,
|
||||
"pm_approved": true
|
||||
},
|
||||
{
|
||||
"id": "PAY-567",
|
||||
"title": "Stripe Payment Integration",
|
||||
"description": "Add Stripe as payment processor for premium features",
|
||||
"type": "feature",
|
||||
"assignee": "rachel.green@example.com",
|
||||
"status": "ready",
|
||||
"pull_request_url": "https://github.com/ourapp/backend/pull/678",
|
||||
"issue_url": "https://github.com/ourapp/backend/issues/567",
|
||||
"risk_level": "high",
|
||||
"test_coverage_required": 95.0,
|
||||
"test_coverage_actual": 97.2,
|
||||
"requires_migration": true,
|
||||
"migration_complexity": "complex",
|
||||
"breaking_changes": [],
|
||||
"dependencies": ["SEC-890"],
|
||||
"qa_approved": true,
|
||||
"security_approved": true,
|
||||
"pm_approved": true
|
||||
},
|
||||
{
|
||||
"id": "SEARCH-789",
|
||||
"title": "Elasticsearch Fuzzy Search",
|
||||
"description": "Implement fuzzy search functionality with Elasticsearch",
|
||||
"type": "feature",
|
||||
"assignee": "kevin.park@example.com",
|
||||
"status": "in_progress",
|
||||
"pull_request_url": "https://github.com/ourapp/backend/pull/890",
|
||||
"issue_url": "https://github.com/ourapp/backend/issues/789",
|
||||
"risk_level": "medium",
|
||||
"test_coverage_required": 80.0,
|
||||
"test_coverage_actual": 76.5,
|
||||
"requires_migration": true,
|
||||
"migration_complexity": "moderate",
|
||||
"breaking_changes": [],
|
||||
"dependencies": ["INFRA-234"],
|
||||
"qa_approved": false,
|
||||
"security_approved": true,
|
||||
"pm_approved": true
|
||||
},
|
||||
{
|
||||
"id": "MOBILE-456",
|
||||
"title": "Biometric Authentication",
|
||||
"description": "Add fingerprint and face ID support for mobile apps",
|
||||
"type": "feature",
|
||||
"assignee": "alex.thompson@example.com",
|
||||
"status": "blocked",
|
||||
"pull_request_url": null,
|
||||
"issue_url": "https://github.com/ourapp/mobile/issues/456",
|
||||
"risk_level": "medium",
|
||||
"test_coverage_required": 85.0,
|
||||
"test_coverage_actual": null,
|
||||
"requires_migration": false,
|
||||
"breaking_changes": [],
|
||||
"dependencies": ["AUTH-123"],
|
||||
"qa_approved": false,
|
||||
"security_approved": false,
|
||||
"pm_approved": true
|
||||
},
|
||||
{
|
||||
"id": "PERF-678",
|
||||
"title": "Redis Caching Implementation",
|
||||
"description": "Implement Redis caching for frequently accessed data",
|
||||
"type": "performance",
|
||||
"assignee": "lisa.rodriguez@example.com",
|
||||
"status": "ready",
|
||||
"pull_request_url": "https://github.com/ourapp/backend/pull/901",
|
||||
"issue_url": "https://github.com/ourapp/backend/issues/678",
|
||||
"risk_level": "low",
|
||||
"test_coverage_required": 75.0,
|
||||
"test_coverage_actual": 82.1,
|
||||
"requires_migration": false,
|
||||
"breaking_changes": [],
|
||||
"dependencies": [],
|
||||
"qa_approved": true,
|
||||
"security_approved": false,
|
||||
"pm_approved": true
|
||||
}
|
||||
],
|
||||
"quality_gates": [
|
||||
{
|
||||
"name": "Unit Test Coverage",
|
||||
"required": true,
|
||||
"status": "ready",
|
||||
"details": "Overall test coverage above 85% threshold",
|
||||
"threshold": 85.0,
|
||||
"actual_value": 87.3
|
||||
},
|
||||
{
|
||||
"name": "Integration Tests",
|
||||
"required": true,
|
||||
"status": "ready",
|
||||
"details": "All integration tests passing"
|
||||
},
|
||||
{
|
||||
"name": "Security Scan",
|
||||
"required": true,
|
||||
"status": "pending",
|
||||
"details": "Waiting for security team review of payment integration"
|
||||
},
|
||||
{
|
||||
"name": "Performance Testing",
|
||||
"required": true,
|
||||
"status": "ready",
|
||||
"details": "Load testing shows 99th percentile response time under 500ms"
|
||||
},
|
||||
{
|
||||
"name": "Documentation Review",
|
||||
"required": true,
|
||||
"status": "pending",
|
||||
"details": "API documentation needs update for dashboard changes"
|
||||
},
|
||||
{
|
||||
"name": "Dependency Audit",
|
||||
"required": true,
|
||||
"status": "ready",
|
||||
"details": "No high or critical vulnerabilities found"
|
||||
}
|
||||
],
|
||||
"stakeholders": [
|
||||
{
|
||||
"name": "Engineering Team",
|
||||
"role": "developer",
|
||||
"contact": "engineering@example.com",
|
||||
"notification_type": "slack",
|
||||
"critical_path": true
|
||||
},
|
||||
{
|
||||
"name": "Product Team",
|
||||
"role": "pm",
|
||||
"contact": "product@example.com",
|
||||
"notification_type": "email",
|
||||
"critical_path": true
|
||||
},
|
||||
{
|
||||
"name": "QA Team",
|
||||
"role": "qa",
|
||||
"contact": "qa@example.com",
|
||||
"notification_type": "slack",
|
||||
"critical_path": true
|
||||
},
|
||||
{
|
||||
"name": "Security Team",
|
||||
"role": "security",
|
||||
"contact": "security@example.com",
|
||||
"notification_type": "email",
|
||||
"critical_path": false
|
||||
},
|
||||
{
|
||||
"name": "Customer Support",
|
||||
"role": "support",
|
||||
"contact": "support@example.com",
|
||||
"notification_type": "email",
|
||||
"critical_path": false
|
||||
},
|
||||
{
|
||||
"name": "Sales Team",
|
||||
"role": "sales",
|
||||
"contact": "sales@example.com",
|
||||
"notification_type": "email",
|
||||
"critical_path": false
|
||||
},
|
||||
{
|
||||
"name": "Beta Users",
|
||||
"role": "customer",
|
||||
"contact": "beta-users@example.com",
|
||||
"notification_type": "email",
|
||||
"critical_path": false
|
||||
}
|
||||
],
|
||||
"rollback_steps": [
|
||||
{
|
||||
"order": 1,
|
||||
"description": "Alert incident response team and stakeholders",
|
||||
"estimated_time": "2 minutes",
|
||||
"risk_level": "low",
|
||||
"verification": "Confirm team is aware and responding via Slack"
|
||||
},
|
||||
{
|
||||
"order": 2,
|
||||
"description": "Switch load balancer to previous version",
|
||||
"command": "kubectl patch service app --patch '{\"spec\": {\"selector\": {\"version\": \"v2.2.1\"}}}'",
|
||||
"estimated_time": "30 seconds",
|
||||
"risk_level": "low",
|
||||
"verification": "Check traffic routing to previous version via monitoring dashboard"
|
||||
},
|
||||
{
|
||||
"order": 3,
|
||||
"description": "Disable new feature flags",
|
||||
"command": "curl -X POST https://api.example.com/feature-flags/oauth2/disable",
|
||||
"estimated_time": "1 minute",
|
||||
"risk_level": "low",
|
||||
"verification": "Verify feature flags are disabled in admin panel"
|
||||
},
|
||||
{
|
||||
"order": 4,
|
||||
"description": "Roll back database migrations",
|
||||
"command": "python manage.py migrate app 0042",
|
||||
"estimated_time": "10 minutes",
|
||||
"risk_level": "high",
|
||||
"verification": "Verify database schema and run data integrity checks"
|
||||
},
|
||||
{
|
||||
"order": 5,
|
||||
"description": "Clear Redis cache",
|
||||
"command": "redis-cli FLUSHALL",
|
||||
"estimated_time": "30 seconds",
|
||||
"risk_level": "medium",
|
||||
"verification": "Confirm cache is cleared and application rebuilds cache properly"
|
||||
},
|
||||
{
|
||||
"order": 6,
|
||||
"description": "Verify application health",
|
||||
"estimated_time": "5 minutes",
|
||||
"risk_level": "low",
|
||||
"verification": "Check health endpoints, error rates, and core user workflows"
|
||||
},
|
||||
{
|
||||
"order": 7,
|
||||
"description": "Update status page and notify users",
|
||||
"estimated_time": "5 minutes",
|
||||
"risk_level": "low",
|
||||
"verification": "Confirm status page updated and notifications sent"
|
||||
}
|
||||
]
|
||||
}
|
||||
504
engineering/release-manager/changelog_generator.py
Normal file
504
engineering/release-manager/changelog_generator.py
Normal file
@@ -0,0 +1,504 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Changelog Generator
|
||||
|
||||
Parses git log output in conventional commits format and generates structured changelogs
|
||||
in multiple formats (Markdown, Keep a Changelog). Groups commits by type, extracts scope,
|
||||
links to PRs/issues, and highlights breaking changes.
|
||||
|
||||
Input: git log text (piped from git log) or JSON array of commits
|
||||
Output: formatted CHANGELOG.md section + release summary stats
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict, Counter
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
|
||||
class ConventionalCommit:
|
||||
"""Represents a parsed conventional commit."""
|
||||
|
||||
def __init__(self, raw_message: str, commit_hash: str = "", author: str = "",
|
||||
date: str = "", merge_info: Optional[str] = None):
|
||||
self.raw_message = raw_message
|
||||
self.commit_hash = commit_hash
|
||||
self.author = author
|
||||
self.date = date
|
||||
self.merge_info = merge_info
|
||||
|
||||
# Parse the commit message
|
||||
self.type = ""
|
||||
self.scope = ""
|
||||
self.description = ""
|
||||
self.body = ""
|
||||
self.footers = []
|
||||
self.is_breaking = False
|
||||
self.breaking_change_description = ""
|
||||
|
||||
self._parse_commit_message()
|
||||
|
||||
def _parse_commit_message(self):
|
||||
"""Parse conventional commit format."""
|
||||
lines = self.raw_message.split('\n')
|
||||
header = lines[0] if lines else ""
|
||||
|
||||
# Parse header: type(scope): description
|
||||
header_pattern = r'^(\w+)(\([^)]+\))?(!)?:\s*(.+)$'
|
||||
match = re.match(header_pattern, header)
|
||||
|
||||
if match:
|
||||
self.type = match.group(1).lower()
|
||||
scope_match = match.group(2)
|
||||
self.scope = scope_match[1:-1] if scope_match else "" # Remove parentheses
|
||||
self.is_breaking = bool(match.group(3)) # ! indicates breaking change
|
||||
self.description = match.group(4).strip()
|
||||
else:
|
||||
# Fallback for non-conventional commits
|
||||
self.type = "chore"
|
||||
self.description = header
|
||||
|
||||
# Parse body and footers
|
||||
if len(lines) > 1:
|
||||
body_lines = []
|
||||
footer_lines = []
|
||||
in_footer = False
|
||||
|
||||
for line in lines[1:]:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# Check if this is a footer (KEY: value or KEY #value format)
|
||||
footer_pattern = r'^([A-Z-]+):\s*(.+)$|^([A-Z-]+)\s+#(\d+)$'
|
||||
if re.match(footer_pattern, line):
|
||||
in_footer = True
|
||||
footer_lines.append(line)
|
||||
|
||||
# Check for breaking change
|
||||
if line.startswith('BREAKING CHANGE:'):
|
||||
self.is_breaking = True
|
||||
self.breaking_change_description = line[16:].strip()
|
||||
else:
|
||||
if in_footer:
|
||||
# Continuation of footer
|
||||
footer_lines.append(line)
|
||||
else:
|
||||
body_lines.append(line)
|
||||
|
||||
self.body = '\n'.join(body_lines).strip()
|
||||
self.footers = footer_lines
|
||||
|
||||
def extract_issue_references(self) -> List[str]:
|
||||
"""Extract issue/PR references like #123, fixes #456, etc."""
|
||||
text = f"{self.description} {self.body} {' '.join(self.footers)}"
|
||||
|
||||
# Common patterns for issue references
|
||||
patterns = [
|
||||
r'#(\d+)', # Simple #123
|
||||
r'(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+#(\d+)', # closes #123
|
||||
r'(?:close[sd]?|fix(?:e[sd])?|resolve[sd]?)\s+(\w+/\w+)?#(\d+)' # fixes repo#123
|
||||
]
|
||||
|
||||
references = []
|
||||
for pattern in patterns:
|
||||
matches = re.findall(pattern, text, re.IGNORECASE)
|
||||
for match in matches:
|
||||
if isinstance(match, tuple):
|
||||
# Handle tuple results from more complex patterns
|
||||
ref = match[-1] if match[-1] else match[0]
|
||||
else:
|
||||
ref = match
|
||||
if ref and ref not in references:
|
||||
references.append(ref)
|
||||
|
||||
return references
|
||||
|
||||
def get_changelog_category(self) -> str:
|
||||
"""Map commit type to changelog category."""
|
||||
category_map = {
|
||||
'feat': 'Added',
|
||||
'add': 'Added',
|
||||
'fix': 'Fixed',
|
||||
'bugfix': 'Fixed',
|
||||
'security': 'Security',
|
||||
'perf': 'Fixed', # Performance improvements go to Fixed
|
||||
'refactor': 'Changed',
|
||||
'style': 'Changed',
|
||||
'docs': 'Changed',
|
||||
'test': None, # Tests don't appear in user-facing changelog
|
||||
'ci': None,
|
||||
'build': None,
|
||||
'chore': None,
|
||||
'revert': 'Fixed',
|
||||
'remove': 'Removed',
|
||||
'deprecate': 'Deprecated'
|
||||
}
|
||||
|
||||
return category_map.get(self.type, 'Changed')
|
||||
|
||||
|
||||
class ChangelogGenerator:
|
||||
"""Main changelog generator class."""
|
||||
|
||||
def __init__(self):
|
||||
self.commits: List[ConventionalCommit] = []
|
||||
self.version = "Unreleased"
|
||||
self.date = datetime.now().strftime("%Y-%m-%d")
|
||||
self.base_url = ""
|
||||
|
||||
def parse_git_log_output(self, git_log_text: str):
|
||||
"""Parse git log output into ConventionalCommit objects."""
|
||||
# Try to detect format based on patterns in the text
|
||||
lines = git_log_text.strip().split('\n')
|
||||
|
||||
if not lines or not lines[0]:
|
||||
return
|
||||
|
||||
# Format 1: Simple oneline format (hash message)
|
||||
oneline_pattern = r'^([a-f0-9]{7,40})\s+(.+)$'
|
||||
|
||||
# Format 2: Full format with metadata
|
||||
full_pattern = r'^commit\s+([a-f0-9]+)'
|
||||
|
||||
current_commit = None
|
||||
commit_buffer = []
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Check if this is a new commit (oneline format)
|
||||
oneline_match = re.match(oneline_pattern, line)
|
||||
if oneline_match:
|
||||
# Process previous commit
|
||||
if current_commit:
|
||||
self.commits.append(current_commit)
|
||||
|
||||
# Start new commit
|
||||
commit_hash = oneline_match.group(1)
|
||||
message = oneline_match.group(2)
|
||||
current_commit = ConventionalCommit(message, commit_hash)
|
||||
continue
|
||||
|
||||
# Check if this is a new commit (full format)
|
||||
full_match = re.match(full_pattern, line)
|
||||
if full_match:
|
||||
# Process previous commit
|
||||
if current_commit:
|
||||
commit_message = '\n'.join(commit_buffer).strip()
|
||||
if commit_message:
|
||||
current_commit = ConventionalCommit(commit_message, current_commit.commit_hash,
|
||||
current_commit.author, current_commit.date)
|
||||
self.commits.append(current_commit)
|
||||
|
||||
# Start new commit
|
||||
commit_hash = full_match.group(1)
|
||||
current_commit = ConventionalCommit("", commit_hash)
|
||||
commit_buffer = []
|
||||
continue
|
||||
|
||||
# Parse metadata lines in full format
|
||||
if current_commit and not current_commit.raw_message:
|
||||
if line.startswith('Author:'):
|
||||
current_commit.author = line[7:].strip()
|
||||
elif line.startswith('Date:'):
|
||||
current_commit.date = line[5:].strip()
|
||||
elif line.startswith('Merge:'):
|
||||
current_commit.merge_info = line[6:].strip()
|
||||
elif line.startswith(' '):
|
||||
# Commit message line (indented)
|
||||
commit_buffer.append(line[4:]) # Remove 4-space indent
|
||||
|
||||
# Process final commit
|
||||
if current_commit:
|
||||
if commit_buffer:
|
||||
commit_message = '\n'.join(commit_buffer).strip()
|
||||
current_commit = ConventionalCommit(commit_message, current_commit.commit_hash,
|
||||
current_commit.author, current_commit.date)
|
||||
self.commits.append(current_commit)
|
||||
|
||||
def parse_json_commits(self, json_data: Union[str, List[Dict]]):
|
||||
"""Parse commits from JSON format."""
|
||||
if isinstance(json_data, str):
|
||||
data = json.loads(json_data)
|
||||
else:
|
||||
data = json_data
|
||||
|
||||
for commit_data in data:
|
||||
commit = ConventionalCommit(
|
||||
raw_message=commit_data.get('message', ''),
|
||||
commit_hash=commit_data.get('hash', ''),
|
||||
author=commit_data.get('author', ''),
|
||||
date=commit_data.get('date', '')
|
||||
)
|
||||
self.commits.append(commit)
|
||||
|
||||
def group_commits_by_category(self) -> Dict[str, List[ConventionalCommit]]:
|
||||
"""Group commits by changelog category."""
|
||||
categories = defaultdict(list)
|
||||
|
||||
for commit in self.commits:
|
||||
category = commit.get_changelog_category()
|
||||
if category: # Skip None categories (internal changes)
|
||||
categories[category].append(commit)
|
||||
|
||||
return dict(categories)
|
||||
|
||||
def generate_markdown_changelog(self, include_unreleased: bool = True) -> str:
|
||||
"""Generate Keep a Changelog format markdown."""
|
||||
grouped_commits = self.group_commits_by_category()
|
||||
|
||||
if not grouped_commits:
|
||||
return "No notable changes.\n"
|
||||
|
||||
# Start with header
|
||||
changelog = []
|
||||
if include_unreleased and self.version == "Unreleased":
|
||||
changelog.append(f"## [{self.version}]")
|
||||
else:
|
||||
changelog.append(f"## [{self.version}] - {self.date}")
|
||||
|
||||
changelog.append("")
|
||||
|
||||
# Order categories logically
|
||||
category_order = ['Added', 'Changed', 'Deprecated', 'Removed', 'Fixed', 'Security']
|
||||
|
||||
# Separate breaking changes
|
||||
breaking_changes = [commit for commit in self.commits if commit.is_breaking]
|
||||
|
||||
# Add breaking changes section first if any exist
|
||||
if breaking_changes:
|
||||
changelog.append("### Breaking Changes")
|
||||
for commit in breaking_changes:
|
||||
line = self._format_commit_line(commit, show_breaking=True)
|
||||
changelog.append(f"- {line}")
|
||||
changelog.append("")
|
||||
|
||||
# Add regular categories
|
||||
for category in category_order:
|
||||
if category not in grouped_commits:
|
||||
continue
|
||||
|
||||
changelog.append(f"### {category}")
|
||||
|
||||
# Group by scope for better organization
|
||||
scoped_commits = defaultdict(list)
|
||||
for commit in grouped_commits[category]:
|
||||
scope = commit.scope if commit.scope else "general"
|
||||
scoped_commits[scope].append(commit)
|
||||
|
||||
# Sort scopes, with 'general' last
|
||||
scopes = sorted(scoped_commits.keys())
|
||||
if "general" in scopes:
|
||||
scopes.remove("general")
|
||||
scopes.append("general")
|
||||
|
||||
for scope in scopes:
|
||||
if len(scoped_commits) > 1 and scope != "general":
|
||||
changelog.append(f"#### {scope.title()}")
|
||||
|
||||
for commit in scoped_commits[scope]:
|
||||
line = self._format_commit_line(commit)
|
||||
changelog.append(f"- {line}")
|
||||
|
||||
changelog.append("")
|
||||
|
||||
return '\n'.join(changelog)
|
||||
|
||||
def _format_commit_line(self, commit: ConventionalCommit, show_breaking: bool = False) -> str:
|
||||
"""Format a single commit line for the changelog."""
|
||||
# Start with description
|
||||
line = commit.description.capitalize()
|
||||
|
||||
# Add scope if present and not already in description
|
||||
if commit.scope and commit.scope.lower() not in line.lower():
|
||||
line = f"{commit.scope}: {line}"
|
||||
|
||||
# Add issue references
|
||||
issue_refs = commit.extract_issue_references()
|
||||
if issue_refs:
|
||||
refs_str = ', '.join(f"#{ref}" for ref in issue_refs)
|
||||
line += f" ({refs_str})"
|
||||
|
||||
# Add commit hash if available
|
||||
if commit.commit_hash:
|
||||
short_hash = commit.commit_hash[:7]
|
||||
line += f" [{short_hash}]"
|
||||
|
||||
if self.base_url:
|
||||
line += f"({self.base_url}/commit/{commit.commit_hash})"
|
||||
|
||||
# Add breaking change indicator
|
||||
if show_breaking and commit.breaking_change_description:
|
||||
line += f" - {commit.breaking_change_description}"
|
||||
elif commit.is_breaking and not show_breaking:
|
||||
line += " ⚠️ BREAKING"
|
||||
|
||||
return line
|
||||
|
||||
def generate_release_summary(self) -> Dict:
|
||||
"""Generate summary statistics for the release."""
|
||||
if not self.commits:
|
||||
return {
|
||||
'version': self.version,
|
||||
'date': self.date,
|
||||
'total_commits': 0,
|
||||
'by_type': {},
|
||||
'by_author': {},
|
||||
'breaking_changes': 0,
|
||||
'notable_changes': 0
|
||||
}
|
||||
|
||||
# Count by type
|
||||
type_counts = Counter(commit.type for commit in self.commits)
|
||||
|
||||
# Count by author
|
||||
author_counts = Counter(commit.author for commit in self.commits if commit.author)
|
||||
|
||||
# Count breaking changes
|
||||
breaking_count = sum(1 for commit in self.commits if commit.is_breaking)
|
||||
|
||||
# Count notable changes (excluding chore, ci, build, test)
|
||||
notable_types = {'feat', 'fix', 'security', 'perf', 'refactor', 'remove', 'deprecate'}
|
||||
notable_count = sum(1 for commit in self.commits if commit.type in notable_types)
|
||||
|
||||
return {
|
||||
'version': self.version,
|
||||
'date': self.date,
|
||||
'total_commits': len(self.commits),
|
||||
'by_type': dict(type_counts.most_common()),
|
||||
'by_author': dict(author_counts.most_common(10)), # Top 10 contributors
|
||||
'breaking_changes': breaking_count,
|
||||
'notable_changes': notable_count,
|
||||
'scopes': list(set(commit.scope for commit in self.commits if commit.scope)),
|
||||
'issue_references': len(set().union(*(commit.extract_issue_references() for commit in self.commits)))
|
||||
}
|
||||
|
||||
def generate_json_output(self) -> str:
|
||||
"""Generate JSON representation of the changelog data."""
|
||||
grouped_commits = self.group_commits_by_category()
|
||||
|
||||
# Convert commits to serializable format
|
||||
json_data = {
|
||||
'version': self.version,
|
||||
'date': self.date,
|
||||
'summary': self.generate_release_summary(),
|
||||
'categories': {}
|
||||
}
|
||||
|
||||
for category, commits in grouped_commits.items():
|
||||
json_data['categories'][category] = []
|
||||
for commit in commits:
|
||||
commit_data = {
|
||||
'type': commit.type,
|
||||
'scope': commit.scope,
|
||||
'description': commit.description,
|
||||
'hash': commit.commit_hash,
|
||||
'author': commit.author,
|
||||
'date': commit.date,
|
||||
'breaking': commit.is_breaking,
|
||||
'breaking_description': commit.breaking_change_description,
|
||||
'issue_references': commit.extract_issue_references()
|
||||
}
|
||||
json_data['categories'][category].append(commit_data)
|
||||
|
||||
return json.dumps(json_data, indent=2)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point with CLI argument parsing."""
|
||||
parser = argparse.ArgumentParser(description="Generate changelog from conventional commits")
|
||||
parser.add_argument('--input', '-i', type=str, help='Input file (default: stdin)')
|
||||
parser.add_argument('--format', '-f', choices=['markdown', 'json', 'both'],
|
||||
default='markdown', help='Output format')
|
||||
parser.add_argument('--version', '-v', type=str, default='Unreleased',
|
||||
help='Version for this release')
|
||||
parser.add_argument('--date', '-d', type=str,
|
||||
default=datetime.now().strftime("%Y-%m-%d"),
|
||||
help='Release date (YYYY-MM-DD format)')
|
||||
parser.add_argument('--base-url', '-u', type=str, default='',
|
||||
help='Base URL for commit links')
|
||||
parser.add_argument('--input-format', choices=['git-log', 'json'],
|
||||
default='git-log', help='Input format')
|
||||
parser.add_argument('--output', '-o', type=str, help='Output file (default: stdout)')
|
||||
parser.add_argument('--summary', '-s', action='store_true',
|
||||
help='Include release summary statistics')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Read input
|
||||
if args.input:
|
||||
with open(args.input, 'r', encoding='utf-8') as f:
|
||||
input_data = f.read()
|
||||
else:
|
||||
input_data = sys.stdin.read()
|
||||
|
||||
if not input_data.strip():
|
||||
print("No input data provided", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize generator
|
||||
generator = ChangelogGenerator()
|
||||
generator.version = args.version
|
||||
generator.date = args.date
|
||||
generator.base_url = args.base_url
|
||||
|
||||
# Parse input
|
||||
try:
|
||||
if args.input_format == 'json':
|
||||
generator.parse_json_commits(input_data)
|
||||
else:
|
||||
generator.parse_git_log_output(input_data)
|
||||
except Exception as e:
|
||||
print(f"Error parsing input: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if not generator.commits:
|
||||
print("No valid commits found in input", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Generate output
|
||||
output_lines = []
|
||||
|
||||
if args.format in ['markdown', 'both']:
|
||||
changelog_md = generator.generate_markdown_changelog()
|
||||
if args.format == 'both':
|
||||
output_lines.append("# Markdown Changelog\n")
|
||||
output_lines.append(changelog_md)
|
||||
|
||||
if args.format in ['json', 'both']:
|
||||
changelog_json = generator.generate_json_output()
|
||||
if args.format == 'both':
|
||||
output_lines.append("\n# JSON Output\n")
|
||||
output_lines.append(changelog_json)
|
||||
|
||||
if args.summary:
|
||||
summary = generator.generate_release_summary()
|
||||
output_lines.append(f"\n# Release Summary")
|
||||
output_lines.append(f"- **Version:** {summary['version']}")
|
||||
output_lines.append(f"- **Total Commits:** {summary['total_commits']}")
|
||||
output_lines.append(f"- **Notable Changes:** {summary['notable_changes']}")
|
||||
output_lines.append(f"- **Breaking Changes:** {summary['breaking_changes']}")
|
||||
output_lines.append(f"- **Issue References:** {summary['issue_references']}")
|
||||
|
||||
if summary['by_type']:
|
||||
output_lines.append("- **By Type:**")
|
||||
for commit_type, count in summary['by_type'].items():
|
||||
output_lines.append(f" - {commit_type}: {count}")
|
||||
|
||||
# Write output
|
||||
final_output = '\n'.join(output_lines)
|
||||
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(final_output)
|
||||
else:
|
||||
print(final_output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,37 @@
|
||||
# Expected Changelog Output
|
||||
|
||||
## [2.3.0] - 2024-01-15
|
||||
|
||||
### Breaking Changes
|
||||
- ui: redesign dashboard with new component library - The dashboard API endpoints have changed structure. Frontend clients must update to use the new /v2/dashboard endpoints. The legacy /v1/dashboard endpoints will be removed in version 3.0.0. (#345, #367, #389) [m1n2o3p]
|
||||
|
||||
### Added
|
||||
- auth: add OAuth2 integration with Google and GitHub (#123, #145) [a1b2c3d]
|
||||
- payment: add Stripe payment processor integration (#567) [g7h8i9j]
|
||||
- search: implement fuzzy search with Elasticsearch (#789) [s7t8u9v]
|
||||
|
||||
### Fixed
|
||||
- api: resolve race condition in user creation endpoint (#234) [e4f5g6h]
|
||||
- db: optimize slow query in user search functionality (#456) [q4r5s6t]
|
||||
- ui: resolve mobile navigation menu overflow issue (#678) [k1l2m3n]
|
||||
- security: patch SQL injection vulnerability in reports [w1x2y3z] ⚠️ BREAKING
|
||||
|
||||
### Changed
|
||||
- image: implement WebP compression reducing size by 40% [c4d5e6f]
|
||||
- api: extract validation logic into reusable middleware [o4p5q6r]
|
||||
- readme: update installation and deployment instructions [i7j8k9l]
|
||||
|
||||
# Release Summary
|
||||
- **Version:** 2.3.0
|
||||
- **Total Commits:** 13
|
||||
- **Notable Changes:** 9
|
||||
- **Breaking Changes:** 2
|
||||
- **Issue References:** 8
|
||||
- **By Type:**
|
||||
- feat: 4
|
||||
- fix: 4
|
||||
- perf: 1
|
||||
- refactor: 1
|
||||
- docs: 1
|
||||
- test: 1
|
||||
- chore: 1
|
||||
@@ -0,0 +1,30 @@
|
||||
Release Readiness Report
|
||||
========================
|
||||
Release: Winter 2024 Release v2.3.0
|
||||
Status: AT_RISK
|
||||
Readiness Score: 73.3%
|
||||
|
||||
WARNINGS:
|
||||
|
||||
⚠️ Feature 'Elasticsearch Fuzzy Search' (SEARCH-789) still in progress
|
||||
⚠️ Feature 'Elasticsearch Fuzzy Search' has low test coverage: 76.5% < 80.0%
|
||||
⚠️ Required quality gate 'Security Scan' is pending
|
||||
⚠️ Required quality gate 'Documentation Review' is pending
|
||||
|
||||
BLOCKING ISSUES:
|
||||
|
||||
❌ Feature 'Biometric Authentication' (MOBILE-456) is blocked
|
||||
❌ Feature 'Biometric Authentication' missing approvals: QA approval, Security approval
|
||||
|
||||
RECOMMENDATIONS:
|
||||
|
||||
💡 Obtain required approvals for pending features
|
||||
💡 Improve test coverage for features below threshold
|
||||
💡 Complete pending quality gate validations
|
||||
|
||||
FEATURE SUMMARY:
|
||||
Total: 6 | Ready: 3 | Blocked: 1
|
||||
Breaking Changes: 1 | Missing Approvals: 1
|
||||
|
||||
QUALITY GATES:
|
||||
Total: 6 | Passed: 3 | Failed: 0
|
||||
@@ -0,0 +1,31 @@
|
||||
Current Version: 2.2.5
|
||||
Recommended Version: 3.0.0
|
||||
With v prefix: v3.0.0
|
||||
Bump Type: major
|
||||
|
||||
Commit Analysis:
|
||||
- Total commits: 13
|
||||
- Breaking changes: 2
|
||||
- New features: 4
|
||||
- Bug fixes: 4
|
||||
- Ignored commits: 3
|
||||
|
||||
Breaking Changes:
|
||||
- feat(ui): redesign dashboard with new component library
|
||||
- fix(security): patch SQL injection vulnerability in reports
|
||||
|
||||
Bump Commands:
|
||||
npm:
|
||||
npm version 3.0.0 --no-git-tag-version
|
||||
python:
|
||||
# Update version in setup.py, __init__.py, or pyproject.toml
|
||||
# pyproject.toml: version = "3.0.0"
|
||||
rust:
|
||||
# Update Cargo.toml
|
||||
# version = "3.0.0"
|
||||
git:
|
||||
git tag -a v3.0.0 -m 'Release v3.0.0'
|
||||
git push origin v3.0.0
|
||||
docker:
|
||||
docker build -t myapp:3.0.0 .
|
||||
docker tag myapp:3.0.0 myapp:latest
|
||||
@@ -0,0 +1,341 @@
|
||||
# Conventional Commits Guide
|
||||
|
||||
## Overview
|
||||
|
||||
Conventional Commits is a specification for adding human and machine readable meaning to commit messages. The specification provides an easy set of rules for creating an explicit commit history, which makes it easier to write automated tools for version management, changelog generation, and release planning.
|
||||
|
||||
## Basic Format
|
||||
|
||||
```
|
||||
<type>[optional scope]: <description>
|
||||
|
||||
[optional body]
|
||||
|
||||
[optional footer(s)]
|
||||
```
|
||||
|
||||
## Commit Types
|
||||
|
||||
### Primary Types
|
||||
|
||||
- **feat**: A new feature for the user (correlates with MINOR in semantic versioning)
|
||||
- **fix**: A bug fix for the user (correlates with PATCH in semantic versioning)
|
||||
|
||||
### Secondary Types
|
||||
|
||||
- **build**: Changes that affect the build system or external dependencies (webpack, npm, etc.)
|
||||
- **ci**: Changes to CI configuration files and scripts (Travis, Circle, BrowserStack, SauceLabs)
|
||||
- **docs**: Documentation only changes
|
||||
- **perf**: A code change that improves performance
|
||||
- **refactor**: A code change that neither fixes a bug nor adds a feature
|
||||
- **style**: Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc.)
|
||||
- **test**: Adding missing tests or correcting existing tests
|
||||
- **chore**: Other changes that don't modify src or test files
|
||||
- **revert**: Reverts a previous commit
|
||||
|
||||
### Breaking Changes
|
||||
|
||||
Any commit can introduce a breaking change by:
|
||||
1. Adding `!` after the type: `feat!: remove deprecated API`
|
||||
2. Including `BREAKING CHANGE:` in the footer
|
||||
|
||||
## Scopes
|
||||
|
||||
Scopes provide additional contextual information about the change. They should be noun describing a section of the codebase:
|
||||
|
||||
- `auth` - Authentication and authorization
|
||||
- `api` - API changes
|
||||
- `ui` - User interface
|
||||
- `db` - Database related changes
|
||||
- `config` - Configuration changes
|
||||
- `deps` - Dependency updates
|
||||
|
||||
## Examples
|
||||
|
||||
### Simple Feature
|
||||
```
|
||||
feat(auth): add OAuth2 integration
|
||||
|
||||
Integrate OAuth2 authentication with Google and GitHub providers.
|
||||
Users can now log in using their existing social media accounts.
|
||||
```
|
||||
|
||||
### Bug Fix
|
||||
```
|
||||
fix(api): resolve race condition in user creation
|
||||
|
||||
When multiple requests tried to create users with the same email
|
||||
simultaneously, duplicate records were sometimes created. Added
|
||||
proper database constraints and error handling.
|
||||
|
||||
Fixes #234
|
||||
```
|
||||
|
||||
### Breaking Change with !
|
||||
```
|
||||
feat(api)!: remove deprecated /v1/users endpoint
|
||||
|
||||
The deprecated /v1/users endpoint has been removed. All clients
|
||||
should migrate to /v2/users which provides better performance
|
||||
and additional features.
|
||||
|
||||
BREAKING CHANGE: /v1/users endpoint removed, use /v2/users instead
|
||||
```
|
||||
|
||||
### Breaking Change with Footer
|
||||
```
|
||||
feat(auth): implement new authentication flow
|
||||
|
||||
Add support for multi-factor authentication and improved session
|
||||
management. This change requires all users to re-authenticate.
|
||||
|
||||
BREAKING CHANGE: Authentication tokens issued before this release
|
||||
are no longer valid. Users must log in again.
|
||||
```
|
||||
|
||||
### Performance Improvement
|
||||
```
|
||||
perf(image): optimize image compression algorithm
|
||||
|
||||
Replaced PNG compression with WebP format, reducing image sizes
|
||||
by 40% on average while maintaining visual quality.
|
||||
|
||||
Closes #456
|
||||
```
|
||||
|
||||
### Dependency Update
|
||||
```
|
||||
build(deps): upgrade React to version 18.2.0
|
||||
|
||||
Updates React and related packages to latest stable versions.
|
||||
Includes performance improvements and new concurrent features.
|
||||
```
|
||||
|
||||
### Documentation
|
||||
```
|
||||
docs(readme): add deployment instructions
|
||||
|
||||
Added comprehensive deployment guide including Docker setup,
|
||||
environment variables configuration, and troubleshooting tips.
|
||||
```
|
||||
|
||||
### Revert
|
||||
```
|
||||
revert: feat(payment): add cryptocurrency support
|
||||
|
||||
This reverts commit 667ecc1654a317a13331b17617d973392f415f02.
|
||||
|
||||
Reverting due to security concerns identified in code review.
|
||||
The feature will be re-implemented with proper security measures.
|
||||
```
|
||||
|
||||
## Multi-paragraph Body
|
||||
|
||||
For complex changes, use multiple paragraphs in the body:
|
||||
|
||||
```
|
||||
feat(search): implement advanced search functionality
|
||||
|
||||
Add support for complex search queries including:
|
||||
- Boolean operators (AND, OR, NOT)
|
||||
- Field-specific searches (title:, author:, date:)
|
||||
- Fuzzy matching with configurable threshold
|
||||
- Search result highlighting
|
||||
|
||||
The search index has been restructured to support these new
|
||||
features while maintaining backward compatibility with existing
|
||||
simple search queries.
|
||||
|
||||
Performance testing shows less than 10ms impact on search
|
||||
response times even with complex queries.
|
||||
|
||||
Closes #789, #823, #901
|
||||
```
|
||||
|
||||
## Footers
|
||||
|
||||
### Issue References
|
||||
```
|
||||
Fixes #123
|
||||
Closes #234, #345
|
||||
Resolves #456
|
||||
```
|
||||
|
||||
### Breaking Changes
|
||||
```
|
||||
BREAKING CHANGE: The `authenticate` function now requires a second
|
||||
parameter for the authentication method. Update all calls from
|
||||
`authenticate(token)` to `authenticate(token, 'bearer')`.
|
||||
```
|
||||
|
||||
### Co-authors
|
||||
```
|
||||
Co-authored-by: Jane Doe <jane@example.com>
|
||||
Co-authored-by: John Smith <john@example.com>
|
||||
```
|
||||
|
||||
### Reviewed By
|
||||
```
|
||||
Reviewed-by: Senior Developer <senior@example.com>
|
||||
Acked-by: Tech Lead <lead@example.com>
|
||||
```
|
||||
|
||||
## Automation Benefits
|
||||
|
||||
Using conventional commits enables:
|
||||
|
||||
### Automatic Version Bumping
|
||||
- `fix` commits trigger PATCH version bump (1.0.0 → 1.0.1)
|
||||
- `feat` commits trigger MINOR version bump (1.0.0 → 1.1.0)
|
||||
- `BREAKING CHANGE` triggers MAJOR version bump (1.0.0 → 2.0.0)
|
||||
|
||||
### Changelog Generation
|
||||
```markdown
|
||||
## [1.2.0] - 2024-01-15
|
||||
|
||||
### Added
|
||||
- OAuth2 integration (auth)
|
||||
- Advanced search functionality (search)
|
||||
|
||||
### Fixed
|
||||
- Race condition in user creation (api)
|
||||
- Memory leak in image processing (image)
|
||||
|
||||
### Breaking Changes
|
||||
- Authentication tokens issued before this release are no longer valid
|
||||
```
|
||||
|
||||
### Release Notes
|
||||
Generate user-friendly release notes automatically from commit history, filtering out internal changes and highlighting user-facing improvements.
|
||||
|
||||
## Best Practices
|
||||
|
||||
### Writing Good Descriptions
|
||||
- Use imperative mood: "add feature" not "added feature"
|
||||
- Start with lowercase letter
|
||||
- No period at the end
|
||||
- Limit to 50 characters when possible
|
||||
- Be specific and descriptive
|
||||
|
||||
### Good Examples
|
||||
```
|
||||
feat(auth): add password reset functionality
|
||||
fix(ui): resolve mobile navigation menu overflow
|
||||
perf(db): optimize user query with proper indexing
|
||||
```
|
||||
|
||||
### Bad Examples
|
||||
```
|
||||
feat: stuff
|
||||
fix: bug
|
||||
update: changes
|
||||
```
|
||||
|
||||
### Body Guidelines
|
||||
- Separate subject from body with blank line
|
||||
- Wrap body at 72 characters
|
||||
- Use body to explain what and why, not how
|
||||
- Reference issues and PRs when relevant
|
||||
|
||||
### Scope Guidelines
|
||||
- Use consistent scope naming across the team
|
||||
- Keep scopes short and meaningful
|
||||
- Document your team's scope conventions
|
||||
- Consider using scopes that match your codebase structure
|
||||
|
||||
## Tools and Integration
|
||||
|
||||
### Git Hooks
|
||||
Use tools like `commitizen` or `husky` to enforce conventional commit format:
|
||||
|
||||
```bash
|
||||
# Install commitizen
|
||||
npm install -g commitizen cz-conventional-changelog
|
||||
|
||||
# Configure
|
||||
echo '{ "path": "cz-conventional-changelog" }' > ~/.czrc
|
||||
|
||||
# Use
|
||||
git cz
|
||||
```
|
||||
|
||||
### Automated Validation
|
||||
Add commit message validation to prevent non-conventional commits:
|
||||
|
||||
```javascript
|
||||
// commitlint.config.js
|
||||
module.exports = {
|
||||
extends: ['@commitlint/config-conventional'],
|
||||
rules: {
|
||||
'type-enum': [
|
||||
2, 'always',
|
||||
['feat', 'fix', 'docs', 'style', 'refactor', 'perf', 'test', 'build', 'ci', 'chore', 'revert']
|
||||
],
|
||||
'subject-case': [2, 'always', 'lower-case'],
|
||||
'subject-max-length': [2, 'always', 50]
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
### CI/CD Integration
|
||||
Integrate with release automation tools:
|
||||
- **semantic-release**: Automated version management and package publishing
|
||||
- **standard-version**: Generate changelog and tag releases
|
||||
- **release-please**: Google's release automation tool
|
||||
|
||||
## Common Mistakes
|
||||
|
||||
### Mixing Multiple Changes
|
||||
```
|
||||
# Bad: Multiple unrelated changes
|
||||
feat: add login page and fix CSS bug and update dependencies
|
||||
|
||||
# Good: Separate commits
|
||||
feat(auth): add login page
|
||||
fix(ui): resolve CSS styling issue
|
||||
build(deps): update React to version 18
|
||||
```
|
||||
|
||||
### Vague Descriptions
|
||||
```
|
||||
# Bad: Not descriptive
|
||||
fix: bug in code
|
||||
feat: new stuff
|
||||
|
||||
# Good: Specific and clear
|
||||
fix(api): resolve null pointer exception in user validation
|
||||
feat(search): implement fuzzy matching algorithm
|
||||
```
|
||||
|
||||
### Missing Breaking Change Indicators
|
||||
```
|
||||
# Bad: Breaking change not marked
|
||||
feat(api): update user authentication
|
||||
|
||||
# Good: Properly marked breaking change
|
||||
feat(api)!: update user authentication
|
||||
|
||||
BREAKING CHANGE: All API clients must now include authentication
|
||||
headers in every request. Anonymous access is no longer supported.
|
||||
```
|
||||
|
||||
## Team Guidelines
|
||||
|
||||
### Establishing Conventions
|
||||
1. **Define scope vocabulary**: Create a list of approved scopes for your project
|
||||
2. **Document examples**: Provide team-specific examples of good commits
|
||||
3. **Set up tooling**: Use linters and hooks to enforce standards
|
||||
4. **Review process**: Include commit message quality in code reviews
|
||||
5. **Training**: Ensure all team members understand the format
|
||||
|
||||
### Scope Examples by Project Type
|
||||
**Web Application:**
|
||||
- `auth`, `ui`, `api`, `db`, `config`, `deploy`
|
||||
|
||||
**Library/SDK:**
|
||||
- `core`, `utils`, `docs`, `examples`, `tests`
|
||||
|
||||
**Mobile App:**
|
||||
- `ios`, `android`, `shared`, `ui`, `network`, `storage`
|
||||
|
||||
By following conventional commits consistently, your team will have a clear, searchable commit history that enables powerful automation and improves the overall development workflow.
|
||||
592
engineering/release-manager/references/hotfix-procedures.md
Normal file
592
engineering/release-manager/references/hotfix-procedures.md
Normal file
@@ -0,0 +1,592 @@
|
||||
# Hotfix Procedures
|
||||
|
||||
## Overview
|
||||
|
||||
Hotfixes are emergency releases designed to address critical production issues that cannot wait for the regular release cycle. This document outlines classification, procedures, and best practices for managing hotfixes across different development workflows.
|
||||
|
||||
## Severity Classification
|
||||
|
||||
### P0 - Critical (Production Down)
|
||||
**Definition:** Complete system outage, data corruption, or security breach affecting all users.
|
||||
|
||||
**Examples:**
|
||||
- Server crashes preventing any user access
|
||||
- Database corruption causing data loss
|
||||
- Security vulnerability being actively exploited
|
||||
- Payment system completely non-functional
|
||||
- Authentication system failure preventing all logins
|
||||
|
||||
**Response Requirements:**
|
||||
- **Timeline:** Fix deployed within 2 hours
|
||||
- **Approval:** Engineering Lead + On-call Manager (verbal approval acceptable)
|
||||
- **Process:** Emergency deployment bypassing normal gates
|
||||
- **Communication:** Immediate notification to all stakeholders
|
||||
- **Documentation:** Post-incident review required within 24 hours
|
||||
|
||||
**Escalation:**
|
||||
- Page on-call engineer immediately
|
||||
- Escalate to Engineering Lead within 15 minutes
|
||||
- Notify CEO/CTO if resolution exceeds 4 hours
|
||||
|
||||
### P1 - High (Major Feature Broken)
|
||||
**Definition:** Critical functionality broken affecting significant portion of users.
|
||||
|
||||
**Examples:**
|
||||
- Core user workflow completely broken
|
||||
- Payment processing failures affecting >50% of transactions
|
||||
- Search functionality returning no results
|
||||
- Mobile app crashes on startup
|
||||
- API returning 500 errors for main endpoints
|
||||
|
||||
**Response Requirements:**
|
||||
- **Timeline:** Fix deployed within 24 hours
|
||||
- **Approval:** Engineering Lead + Product Manager
|
||||
- **Process:** Expedited review and testing
|
||||
- **Communication:** Stakeholder notification within 1 hour
|
||||
- **Documentation:** Root cause analysis within 48 hours
|
||||
|
||||
**Escalation:**
|
||||
- Notify on-call engineer within 30 minutes
|
||||
- Escalate to Engineering Lead within 2 hours
|
||||
- Daily updates to Product/Business stakeholders
|
||||
|
||||
### P2 - Medium (Minor Feature Issues)
|
||||
**Definition:** Non-critical functionality issues with limited user impact.
|
||||
|
||||
**Examples:**
|
||||
- Cosmetic UI issues affecting user experience
|
||||
- Non-essential features not working properly
|
||||
- Performance degradation not affecting core workflows
|
||||
- Minor API inconsistencies
|
||||
- Reporting/analytics data inaccuracies
|
||||
|
||||
**Response Requirements:**
|
||||
- **Timeline:** Include in next regular release
|
||||
- **Approval:** Standard pull request review process
|
||||
- **Process:** Normal development and testing cycle
|
||||
- **Communication:** Include in regular release notes
|
||||
- **Documentation:** Standard issue tracking
|
||||
|
||||
**Escalation:**
|
||||
- Create ticket in normal backlog
|
||||
- No special escalation required
|
||||
- Include in release planning discussions
|
||||
|
||||
## Hotfix Workflows by Development Model
|
||||
|
||||
### Git Flow Hotfix Process
|
||||
|
||||
#### Branch Structure
|
||||
```
|
||||
main (v1.2.3) ← hotfix/security-patch → main (v1.2.4)
|
||||
→ develop
|
||||
```
|
||||
|
||||
#### Step-by-Step Process
|
||||
1. **Create Hotfix Branch**
|
||||
```bash
|
||||
git checkout main
|
||||
git pull origin main
|
||||
git checkout -b hotfix/security-patch
|
||||
```
|
||||
|
||||
2. **Implement Fix**
|
||||
- Make minimal changes addressing only the specific issue
|
||||
- Include tests to prevent regression
|
||||
- Update version number (patch increment)
|
||||
```bash
|
||||
# Fix the issue
|
||||
git add .
|
||||
git commit -m "fix: resolve SQL injection vulnerability"
|
||||
|
||||
# Version bump
|
||||
echo "1.2.4" > VERSION
|
||||
git add VERSION
|
||||
git commit -m "chore: bump version to 1.2.4"
|
||||
```
|
||||
|
||||
3. **Test Fix**
|
||||
- Run automated test suite
|
||||
- Manual testing of affected functionality
|
||||
- Security review if applicable
|
||||
```bash
|
||||
# Run tests
|
||||
npm test
|
||||
python -m pytest
|
||||
|
||||
# Security scan
|
||||
npm audit
|
||||
bandit -r src/
|
||||
```
|
||||
|
||||
4. **Deploy to Staging**
|
||||
```bash
|
||||
# Deploy hotfix branch to staging
|
||||
git push origin hotfix/security-patch
|
||||
# Trigger staging deployment via CI/CD
|
||||
```
|
||||
|
||||
5. **Merge to Production**
|
||||
```bash
|
||||
# Merge to main
|
||||
git checkout main
|
||||
git merge --no-ff hotfix/security-patch
|
||||
git tag -a v1.2.4 -m "Hotfix: Security vulnerability patch"
|
||||
git push origin main --tags
|
||||
|
||||
# Merge back to develop
|
||||
git checkout develop
|
||||
git merge --no-ff hotfix/security-patch
|
||||
git push origin develop
|
||||
|
||||
# Clean up
|
||||
git branch -d hotfix/security-patch
|
||||
git push origin --delete hotfix/security-patch
|
||||
```
|
||||
|
||||
### GitHub Flow Hotfix Process
|
||||
|
||||
#### Branch Structure
|
||||
```
|
||||
main ← hotfix/critical-fix → main (immediate deploy)
|
||||
```
|
||||
|
||||
#### Step-by-Step Process
|
||||
1. **Create Fix Branch**
|
||||
```bash
|
||||
git checkout main
|
||||
git pull origin main
|
||||
git checkout -b hotfix/payment-gateway-fix
|
||||
```
|
||||
|
||||
2. **Implement and Test**
|
||||
```bash
|
||||
# Make the fix
|
||||
git add .
|
||||
git commit -m "fix(payment): resolve gateway timeout issue"
|
||||
git push origin hotfix/payment-gateway-fix
|
||||
```
|
||||
|
||||
3. **Create Emergency PR**
|
||||
```bash
|
||||
# Use GitHub CLI or web interface
|
||||
gh pr create --title "HOTFIX: Payment gateway timeout" \
|
||||
--body "Critical fix for payment processing failures" \
|
||||
--reviewer engineering-team \
|
||||
--label hotfix
|
||||
```
|
||||
|
||||
4. **Deploy Branch for Testing**
|
||||
```bash
|
||||
# Deploy branch to staging for validation
|
||||
./deploy.sh hotfix/payment-gateway-fix staging
|
||||
# Quick smoke tests
|
||||
```
|
||||
|
||||
5. **Emergency Merge and Deploy**
|
||||
```bash
|
||||
# After approval, merge and deploy
|
||||
gh pr merge --squash
|
||||
# Automatic deployment to production via CI/CD
|
||||
```
|
||||
|
||||
### Trunk-based Hotfix Process
|
||||
|
||||
#### Direct Commit Approach
|
||||
```bash
|
||||
# For small fixes, commit directly to main
|
||||
git checkout main
|
||||
git pull origin main
|
||||
# Make fix
|
||||
git add .
|
||||
git commit -m "fix: resolve memory leak in user session handling"
|
||||
git push origin main
|
||||
# Automatic deployment triggers
|
||||
```
|
||||
|
||||
#### Feature Flag Rollback
|
||||
```bash
|
||||
# For feature-related issues, disable via feature flag
|
||||
curl -X POST api/feature-flags/new-search/disable
|
||||
# Verify issue resolved
|
||||
# Plan proper fix for next deployment
|
||||
```
|
||||
|
||||
## Emergency Response Procedures
|
||||
|
||||
### Incident Declaration Process
|
||||
|
||||
1. **Detection and Assessment** (0-5 minutes)
|
||||
- Monitor alerts or user reports identify issue
|
||||
- Assess severity using classification matrix
|
||||
- Determine if hotfix is required
|
||||
|
||||
2. **Team Assembly** (5-10 minutes)
|
||||
- Page appropriate on-call engineer
|
||||
- Assemble incident response team
|
||||
- Establish communication channel (Slack, Teams)
|
||||
|
||||
3. **Initial Response** (10-30 minutes)
|
||||
- Create incident ticket/document
|
||||
- Begin investigating root cause
|
||||
- Implement immediate mitigations if possible
|
||||
|
||||
4. **Hotfix Development** (30 minutes - 2 hours)
|
||||
- Create hotfix branch
|
||||
- Implement minimal fix
|
||||
- Test fix in isolation
|
||||
|
||||
5. **Deployment** (15-30 minutes)
|
||||
- Deploy to staging for validation
|
||||
- Deploy to production
|
||||
- Monitor for successful resolution
|
||||
|
||||
6. **Verification** (15-30 minutes)
|
||||
- Confirm issue is resolved
|
||||
- Monitor system stability
|
||||
- Update stakeholders
|
||||
|
||||
### Communication Templates
|
||||
|
||||
#### P0 Initial Alert
|
||||
```
|
||||
🚨 CRITICAL INCIDENT - Production Down
|
||||
|
||||
Status: Investigating
|
||||
Impact: Complete service outage
|
||||
Affected Users: All users
|
||||
Started: 2024-01-15 14:30 UTC
|
||||
Incident Commander: @john.doe
|
||||
|
||||
Current Actions:
|
||||
- Investigating root cause
|
||||
- Preparing emergency fix
|
||||
- Will update every 15 minutes
|
||||
|
||||
Status Page: https://status.ourapp.com
|
||||
Incident Channel: #incident-2024-001
|
||||
```
|
||||
|
||||
#### P0 Resolution Notice
|
||||
```
|
||||
✅ RESOLVED - Production Restored
|
||||
|
||||
Status: Resolved
|
||||
Resolution Time: 1h 23m
|
||||
Root Cause: Database connection pool exhaustion
|
||||
Fix: Increased connection limits and restarted services
|
||||
|
||||
Timeline:
|
||||
14:30 UTC - Issue detected
|
||||
14:45 UTC - Root cause identified
|
||||
15:20 UTC - Fix deployed
|
||||
15:35 UTC - Full functionality restored
|
||||
|
||||
Post-incident review scheduled for tomorrow 10:00 AM.
|
||||
Thank you for your patience.
|
||||
```
|
||||
|
||||
#### P1 Status Update
|
||||
```
|
||||
⚠️ Issue Update - Payment Processing
|
||||
|
||||
Status: Fix deployed, monitoring
|
||||
Impact: Payment failures reduced from 45% to <2%
|
||||
ETA: Complete resolution within 2 hours
|
||||
|
||||
Actions taken:
|
||||
- Deployed hotfix to address timeout issues
|
||||
- Increased monitoring on payment gateway
|
||||
- Contacting affected customers
|
||||
|
||||
Next update in 30 minutes or when resolved.
|
||||
```
|
||||
|
||||
### Rollback Procedures
|
||||
|
||||
#### When to Rollback
|
||||
- Fix doesn't resolve the issue
|
||||
- Fix introduces new problems
|
||||
- System stability is compromised
|
||||
- Data corruption is detected
|
||||
|
||||
#### Rollback Process
|
||||
1. **Immediate Assessment** (2-5 minutes)
|
||||
```bash
|
||||
# Check system health
|
||||
curl -f https://api.ourapp.com/health
|
||||
# Review error logs
|
||||
kubectl logs deployment/app --tail=100
|
||||
# Check key metrics
|
||||
```
|
||||
|
||||
2. **Rollback Execution** (5-15 minutes)
|
||||
```bash
|
||||
# Git-based rollback
|
||||
git checkout main
|
||||
git revert HEAD
|
||||
git push origin main
|
||||
|
||||
# Or container-based rollback
|
||||
kubectl rollout undo deployment/app
|
||||
|
||||
# Or load balancer switch
|
||||
aws elbv2 modify-target-group --target-group-arn arn:aws:elasticloadbalancing:us-east-1:123456789012:targetgroup/previous-version
|
||||
```
|
||||
|
||||
3. **Verification** (5-10 minutes)
|
||||
```bash
|
||||
# Confirm rollback successful
|
||||
# Check system health endpoints
|
||||
# Verify core functionality working
|
||||
# Monitor error rates and performance
|
||||
```
|
||||
|
||||
4. **Communication**
|
||||
```
|
||||
🔄 ROLLBACK COMPLETE
|
||||
|
||||
The hotfix has been rolled back due to [reason].
|
||||
System is now stable on previous version.
|
||||
We are investigating the issue and will provide updates.
|
||||
```
|
||||
|
||||
## Testing Strategies for Hotfixes
|
||||
|
||||
### Pre-deployment Testing
|
||||
|
||||
#### Automated Testing
|
||||
```bash
|
||||
# Run full test suite
|
||||
npm test
|
||||
pytest tests/
|
||||
go test ./...
|
||||
|
||||
# Security scanning
|
||||
npm audit --audit-level high
|
||||
bandit -r src/
|
||||
gosec ./...
|
||||
|
||||
# Integration tests
|
||||
./run_integration_tests.sh
|
||||
|
||||
# Load testing (if performance-related)
|
||||
artillery quick --count 100 --num 10 https://staging.ourapp.com
|
||||
```
|
||||
|
||||
#### Manual Testing Checklist
|
||||
- [ ] Core user workflow functions correctly
|
||||
- [ ] Authentication and authorization working
|
||||
- [ ] Payment processing (if applicable)
|
||||
- [ ] Data integrity maintained
|
||||
- [ ] No new error logs or exceptions
|
||||
- [ ] Performance within acceptable range
|
||||
- [ ] Mobile app functionality (if applicable)
|
||||
- [ ] Third-party integrations working
|
||||
|
||||
#### Staging Validation
|
||||
```bash
|
||||
# Deploy to staging
|
||||
./deploy.sh hotfix/critical-fix staging
|
||||
|
||||
# Run smoke tests
|
||||
curl -f https://staging.ourapp.com/api/health
|
||||
./smoke_tests.sh
|
||||
|
||||
# Manual verification of specific issue
|
||||
# Document test results
|
||||
```
|
||||
|
||||
### Post-deployment Monitoring
|
||||
|
||||
#### Immediate Monitoring (First 30 minutes)
|
||||
- Error rate and count
|
||||
- Response time and latency
|
||||
- CPU and memory usage
|
||||
- Database connection counts
|
||||
- Key business metrics
|
||||
|
||||
#### Extended Monitoring (First 24 hours)
|
||||
- User activity patterns
|
||||
- Feature usage statistics
|
||||
- Customer support tickets
|
||||
- Performance trends
|
||||
- Security log analysis
|
||||
|
||||
#### Monitoring Scripts
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# monitor_hotfix.sh - Post-deployment monitoring
|
||||
|
||||
echo "=== Hotfix Deployment Monitoring ==="
|
||||
echo "Deployment time: $(date)"
|
||||
echo
|
||||
|
||||
# Check application health
|
||||
echo "--- Application Health ---"
|
||||
curl -s https://api.ourapp.com/health | jq '.'
|
||||
|
||||
# Check error rates
|
||||
echo "--- Error Rates (last 30min) ---"
|
||||
curl -s "https://api.datadog.com/api/v1/query?query=sum:application.errors{*}" \
|
||||
-H "DD-API-KEY: $DATADOG_API_KEY" | jq '.series[0].pointlist[-1][1]'
|
||||
|
||||
# Check response times
|
||||
echo "--- Response Times ---"
|
||||
curl -s "https://api.datadog.com/api/v1/query?query=avg:application.response_time{*}" \
|
||||
-H "DD-API-KEY: $DATADOG_API_KEY" | jq '.series[0].pointlist[-1][1]'
|
||||
|
||||
# Check database connections
|
||||
echo "--- Database Status ---"
|
||||
psql -h db.ourapp.com -U readonly -c "SELECT count(*) as active_connections FROM pg_stat_activity;"
|
||||
|
||||
echo "=== Monitoring Complete ==="
|
||||
```
|
||||
|
||||
## Documentation and Learning
|
||||
|
||||
### Incident Documentation Template
|
||||
|
||||
```markdown
|
||||
# Incident Report: [Brief Description]
|
||||
|
||||
## Summary
|
||||
- **Incident ID:** INC-2024-001
|
||||
- **Severity:** P0/P1/P2
|
||||
- **Start Time:** 2024-01-15 14:30 UTC
|
||||
- **End Time:** 2024-01-15 15:45 UTC
|
||||
- **Duration:** 1h 15m
|
||||
- **Impact:** [Description of user/business impact]
|
||||
|
||||
## Root Cause
|
||||
[Detailed explanation of what went wrong and why]
|
||||
|
||||
## Timeline
|
||||
| Time | Event |
|
||||
|------|-------|
|
||||
| 14:30 | Issue detected via monitoring alert |
|
||||
| 14:35 | Incident team assembled |
|
||||
| 14:45 | Root cause identified |
|
||||
| 15:00 | Fix developed and tested |
|
||||
| 15:20 | Fix deployed to production |
|
||||
| 15:45 | Issue confirmed resolved |
|
||||
|
||||
## Resolution
|
||||
[What was done to fix the issue]
|
||||
|
||||
## Lessons Learned
|
||||
### What went well
|
||||
- Quick detection through monitoring
|
||||
- Effective team coordination
|
||||
- Minimal user impact
|
||||
|
||||
### What could be improved
|
||||
- Earlier detection possible with better alerting
|
||||
- Testing could have caught this issue
|
||||
- Communication could be more proactive
|
||||
|
||||
## Action Items
|
||||
- [ ] Improve monitoring for [specific area]
|
||||
- [ ] Add automated test for [specific scenario]
|
||||
- [ ] Update documentation for [specific process]
|
||||
- [ ] Training on [specific topic] for team
|
||||
|
||||
## Prevention Measures
|
||||
[How we'll prevent this from happening again]
|
||||
```
|
||||
|
||||
### Post-Incident Review Process
|
||||
|
||||
1. **Schedule Review** (within 24-48 hours)
|
||||
- Involve all key participants
|
||||
- Book 60-90 minute session
|
||||
- Prepare incident timeline
|
||||
|
||||
2. **Blameless Analysis**
|
||||
- Focus on systems and processes, not individuals
|
||||
- Understand contributing factors
|
||||
- Identify improvement opportunities
|
||||
|
||||
3. **Action Plan**
|
||||
- Concrete, assignable tasks
|
||||
- Realistic timelines
|
||||
- Clear success criteria
|
||||
|
||||
4. **Follow-up**
|
||||
- Track action item completion
|
||||
- Share learnings with broader team
|
||||
- Update procedures based on insights
|
||||
|
||||
### Knowledge Sharing
|
||||
|
||||
#### Runbook Updates
|
||||
After each hotfix, update relevant runbooks:
|
||||
- Add new troubleshooting steps
|
||||
- Update contact information
|
||||
- Refine escalation procedures
|
||||
- Document new tools or processes
|
||||
|
||||
#### Team Training
|
||||
- Share incident learnings in team meetings
|
||||
- Conduct tabletop exercises for common scenarios
|
||||
- Update onboarding materials with hotfix procedures
|
||||
- Create decision trees for severity classification
|
||||
|
||||
#### Automation Improvements
|
||||
- Add alerts for new failure modes
|
||||
- Automate manual steps where possible
|
||||
- Improve deployment and rollback processes
|
||||
- Enhance monitoring and observability
|
||||
|
||||
## Common Pitfalls and Best Practices
|
||||
|
||||
### Common Pitfalls
|
||||
|
||||
❌ **Over-engineering the fix**
|
||||
- Making broad changes instead of minimal targeted fix
|
||||
- Adding features while fixing bugs
|
||||
- Refactoring unrelated code
|
||||
|
||||
❌ **Insufficient testing**
|
||||
- Skipping automated tests due to time pressure
|
||||
- Not testing the exact scenario that caused the issue
|
||||
- Deploying without staging validation
|
||||
|
||||
❌ **Poor communication**
|
||||
- Not notifying stakeholders promptly
|
||||
- Unclear or infrequent status updates
|
||||
- Forgetting to announce resolution
|
||||
|
||||
❌ **Inadequate monitoring**
|
||||
- Not watching system health after deployment
|
||||
- Missing secondary effects of the fix
|
||||
- Failing to verify the issue is actually resolved
|
||||
|
||||
### Best Practices
|
||||
|
||||
✅ **Keep fixes minimal and focused**
|
||||
- Address only the specific issue
|
||||
- Avoid scope creep or improvements
|
||||
- Save refactoring for regular releases
|
||||
|
||||
✅ **Maintain clear communication**
|
||||
- Set up dedicated incident channel
|
||||
- Provide regular status updates
|
||||
- Use clear, non-technical language for business stakeholders
|
||||
|
||||
✅ **Test thoroughly but efficiently**
|
||||
- Focus testing on affected functionality
|
||||
- Use automated tests where possible
|
||||
- Validate in staging before production
|
||||
|
||||
✅ **Document everything**
|
||||
- Maintain timeline of events
|
||||
- Record decisions and rationale
|
||||
- Share lessons learned with team
|
||||
|
||||
✅ **Plan for rollback**
|
||||
- Always have a rollback plan ready
|
||||
- Test rollback procedure in advance
|
||||
- Monitor closely after deployment
|
||||
|
||||
By following these procedures and continuously improving based on experience, teams can handle production emergencies effectively while minimizing impact and learning from each incident.
|
||||
@@ -0,0 +1,410 @@
|
||||
# Release Workflow Comparison
|
||||
|
||||
## Overview
|
||||
|
||||
This document compares the three most popular branching and release workflows: Git Flow, GitHub Flow, and Trunk-based Development. Each approach has distinct advantages and trade-offs depending on your team size, deployment frequency, and risk tolerance.
|
||||
|
||||
## Git Flow
|
||||
|
||||
### Structure
|
||||
```
|
||||
main (production)
|
||||
↑
|
||||
release/1.2.0 ← develop (integration) ← feature/user-auth
|
||||
↑ ← feature/payment-api
|
||||
hotfix/critical-fix
|
||||
```
|
||||
|
||||
### Branch Types
|
||||
- **main**: Production-ready code, tagged releases
|
||||
- **develop**: Integration branch for next release
|
||||
- **feature/***: Individual features, merged to develop
|
||||
- **release/X.Y.Z**: Release preparation, branched from develop
|
||||
- **hotfix/***: Critical fixes, branched from main
|
||||
|
||||
### Typical Flow
|
||||
1. Create feature branch from develop: `git checkout -b feature/login develop`
|
||||
2. Work on feature, commit changes
|
||||
3. Merge feature to develop when complete
|
||||
4. When ready for release, create release branch: `git checkout -b release/1.2.0 develop`
|
||||
5. Finalize release (version bump, changelog, bug fixes)
|
||||
6. Merge release branch to both main and develop
|
||||
7. Tag release: `git tag v1.2.0`
|
||||
8. Deploy from main branch
|
||||
|
||||
### Advantages
|
||||
- **Clear separation** between production and development code
|
||||
- **Stable main branch** always represents production state
|
||||
- **Parallel development** of features without interference
|
||||
- **Structured release process** with dedicated release branches
|
||||
- **Hotfix support** without disrupting development work
|
||||
- **Good for scheduled releases** and traditional release cycles
|
||||
|
||||
### Disadvantages
|
||||
- **Complex workflow** with many branch types
|
||||
- **Merge overhead** from multiple integration points
|
||||
- **Delayed feedback** from long-lived feature branches
|
||||
- **Integration conflicts** when merging large features
|
||||
- **Slower deployment** due to process overhead
|
||||
- **Not ideal for continuous deployment**
|
||||
|
||||
### Best For
|
||||
- Large teams (10+ developers)
|
||||
- Products with scheduled release cycles
|
||||
- Enterprise software with formal testing phases
|
||||
- Projects requiring stable release branches
|
||||
- Teams comfortable with complex Git workflows
|
||||
|
||||
### Example Commands
|
||||
```bash
|
||||
# Start new feature
|
||||
git checkout develop
|
||||
git checkout -b feature/user-authentication
|
||||
|
||||
# Finish feature
|
||||
git checkout develop
|
||||
git merge --no-ff feature/user-authentication
|
||||
git branch -d feature/user-authentication
|
||||
|
||||
# Start release
|
||||
git checkout develop
|
||||
git checkout -b release/1.2.0
|
||||
# Version bump and changelog updates
|
||||
git commit -am "Bump version to 1.2.0"
|
||||
|
||||
# Finish release
|
||||
git checkout main
|
||||
git merge --no-ff release/1.2.0
|
||||
git tag -a v1.2.0 -m "Release version 1.2.0"
|
||||
git checkout develop
|
||||
git merge --no-ff release/1.2.0
|
||||
git branch -d release/1.2.0
|
||||
|
||||
# Hotfix
|
||||
git checkout main
|
||||
git checkout -b hotfix/security-patch
|
||||
# Fix the issue
|
||||
git commit -am "Fix security vulnerability"
|
||||
git checkout main
|
||||
git merge --no-ff hotfix/security-patch
|
||||
git tag -a v1.2.1 -m "Hotfix version 1.2.1"
|
||||
git checkout develop
|
||||
git merge --no-ff hotfix/security-patch
|
||||
```
|
||||
|
||||
## GitHub Flow
|
||||
|
||||
### Structure
|
||||
```
|
||||
main ← feature/user-auth
|
||||
← feature/payment-api
|
||||
← hotfix/critical-fix
|
||||
```
|
||||
|
||||
### Branch Types
|
||||
- **main**: Production-ready code, deployed automatically
|
||||
- **feature/***: All changes, regardless of size or type
|
||||
|
||||
### Typical Flow
|
||||
1. Create feature branch from main: `git checkout -b feature/login main`
|
||||
2. Work on feature with regular commits and pushes
|
||||
3. Open pull request when ready for feedback
|
||||
4. Deploy feature branch to staging for testing
|
||||
5. Merge to main when approved and tested
|
||||
6. Deploy main to production automatically
|
||||
7. Delete feature branch
|
||||
|
||||
### Advantages
|
||||
- **Simple workflow** with only two branch types
|
||||
- **Fast deployment** with minimal process overhead
|
||||
- **Continuous integration** with frequent merges to main
|
||||
- **Early feedback** through pull request reviews
|
||||
- **Deploy from branches** allows testing before merge
|
||||
- **Good for continuous deployment**
|
||||
|
||||
### Disadvantages
|
||||
- **Main can be unstable** if testing is insufficient
|
||||
- **No release branches** for coordinating multiple features
|
||||
- **Limited hotfix process** requires careful coordination
|
||||
- **Requires strong testing** and CI/CD infrastructure
|
||||
- **Not suitable for scheduled releases**
|
||||
- **Can be chaotic** with many simultaneous features
|
||||
|
||||
### Best For
|
||||
- Small to medium teams (2-10 developers)
|
||||
- Web applications with continuous deployment
|
||||
- Products with rapid iteration cycles
|
||||
- Teams with strong testing and CI/CD practices
|
||||
- Projects where main is always deployable
|
||||
|
||||
### Example Commands
|
||||
```bash
|
||||
# Start new feature
|
||||
git checkout main
|
||||
git pull origin main
|
||||
git checkout -b feature/user-authentication
|
||||
|
||||
# Regular work
|
||||
git add .
|
||||
git commit -m "feat(auth): add login form validation"
|
||||
git push origin feature/user-authentication
|
||||
|
||||
# Deploy branch for testing
|
||||
# (Usually done through CI/CD)
|
||||
./deploy.sh feature/user-authentication staging
|
||||
|
||||
# Merge when ready
|
||||
git checkout main
|
||||
git merge feature/user-authentication
|
||||
git push origin main
|
||||
git branch -d feature/user-authentication
|
||||
|
||||
# Automatic deployment to production
|
||||
# (Triggered by push to main)
|
||||
```
|
||||
|
||||
## Trunk-based Development
|
||||
|
||||
### Structure
|
||||
```
|
||||
main ← short-feature-branch (1-3 days max)
|
||||
← another-short-branch
|
||||
← direct-commits
|
||||
```
|
||||
|
||||
### Branch Types
|
||||
- **main**: The single source of truth, always deployable
|
||||
- **Short-lived branches**: Optional, for changes taking >1 day
|
||||
|
||||
### Typical Flow
|
||||
1. Commit directly to main for small changes
|
||||
2. Create short-lived branch for larger changes (max 2-3 days)
|
||||
3. Merge to main frequently (multiple times per day)
|
||||
4. Use feature flags to hide incomplete features
|
||||
5. Deploy main to production multiple times per day
|
||||
6. Release by enabling feature flags, not code deployment
|
||||
|
||||
### Advantages
|
||||
- **Simplest workflow** with minimal branching
|
||||
- **Fastest integration** with continuous merges
|
||||
- **Reduced merge conflicts** from short-lived branches
|
||||
- **Always deployable main** through feature flags
|
||||
- **Fastest feedback loop** with immediate integration
|
||||
- **Excellent for CI/CD** and DevOps practices
|
||||
|
||||
### Disadvantages
|
||||
- **Requires discipline** to keep main stable
|
||||
- **Needs feature flags** for incomplete features
|
||||
- **Limited code review** for direct commits
|
||||
- **Can be destabilizing** without proper testing
|
||||
- **Requires advanced CI/CD** infrastructure
|
||||
- **Not suitable for teams** uncomfortable with frequent changes
|
||||
|
||||
### Best For
|
||||
- Expert teams with strong DevOps culture
|
||||
- Products requiring very fast iteration
|
||||
- Microservices architectures
|
||||
- Teams practicing continuous deployment
|
||||
- Organizations with mature testing practices
|
||||
|
||||
### Example Commands
|
||||
```bash
|
||||
# Small change - direct to main
|
||||
git checkout main
|
||||
git pull origin main
|
||||
# Make changes
|
||||
git add .
|
||||
git commit -m "fix(ui): resolve button alignment issue"
|
||||
git push origin main
|
||||
|
||||
# Larger change - short branch
|
||||
git checkout main
|
||||
git pull origin main
|
||||
git checkout -b payment-integration
|
||||
# Work for 1-2 days maximum
|
||||
git add .
|
||||
git commit -m "feat(payment): add Stripe integration"
|
||||
git push origin payment-integration
|
||||
|
||||
# Immediate merge
|
||||
git checkout main
|
||||
git merge payment-integration
|
||||
git push origin main
|
||||
git branch -d payment-integration
|
||||
|
||||
# Feature flag usage
|
||||
if (featureFlags.enabled('stripe_payments', userId)) {
|
||||
return renderStripePayment();
|
||||
} else {
|
||||
return renderLegacyPayment();
|
||||
}
|
||||
```
|
||||
|
||||
## Feature Comparison Matrix
|
||||
|
||||
| Aspect | Git Flow | GitHub Flow | Trunk-based |
|
||||
|--------|----------|-------------|-------------|
|
||||
| **Complexity** | High | Medium | Low |
|
||||
| **Learning Curve** | Steep | Moderate | Gentle |
|
||||
| **Deployment Frequency** | Weekly/Monthly | Daily | Multiple/day |
|
||||
| **Branch Lifetime** | Weeks/Months | Days/Weeks | Hours/Days |
|
||||
| **Main Stability** | Very High | High | High* |
|
||||
| **Release Coordination** | Excellent | Limited | Feature Flags |
|
||||
| **Hotfix Support** | Built-in | Manual | Direct |
|
||||
| **Merge Conflicts** | High | Medium | Low |
|
||||
| **Team Size** | 10+ | 3-10 | Any |
|
||||
| **CI/CD Requirements** | Medium | High | Very High |
|
||||
|
||||
*With proper feature flags and testing
|
||||
|
||||
## Release Strategies by Workflow
|
||||
|
||||
### Git Flow Releases
|
||||
```bash
|
||||
# Scheduled release every 2 weeks
|
||||
git checkout develop
|
||||
git checkout -b release/2.3.0
|
||||
|
||||
# Version management
|
||||
echo "2.3.0" > VERSION
|
||||
npm version 2.3.0 --no-git-tag-version
|
||||
python setup.py --version 2.3.0
|
||||
|
||||
# Changelog generation
|
||||
git log --oneline release/2.2.0..HEAD --pretty=format:"%s" > CHANGELOG_DRAFT.md
|
||||
|
||||
# Testing and bug fixes in release branch
|
||||
git commit -am "fix: resolve issue found in release testing"
|
||||
|
||||
# Finalize release
|
||||
git checkout main
|
||||
git merge --no-ff release/2.3.0
|
||||
git tag -a v2.3.0 -m "Release 2.3.0"
|
||||
|
||||
# Deploy tagged version
|
||||
docker build -t app:2.3.0 .
|
||||
kubectl set image deployment/app app=app:2.3.0
|
||||
```
|
||||
|
||||
### GitHub Flow Releases
|
||||
```bash
|
||||
# Deploy every merge to main
|
||||
git checkout main
|
||||
git merge feature/new-payment-method
|
||||
|
||||
# Automatic deployment via CI/CD
|
||||
# .github/workflows/deploy.yml triggers on push to main
|
||||
|
||||
# Tag releases for tracking (optional)
|
||||
git tag -a v2.3.$(date +%Y%m%d%H%M) -m "Production deployment"
|
||||
|
||||
# Rollback if needed
|
||||
git revert HEAD
|
||||
git push origin main # Triggers automatic rollback deployment
|
||||
```
|
||||
|
||||
### Trunk-based Releases
|
||||
```bash
|
||||
# Continuous deployment with feature flags
|
||||
git checkout main
|
||||
git add feature_flags.json
|
||||
git commit -m "feat: enable new payment method for 10% of users"
|
||||
git push origin main
|
||||
|
||||
# Gradual rollout
|
||||
curl -X POST api/feature-flags/payment-v2/rollout/25 # 25% of users
|
||||
# Monitor metrics...
|
||||
curl -X POST api/feature-flags/payment-v2/rollout/50 # 50% of users
|
||||
# Monitor metrics...
|
||||
curl -X POST api/feature-flags/payment-v2/rollout/100 # Full rollout
|
||||
|
||||
# Remove flag after successful rollout
|
||||
git rm old_payment_code.js
|
||||
git commit -m "cleanup: remove legacy payment code"
|
||||
```
|
||||
|
||||
## Choosing the Right Workflow
|
||||
|
||||
### Decision Matrix
|
||||
|
||||
**Choose Git Flow if:**
|
||||
- ✅ Team size > 10 developers
|
||||
- ✅ Scheduled release cycles (weekly/monthly)
|
||||
- ✅ Multiple versions supported simultaneously
|
||||
- ✅ Formal testing and QA processes
|
||||
- ✅ Complex enterprise software
|
||||
- ❌ Need rapid deployment
|
||||
- ❌ Small team or startup
|
||||
|
||||
**Choose GitHub Flow if:**
|
||||
- ✅ Team size 3-10 developers
|
||||
- ✅ Web applications or APIs
|
||||
- ✅ Strong CI/CD and testing
|
||||
- ✅ Daily or continuous deployment
|
||||
- ✅ Simple release requirements
|
||||
- ❌ Complex release coordination needed
|
||||
- ❌ Multiple release branches required
|
||||
|
||||
**Choose Trunk-based Development if:**
|
||||
- ✅ Expert development team
|
||||
- ✅ Mature DevOps practices
|
||||
- ✅ Microservices architecture
|
||||
- ✅ Feature flag infrastructure
|
||||
- ✅ Multiple deployments per day
|
||||
- ✅ Strong automated testing
|
||||
- ❌ Junior developers
|
||||
- ❌ Complex integration requirements
|
||||
|
||||
### Migration Strategies
|
||||
|
||||
#### From Git Flow to GitHub Flow
|
||||
1. **Simplify branching**: Eliminate develop branch, work directly with main
|
||||
2. **Increase deployment frequency**: Move from scheduled to continuous releases
|
||||
3. **Strengthen testing**: Improve automated test coverage and CI/CD
|
||||
4. **Reduce branch lifetime**: Limit feature branches to 1-2 weeks maximum
|
||||
5. **Train team**: Educate on simpler workflow and increased responsibility
|
||||
|
||||
#### From GitHub Flow to Trunk-based
|
||||
1. **Implement feature flags**: Add feature toggle infrastructure
|
||||
2. **Improve CI/CD**: Ensure all tests run in <10 minutes
|
||||
3. **Increase commit frequency**: Encourage multiple commits per day
|
||||
4. **Reduce branch usage**: Start committing small changes directly to main
|
||||
5. **Monitor stability**: Ensure main remains deployable at all times
|
||||
|
||||
#### From Trunk-based to Git Flow
|
||||
1. **Add structure**: Introduce develop and release branches
|
||||
2. **Reduce deployment frequency**: Move to scheduled release cycles
|
||||
3. **Extend branch lifetime**: Allow longer feature development cycles
|
||||
4. **Formalize process**: Add approval gates and testing phases
|
||||
5. **Coordinate releases**: Plan features for specific release versions
|
||||
|
||||
## Anti-patterns to Avoid
|
||||
|
||||
### Git Flow Anti-patterns
|
||||
- **Long-lived feature branches** (>2 weeks)
|
||||
- **Skipping release branches** for small releases
|
||||
- **Direct commits to main** bypassing develop
|
||||
- **Forgetting to merge back** to develop after hotfixes
|
||||
- **Complex merge conflicts** from delayed integration
|
||||
|
||||
### GitHub Flow Anti-patterns
|
||||
- **Unstable main branch** due to insufficient testing
|
||||
- **Long-lived feature branches** defeating the purpose
|
||||
- **Skipping pull request reviews** for speed
|
||||
- **Direct production deployment** without staging validation
|
||||
- **No rollback plan** when deployments fail
|
||||
|
||||
### Trunk-based Anti-patterns
|
||||
- **Committing broken code** to main branch
|
||||
- **Feature branches lasting weeks** defeating the philosophy
|
||||
- **No feature flags** for incomplete features
|
||||
- **Insufficient automated testing** leading to instability
|
||||
- **Poor CI/CD pipeline** causing deployment delays
|
||||
|
||||
## Conclusion
|
||||
|
||||
The choice of release workflow significantly impacts your team's productivity, code quality, and deployment reliability. Consider your team size, technical maturity, deployment requirements, and organizational culture when making this decision.
|
||||
|
||||
**Start conservative** (Git Flow) and evolve toward more agile approaches (GitHub Flow, Trunk-based) as your team's skills and infrastructure mature. The key is consistency within your team and alignment with your organization's goals and constraints.
|
||||
|
||||
Remember: **The best workflow is the one your team can execute consistently and reliably**.
|
||||
1003
engineering/release-manager/release_planner.py
Normal file
1003
engineering/release-manager/release_planner.py
Normal file
File diff suppressed because it is too large
Load Diff
645
engineering/release-manager/version_bumper.py
Normal file
645
engineering/release-manager/version_bumper.py
Normal file
@@ -0,0 +1,645 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Version Bumper
|
||||
|
||||
Analyzes commits since last tag to determine the correct version bump (major/minor/patch)
|
||||
based on conventional commits. Handles pre-release versions (alpha, beta, rc) and generates
|
||||
version bump commands for various package files.
|
||||
|
||||
Input: current version + commit list JSON or git log
|
||||
Output: recommended new version + bump commands + updated file snippets
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from enum import Enum
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
class BumpType(Enum):
|
||||
"""Version bump types."""
|
||||
NONE = "none"
|
||||
PATCH = "patch"
|
||||
MINOR = "minor"
|
||||
MAJOR = "major"
|
||||
|
||||
|
||||
class PreReleaseType(Enum):
|
||||
"""Pre-release types."""
|
||||
ALPHA = "alpha"
|
||||
BETA = "beta"
|
||||
RC = "rc"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Version:
|
||||
"""Semantic version representation."""
|
||||
major: int
|
||||
minor: int
|
||||
patch: int
|
||||
prerelease_type: Optional[PreReleaseType] = None
|
||||
prerelease_number: Optional[int] = None
|
||||
|
||||
@classmethod
|
||||
def parse(cls, version_str: str) -> 'Version':
|
||||
"""Parse version string into Version object."""
|
||||
# Remove 'v' prefix if present
|
||||
clean_version = version_str.lstrip('v')
|
||||
|
||||
# Pattern for semantic versioning with optional pre-release
|
||||
pattern = r'^(\d+)\.(\d+)\.(\d+)(?:-(\w+)\.?(\d+)?)?$'
|
||||
match = re.match(pattern, clean_version)
|
||||
|
||||
if not match:
|
||||
raise ValueError(f"Invalid version format: {version_str}")
|
||||
|
||||
major, minor, patch = int(match.group(1)), int(match.group(2)), int(match.group(3))
|
||||
|
||||
prerelease_type = None
|
||||
prerelease_number = None
|
||||
|
||||
if match.group(4): # Pre-release identifier
|
||||
prerelease_str = match.group(4).lower()
|
||||
try:
|
||||
prerelease_type = PreReleaseType(prerelease_str)
|
||||
except ValueError:
|
||||
# Handle variations like 'alpha1' -> 'alpha'
|
||||
if prerelease_str.startswith('alpha'):
|
||||
prerelease_type = PreReleaseType.ALPHA
|
||||
elif prerelease_str.startswith('beta'):
|
||||
prerelease_type = PreReleaseType.BETA
|
||||
elif prerelease_str.startswith('rc'):
|
||||
prerelease_type = PreReleaseType.RC
|
||||
else:
|
||||
raise ValueError(f"Unknown pre-release type: {prerelease_str}")
|
||||
|
||||
if match.group(5):
|
||||
prerelease_number = int(match.group(5))
|
||||
else:
|
||||
# Extract number from combined string like 'alpha1'
|
||||
number_match = re.search(r'(\d+)$', prerelease_str)
|
||||
if number_match:
|
||||
prerelease_number = int(number_match.group(1))
|
||||
else:
|
||||
prerelease_number = 1 # Default to 1
|
||||
|
||||
return cls(major, minor, patch, prerelease_type, prerelease_number)
|
||||
|
||||
def to_string(self, include_v_prefix: bool = False) -> str:
|
||||
"""Convert version to string representation."""
|
||||
base = f"{self.major}.{self.minor}.{self.patch}"
|
||||
|
||||
if self.prerelease_type:
|
||||
if self.prerelease_number is not None:
|
||||
base += f"-{self.prerelease_type.value}.{self.prerelease_number}"
|
||||
else:
|
||||
base += f"-{self.prerelease_type.value}"
|
||||
|
||||
return f"v{base}" if include_v_prefix else base
|
||||
|
||||
def bump(self, bump_type: BumpType, prerelease_type: Optional[PreReleaseType] = None) -> 'Version':
|
||||
"""Create new version with specified bump."""
|
||||
if bump_type == BumpType.NONE:
|
||||
return Version(self.major, self.minor, self.patch, self.prerelease_type, self.prerelease_number)
|
||||
|
||||
new_major = self.major
|
||||
new_minor = self.minor
|
||||
new_patch = self.patch
|
||||
new_prerelease_type = None
|
||||
new_prerelease_number = None
|
||||
|
||||
# Handle pre-release versions
|
||||
if prerelease_type:
|
||||
if bump_type == BumpType.MAJOR:
|
||||
new_major += 1
|
||||
new_minor = 0
|
||||
new_patch = 0
|
||||
elif bump_type == BumpType.MINOR:
|
||||
new_minor += 1
|
||||
new_patch = 0
|
||||
elif bump_type == BumpType.PATCH:
|
||||
new_patch += 1
|
||||
|
||||
new_prerelease_type = prerelease_type
|
||||
new_prerelease_number = 1
|
||||
|
||||
# Handle existing pre-release -> next pre-release
|
||||
elif self.prerelease_type:
|
||||
# If we're already in pre-release, increment or promote
|
||||
if prerelease_type is None:
|
||||
# Promote to stable release
|
||||
# Don't change version numbers, just remove pre-release
|
||||
pass
|
||||
else:
|
||||
# Move to next pre-release type or increment
|
||||
if prerelease_type == self.prerelease_type:
|
||||
# Same pre-release type, increment number
|
||||
new_prerelease_type = self.prerelease_type
|
||||
new_prerelease_number = (self.prerelease_number or 0) + 1
|
||||
else:
|
||||
# Different pre-release type
|
||||
new_prerelease_type = prerelease_type
|
||||
new_prerelease_number = 1
|
||||
|
||||
# Handle stable version bumps
|
||||
else:
|
||||
if bump_type == BumpType.MAJOR:
|
||||
new_major += 1
|
||||
new_minor = 0
|
||||
new_patch = 0
|
||||
elif bump_type == BumpType.MINOR:
|
||||
new_minor += 1
|
||||
new_patch = 0
|
||||
elif bump_type == BumpType.PATCH:
|
||||
new_patch += 1
|
||||
|
||||
return Version(new_major, new_minor, new_patch, new_prerelease_type, new_prerelease_number)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConventionalCommit:
|
||||
"""Represents a parsed conventional commit for version analysis."""
|
||||
type: str
|
||||
scope: str
|
||||
description: str
|
||||
is_breaking: bool
|
||||
breaking_description: str
|
||||
hash: str = ""
|
||||
author: str = ""
|
||||
date: str = ""
|
||||
|
||||
@classmethod
|
||||
def parse_message(cls, message: str, commit_hash: str = "",
|
||||
author: str = "", date: str = "") -> 'ConventionalCommit':
|
||||
"""Parse conventional commit message."""
|
||||
lines = message.split('\n')
|
||||
header = lines[0] if lines else ""
|
||||
|
||||
# Parse header: type(scope): description
|
||||
header_pattern = r'^(\w+)(\([^)]+\))?(!)?:\s*(.+)$'
|
||||
match = re.match(header_pattern, header)
|
||||
|
||||
commit_type = "chore"
|
||||
scope = ""
|
||||
description = header
|
||||
is_breaking = False
|
||||
breaking_description = ""
|
||||
|
||||
if match:
|
||||
commit_type = match.group(1).lower()
|
||||
scope_match = match.group(2)
|
||||
scope = scope_match[1:-1] if scope_match else ""
|
||||
is_breaking = bool(match.group(3)) # ! indicates breaking change
|
||||
description = match.group(4).strip()
|
||||
|
||||
# Check for breaking change in body/footers
|
||||
if len(lines) > 1:
|
||||
body_text = '\n'.join(lines[1:])
|
||||
if 'BREAKING CHANGE:' in body_text:
|
||||
is_breaking = True
|
||||
breaking_match = re.search(r'BREAKING CHANGE:\s*(.+)', body_text)
|
||||
if breaking_match:
|
||||
breaking_description = breaking_match.group(1).strip()
|
||||
|
||||
return cls(commit_type, scope, description, is_breaking, breaking_description,
|
||||
commit_hash, author, date)
|
||||
|
||||
|
||||
class VersionBumper:
|
||||
"""Main version bumping logic."""
|
||||
|
||||
def __init__(self):
|
||||
self.current_version: Optional[Version] = None
|
||||
self.commits: List[ConventionalCommit] = []
|
||||
self.custom_rules: Dict[str, BumpType] = {}
|
||||
self.ignore_types: List[str] = ['test', 'ci', 'build', 'chore', 'docs', 'style']
|
||||
|
||||
def set_current_version(self, version_str: str):
|
||||
"""Set the current version."""
|
||||
self.current_version = Version.parse(version_str)
|
||||
|
||||
def add_custom_rule(self, commit_type: str, bump_type: BumpType):
|
||||
"""Add custom rule for commit type to bump type mapping."""
|
||||
self.custom_rules[commit_type] = bump_type
|
||||
|
||||
def parse_commits_from_json(self, json_data: Union[str, List[Dict]]):
|
||||
"""Parse commits from JSON format."""
|
||||
if isinstance(json_data, str):
|
||||
data = json.loads(json_data)
|
||||
else:
|
||||
data = json_data
|
||||
|
||||
self.commits = []
|
||||
for commit_data in data:
|
||||
commit = ConventionalCommit.parse_message(
|
||||
message=commit_data.get('message', ''),
|
||||
commit_hash=commit_data.get('hash', ''),
|
||||
author=commit_data.get('author', ''),
|
||||
date=commit_data.get('date', '')
|
||||
)
|
||||
self.commits.append(commit)
|
||||
|
||||
def parse_commits_from_git_log(self, git_log_text: str):
|
||||
"""Parse commits from git log output."""
|
||||
lines = git_log_text.strip().split('\n')
|
||||
|
||||
if not lines or not lines[0]:
|
||||
return
|
||||
|
||||
# Simple oneline format (hash message)
|
||||
oneline_pattern = r'^([a-f0-9]{7,40})\s+(.+)$'
|
||||
|
||||
self.commits = []
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
match = re.match(oneline_pattern, line)
|
||||
if match:
|
||||
commit_hash = match.group(1)
|
||||
message = match.group(2)
|
||||
commit = ConventionalCommit.parse_message(message, commit_hash)
|
||||
self.commits.append(commit)
|
||||
|
||||
def determine_bump_type(self) -> BumpType:
|
||||
"""Determine version bump type based on commits."""
|
||||
if not self.commits:
|
||||
return BumpType.NONE
|
||||
|
||||
has_breaking = False
|
||||
has_feature = False
|
||||
has_fix = False
|
||||
|
||||
for commit in self.commits:
|
||||
# Check for breaking changes
|
||||
if commit.is_breaking:
|
||||
has_breaking = True
|
||||
continue
|
||||
|
||||
# Apply custom rules first
|
||||
if commit.type in self.custom_rules:
|
||||
bump_type = self.custom_rules[commit.type]
|
||||
if bump_type == BumpType.MAJOR:
|
||||
has_breaking = True
|
||||
elif bump_type == BumpType.MINOR:
|
||||
has_feature = True
|
||||
elif bump_type == BumpType.PATCH:
|
||||
has_fix = True
|
||||
continue
|
||||
|
||||
# Standard rules
|
||||
if commit.type in ['feat', 'add']:
|
||||
has_feature = True
|
||||
elif commit.type in ['fix', 'security', 'perf', 'bugfix']:
|
||||
has_fix = True
|
||||
# Ignore types in ignore_types list
|
||||
|
||||
# Determine bump type by priority
|
||||
if has_breaking:
|
||||
return BumpType.MAJOR
|
||||
elif has_feature:
|
||||
return BumpType.MINOR
|
||||
elif has_fix:
|
||||
return BumpType.PATCH
|
||||
else:
|
||||
return BumpType.NONE
|
||||
|
||||
def recommend_version(self, prerelease_type: Optional[PreReleaseType] = None) -> Version:
|
||||
"""Recommend new version based on commits."""
|
||||
if not self.current_version:
|
||||
raise ValueError("Current version not set")
|
||||
|
||||
bump_type = self.determine_bump_type()
|
||||
return self.current_version.bump(bump_type, prerelease_type)
|
||||
|
||||
def generate_bump_commands(self, new_version: Version) -> Dict[str, List[str]]:
|
||||
"""Generate version bump commands for different package managers."""
|
||||
version_str = new_version.to_string()
|
||||
version_with_v = new_version.to_string(include_v_prefix=True)
|
||||
|
||||
commands = {
|
||||
'npm': [
|
||||
f"npm version {version_str} --no-git-tag-version",
|
||||
f"# Or manually edit package.json version field to '{version_str}'"
|
||||
],
|
||||
'python': [
|
||||
f"# Update version in setup.py, __init__.py, or pyproject.toml",
|
||||
f"# setup.py: version='{version_str}'",
|
||||
f"# pyproject.toml: version = '{version_str}'",
|
||||
f"# __init__.py: __version__ = '{version_str}'"
|
||||
],
|
||||
'rust': [
|
||||
f"# Update Cargo.toml",
|
||||
f"# [package]",
|
||||
f"# version = '{version_str}'"
|
||||
],
|
||||
'git': [
|
||||
f"git tag -a {version_with_v} -m 'Release {version_with_v}'",
|
||||
f"git push origin {version_with_v}"
|
||||
],
|
||||
'docker': [
|
||||
f"docker build -t myapp:{version_str} .",
|
||||
f"docker tag myapp:{version_str} myapp:latest"
|
||||
]
|
||||
}
|
||||
|
||||
return commands
|
||||
|
||||
def generate_file_updates(self, new_version: Version) -> Dict[str, str]:
|
||||
"""Generate file update snippets for common package files."""
|
||||
version_str = new_version.to_string()
|
||||
|
||||
updates = {}
|
||||
|
||||
# package.json
|
||||
updates['package.json'] = json.dumps({
|
||||
"name": "your-package",
|
||||
"version": version_str,
|
||||
"description": "Your package description",
|
||||
"main": "index.js"
|
||||
}, indent=2)
|
||||
|
||||
# pyproject.toml
|
||||
updates['pyproject.toml'] = f'''[build-system]
|
||||
requires = ["setuptools>=61.0", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "your-package"
|
||||
version = "{version_str}"
|
||||
description = "Your package description"
|
||||
authors = [
|
||||
{{name = "Your Name", email = "your.email@example.com"}},
|
||||
]
|
||||
'''
|
||||
|
||||
# setup.py
|
||||
updates['setup.py'] = f'''from setuptools import setup, find_packages
|
||||
|
||||
setup(
|
||||
name="your-package",
|
||||
version="{version_str}",
|
||||
description="Your package description",
|
||||
packages=find_packages(),
|
||||
python_requires=">=3.8",
|
||||
)
|
||||
'''
|
||||
|
||||
# Cargo.toml
|
||||
updates['Cargo.toml'] = f'''[package]
|
||||
name = "your-package"
|
||||
version = "{version_str}"
|
||||
edition = "2021"
|
||||
description = "Your package description"
|
||||
'''
|
||||
|
||||
# __init__.py
|
||||
updates['__init__.py'] = f'''"""Your package."""
|
||||
|
||||
__version__ = "{version_str}"
|
||||
__author__ = "Your Name"
|
||||
__email__ = "your.email@example.com"
|
||||
'''
|
||||
|
||||
return updates
|
||||
|
||||
def analyze_commits(self) -> Dict:
|
||||
"""Provide detailed analysis of commits for version bumping."""
|
||||
if not self.commits:
|
||||
return {
|
||||
'total_commits': 0,
|
||||
'by_type': {},
|
||||
'breaking_changes': [],
|
||||
'features': [],
|
||||
'fixes': [],
|
||||
'ignored': []
|
||||
}
|
||||
|
||||
analysis = {
|
||||
'total_commits': len(self.commits),
|
||||
'by_type': {},
|
||||
'breaking_changes': [],
|
||||
'features': [],
|
||||
'fixes': [],
|
||||
'ignored': []
|
||||
}
|
||||
|
||||
type_counts = {}
|
||||
for commit in self.commits:
|
||||
type_counts[commit.type] = type_counts.get(commit.type, 0) + 1
|
||||
|
||||
if commit.is_breaking:
|
||||
analysis['breaking_changes'].append({
|
||||
'type': commit.type,
|
||||
'scope': commit.scope,
|
||||
'description': commit.description,
|
||||
'breaking_description': commit.breaking_description,
|
||||
'hash': commit.hash
|
||||
})
|
||||
elif commit.type in ['feat', 'add']:
|
||||
analysis['features'].append({
|
||||
'scope': commit.scope,
|
||||
'description': commit.description,
|
||||
'hash': commit.hash
|
||||
})
|
||||
elif commit.type in ['fix', 'security', 'perf', 'bugfix']:
|
||||
analysis['fixes'].append({
|
||||
'scope': commit.scope,
|
||||
'description': commit.description,
|
||||
'hash': commit.hash
|
||||
})
|
||||
elif commit.type in self.ignore_types:
|
||||
analysis['ignored'].append({
|
||||
'type': commit.type,
|
||||
'scope': commit.scope,
|
||||
'description': commit.description,
|
||||
'hash': commit.hash
|
||||
})
|
||||
|
||||
analysis['by_type'] = type_counts
|
||||
return analysis
|
||||
|
||||
|
||||
def main():
|
||||
"""Main CLI entry point."""
|
||||
parser = argparse.ArgumentParser(description="Determine version bump based on conventional commits")
|
||||
parser.add_argument('--current-version', '-c', required=True,
|
||||
help='Current version (e.g., 1.2.3, v1.2.3)')
|
||||
parser.add_argument('--input', '-i', type=str,
|
||||
help='Input file with commits (default: stdin)')
|
||||
parser.add_argument('--input-format', choices=['git-log', 'json'],
|
||||
default='git-log', help='Input format')
|
||||
parser.add_argument('--prerelease', '-p',
|
||||
choices=['alpha', 'beta', 'rc'],
|
||||
help='Generate pre-release version')
|
||||
parser.add_argument('--output-format', '-f',
|
||||
choices=['text', 'json', 'commands'],
|
||||
default='text', help='Output format')
|
||||
parser.add_argument('--output', '-o', type=str,
|
||||
help='Output file (default: stdout)')
|
||||
parser.add_argument('--include-commands', action='store_true',
|
||||
help='Include bump commands in output')
|
||||
parser.add_argument('--include-files', action='store_true',
|
||||
help='Include file update snippets')
|
||||
parser.add_argument('--custom-rules', type=str,
|
||||
help='JSON string with custom type->bump rules')
|
||||
parser.add_argument('--ignore-types', type=str,
|
||||
help='Comma-separated list of types to ignore')
|
||||
parser.add_argument('--analysis', '-a', action='store_true',
|
||||
help='Include detailed commit analysis')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Read input
|
||||
if args.input:
|
||||
with open(args.input, 'r', encoding='utf-8') as f:
|
||||
input_data = f.read()
|
||||
else:
|
||||
input_data = sys.stdin.read()
|
||||
|
||||
if not input_data.strip():
|
||||
print("No input data provided", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Initialize version bumper
|
||||
bumper = VersionBumper()
|
||||
|
||||
try:
|
||||
bumper.set_current_version(args.current_version)
|
||||
except ValueError as e:
|
||||
print(f"Invalid current version: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Apply custom rules
|
||||
if args.custom_rules:
|
||||
try:
|
||||
custom_rules = json.loads(args.custom_rules)
|
||||
for commit_type, bump_type_str in custom_rules.items():
|
||||
bump_type = BumpType(bump_type_str.lower())
|
||||
bumper.add_custom_rule(commit_type, bump_type)
|
||||
except Exception as e:
|
||||
print(f"Invalid custom rules: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Set ignore types
|
||||
if args.ignore_types:
|
||||
bumper.ignore_types = [t.strip() for t in args.ignore_types.split(',')]
|
||||
|
||||
# Parse commits
|
||||
try:
|
||||
if args.input_format == 'json':
|
||||
bumper.parse_commits_from_json(input_data)
|
||||
else:
|
||||
bumper.parse_commits_from_git_log(input_data)
|
||||
except Exception as e:
|
||||
print(f"Error parsing commits: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Determine pre-release type
|
||||
prerelease_type = None
|
||||
if args.prerelease:
|
||||
prerelease_type = PreReleaseType(args.prerelease)
|
||||
|
||||
# Generate recommendation
|
||||
try:
|
||||
recommended_version = bumper.recommend_version(prerelease_type)
|
||||
bump_type = bumper.determine_bump_type()
|
||||
except Exception as e:
|
||||
print(f"Error determining version: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
# Generate output
|
||||
output_data = {}
|
||||
|
||||
if args.output_format == 'json':
|
||||
output_data = {
|
||||
'current_version': args.current_version,
|
||||
'recommended_version': recommended_version.to_string(),
|
||||
'recommended_version_with_v': recommended_version.to_string(include_v_prefix=True),
|
||||
'bump_type': bump_type.value,
|
||||
'prerelease': args.prerelease
|
||||
}
|
||||
|
||||
if args.analysis:
|
||||
output_data['analysis'] = bumper.analyze_commits()
|
||||
|
||||
if args.include_commands:
|
||||
output_data['commands'] = bumper.generate_bump_commands(recommended_version)
|
||||
|
||||
if args.include_files:
|
||||
output_data['file_updates'] = bumper.generate_file_updates(recommended_version)
|
||||
|
||||
output_text = json.dumps(output_data, indent=2)
|
||||
|
||||
elif args.output_format == 'commands':
|
||||
commands = bumper.generate_bump_commands(recommended_version)
|
||||
output_lines = [
|
||||
f"# Version Bump Commands",
|
||||
f"# Current: {args.current_version}",
|
||||
f"# New: {recommended_version.to_string()}",
|
||||
f"# Bump Type: {bump_type.value}",
|
||||
""
|
||||
]
|
||||
|
||||
for category, cmd_list in commands.items():
|
||||
output_lines.append(f"## {category.upper()}")
|
||||
for cmd in cmd_list:
|
||||
output_lines.append(cmd)
|
||||
output_lines.append("")
|
||||
|
||||
output_text = '\n'.join(output_lines)
|
||||
|
||||
else: # text format
|
||||
output_lines = [
|
||||
f"Current Version: {args.current_version}",
|
||||
f"Recommended Version: {recommended_version.to_string()}",
|
||||
f"With v prefix: {recommended_version.to_string(include_v_prefix=True)}",
|
||||
f"Bump Type: {bump_type.value}",
|
||||
""
|
||||
]
|
||||
|
||||
if args.analysis:
|
||||
analysis = bumper.analyze_commits()
|
||||
output_lines.extend([
|
||||
"Commit Analysis:",
|
||||
f"- Total commits: {analysis['total_commits']}",
|
||||
f"- Breaking changes: {len(analysis['breaking_changes'])}",
|
||||
f"- New features: {len(analysis['features'])}",
|
||||
f"- Bug fixes: {len(analysis['fixes'])}",
|
||||
f"- Ignored commits: {len(analysis['ignored'])}",
|
||||
""
|
||||
])
|
||||
|
||||
if analysis['breaking_changes']:
|
||||
output_lines.append("Breaking Changes:")
|
||||
for change in analysis['breaking_changes']:
|
||||
scope = f"({change['scope']})" if change['scope'] else ""
|
||||
output_lines.append(f" - {change['type']}{scope}: {change['description']}")
|
||||
output_lines.append("")
|
||||
|
||||
if args.include_commands:
|
||||
commands = bumper.generate_bump_commands(recommended_version)
|
||||
output_lines.append("Bump Commands:")
|
||||
for category, cmd_list in commands.items():
|
||||
output_lines.append(f" {category}:")
|
||||
for cmd in cmd_list:
|
||||
if not cmd.startswith('#'):
|
||||
output_lines.append(f" {cmd}")
|
||||
output_lines.append("")
|
||||
|
||||
output_text = '\n'.join(output_lines)
|
||||
|
||||
# Write output
|
||||
if args.output:
|
||||
with open(args.output, 'w', encoding='utf-8') as f:
|
||||
f.write(output_text)
|
||||
else:
|
||||
print(output_text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
229
engineering/skill-tester/README.md
Normal file
229
engineering/skill-tester/README.md
Normal file
@@ -0,0 +1,229 @@
|
||||
# Skill Tester - Quality Assurance Meta-Skill
|
||||
|
||||
A POWERFUL-tier skill that provides comprehensive validation, testing, and quality scoring for skills in the claude-skills ecosystem.
|
||||
|
||||
## Overview
|
||||
|
||||
The Skill Tester is a meta-skill that ensures quality and consistency across all skills in the repository through:
|
||||
|
||||
- **Structure Validation** - Verifies directory structure, file presence, and documentation standards
|
||||
- **Script Testing** - Tests Python scripts for syntax, functionality, and compliance
|
||||
- **Quality Scoring** - Provides comprehensive quality assessment across multiple dimensions
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Validate a Skill
|
||||
```bash
|
||||
# Basic validation
|
||||
python scripts/skill_validator.py engineering/my-skill
|
||||
|
||||
# Validate against specific tier
|
||||
python scripts/skill_validator.py engineering/my-skill --tier POWERFUL --json
|
||||
```
|
||||
|
||||
### Test Scripts
|
||||
```bash
|
||||
# Test all scripts in a skill
|
||||
python scripts/script_tester.py engineering/my-skill
|
||||
|
||||
# Test with custom timeout
|
||||
python scripts/script_tester.py engineering/my-skill --timeout 60 --json
|
||||
```
|
||||
|
||||
### Score Quality
|
||||
```bash
|
||||
# Get quality assessment
|
||||
python scripts/quality_scorer.py engineering/my-skill
|
||||
|
||||
# Detailed scoring with improvement suggestions
|
||||
python scripts/quality_scorer.py engineering/my-skill --detailed --json
|
||||
```
|
||||
|
||||
## Components
|
||||
|
||||
### Scripts
|
||||
- **skill_validator.py** (700+ LOC) - Validates skill structure and compliance
|
||||
- **script_tester.py** (800+ LOC) - Tests script functionality and quality
|
||||
- **quality_scorer.py** (1100+ LOC) - Multi-dimensional quality assessment
|
||||
|
||||
### Reference Documentation
|
||||
- **skill-structure-specification.md** - Complete structural requirements
|
||||
- **tier-requirements-matrix.md** - Tier-specific quality standards
|
||||
- **quality-scoring-rubric.md** - Detailed scoring methodology
|
||||
|
||||
### Sample Assets
|
||||
- **sample-skill/** - Complete sample skill for testing the tester itself
|
||||
|
||||
## Features
|
||||
|
||||
### Validation Capabilities
|
||||
- SKILL.md format and content validation
|
||||
- Directory structure compliance checking
|
||||
- Python script syntax and import validation
|
||||
- Argparse implementation verification
|
||||
- Tier-specific requirement enforcement
|
||||
|
||||
### Testing Framework
|
||||
- Syntax validation using AST parsing
|
||||
- Import analysis for external dependencies
|
||||
- Runtime execution testing with timeout protection
|
||||
- Help functionality verification
|
||||
- Sample data processing validation
|
||||
- Output format compliance checking
|
||||
|
||||
### Quality Assessment
|
||||
- Documentation quality scoring (25%)
|
||||
- Code quality evaluation (25%)
|
||||
- Completeness assessment (25%)
|
||||
- Usability analysis (25%)
|
||||
- Letter grade assignment (A+ to F)
|
||||
- Tier recommendation generation
|
||||
- Improvement roadmap creation
|
||||
|
||||
## CI/CD Integration
|
||||
|
||||
### GitHub Actions Example
|
||||
```yaml
|
||||
name: Skill Quality Gate
|
||||
on:
|
||||
pull_request:
|
||||
paths: ['engineering/**']
|
||||
|
||||
jobs:
|
||||
validate-skills:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Validate Skills
|
||||
run: |
|
||||
for skill in $(git diff --name-only ${{ github.event.before }} | grep -E '^engineering/[^/]+/' | cut -d'/' -f1-2 | sort -u); do
|
||||
python engineering/skill-tester/scripts/skill_validator.py $skill --json
|
||||
python engineering/skill-tester/scripts/script_tester.py $skill
|
||||
python engineering/skill-tester/scripts/quality_scorer.py $skill --minimum-score 75
|
||||
done
|
||||
```
|
||||
|
||||
### Pre-commit Hook
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# .git/hooks/pre-commit
|
||||
python engineering/skill-tester/scripts/skill_validator.py engineering/my-skill --tier STANDARD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Skill validation failed. Commit blocked."
|
||||
exit 1
|
||||
fi
|
||||
```
|
||||
|
||||
## Quality Standards
|
||||
|
||||
### All Scripts
|
||||
- **Zero External Dependencies** - Python standard library only
|
||||
- **Comprehensive Error Handling** - Meaningful error messages and recovery
|
||||
- **Dual Output Support** - Both JSON and human-readable formats
|
||||
- **Proper Documentation** - Comprehensive docstrings and comments
|
||||
- **CLI Best Practices** - Full argparse implementation with help text
|
||||
|
||||
### Validation Accuracy
|
||||
- **Structure Checks** - 100% accurate directory and file validation
|
||||
- **Content Analysis** - Deep parsing of SKILL.md and documentation
|
||||
- **Code Analysis** - AST-based Python code validation
|
||||
- **Compliance Scoring** - Objective, repeatable quality assessment
|
||||
|
||||
## Self-Testing
|
||||
|
||||
The skill-tester can validate itself:
|
||||
|
||||
```bash
|
||||
# Validate the skill-tester structure
|
||||
python scripts/skill_validator.py . --tier POWERFUL
|
||||
|
||||
# Test the skill-tester scripts
|
||||
python scripts/script_tester.py .
|
||||
|
||||
# Score the skill-tester quality
|
||||
python scripts/quality_scorer.py . --detailed
|
||||
```
|
||||
|
||||
## Advanced Usage
|
||||
|
||||
### Batch Validation
|
||||
```bash
|
||||
# Validate all skills in repository
|
||||
find engineering/ -maxdepth 1 -type d | while read skill; do
|
||||
echo "Validating $skill..."
|
||||
python engineering/skill-tester/scripts/skill_validator.py "$skill"
|
||||
done
|
||||
```
|
||||
|
||||
### Quality Monitoring
|
||||
```bash
|
||||
# Generate quality report for all skills
|
||||
python engineering/skill-tester/scripts/quality_scorer.py engineering/ \
|
||||
--batch --json > quality_report.json
|
||||
```
|
||||
|
||||
### Custom Scoring Thresholds
|
||||
```bash
|
||||
# Enforce minimum quality scores
|
||||
python scripts/quality_scorer.py engineering/my-skill --minimum-score 80
|
||||
# Exit code 0 = passed, 1 = failed, 2 = needs improvement
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
All scripts provide comprehensive error handling:
|
||||
- **File System Errors** - Missing files, permission issues, invalid paths
|
||||
- **Content Errors** - Malformed YAML, invalid JSON, encoding issues
|
||||
- **Execution Errors** - Script timeouts, runtime failures, import errors
|
||||
- **Validation Errors** - Standards violations, compliance failures
|
||||
|
||||
## Output Formats
|
||||
|
||||
### Human-Readable
|
||||
```
|
||||
=== SKILL VALIDATION REPORT ===
|
||||
Skill: engineering/my-skill
|
||||
Overall Score: 85.2/100 (B+)
|
||||
Tier Recommendation: STANDARD
|
||||
|
||||
STRUCTURE VALIDATION:
|
||||
✓ PASS: SKILL.md found
|
||||
✓ PASS: README.md found
|
||||
✓ PASS: scripts/ directory found
|
||||
|
||||
SUGGESTIONS:
|
||||
• Add references/ directory
|
||||
• Improve error handling in main.py
|
||||
```
|
||||
|
||||
### JSON Format
|
||||
```json
|
||||
{
|
||||
"skill_path": "engineering/my-skill",
|
||||
"overall_score": 85.2,
|
||||
"letter_grade": "B+",
|
||||
"tier_recommendation": "STANDARD",
|
||||
"dimensions": {
|
||||
"Documentation": {"score": 88.5, "weight": 0.25},
|
||||
"Code Quality": {"score": 82.0, "weight": 0.25},
|
||||
"Completeness": {"score": 85.5, "weight": 0.25},
|
||||
"Usability": {"score": 84.8, "weight": 0.25}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Requirements
|
||||
|
||||
- **Python 3.7+** - No external dependencies required
|
||||
- **File System Access** - Read access to skill directories
|
||||
- **Execution Permissions** - Ability to run Python scripts for testing
|
||||
|
||||
## Contributing
|
||||
|
||||
See [SKILL.md](SKILL.md) for comprehensive documentation and contribution guidelines.
|
||||
|
||||
The skill-tester itself serves as a reference implementation of POWERFUL-tier quality standards.
|
||||
385
engineering/skill-tester/SKILL.md
Normal file
385
engineering/skill-tester/SKILL.md
Normal file
@@ -0,0 +1,385 @@
|
||||
# Skill Tester
|
||||
|
||||
---
|
||||
|
||||
**Name**: skill-tester
|
||||
**Tier**: POWERFUL
|
||||
**Category**: Engineering Quality Assurance
|
||||
**Dependencies**: None (Python Standard Library Only)
|
||||
**Author**: Claude Skills Engineering Team
|
||||
**Version**: 1.0.0
|
||||
**Last Updated**: 2026-02-16
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
|
||||
The Skill Tester is a comprehensive meta-skill designed to validate, test, and score the quality of skills within the claude-skills ecosystem. This powerful quality assurance tool ensures that all skills meet the rigorous standards required for BASIC, STANDARD, and POWERFUL tier classifications through automated validation, testing, and scoring mechanisms.
|
||||
|
||||
As the gatekeeping system for skill quality, this meta-skill provides three core capabilities:
|
||||
1. **Structure Validation** - Ensures skills conform to required directory structures, file formats, and documentation standards
|
||||
2. **Script Testing** - Validates Python scripts for syntax, imports, functionality, and output format compliance
|
||||
3. **Quality Scoring** - Provides comprehensive quality assessment across multiple dimensions with letter grades and improvement recommendations
|
||||
|
||||
This skill is essential for maintaining ecosystem consistency, enabling automated CI/CD integration, and supporting both manual and automated quality assurance workflows. It serves as the foundation for pre-commit hooks, pull request validation, and continuous integration processes that maintain the high-quality standards of the claude-skills repository.
|
||||
|
||||
## Core Features
|
||||
|
||||
### Comprehensive Skill Validation
|
||||
- **Structure Compliance**: Validates directory structure, required files (SKILL.md, README.md, scripts/, references/, assets/, expected_outputs/)
|
||||
- **Documentation Standards**: Checks SKILL.md frontmatter, section completeness, minimum line counts per tier
|
||||
- **File Format Validation**: Ensures proper Markdown formatting, YAML frontmatter syntax, and file naming conventions
|
||||
|
||||
### Advanced Script Testing
|
||||
- **Syntax Validation**: Compiles Python scripts to detect syntax errors before execution
|
||||
- **Import Analysis**: Enforces standard library only policy, identifies external dependencies
|
||||
- **Runtime Testing**: Executes scripts with sample data, validates argparse implementation, tests --help functionality
|
||||
- **Output Format Compliance**: Verifies dual output support (JSON + human-readable), proper error handling
|
||||
|
||||
### Multi-Dimensional Quality Scoring
|
||||
- **Documentation Quality (25%)**: SKILL.md depth and completeness, README clarity, reference documentation quality
|
||||
- **Code Quality (25%)**: Script complexity, error handling robustness, output format consistency, maintainability
|
||||
- **Completeness (25%)**: Required directory presence, sample data adequacy, expected output verification
|
||||
- **Usability (25%)**: Example clarity, argparse help text quality, installation simplicity, user experience
|
||||
|
||||
### Tier Classification System
|
||||
Automatically classifies skills based on complexity and functionality:
|
||||
|
||||
#### BASIC Tier Requirements
|
||||
- Minimum 100 lines in SKILL.md
|
||||
- At least 1 Python script (100-300 LOC)
|
||||
- Basic argparse implementation
|
||||
- Simple input/output handling
|
||||
- Essential documentation coverage
|
||||
|
||||
#### STANDARD Tier Requirements
|
||||
- Minimum 200 lines in SKILL.md
|
||||
- 1-2 Python scripts (300-500 LOC each)
|
||||
- Advanced argparse with subcommands
|
||||
- JSON + text output formats
|
||||
- Comprehensive examples and references
|
||||
- Error handling and edge case management
|
||||
|
||||
#### POWERFUL Tier Requirements
|
||||
- Minimum 300 lines in SKILL.md
|
||||
- 2-3 Python scripts (500-800 LOC each)
|
||||
- Complex argparse with multiple modes
|
||||
- Sophisticated output formatting and validation
|
||||
- Extensive documentation and reference materials
|
||||
- Advanced error handling and recovery mechanisms
|
||||
- CI/CD integration capabilities
|
||||
|
||||
## Architecture & Design
|
||||
|
||||
### Modular Design Philosophy
|
||||
The skill-tester follows a modular architecture where each component serves a specific validation purpose:
|
||||
|
||||
- **skill_validator.py**: Core structural and documentation validation engine
|
||||
- **script_tester.py**: Runtime testing and execution validation framework
|
||||
- **quality_scorer.py**: Multi-dimensional quality assessment and scoring system
|
||||
|
||||
### Standards Enforcement
|
||||
All validation is performed against well-defined standards documented in the references/ directory:
|
||||
- **Skill Structure Specification**: Defines mandatory and optional components
|
||||
- **Tier Requirements Matrix**: Detailed requirements for each skill tier
|
||||
- **Quality Scoring Rubric**: Comprehensive scoring methodology and weightings
|
||||
|
||||
### Integration Capabilities
|
||||
Designed for seamless integration into existing development workflows:
|
||||
- **Pre-commit Hooks**: Prevents substandard skills from being committed
|
||||
- **CI/CD Pipelines**: Automated quality gates in pull request workflows
|
||||
- **Manual Validation**: Interactive command-line tools for development-time validation
|
||||
- **Batch Processing**: Bulk validation and scoring of existing skill repositories
|
||||
|
||||
## Implementation Details
|
||||
|
||||
### skill_validator.py Core Functions
|
||||
```python
|
||||
# Primary validation workflow
|
||||
validate_skill_structure() -> ValidationReport
|
||||
check_skill_md_compliance() -> DocumentationReport
|
||||
validate_python_scripts() -> ScriptReport
|
||||
generate_compliance_score() -> float
|
||||
```
|
||||
|
||||
Key validation checks include:
|
||||
- SKILL.md frontmatter parsing and validation
|
||||
- Required section presence (Description, Features, Usage, etc.)
|
||||
- Minimum line count enforcement per tier
|
||||
- Python script argparse implementation verification
|
||||
- Standard library import enforcement
|
||||
- Directory structure compliance
|
||||
- README.md quality assessment
|
||||
|
||||
### script_tester.py Testing Framework
|
||||
```python
|
||||
# Core testing functions
|
||||
syntax_validation() -> SyntaxReport
|
||||
import_validation() -> ImportReport
|
||||
runtime_testing() -> RuntimeReport
|
||||
output_format_validation() -> OutputReport
|
||||
```
|
||||
|
||||
Testing capabilities encompass:
|
||||
- Python AST-based syntax validation
|
||||
- Import statement analysis and external dependency detection
|
||||
- Controlled script execution with timeout protection
|
||||
- Argparse --help functionality verification
|
||||
- Sample data processing and output validation
|
||||
- Expected output comparison and difference reporting
|
||||
|
||||
### quality_scorer.py Scoring System
|
||||
```python
|
||||
# Multi-dimensional scoring
|
||||
score_documentation() -> float # 25% weight
|
||||
score_code_quality() -> float # 25% weight
|
||||
score_completeness() -> float # 25% weight
|
||||
score_usability() -> float # 25% weight
|
||||
calculate_overall_grade() -> str # A-F grade
|
||||
```
|
||||
|
||||
Scoring dimensions include:
|
||||
- **Documentation**: Completeness, clarity, examples, reference quality
|
||||
- **Code Quality**: Complexity, maintainability, error handling, output consistency
|
||||
- **Completeness**: Required files, sample data, expected outputs, test coverage
|
||||
- **Usability**: Help text quality, example clarity, installation simplicity
|
||||
|
||||
## Usage Scenarios
|
||||
|
||||
### Development Workflow Integration
|
||||
```bash
|
||||
# Pre-commit hook validation
|
||||
skill_validator.py path/to/skill --tier POWERFUL --json
|
||||
|
||||
# Comprehensive skill testing
|
||||
script_tester.py path/to/skill --timeout 30 --sample-data
|
||||
|
||||
# Quality assessment and scoring
|
||||
quality_scorer.py path/to/skill --detailed --recommendations
|
||||
```
|
||||
|
||||
### CI/CD Pipeline Integration
|
||||
```yaml
|
||||
# GitHub Actions workflow example
|
||||
- name: Validate Skill Quality
|
||||
run: |
|
||||
python skill_validator.py engineering/${{ matrix.skill }} --json | tee validation.json
|
||||
python script_tester.py engineering/${{ matrix.skill }} | tee testing.json
|
||||
python quality_scorer.py engineering/${{ matrix.skill }} --json | tee scoring.json
|
||||
```
|
||||
|
||||
### Batch Repository Analysis
|
||||
```bash
|
||||
# Validate all skills in repository
|
||||
find engineering/ -type d -maxdepth 1 | xargs -I {} skill_validator.py {}
|
||||
|
||||
# Generate repository quality report
|
||||
quality_scorer.py engineering/ --batch --output-format json > repo_quality.json
|
||||
```
|
||||
|
||||
## Output Formats & Reporting
|
||||
|
||||
### Dual Output Support
|
||||
All tools provide both human-readable and machine-parseable output:
|
||||
|
||||
#### Human-Readable Format
|
||||
```
|
||||
=== SKILL VALIDATION REPORT ===
|
||||
Skill: engineering/example-skill
|
||||
Tier: STANDARD
|
||||
Overall Score: 85/100 (B)
|
||||
|
||||
Structure Validation: ✓ PASS
|
||||
├─ SKILL.md: ✓ EXISTS (247 lines)
|
||||
├─ README.md: ✓ EXISTS
|
||||
├─ scripts/: ✓ EXISTS (2 files)
|
||||
└─ references/: ⚠ MISSING (recommended)
|
||||
|
||||
Documentation Quality: 22/25 (88%)
|
||||
Code Quality: 20/25 (80%)
|
||||
Completeness: 18/25 (72%)
|
||||
Usability: 21/25 (84%)
|
||||
|
||||
Recommendations:
|
||||
• Add references/ directory with documentation
|
||||
• Improve error handling in main.py
|
||||
• Include more comprehensive examples
|
||||
```
|
||||
|
||||
#### JSON Format
|
||||
```json
|
||||
{
|
||||
"skill_path": "engineering/example-skill",
|
||||
"timestamp": "2026-02-16T16:41:00Z",
|
||||
"validation_results": {
|
||||
"structure_compliance": {
|
||||
"score": 0.95,
|
||||
"checks": {
|
||||
"skill_md_exists": true,
|
||||
"readme_exists": true,
|
||||
"scripts_directory": true,
|
||||
"references_directory": false
|
||||
}
|
||||
},
|
||||
"overall_score": 85,
|
||||
"letter_grade": "B",
|
||||
"tier_recommendation": "STANDARD",
|
||||
"improvement_suggestions": [
|
||||
"Add references/ directory",
|
||||
"Improve error handling",
|
||||
"Include comprehensive examples"
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Quality Assurance Standards
|
||||
|
||||
### Code Quality Requirements
|
||||
- **Standard Library Only**: No external dependencies (pip packages)
|
||||
- **Error Handling**: Comprehensive exception handling with meaningful error messages
|
||||
- **Output Consistency**: Standardized JSON schema and human-readable formatting
|
||||
- **Performance**: Efficient validation algorithms with reasonable execution time
|
||||
- **Maintainability**: Clear code structure, comprehensive docstrings, type hints where appropriate
|
||||
|
||||
### Testing Standards
|
||||
- **Self-Testing**: The skill-tester validates itself (meta-validation)
|
||||
- **Sample Data Coverage**: Comprehensive test cases covering edge cases and error conditions
|
||||
- **Expected Output Verification**: All sample runs produce verifiable, reproducible outputs
|
||||
- **Timeout Protection**: Safe execution of potentially problematic scripts with timeout limits
|
||||
|
||||
### Documentation Standards
|
||||
- **Comprehensive Coverage**: All functions, classes, and modules documented
|
||||
- **Usage Examples**: Clear, practical examples for all use cases
|
||||
- **Integration Guides**: Step-by-step CI/CD and workflow integration instructions
|
||||
- **Reference Materials**: Complete specification documents for standards and requirements
|
||||
|
||||
## Integration Examples
|
||||
|
||||
### Pre-Commit Hook Setup
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# .git/hooks/pre-commit
|
||||
echo "Running skill validation..."
|
||||
python engineering/skill-tester/scripts/skill_validator.py engineering/new-skill --tier STANDARD
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Skill validation failed. Commit blocked."
|
||||
exit 1
|
||||
fi
|
||||
echo "Validation passed. Proceeding with commit."
|
||||
```
|
||||
|
||||
### GitHub Actions Workflow
|
||||
```yaml
|
||||
name: Skill Quality Gate
|
||||
on:
|
||||
pull_request:
|
||||
paths: ['engineering/**']
|
||||
|
||||
jobs:
|
||||
validate-skills:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Setup Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.11'
|
||||
- name: Validate Changed Skills
|
||||
run: |
|
||||
changed_skills=$(git diff --name-only ${{ github.event.before }} | grep -E '^engineering/[^/]+/' | cut -d'/' -f1-2 | sort -u)
|
||||
for skill in $changed_skills; do
|
||||
echo "Validating $skill..."
|
||||
python engineering/skill-tester/scripts/skill_validator.py $skill --json
|
||||
python engineering/skill-tester/scripts/script_tester.py $skill
|
||||
python engineering/skill-tester/scripts/quality_scorer.py $skill --minimum-score 75
|
||||
done
|
||||
```
|
||||
|
||||
### Continuous Quality Monitoring
|
||||
```bash
|
||||
#!/bin/bash
|
||||
# Daily quality report generation
|
||||
echo "Generating daily skill quality report..."
|
||||
timestamp=$(date +"%Y-%m-%d")
|
||||
python engineering/skill-tester/scripts/quality_scorer.py engineering/ \
|
||||
--batch --json > "reports/quality_report_${timestamp}.json"
|
||||
|
||||
echo "Quality trends analysis..."
|
||||
python engineering/skill-tester/scripts/trend_analyzer.py reports/ \
|
||||
--days 30 > "reports/quality_trends_${timestamp}.md"
|
||||
```
|
||||
|
||||
## Performance & Scalability
|
||||
|
||||
### Execution Performance
|
||||
- **Fast Validation**: Structure validation completes in <1 second per skill
|
||||
- **Efficient Testing**: Script testing with timeout protection (configurable, default 30s)
|
||||
- **Batch Processing**: Optimized for repository-wide analysis with parallel processing support
|
||||
- **Memory Efficiency**: Minimal memory footprint for large-scale repository analysis
|
||||
|
||||
### Scalability Considerations
|
||||
- **Repository Size**: Designed to handle repositories with 100+ skills
|
||||
- **Concurrent Execution**: Thread-safe implementation supports parallel validation
|
||||
- **Resource Management**: Automatic cleanup of temporary files and subprocess resources
|
||||
- **Configuration Flexibility**: Configurable timeouts, memory limits, and validation strictness
|
||||
|
||||
## Security & Safety
|
||||
|
||||
### Safe Execution Environment
|
||||
- **Sandboxed Testing**: Scripts execute in controlled environment with timeout protection
|
||||
- **Resource Limits**: Memory and CPU usage monitoring to prevent resource exhaustion
|
||||
- **Input Validation**: All inputs sanitized and validated before processing
|
||||
- **No Network Access**: Offline operation ensures no external dependencies or network calls
|
||||
|
||||
### Security Best Practices
|
||||
- **No Code Injection**: Static analysis only, no dynamic code generation
|
||||
- **Path Traversal Protection**: Secure file system access with path validation
|
||||
- **Minimal Privileges**: Operates with minimal required file system permissions
|
||||
- **Audit Logging**: Comprehensive logging for security monitoring and troubleshooting
|
||||
|
||||
## Troubleshooting & Support
|
||||
|
||||
### Common Issues & Solutions
|
||||
|
||||
#### Validation Failures
|
||||
- **Missing Files**: Check directory structure against tier requirements
|
||||
- **Import Errors**: Ensure only standard library imports are used
|
||||
- **Documentation Issues**: Verify SKILL.md frontmatter and section completeness
|
||||
|
||||
#### Script Testing Problems
|
||||
- **Timeout Errors**: Increase timeout limit or optimize script performance
|
||||
- **Execution Failures**: Check script syntax and import statement validity
|
||||
- **Output Format Issues**: Ensure proper JSON formatting and dual output support
|
||||
|
||||
#### Quality Scoring Discrepancies
|
||||
- **Low Scores**: Review scoring rubric and improvement recommendations
|
||||
- **Tier Misclassification**: Verify skill complexity against tier requirements
|
||||
- **Inconsistent Results**: Check for recent changes in quality standards or scoring weights
|
||||
|
||||
### Debugging Support
|
||||
- **Verbose Mode**: Detailed logging and execution tracing available
|
||||
- **Dry Run Mode**: Validation without execution for debugging purposes
|
||||
- **Debug Output**: Comprehensive error reporting with file locations and suggestions
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
### Planned Features
|
||||
- **Machine Learning Quality Prediction**: AI-powered quality assessment using historical data
|
||||
- **Performance Benchmarking**: Execution time and resource usage tracking across skills
|
||||
- **Dependency Analysis**: Automated detection and validation of skill interdependencies
|
||||
- **Quality Trend Analysis**: Historical quality tracking and regression detection
|
||||
|
||||
### Integration Roadmap
|
||||
- **IDE Plugins**: Real-time validation in popular development environments
|
||||
- **Web Dashboard**: Centralized quality monitoring and reporting interface
|
||||
- **API Endpoints**: RESTful API for external integration and automation
|
||||
- **Notification Systems**: Automated alerts for quality degradation or validation failures
|
||||
|
||||
## Conclusion
|
||||
|
||||
The Skill Tester represents a critical infrastructure component for maintaining the high-quality standards of the claude-skills ecosystem. By providing comprehensive validation, testing, and scoring capabilities, it ensures that all skills meet or exceed the rigorous requirements for their respective tiers.
|
||||
|
||||
This meta-skill not only serves as a quality gate but also as a development tool that guides skill authors toward best practices and helps maintain consistency across the entire repository. Through its integration capabilities and comprehensive reporting, it enables both manual and automated quality assurance workflows that scale with the growing claude-skills ecosystem.
|
||||
|
||||
The combination of structural validation, runtime testing, and multi-dimensional quality scoring provides unparalleled visibility into skill quality while maintaining the flexibility needed for diverse skill types and complexity levels. As the claude-skills repository continues to grow, the Skill Tester will remain the cornerstone of quality assurance and ecosystem integrity.
|
||||
40
engineering/skill-tester/assets/sample-skill/README.md
Normal file
40
engineering/skill-tester/assets/sample-skill/README.md
Normal file
@@ -0,0 +1,40 @@
|
||||
# Sample Text Processor
|
||||
|
||||
A basic text processing skill that demonstrates BASIC tier requirements for the claude-skills ecosystem.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Analyze a text file
|
||||
python scripts/text_processor.py analyze sample.txt
|
||||
|
||||
# Get JSON output
|
||||
python scripts/text_processor.py analyze sample.txt --format json
|
||||
|
||||
# Transform text to uppercase
|
||||
python scripts/text_processor.py transform sample.txt --mode upper
|
||||
|
||||
# Process multiple files
|
||||
python scripts/text_processor.py batch text_files/ --verbose
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
- Word count and text statistics
|
||||
- Text transformations (upper, lower, title, reverse)
|
||||
- Batch file processing
|
||||
- JSON and human-readable output formats
|
||||
- Comprehensive error handling
|
||||
|
||||
## Requirements
|
||||
|
||||
- Python 3.7 or later
|
||||
- No external dependencies (standard library only)
|
||||
|
||||
## Usage
|
||||
|
||||
See [SKILL.md](SKILL.md) for comprehensive documentation and examples.
|
||||
|
||||
## Testing
|
||||
|
||||
Sample data files are provided in the `assets/` directory for testing the functionality.
|
||||
163
engineering/skill-tester/assets/sample-skill/SKILL.md
Normal file
163
engineering/skill-tester/assets/sample-skill/SKILL.md
Normal file
@@ -0,0 +1,163 @@
|
||||
# Sample Text Processor
|
||||
|
||||
---
|
||||
|
||||
**Name**: sample-text-processor
|
||||
**Tier**: BASIC
|
||||
**Category**: Text Processing
|
||||
**Dependencies**: None (Python Standard Library Only)
|
||||
**Author**: Claude Skills Engineering Team
|
||||
**Version**: 1.0.0
|
||||
**Last Updated**: 2026-02-16
|
||||
|
||||
---
|
||||
|
||||
## Description
|
||||
|
||||
The Sample Text Processor is a simple skill designed to demonstrate the basic structure and functionality expected in the claude-skills ecosystem. This skill provides fundamental text processing capabilities including word counting, character analysis, and basic text transformations.
|
||||
|
||||
This skill serves as a reference implementation for BASIC tier requirements and can be used as a template for creating new skills. It demonstrates proper file structure, documentation standards, and implementation patterns that align with ecosystem best practices.
|
||||
|
||||
The skill processes text files and provides statistics and transformations in both human-readable and JSON formats, showcasing the dual output requirement for skills in the claude-skills repository.
|
||||
|
||||
## Features
|
||||
|
||||
### Core Functionality
|
||||
- **Word Count Analysis**: Count total words, unique words, and word frequency
|
||||
- **Character Statistics**: Analyze character count, line count, and special characters
|
||||
- **Text Transformations**: Convert text to uppercase, lowercase, or title case
|
||||
- **File Processing**: Process single text files or batch process directories
|
||||
- **Dual Output Formats**: Generate results in both JSON and human-readable formats
|
||||
|
||||
### Technical Features
|
||||
- Command-line interface with comprehensive argument parsing
|
||||
- Error handling for common file and processing issues
|
||||
- Progress reporting for batch operations
|
||||
- Configurable output formatting and verbosity levels
|
||||
- Cross-platform compatibility with standard library only dependencies
|
||||
|
||||
## Usage
|
||||
|
||||
### Basic Text Analysis
|
||||
```bash
|
||||
python text_processor.py analyze document.txt
|
||||
python text_processor.py analyze document.txt --output results.json
|
||||
```
|
||||
|
||||
### Text Transformation
|
||||
```bash
|
||||
python text_processor.py transform document.txt --mode uppercase
|
||||
python text_processor.py transform document.txt --mode title --output transformed.txt
|
||||
```
|
||||
|
||||
### Batch Processing
|
||||
```bash
|
||||
python text_processor.py batch text_files/ --output results/
|
||||
python text_processor.py batch text_files/ --format json --output batch_results.json
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
### Example 1: Basic Word Count
|
||||
```bash
|
||||
$ python text_processor.py analyze sample.txt
|
||||
=== TEXT ANALYSIS RESULTS ===
|
||||
File: sample.txt
|
||||
Total words: 150
|
||||
Unique words: 85
|
||||
Total characters: 750
|
||||
Lines: 12
|
||||
Most frequent word: "the" (8 occurrences)
|
||||
```
|
||||
|
||||
### Example 2: JSON Output
|
||||
```bash
|
||||
$ python text_processor.py analyze sample.txt --format json
|
||||
{
|
||||
"file": "sample.txt",
|
||||
"statistics": {
|
||||
"total_words": 150,
|
||||
"unique_words": 85,
|
||||
"total_characters": 750,
|
||||
"lines": 12,
|
||||
"most_frequent": {
|
||||
"word": "the",
|
||||
"count": 8
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Example 3: Text Transformation
|
||||
```bash
|
||||
$ python text_processor.py transform sample.txt --mode title
|
||||
Original: "hello world from the text processor"
|
||||
Transformed: "Hello World From The Text Processor"
|
||||
```
|
||||
|
||||
## Installation
|
||||
|
||||
This skill requires only Python 3.7 or later with the standard library. No external dependencies are required.
|
||||
|
||||
1. Clone or download the skill directory
|
||||
2. Navigate to the scripts directory
|
||||
3. Run the text processor directly with Python
|
||||
|
||||
```bash
|
||||
cd scripts/
|
||||
python text_processor.py --help
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The text processor supports various configuration options through command-line arguments:
|
||||
|
||||
- `--format`: Output format (json, text)
|
||||
- `--verbose`: Enable verbose output and progress reporting
|
||||
- `--output`: Specify output file or directory
|
||||
- `--encoding`: Specify text file encoding (default: utf-8)
|
||||
|
||||
## Architecture
|
||||
|
||||
The skill follows a simple modular architecture:
|
||||
|
||||
- **TextProcessor Class**: Core processing logic and statistics calculation
|
||||
- **OutputFormatter Class**: Handles dual output format generation
|
||||
- **FileManager Class**: Manages file I/O operations and batch processing
|
||||
- **CLI Interface**: Command-line argument parsing and user interaction
|
||||
|
||||
## Error Handling
|
||||
|
||||
The skill includes comprehensive error handling for:
|
||||
- File not found or permission errors
|
||||
- Invalid encoding or corrupted text files
|
||||
- Memory limitations for very large files
|
||||
- Output directory creation and write permissions
|
||||
- Invalid command-line arguments and parameters
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
- Efficient memory usage for large text files through streaming
|
||||
- Optimized word counting using dictionary lookups
|
||||
- Batch processing with progress reporting for large datasets
|
||||
- Configurable encoding detection for international text
|
||||
|
||||
## Contributing
|
||||
|
||||
This skill serves as a reference implementation and contributions are welcome to demonstrate best practices:
|
||||
|
||||
1. Follow PEP 8 coding standards
|
||||
2. Include comprehensive docstrings
|
||||
3. Add test cases with sample data
|
||||
4. Update documentation for any new features
|
||||
5. Ensure backward compatibility
|
||||
|
||||
## Limitations
|
||||
|
||||
As a BASIC tier skill, some advanced features are intentionally omitted:
|
||||
- Complex text analysis (sentiment, language detection)
|
||||
- Advanced file format support (PDF, Word documents)
|
||||
- Database integration or external API calls
|
||||
- Parallel processing for very large datasets
|
||||
|
||||
This skill demonstrates the essential structure and quality standards required for BASIC tier skills in the claude-skills ecosystem while remaining simple and focused on core functionality.
|
||||
@@ -0,0 +1,23 @@
|
||||
This is a sample text file for testing the text processor skill.
|
||||
It contains multiple lines of text with various words and punctuation.
|
||||
The quick brown fox jumps over the lazy dog.
|
||||
This sentence contains all 26 letters of the English alphabet.
|
||||
|
||||
Some additional content:
|
||||
- Numbers: 123, 456, 789
|
||||
- Special characters: !@#$%^&*()
|
||||
- Mixed case: CamelCase, snake_case, PascalCase
|
||||
|
||||
Lorem ipsum dolor sit amet, consectetur adipiscing elit.
|
||||
Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
|
||||
Ut enim ad minim veniam, quis nostrud exercitation ullamco.
|
||||
|
||||
This file serves as a basic test case for:
|
||||
1. Word counting functionality
|
||||
2. Character analysis
|
||||
3. Line counting
|
||||
4. Text transformations
|
||||
5. Statistical analysis
|
||||
|
||||
The text processor should handle this content correctly and produce
|
||||
meaningful statistics and transformations for testing purposes.
|
||||
@@ -0,0 +1,16 @@
|
||||
name,age,city,country
|
||||
John Doe,25,New York,USA
|
||||
Jane Smith,30,London,UK
|
||||
Bob Johnson,22,Toronto,Canada
|
||||
Alice Brown,28,Sydney,Australia
|
||||
Charlie Wilson,35,Berlin,Germany
|
||||
|
||||
This CSV file contains sample data with headers and multiple rows.
|
||||
It can be used to test the text processor's ability to handle
|
||||
structured data formats and count words across different content types.
|
||||
|
||||
The file includes:
|
||||
- Header row with column names
|
||||
- Data rows with mixed text and numbers
|
||||
- Various city and country names
|
||||
- Different age values for statistical analysis
|
||||
|
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"file": "assets/sample_text.txt",
|
||||
"file_size": 855,
|
||||
"total_words": 116,
|
||||
"unique_words": 87,
|
||||
"total_characters": 855,
|
||||
"lines": 19,
|
||||
"average_word_length": 4.7,
|
||||
"most_frequent": {
|
||||
"word": "the",
|
||||
"count": 5
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
# Text Processor API Reference
|
||||
|
||||
## Classes
|
||||
|
||||
### TextProcessor
|
||||
|
||||
Main class for text processing operations.
|
||||
|
||||
#### `__init__(self, encoding: str = 'utf-8')`
|
||||
|
||||
Initialize the text processor with specified encoding.
|
||||
|
||||
**Parameters:**
|
||||
- `encoding` (str): Character encoding for file operations. Default: 'utf-8'
|
||||
|
||||
#### `analyze_text(self, text: str) -> Dict[str, Any]`
|
||||
|
||||
Analyze text and return comprehensive statistics.
|
||||
|
||||
**Parameters:**
|
||||
- `text` (str): Text content to analyze
|
||||
|
||||
**Returns:**
|
||||
- `dict`: Statistics including word count, character count, lines, most frequent word
|
||||
|
||||
**Example:**
|
||||
```python
|
||||
processor = TextProcessor()
|
||||
stats = processor.analyze_text("Hello world")
|
||||
# Returns: {'total_words': 2, 'unique_words': 2, ...}
|
||||
```
|
||||
|
||||
#### `transform_text(self, text: str, mode: str) -> str`
|
||||
|
||||
Transform text according to specified mode.
|
||||
|
||||
**Parameters:**
|
||||
- `text` (str): Text to transform
|
||||
- `mode` (str): Transformation mode ('upper', 'lower', 'title', 'reverse')
|
||||
|
||||
**Returns:**
|
||||
- `str`: Transformed text
|
||||
|
||||
**Raises:**
|
||||
- `ValueError`: If mode is not supported
|
||||
|
||||
### OutputFormatter
|
||||
|
||||
Static methods for output formatting.
|
||||
|
||||
#### `format_json(data: Dict[str, Any]) -> str`
|
||||
|
||||
Format data as JSON string.
|
||||
|
||||
#### `format_human_readable(data: Dict[str, Any]) -> str`
|
||||
|
||||
Format data as human-readable text.
|
||||
|
||||
### FileManager
|
||||
|
||||
Handles file operations and batch processing.
|
||||
|
||||
#### `find_text_files(self, directory: str) -> List[str]`
|
||||
|
||||
Find all text files in a directory recursively.
|
||||
|
||||
**Supported Extensions:**
|
||||
- .txt
|
||||
- .md
|
||||
- .rst
|
||||
- .csv
|
||||
- .log
|
||||
|
||||
## Command Line Interface
|
||||
|
||||
### Commands
|
||||
|
||||
#### `analyze`
|
||||
Analyze text file statistics.
|
||||
|
||||
```bash
|
||||
python text_processor.py analyze <file> [options]
|
||||
```
|
||||
|
||||
#### `transform`
|
||||
Transform text file content.
|
||||
|
||||
```bash
|
||||
python text_processor.py transform <file> --mode <mode> [options]
|
||||
```
|
||||
|
||||
#### `batch`
|
||||
Process multiple files in a directory.
|
||||
|
||||
```bash
|
||||
python text_processor.py batch <directory> [options]
|
||||
```
|
||||
|
||||
### Global Options
|
||||
|
||||
- `--format {json,text}`: Output format (default: text)
|
||||
- `--output FILE`: Output file path (default: stdout)
|
||||
- `--encoding ENCODING`: Text file encoding (default: utf-8)
|
||||
- `--verbose`: Enable verbose output
|
||||
|
||||
## Error Handling
|
||||
|
||||
The text processor handles several error conditions:
|
||||
|
||||
- **FileNotFoundError**: When input file doesn't exist
|
||||
- **UnicodeDecodeError**: When file encoding doesn't match specified encoding
|
||||
- **PermissionError**: When file access is denied
|
||||
- **ValueError**: When invalid transformation mode is specified
|
||||
|
||||
All errors are reported to stderr with descriptive messages.
|
||||
@@ -0,0 +1,382 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Sample Text Processor - Basic text analysis and transformation tool
|
||||
|
||||
This script demonstrates the basic structure and functionality expected in
|
||||
BASIC tier skills. It provides text processing capabilities with proper
|
||||
argument parsing, error handling, and dual output formats.
|
||||
|
||||
Usage:
|
||||
python text_processor.py analyze <file> [options]
|
||||
python text_processor.py transform <file> --mode <mode> [options]
|
||||
python text_processor.py batch <directory> [options]
|
||||
|
||||
Author: Claude Skills Engineering Team
|
||||
Version: 1.0.0
|
||||
Dependencies: Python Standard Library Only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional
|
||||
|
||||
|
||||
class TextProcessor:
|
||||
"""Core text processing functionality"""
|
||||
|
||||
def __init__(self, encoding: str = 'utf-8'):
|
||||
self.encoding = encoding
|
||||
|
||||
def analyze_text(self, text: str) -> Dict[str, Any]:
|
||||
"""Analyze text and return statistics"""
|
||||
lines = text.split('\n')
|
||||
words = text.lower().split()
|
||||
|
||||
# Calculate basic statistics
|
||||
stats = {
|
||||
'total_words': len(words),
|
||||
'unique_words': len(set(words)),
|
||||
'total_characters': len(text),
|
||||
'lines': len(lines),
|
||||
'average_word_length': sum(len(word) for word in words) / len(words) if words else 0
|
||||
}
|
||||
|
||||
# Find most frequent word
|
||||
if words:
|
||||
word_counts = Counter(words)
|
||||
most_common = word_counts.most_common(1)[0]
|
||||
stats['most_frequent'] = {
|
||||
'word': most_common[0],
|
||||
'count': most_common[1]
|
||||
}
|
||||
else:
|
||||
stats['most_frequent'] = {'word': '', 'count': 0}
|
||||
|
||||
return stats
|
||||
|
||||
def transform_text(self, text: str, mode: str) -> str:
|
||||
"""Transform text according to specified mode"""
|
||||
if mode == 'upper':
|
||||
return text.upper()
|
||||
elif mode == 'lower':
|
||||
return text.lower()
|
||||
elif mode == 'title':
|
||||
return text.title()
|
||||
elif mode == 'reverse':
|
||||
return text[::-1]
|
||||
else:
|
||||
raise ValueError(f"Unknown transformation mode: {mode}")
|
||||
|
||||
def process_file(self, file_path: str) -> Dict[str, Any]:
|
||||
"""Process a single text file"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding=self.encoding) as file:
|
||||
content = file.read()
|
||||
|
||||
stats = self.analyze_text(content)
|
||||
stats['file'] = file_path
|
||||
stats['file_size'] = os.path.getsize(file_path)
|
||||
|
||||
return stats
|
||||
|
||||
except FileNotFoundError:
|
||||
raise FileNotFoundError(f"File not found: {file_path}")
|
||||
except UnicodeDecodeError:
|
||||
raise UnicodeDecodeError(f"Cannot decode file with {self.encoding} encoding: {file_path}")
|
||||
except PermissionError:
|
||||
raise PermissionError(f"Permission denied accessing file: {file_path}")
|
||||
|
||||
|
||||
class OutputFormatter:
|
||||
"""Handles dual output format generation"""
|
||||
|
||||
@staticmethod
|
||||
def format_json(data: Dict[str, Any]) -> str:
|
||||
"""Format data as JSON"""
|
||||
return json.dumps(data, indent=2, ensure_ascii=False)
|
||||
|
||||
@staticmethod
|
||||
def format_human_readable(data: Dict[str, Any]) -> str:
|
||||
"""Format data as human-readable text"""
|
||||
lines = []
|
||||
lines.append("=== TEXT ANALYSIS RESULTS ===")
|
||||
lines.append(f"File: {data.get('file', 'Unknown')}")
|
||||
lines.append(f"File size: {data.get('file_size', 0)} bytes")
|
||||
lines.append(f"Total words: {data.get('total_words', 0)}")
|
||||
lines.append(f"Unique words: {data.get('unique_words', 0)}")
|
||||
lines.append(f"Total characters: {data.get('total_characters', 0)}")
|
||||
lines.append(f"Lines: {data.get('lines', 0)}")
|
||||
lines.append(f"Average word length: {data.get('average_word_length', 0):.1f}")
|
||||
|
||||
most_frequent = data.get('most_frequent', {})
|
||||
lines.append(f"Most frequent word: \"{most_frequent.get('word', '')}\" ({most_frequent.get('count', 0)} occurrences)")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class FileManager:
|
||||
"""Manages file I/O operations and batch processing"""
|
||||
|
||||
def __init__(self, verbose: bool = False):
|
||||
self.verbose = verbose
|
||||
|
||||
def log_verbose(self, message: str):
|
||||
"""Log verbose message if verbose mode enabled"""
|
||||
if self.verbose:
|
||||
print(f"[INFO] {message}", file=sys.stderr)
|
||||
|
||||
def find_text_files(self, directory: str) -> List[str]:
|
||||
"""Find all text files in directory"""
|
||||
text_extensions = {'.txt', '.md', '.rst', '.csv', '.log'}
|
||||
text_files = []
|
||||
|
||||
try:
|
||||
for file_path in Path(directory).rglob('*'):
|
||||
if file_path.is_file() and file_path.suffix.lower() in text_extensions:
|
||||
text_files.append(str(file_path))
|
||||
|
||||
except PermissionError:
|
||||
raise PermissionError(f"Permission denied accessing directory: {directory}")
|
||||
|
||||
return text_files
|
||||
|
||||
def write_output(self, content: str, output_path: Optional[str] = None):
|
||||
"""Write content to file or stdout"""
|
||||
if output_path:
|
||||
try:
|
||||
# Create directory if needed
|
||||
output_dir = os.path.dirname(output_path)
|
||||
if output_dir and not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as file:
|
||||
file.write(content)
|
||||
|
||||
self.log_verbose(f"Output written to: {output_path}")
|
||||
|
||||
except PermissionError:
|
||||
raise PermissionError(f"Permission denied writing to: {output_path}")
|
||||
else:
|
||||
print(content)
|
||||
|
||||
|
||||
def analyze_command(args: argparse.Namespace) -> int:
|
||||
"""Handle analyze command"""
|
||||
try:
|
||||
processor = TextProcessor(args.encoding)
|
||||
file_manager = FileManager(args.verbose)
|
||||
|
||||
file_manager.log_verbose(f"Analyzing file: {args.file}")
|
||||
|
||||
# Process the file
|
||||
results = processor.process_file(args.file)
|
||||
|
||||
# Format output
|
||||
if args.format == 'json':
|
||||
output = OutputFormatter.format_json(results)
|
||||
else:
|
||||
output = OutputFormatter.format_human_readable(results)
|
||||
|
||||
# Write output
|
||||
file_manager.write_output(output, args.output)
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
except UnicodeDecodeError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
print(f"Try using --encoding option with different encoding", file=sys.stderr)
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def transform_command(args: argparse.Namespace) -> int:
|
||||
"""Handle transform command"""
|
||||
try:
|
||||
processor = TextProcessor(args.encoding)
|
||||
file_manager = FileManager(args.verbose)
|
||||
|
||||
file_manager.log_verbose(f"Transforming file: {args.file}")
|
||||
|
||||
# Read and transform the file
|
||||
with open(args.file, 'r', encoding=args.encoding) as file:
|
||||
content = file.read()
|
||||
|
||||
transformed = processor.transform_text(content, args.mode)
|
||||
|
||||
# Write transformed content
|
||||
file_manager.write_output(transformed, args.output)
|
||||
|
||||
return 0
|
||||
|
||||
except FileNotFoundError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
except ValueError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def batch_command(args: argparse.Namespace) -> int:
|
||||
"""Handle batch command"""
|
||||
try:
|
||||
processor = TextProcessor(args.encoding)
|
||||
file_manager = FileManager(args.verbose)
|
||||
|
||||
file_manager.log_verbose(f"Finding text files in: {args.directory}")
|
||||
|
||||
# Find all text files
|
||||
text_files = file_manager.find_text_files(args.directory)
|
||||
|
||||
if not text_files:
|
||||
print(f"No text files found in directory: {args.directory}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
file_manager.log_verbose(f"Found {len(text_files)} text files")
|
||||
|
||||
# Process all files
|
||||
all_results = []
|
||||
for i, file_path in enumerate(text_files, 1):
|
||||
try:
|
||||
file_manager.log_verbose(f"Processing {i}/{len(text_files)}: {file_path}")
|
||||
results = processor.process_file(file_path)
|
||||
all_results.append(results)
|
||||
except Exception as e:
|
||||
print(f"Warning: Failed to process {file_path}: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
if not all_results:
|
||||
print("Error: No files could be processed successfully", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
# Format batch results
|
||||
batch_summary = {
|
||||
'total_files': len(all_results),
|
||||
'total_words': sum(r.get('total_words', 0) for r in all_results),
|
||||
'total_characters': sum(r.get('total_characters', 0) for r in all_results),
|
||||
'files': all_results
|
||||
}
|
||||
|
||||
if args.format == 'json':
|
||||
output = OutputFormatter.format_json(batch_summary)
|
||||
else:
|
||||
lines = []
|
||||
lines.append("=== BATCH PROCESSING RESULTS ===")
|
||||
lines.append(f"Total files processed: {batch_summary['total_files']}")
|
||||
lines.append(f"Total words across all files: {batch_summary['total_words']}")
|
||||
lines.append(f"Total characters across all files: {batch_summary['total_characters']}")
|
||||
lines.append("")
|
||||
lines.append("Individual file results:")
|
||||
for result in all_results:
|
||||
lines.append(f" {result['file']}: {result['total_words']} words")
|
||||
output = "\n".join(lines)
|
||||
|
||||
# Write output
|
||||
file_manager.write_output(output, args.output)
|
||||
|
||||
return 0
|
||||
|
||||
except PermissionError as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
except Exception as e:
|
||||
print(f"Error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point with argument parsing"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Sample Text Processor - Basic text analysis and transformation",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
Analysis:
|
||||
python text_processor.py analyze document.txt
|
||||
python text_processor.py analyze document.txt --format json --output results.json
|
||||
|
||||
Transformation:
|
||||
python text_processor.py transform document.txt --mode upper
|
||||
python text_processor.py transform document.txt --mode title --output transformed.txt
|
||||
|
||||
Batch processing:
|
||||
python text_processor.py batch text_files/ --verbose
|
||||
python text_processor.py batch text_files/ --format json --output batch_results.json
|
||||
|
||||
Transformation modes:
|
||||
upper - Convert to uppercase
|
||||
lower - Convert to lowercase
|
||||
title - Convert to title case
|
||||
reverse - Reverse the text
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('--format',
|
||||
choices=['json', 'text'],
|
||||
default='text',
|
||||
help='Output format (default: text)')
|
||||
parser.add_argument('--output',
|
||||
help='Output file path (default: stdout)')
|
||||
parser.add_argument('--encoding',
|
||||
default='utf-8',
|
||||
help='Text file encoding (default: utf-8)')
|
||||
parser.add_argument('--verbose',
|
||||
action='store_true',
|
||||
help='Enable verbose output')
|
||||
|
||||
subparsers = parser.add_subparsers(dest='command', help='Available commands')
|
||||
|
||||
# Analyze subcommand
|
||||
analyze_parser = subparsers.add_parser('analyze', help='Analyze text file statistics')
|
||||
analyze_parser.add_argument('file', help='Text file to analyze')
|
||||
|
||||
# Transform subcommand
|
||||
transform_parser = subparsers.add_parser('transform', help='Transform text file')
|
||||
transform_parser.add_argument('file', help='Text file to transform')
|
||||
transform_parser.add_argument('--mode',
|
||||
required=True,
|
||||
choices=['upper', 'lower', 'title', 'reverse'],
|
||||
help='Transformation mode')
|
||||
|
||||
# Batch subcommand
|
||||
batch_parser = subparsers.add_parser('batch', help='Process multiple files')
|
||||
batch_parser.add_argument('directory', help='Directory containing text files')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.command:
|
||||
parser.print_help()
|
||||
return 1
|
||||
|
||||
try:
|
||||
if args.command == 'analyze':
|
||||
return analyze_command(args)
|
||||
elif args.command == 'transform':
|
||||
return transform_command(args)
|
||||
elif args.command == 'batch':
|
||||
return batch_command(args)
|
||||
else:
|
||||
print(f"Unknown command: {args.command}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nOperation interrupted by user", file=sys.stderr)
|
||||
return 130
|
||||
except Exception as e:
|
||||
print(f"Unexpected error: {e}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"skill_path": "assets/sample-skill",
|
||||
"timestamp": "2026-02-16T16:41:00Z",
|
||||
"overall_score": 85.0,
|
||||
"compliance_level": "GOOD",
|
||||
"checks": {
|
||||
"skill_md_exists": {
|
||||
"passed": true,
|
||||
"message": "SKILL.md found",
|
||||
"score": 1.0
|
||||
},
|
||||
"readme_exists": {
|
||||
"passed": true,
|
||||
"message": "README.md found",
|
||||
"score": 1.0
|
||||
},
|
||||
"skill_md_length": {
|
||||
"passed": true,
|
||||
"message": "SKILL.md has 145 lines (≥100)",
|
||||
"score": 1.0
|
||||
},
|
||||
"frontmatter_complete": {
|
||||
"passed": true,
|
||||
"message": "All required frontmatter fields present",
|
||||
"score": 1.0
|
||||
},
|
||||
"required_sections": {
|
||||
"passed": true,
|
||||
"message": "All required sections present",
|
||||
"score": 1.0
|
||||
},
|
||||
"dir_scripts_exists": {
|
||||
"passed": true,
|
||||
"message": "scripts/ directory found",
|
||||
"score": 1.0
|
||||
},
|
||||
"min_scripts_count": {
|
||||
"passed": true,
|
||||
"message": "Found 1 Python scripts (≥1)",
|
||||
"score": 1.0
|
||||
},
|
||||
"script_syntax_text_processor.py": {
|
||||
"passed": true,
|
||||
"message": "text_processor.py has valid Python syntax",
|
||||
"score": 1.0
|
||||
},
|
||||
"script_argparse_text_processor.py": {
|
||||
"passed": true,
|
||||
"message": "Uses argparse in text_processor.py",
|
||||
"score": 1.0
|
||||
},
|
||||
"script_main_guard_text_processor.py": {
|
||||
"passed": true,
|
||||
"message": "Has main guard in text_processor.py",
|
||||
"score": 1.0
|
||||
},
|
||||
"tier_compliance": {
|
||||
"passed": true,
|
||||
"message": "Meets BASIC tier requirements",
|
||||
"score": 1.0
|
||||
}
|
||||
},
|
||||
"warnings": [],
|
||||
"errors": [],
|
||||
"suggestions": [
|
||||
"Consider adding optional directories: references, expected_outputs"
|
||||
]
|
||||
}
|
||||
405
engineering/skill-tester/references/quality-scoring-rubric.md
Normal file
405
engineering/skill-tester/references/quality-scoring-rubric.md
Normal file
@@ -0,0 +1,405 @@
|
||||
# Quality Scoring Rubric
|
||||
|
||||
**Version**: 1.0.0
|
||||
**Last Updated**: 2026-02-16
|
||||
**Authority**: Claude Skills Engineering Team
|
||||
|
||||
## Overview
|
||||
|
||||
This document defines the comprehensive quality scoring methodology used to assess skills within the claude-skills ecosystem. The scoring system evaluates four key dimensions, each weighted equally at 25%, to provide an objective and consistent measure of skill quality.
|
||||
|
||||
## Scoring Framework
|
||||
|
||||
### Overall Scoring Scale
|
||||
- **A+ (95-100)**: Exceptional quality, exceeds all standards
|
||||
- **A (90-94)**: Excellent quality, meets highest standards consistently
|
||||
- **A- (85-89)**: Very good quality, minor areas for improvement
|
||||
- **B+ (80-84)**: Good quality, meets most standards well
|
||||
- **B (75-79)**: Satisfactory quality, meets standards adequately
|
||||
- **B- (70-74)**: Below average, several areas need improvement
|
||||
- **C+ (65-69)**: Poor quality, significant improvements needed
|
||||
- **C (60-64)**: Minimal acceptable quality, major improvements required
|
||||
- **C- (55-59)**: Unacceptable quality, extensive rework needed
|
||||
- **D (50-54)**: Very poor quality, fundamental issues present
|
||||
- **F (0-49)**: Failing quality, does not meet basic standards
|
||||
|
||||
### Dimension Weights
|
||||
Each dimension contributes equally to the overall score:
|
||||
- **Documentation Quality**: 25%
|
||||
- **Code Quality**: 25%
|
||||
- **Completeness**: 25%
|
||||
- **Usability**: 25%
|
||||
|
||||
## Documentation Quality (25% Weight)
|
||||
|
||||
### Scoring Components
|
||||
|
||||
#### SKILL.md Quality (40% of Documentation Score)
|
||||
**Component Breakdown:**
|
||||
- **Length and Depth (25%)**: Line count and content substance
|
||||
- **Frontmatter Quality (25%)**: Completeness and accuracy of YAML metadata
|
||||
- **Section Coverage (25%)**: Required and recommended section presence
|
||||
- **Content Depth (25%)**: Technical detail and comprehensiveness
|
||||
|
||||
**Scoring Criteria:**
|
||||
|
||||
| Score Range | Length | Frontmatter | Sections | Depth |
|
||||
|-------------|--------|-------------|----------|-------|
|
||||
| 90-100 | 400+ lines | All fields complete + extras | All required + 4+ recommended | Rich technical detail, examples |
|
||||
| 80-89 | 300-399 lines | All required fields complete | All required + 2-3 recommended | Good technical coverage |
|
||||
| 70-79 | 200-299 lines | Most required fields | All required + 1 recommended | Adequate technical content |
|
||||
| 60-69 | 150-199 lines | Some required fields | Most required sections | Basic technical information |
|
||||
| 50-59 | 100-149 lines | Minimal frontmatter | Some required sections | Limited technical detail |
|
||||
| Below 50 | <100 lines | Missing/invalid frontmatter | Few/no required sections | Insufficient content |
|
||||
|
||||
#### README.md Quality (25% of Documentation Score)
|
||||
**Scoring Criteria:**
|
||||
- **Excellent (90-100)**: 1000+ chars, comprehensive usage guide, examples, troubleshooting
|
||||
- **Good (75-89)**: 500-999 chars, clear usage instructions, basic examples
|
||||
- **Satisfactory (60-74)**: 200-499 chars, minimal usage information
|
||||
- **Poor (40-59)**: <200 chars or confusing content
|
||||
- **Failing (0-39)**: Missing or completely inadequate
|
||||
|
||||
#### Reference Documentation (20% of Documentation Score)
|
||||
**Scoring Criteria:**
|
||||
- **Excellent (90-100)**: Multiple comprehensive reference docs (2000+ chars total)
|
||||
- **Good (75-89)**: 2-3 reference files with substantial content
|
||||
- **Satisfactory (60-74)**: 1-2 reference files with adequate content
|
||||
- **Poor (40-59)**: Minimal reference content or poor quality
|
||||
- **Failing (0-39)**: No reference documentation
|
||||
|
||||
#### Examples and Usage Clarity (15% of Documentation Score)
|
||||
**Scoring Criteria:**
|
||||
- **Excellent (90-100)**: 5+ diverse examples, clear usage patterns
|
||||
- **Good (75-89)**: 3-4 examples covering different scenarios
|
||||
- **Satisfactory (60-74)**: 2-3 basic examples
|
||||
- **Poor (40-59)**: 1-2 minimal examples
|
||||
- **Failing (0-39)**: No examples or unclear usage
|
||||
|
||||
## Code Quality (25% Weight)
|
||||
|
||||
### Scoring Components
|
||||
|
||||
#### Script Complexity and Architecture (25% of Code Score)
|
||||
**Evaluation Criteria:**
|
||||
- Lines of code per script relative to tier requirements
|
||||
- Function and class organization
|
||||
- Code modularity and reusability
|
||||
- Algorithm sophistication
|
||||
|
||||
**Scoring Matrix:**
|
||||
|
||||
| Tier | Excellent (90-100) | Good (75-89) | Satisfactory (60-74) | Poor (Below 60) |
|
||||
|------|-------------------|--------------|---------------------|-----------------|
|
||||
| BASIC | 200-300 LOC, well-structured | 150-199 LOC, organized | 100-149 LOC, basic | <100 LOC, minimal |
|
||||
| STANDARD | 400-500 LOC, modular | 350-399 LOC, structured | 300-349 LOC, adequate | <300 LOC, basic |
|
||||
| POWERFUL | 600-800 LOC, sophisticated | 550-599 LOC, advanced | 500-549 LOC, solid | <500 LOC, simple |
|
||||
|
||||
#### Error Handling Quality (25% of Code Score)
|
||||
**Scoring Criteria:**
|
||||
- **Excellent (90-100)**: Comprehensive exception handling, specific error types, recovery mechanisms
|
||||
- **Good (75-89)**: Good exception handling, meaningful error messages, logging
|
||||
- **Satisfactory (60-74)**: Basic try/except blocks, simple error messages
|
||||
- **Poor (40-59)**: Minimal error handling, generic exceptions
|
||||
- **Failing (0-39)**: No error handling or inappropriate handling
|
||||
|
||||
**Error Handling Checklist:**
|
||||
- [ ] Try/except blocks for risky operations
|
||||
- [ ] Specific exception types (not just Exception)
|
||||
- [ ] Meaningful error messages for users
|
||||
- [ ] Proper error logging or reporting
|
||||
- [ ] Graceful degradation where possible
|
||||
- [ ] Input validation and sanitization
|
||||
|
||||
#### Code Structure and Organization (25% of Code Score)
|
||||
**Evaluation Elements:**
|
||||
- Function decomposition and single responsibility
|
||||
- Class design and inheritance patterns
|
||||
- Import organization and dependency management
|
||||
- Documentation and comments quality
|
||||
- Consistent naming conventions
|
||||
- PEP 8 compliance
|
||||
|
||||
**Scoring Guidelines:**
|
||||
- **Excellent (90-100)**: Exemplary structure, comprehensive docstrings, perfect style
|
||||
- **Good (75-89)**: Well-organized, good documentation, minor style issues
|
||||
- **Satisfactory (60-74)**: Adequate structure, basic documentation, some style issues
|
||||
- **Poor (40-59)**: Poor organization, minimal documentation, style problems
|
||||
- **Failing (0-39)**: No clear structure, no documentation, major style violations
|
||||
|
||||
#### Output Format Support (25% of Code Score)
|
||||
**Required Capabilities:**
|
||||
- JSON output format support
|
||||
- Human-readable output format
|
||||
- Proper data serialization
|
||||
- Consistent output structure
|
||||
- Error output handling
|
||||
|
||||
**Scoring Criteria:**
|
||||
- **Excellent (90-100)**: Dual format + custom formats, perfect serialization
|
||||
- **Good (75-89)**: Dual format support, good serialization
|
||||
- **Satisfactory (60-74)**: Single format well-implemented
|
||||
- **Poor (40-59)**: Basic output, formatting issues
|
||||
- **Failing (0-39)**: Poor or no structured output
|
||||
|
||||
## Completeness (25% Weight)
|
||||
|
||||
### Scoring Components
|
||||
|
||||
#### Directory Structure Compliance (25% of Completeness Score)
|
||||
**Required Directories by Tier:**
|
||||
- **BASIC**: scripts/ (required), assets/ + references/ (recommended)
|
||||
- **STANDARD**: scripts/ + assets/ + references/ (required), expected_outputs/ (recommended)
|
||||
- **POWERFUL**: scripts/ + assets/ + references/ + expected_outputs/ (all required)
|
||||
|
||||
**Scoring Calculation:**
|
||||
```
|
||||
Structure Score = (Required Present / Required Total) * 0.6 +
|
||||
(Recommended Present / Recommended Total) * 0.4
|
||||
```
|
||||
|
||||
#### Asset Availability and Quality (25% of Completeness Score)
|
||||
**Scoring Criteria:**
|
||||
- **Excellent (90-100)**: 5+ diverse assets, multiple file types, realistic data
|
||||
- **Good (75-89)**: 3-4 assets, some diversity, good quality
|
||||
- **Satisfactory (60-74)**: 2-3 assets, basic variety
|
||||
- **Poor (40-59)**: 1-2 minimal assets
|
||||
- **Failing (0-39)**: No assets or unusable assets
|
||||
|
||||
**Asset Quality Factors:**
|
||||
- File diversity (JSON, CSV, YAML, etc.)
|
||||
- Data realism and complexity
|
||||
- Coverage of use cases
|
||||
- File size appropriateness
|
||||
- Documentation of asset purpose
|
||||
|
||||
#### Expected Output Coverage (25% of Completeness Score)
|
||||
**Evaluation Criteria:**
|
||||
- Correspondence with asset files
|
||||
- Coverage of success and error scenarios
|
||||
- Output format variety
|
||||
- Reproducibility and accuracy
|
||||
|
||||
**Scoring Matrix:**
|
||||
- **Excellent (90-100)**: Complete output coverage, all scenarios, verified accuracy
|
||||
- **Good (75-89)**: Good coverage, most scenarios, mostly accurate
|
||||
- **Satisfactory (60-74)**: Basic coverage, main scenarios
|
||||
- **Poor (40-59)**: Minimal coverage, some inaccuracies
|
||||
- **Failing (0-39)**: No expected outputs or completely inaccurate
|
||||
|
||||
#### Test Coverage and Validation (25% of Completeness Score)
|
||||
**Assessment Areas:**
|
||||
- Sample data processing capability
|
||||
- Output verification mechanisms
|
||||
- Edge case handling
|
||||
- Error condition testing
|
||||
- Integration test scenarios
|
||||
|
||||
**Scoring Guidelines:**
|
||||
- **Excellent (90-100)**: Comprehensive test coverage, automated validation
|
||||
- **Good (75-89)**: Good test coverage, manual validation possible
|
||||
- **Satisfactory (60-74)**: Basic testing capability
|
||||
- **Poor (40-59)**: Minimal testing support
|
||||
- **Failing (0-39)**: No testing or validation capability
|
||||
|
||||
## Usability (25% Weight)
|
||||
|
||||
### Scoring Components
|
||||
|
||||
#### Installation and Setup Simplicity (25% of Usability Score)
|
||||
**Evaluation Factors:**
|
||||
- Dependency requirements (Python stdlib preferred)
|
||||
- Setup complexity
|
||||
- Environment requirements
|
||||
- Installation documentation clarity
|
||||
|
||||
**Scoring Criteria:**
|
||||
- **Excellent (90-100)**: Zero external dependencies, single-file execution
|
||||
- **Good (75-89)**: Minimal dependencies, simple setup
|
||||
- **Satisfactory (60-74)**: Some dependencies, documented setup
|
||||
- **Poor (40-59)**: Complex dependencies, unclear setup
|
||||
- **Failing (0-39)**: Unable to install or excessive complexity
|
||||
|
||||
#### Usage Clarity and Help Quality (25% of Usability Score)
|
||||
**Assessment Elements:**
|
||||
- Command-line help comprehensiveness
|
||||
- Usage example clarity
|
||||
- Parameter documentation quality
|
||||
- Error message helpfulness
|
||||
|
||||
**Help Quality Checklist:**
|
||||
- [ ] Comprehensive --help output
|
||||
- [ ] Clear parameter descriptions
|
||||
- [ ] Usage examples included
|
||||
- [ ] Error messages are actionable
|
||||
- [ ] Progress indicators where appropriate
|
||||
|
||||
**Scoring Matrix:**
|
||||
- **Excellent (90-100)**: Exemplary help, multiple examples, perfect error messages
|
||||
- **Good (75-89)**: Good help quality, clear examples, helpful errors
|
||||
- **Satisfactory (60-74)**: Adequate help, basic examples
|
||||
- **Poor (40-59)**: Minimal help, confusing interface
|
||||
- **Failing (0-39)**: No help or completely unclear interface
|
||||
|
||||
#### Documentation Accessibility (25% of Usability Score)
|
||||
**Evaluation Criteria:**
|
||||
- README quick start effectiveness
|
||||
- SKILL.md navigation and structure
|
||||
- Reference material organization
|
||||
- Learning curve considerations
|
||||
|
||||
**Accessibility Factors:**
|
||||
- Information hierarchy clarity
|
||||
- Cross-reference quality
|
||||
- Beginner-friendly explanations
|
||||
- Advanced user shortcuts
|
||||
- Troubleshooting guidance
|
||||
|
||||
#### Practical Example Quality (25% of Usability Score)
|
||||
**Assessment Areas:**
|
||||
- Example realism and relevance
|
||||
- Complexity progression (simple to advanced)
|
||||
- Output demonstration
|
||||
- Common use case coverage
|
||||
- Integration scenarios
|
||||
|
||||
**Scoring Guidelines:**
|
||||
- **Excellent (90-100)**: 5+ examples, perfect progression, real-world scenarios
|
||||
- **Good (75-89)**: 3-4 examples, good variety, practical scenarios
|
||||
- **Satisfactory (60-74)**: 2-3 examples, adequate coverage
|
||||
- **Poor (40-59)**: 1-2 examples, limited practical value
|
||||
- **Failing (0-39)**: No examples or completely impractical
|
||||
|
||||
## Scoring Calculations
|
||||
|
||||
### Dimension Score Calculation
|
||||
Each dimension score is calculated as a weighted average of its components:
|
||||
|
||||
```python
|
||||
def calculate_dimension_score(components):
|
||||
total_weighted_score = 0
|
||||
total_weight = 0
|
||||
|
||||
for component_name, component_data in components.items():
|
||||
score = component_data['score']
|
||||
weight = component_data['weight']
|
||||
total_weighted_score += score * weight
|
||||
total_weight += weight
|
||||
|
||||
return total_weighted_score / total_weight if total_weight > 0 else 0
|
||||
```
|
||||
|
||||
### Overall Score Calculation
|
||||
The overall score combines all dimensions with equal weighting:
|
||||
|
||||
```python
|
||||
def calculate_overall_score(dimensions):
|
||||
return sum(dimension.score * 0.25 for dimension in dimensions.values())
|
||||
```
|
||||
|
||||
### Letter Grade Assignment
|
||||
```python
|
||||
def assign_letter_grade(overall_score):
|
||||
if overall_score >= 95: return "A+"
|
||||
elif overall_score >= 90: return "A"
|
||||
elif overall_score >= 85: return "A-"
|
||||
elif overall_score >= 80: return "B+"
|
||||
elif overall_score >= 75: return "B"
|
||||
elif overall_score >= 70: return "B-"
|
||||
elif overall_score >= 65: return "C+"
|
||||
elif overall_score >= 60: return "C"
|
||||
elif overall_score >= 55: return "C-"
|
||||
elif overall_score >= 50: return "D"
|
||||
else: return "F"
|
||||
```
|
||||
|
||||
## Quality Improvement Recommendations
|
||||
|
||||
### Score-Based Recommendations
|
||||
|
||||
#### For Scores Below 60 (C- or Lower)
|
||||
**Priority Actions:**
|
||||
1. Address fundamental structural issues
|
||||
2. Implement basic error handling
|
||||
3. Add essential documentation sections
|
||||
4. Create minimal viable examples
|
||||
5. Fix critical functionality issues
|
||||
|
||||
#### For Scores 60-74 (C+ to B-)
|
||||
**Improvement Areas:**
|
||||
1. Expand documentation comprehensiveness
|
||||
2. Enhance error handling sophistication
|
||||
3. Add more diverse examples and use cases
|
||||
4. Improve code organization and structure
|
||||
5. Increase test coverage and validation
|
||||
|
||||
#### For Scores 75-84 (B to B+)
|
||||
**Enhancement Opportunities:**
|
||||
1. Refine documentation for expert-level quality
|
||||
2. Implement advanced error recovery mechanisms
|
||||
3. Add comprehensive reference materials
|
||||
4. Optimize code architecture and performance
|
||||
5. Develop extensive example library
|
||||
|
||||
#### For Scores 85+ (A- or Higher)
|
||||
**Excellence Maintenance:**
|
||||
1. Regular quality audits and updates
|
||||
2. Community feedback integration
|
||||
3. Best practice evolution tracking
|
||||
4. Mentoring lower-quality skills
|
||||
5. Innovation and cutting-edge feature adoption
|
||||
|
||||
### Dimension-Specific Improvement Strategies
|
||||
|
||||
#### Low Documentation Scores
|
||||
- Expand SKILL.md with technical details
|
||||
- Add comprehensive API reference
|
||||
- Include architecture diagrams and explanations
|
||||
- Develop troubleshooting guides
|
||||
- Create contributor documentation
|
||||
|
||||
#### Low Code Quality Scores
|
||||
- Refactor for better modularity
|
||||
- Implement comprehensive error handling
|
||||
- Add extensive code documentation
|
||||
- Apply advanced design patterns
|
||||
- Optimize performance and efficiency
|
||||
|
||||
#### Low Completeness Scores
|
||||
- Add missing directories and files
|
||||
- Develop comprehensive sample datasets
|
||||
- Create expected output libraries
|
||||
- Implement automated testing
|
||||
- Add integration examples
|
||||
|
||||
#### Low Usability Scores
|
||||
- Simplify installation process
|
||||
- Improve command-line interface design
|
||||
- Enhance help text and documentation
|
||||
- Create beginner-friendly tutorials
|
||||
- Add interactive examples
|
||||
|
||||
## Quality Assurance Process
|
||||
|
||||
### Automated Scoring
|
||||
The quality scorer runs automated assessments based on this rubric:
|
||||
1. File system analysis for structure compliance
|
||||
2. Content analysis for documentation quality
|
||||
3. Code analysis for quality metrics
|
||||
4. Asset inventory and quality assessment
|
||||
|
||||
### Manual Review Process
|
||||
Human reviewers validate automated scores and provide qualitative insights:
|
||||
1. Content quality assessment beyond automated metrics
|
||||
2. Usability testing with real-world scenarios
|
||||
3. Technical accuracy verification
|
||||
4. Community value assessment
|
||||
|
||||
### Continuous Improvement
|
||||
The scoring rubric evolves based on:
|
||||
- Community feedback and usage patterns
|
||||
- Industry best practice changes
|
||||
- Tool capability enhancements
|
||||
- Quality trend analysis
|
||||
|
||||
This quality scoring rubric ensures consistent, objective, and comprehensive assessment of all skills within the claude-skills ecosystem while providing clear guidance for quality improvement.
|
||||
@@ -0,0 +1,355 @@
|
||||
# Skill Structure Specification
|
||||
|
||||
**Version**: 1.0.0
|
||||
**Last Updated**: 2026-02-16
|
||||
**Authority**: Claude Skills Engineering Team
|
||||
|
||||
## Overview
|
||||
|
||||
This document defines the mandatory and optional components that constitute a well-formed skill within the claude-skills ecosystem. All skills must adhere to these structural requirements to ensure consistency, maintainability, and quality across the repository.
|
||||
|
||||
## Directory Structure
|
||||
|
||||
### Mandatory Components
|
||||
|
||||
```
|
||||
skill-name/
|
||||
├── SKILL.md # Primary skill documentation (REQUIRED)
|
||||
├── README.md # Usage instructions and quick start (REQUIRED)
|
||||
└── scripts/ # Python implementation scripts (REQUIRED)
|
||||
└── *.py # At least one Python script
|
||||
```
|
||||
|
||||
### Recommended Components
|
||||
|
||||
```
|
||||
skill-name/
|
||||
├── SKILL.md
|
||||
├── README.md
|
||||
├── scripts/
|
||||
│ └── *.py
|
||||
├── assets/ # Sample data and input files (RECOMMENDED)
|
||||
│ ├── samples/
|
||||
│ ├── examples/
|
||||
│ └── data/
|
||||
├── references/ # Reference documentation (RECOMMENDED)
|
||||
│ ├── api-reference.md
|
||||
│ ├── specifications.md
|
||||
│ └── external-links.md
|
||||
└── expected_outputs/ # Expected results for testing (RECOMMENDED)
|
||||
├── sample_output.json
|
||||
├── example_results.txt
|
||||
└── test_cases/
|
||||
```
|
||||
|
||||
### Optional Components
|
||||
|
||||
```
|
||||
skill-name/
|
||||
├── [mandatory and recommended components]
|
||||
├── tests/ # Unit tests and validation scripts
|
||||
├── examples/ # Extended examples and tutorials
|
||||
├── docs/ # Additional documentation
|
||||
├── config/ # Configuration files
|
||||
└── templates/ # Template files for code generation
|
||||
```
|
||||
|
||||
## File Requirements
|
||||
|
||||
### SKILL.md Requirements
|
||||
|
||||
The `SKILL.md` file serves as the primary documentation for the skill and must contain:
|
||||
|
||||
#### Mandatory YAML Frontmatter
|
||||
```yaml
|
||||
---
|
||||
Name: skill-name
|
||||
Tier: [BASIC|STANDARD|POWERFUL]
|
||||
Category: [Category Name]
|
||||
Dependencies: [None|List of dependencies]
|
||||
Author: [Author Name]
|
||||
Version: [Semantic Version]
|
||||
Last Updated: [YYYY-MM-DD]
|
||||
---
|
||||
```
|
||||
|
||||
#### Required Sections
|
||||
- **Description**: Comprehensive overview of the skill's purpose and capabilities
|
||||
- **Features**: Detailed list of key features and functionality
|
||||
- **Usage**: Instructions for using the skill and its components
|
||||
- **Examples**: Practical usage examples with expected outcomes
|
||||
|
||||
#### Recommended Sections
|
||||
- **Architecture**: Technical architecture and design decisions
|
||||
- **Installation**: Setup and installation instructions
|
||||
- **Configuration**: Configuration options and parameters
|
||||
- **Troubleshooting**: Common issues and solutions
|
||||
- **Contributing**: Guidelines for contributors
|
||||
- **Changelog**: Version history and changes
|
||||
|
||||
#### Content Requirements by Tier
|
||||
- **BASIC**: Minimum 100 lines of substantial content
|
||||
- **STANDARD**: Minimum 200 lines of substantial content
|
||||
- **POWERFUL**: Minimum 300 lines of substantial content
|
||||
|
||||
### README.md Requirements
|
||||
|
||||
The `README.md` file provides quick start instructions and must include:
|
||||
|
||||
#### Mandatory Content
|
||||
- Brief description of the skill
|
||||
- Quick start instructions
|
||||
- Basic usage examples
|
||||
- Link to full SKILL.md documentation
|
||||
|
||||
#### Recommended Content
|
||||
- Installation instructions
|
||||
- Prerequisites and dependencies
|
||||
- Command-line usage examples
|
||||
- Troubleshooting section
|
||||
- Contributing guidelines
|
||||
|
||||
#### Length Requirements
|
||||
- Minimum 200 characters of substantial content
|
||||
- Recommended 500+ characters for comprehensive coverage
|
||||
|
||||
### Scripts Directory Requirements
|
||||
|
||||
The `scripts/` directory contains all Python implementation files:
|
||||
|
||||
#### Mandatory Requirements
|
||||
- At least one Python (.py) file
|
||||
- All scripts must be executable Python 3.7+
|
||||
- No external dependencies outside Python standard library
|
||||
- Proper file naming conventions (lowercase, hyphens for separation)
|
||||
|
||||
#### Script Content Requirements
|
||||
- **Shebang line**: `#!/usr/bin/env python3`
|
||||
- **Module docstring**: Comprehensive description of script purpose
|
||||
- **Argparse implementation**: Command-line argument parsing
|
||||
- **Main guard**: `if __name__ == "__main__":` protection
|
||||
- **Error handling**: Appropriate exception handling and user feedback
|
||||
- **Dual output**: Support for both JSON and human-readable output formats
|
||||
|
||||
#### Script Size Requirements by Tier
|
||||
- **BASIC**: 100-300 lines of code per script
|
||||
- **STANDARD**: 300-500 lines of code per script
|
||||
- **POWERFUL**: 500-800 lines of code per script
|
||||
|
||||
### Assets Directory Structure
|
||||
|
||||
The `assets/` directory contains sample data and supporting files:
|
||||
|
||||
```
|
||||
assets/
|
||||
├── samples/ # Sample input data
|
||||
│ ├── simple_example.json
|
||||
│ ├── complex_dataset.csv
|
||||
│ └── test_configuration.yaml
|
||||
├── examples/ # Example files demonstrating usage
|
||||
│ ├── basic_workflow.py
|
||||
│ ├── advanced_usage.sh
|
||||
│ └── integration_example.md
|
||||
└── data/ # Static data files
|
||||
├── reference_data.json
|
||||
├── lookup_tables.csv
|
||||
└── configuration_templates/
|
||||
```
|
||||
|
||||
#### Content Requirements
|
||||
- At least 2 sample files demonstrating different use cases
|
||||
- Files should represent realistic usage scenarios
|
||||
- Include both simple and complex examples where applicable
|
||||
- Provide diverse file formats (JSON, CSV, YAML, etc.)
|
||||
|
||||
### References Directory Structure
|
||||
|
||||
The `references/` directory contains detailed reference documentation:
|
||||
|
||||
```
|
||||
references/
|
||||
├── api-reference.md # Complete API documentation
|
||||
├── specifications.md # Technical specifications and requirements
|
||||
├── external-links.md # Links to related resources
|
||||
├── algorithms.md # Algorithm descriptions and implementations
|
||||
└── best-practices.md # Usage best practices and patterns
|
||||
```
|
||||
|
||||
#### Content Requirements
|
||||
- Each file should contain substantial technical content (500+ words)
|
||||
- Include code examples and technical specifications
|
||||
- Provide external references and links where appropriate
|
||||
- Maintain consistent documentation format and style
|
||||
|
||||
### Expected Outputs Directory Structure
|
||||
|
||||
The `expected_outputs/` directory contains reference outputs for testing:
|
||||
|
||||
```
|
||||
expected_outputs/
|
||||
├── basic_example_output.json
|
||||
├── complex_scenario_result.txt
|
||||
├── error_cases/
|
||||
│ ├── invalid_input_error.json
|
||||
│ └── timeout_error.txt
|
||||
└── test_cases/
|
||||
├── unit_test_outputs/
|
||||
└── integration_test_results/
|
||||
```
|
||||
|
||||
#### Content Requirements
|
||||
- Outputs correspond to sample inputs in assets/ directory
|
||||
- Include both successful and error case examples
|
||||
- Provide outputs in multiple formats (JSON, text, CSV)
|
||||
- Ensure outputs are reproducible and verifiable
|
||||
|
||||
## Naming Conventions
|
||||
|
||||
### Directory Names
|
||||
- Use lowercase letters only
|
||||
- Use hyphens (-) to separate words
|
||||
- Keep names concise but descriptive
|
||||
- Avoid special characters and spaces
|
||||
|
||||
Examples: `data-processor`, `api-client`, `ml-trainer`
|
||||
|
||||
### File Names
|
||||
- Use lowercase letters for Python scripts
|
||||
- Use hyphens (-) to separate words in script names
|
||||
- Use underscores (_) only when required by Python conventions
|
||||
- Use descriptive names that indicate purpose
|
||||
|
||||
Examples: `data-processor.py`, `api-client.py`, `quality_scorer.py`
|
||||
|
||||
### Script Internal Naming
|
||||
- Use PascalCase for class names
|
||||
- Use snake_case for function and variable names
|
||||
- Use UPPER_CASE for constants
|
||||
- Use descriptive names that indicate purpose
|
||||
|
||||
## Quality Standards
|
||||
|
||||
### Documentation Standards
|
||||
- All documentation must be written in clear, professional English
|
||||
- Use proper Markdown formatting and structure
|
||||
- Include code examples with syntax highlighting
|
||||
- Provide comprehensive coverage of all features
|
||||
- Maintain consistent terminology throughout
|
||||
|
||||
### Code Standards
|
||||
- Follow PEP 8 Python style guidelines
|
||||
- Include comprehensive docstrings for all functions and classes
|
||||
- Implement proper error handling with meaningful error messages
|
||||
- Use type hints where appropriate
|
||||
- Maintain reasonable code complexity and readability
|
||||
|
||||
### Testing Standards
|
||||
- Provide sample data that exercises all major functionality
|
||||
- Include expected outputs for verification
|
||||
- Cover both successful and error scenarios
|
||||
- Ensure reproducible results across different environments
|
||||
|
||||
## Validation Criteria
|
||||
|
||||
Skills are validated against the following criteria:
|
||||
|
||||
### Structural Validation
|
||||
- All mandatory files and directories present
|
||||
- Proper file naming conventions followed
|
||||
- Directory structure matches specification
|
||||
- File permissions and accessibility correct
|
||||
|
||||
### Content Validation
|
||||
- SKILL.md meets minimum length and section requirements
|
||||
- README.md provides adequate quick start information
|
||||
- Scripts contain required components (argparse, main guard, etc.)
|
||||
- Sample data and expected outputs are complete and realistic
|
||||
|
||||
### Quality Validation
|
||||
- Documentation is comprehensive and accurate
|
||||
- Code follows established style and quality guidelines
|
||||
- Examples are practical and demonstrate real usage
|
||||
- Error handling is appropriate and user-friendly
|
||||
|
||||
## Compliance Levels
|
||||
|
||||
### Full Compliance
|
||||
- All mandatory components present and complete
|
||||
- All recommended components present with substantial content
|
||||
- Exceeds minimum quality thresholds for tier
|
||||
- Demonstrates best practices throughout
|
||||
|
||||
### Partial Compliance
|
||||
- All mandatory components present
|
||||
- Most recommended components present
|
||||
- Meets minimum quality thresholds for tier
|
||||
- Generally follows established patterns
|
||||
|
||||
### Non-Compliance
|
||||
- Missing mandatory components
|
||||
- Inadequate content quality or length
|
||||
- Does not meet minimum tier requirements
|
||||
- Significant deviations from established standards
|
||||
|
||||
## Migration and Updates
|
||||
|
||||
### Existing Skills
|
||||
Skills created before this specification should be updated to comply within:
|
||||
- **POWERFUL tier**: 30 days
|
||||
- **STANDARD tier**: 60 days
|
||||
- **BASIC tier**: 90 days
|
||||
|
||||
### Specification Updates
|
||||
- Changes to this specification require team consensus
|
||||
- Breaking changes must provide 90-day migration period
|
||||
- All changes must be documented with rationale and examples
|
||||
- Automated validation tools must be updated accordingly
|
||||
|
||||
## Tools and Automation
|
||||
|
||||
### Validation Tools
|
||||
- `skill_validator.py` - Validates structure and content compliance
|
||||
- `script_tester.py` - Tests script functionality and quality
|
||||
- `quality_scorer.py` - Provides comprehensive quality assessment
|
||||
|
||||
### Integration Points
|
||||
- Pre-commit hooks for basic validation
|
||||
- CI/CD pipeline integration for pull request validation
|
||||
- Automated quality reporting and tracking
|
||||
- Integration with code review processes
|
||||
|
||||
## Examples and Templates
|
||||
|
||||
### Minimal BASIC Tier Example
|
||||
```
|
||||
basic-skill/
|
||||
├── SKILL.md # 100+ lines
|
||||
├── README.md # Basic usage instructions
|
||||
└── scripts/
|
||||
└── main.py # 100-300 lines with argparse
|
||||
```
|
||||
|
||||
### Complete POWERFUL Tier Example
|
||||
```
|
||||
powerful-skill/
|
||||
├── SKILL.md # 300+ lines with comprehensive sections
|
||||
├── README.md # Detailed usage and setup
|
||||
├── scripts/ # Multiple sophisticated scripts
|
||||
│ ├── main_processor.py # 500-800 lines
|
||||
│ ├── data_analyzer.py # 500-800 lines
|
||||
│ └── report_generator.py # 500-800 lines
|
||||
├── assets/ # Diverse sample data
|
||||
│ ├── samples/
|
||||
│ ├── examples/
|
||||
│ └── data/
|
||||
├── references/ # Comprehensive documentation
|
||||
│ ├── api-reference.md
|
||||
│ ├── specifications.md
|
||||
│ └── best-practices.md
|
||||
└── expected_outputs/ # Complete test outputs
|
||||
├── json_outputs/
|
||||
├── text_reports/
|
||||
└── error_cases/
|
||||
```
|
||||
|
||||
This specification serves as the authoritative guide for skill structure within the claude-skills ecosystem. Adherence to these standards ensures consistency, quality, and maintainability across all skills in the repository.
|
||||
375
engineering/skill-tester/references/tier-requirements-matrix.md
Normal file
375
engineering/skill-tester/references/tier-requirements-matrix.md
Normal file
@@ -0,0 +1,375 @@
|
||||
# Tier Requirements Matrix
|
||||
|
||||
**Version**: 1.0.0
|
||||
**Last Updated**: 2026-02-16
|
||||
**Authority**: Claude Skills Engineering Team
|
||||
|
||||
## Overview
|
||||
|
||||
This document provides a comprehensive matrix of requirements for each skill tier within the claude-skills ecosystem. Skills are classified into three tiers based on complexity, functionality, and comprehensiveness: BASIC, STANDARD, and POWERFUL.
|
||||
|
||||
## Tier Classification Philosophy
|
||||
|
||||
### BASIC Tier
|
||||
Entry-level skills that provide fundamental functionality with minimal complexity. Suitable for simple automation tasks, basic data processing, or straightforward utilities.
|
||||
|
||||
### STANDARD Tier
|
||||
Intermediate skills that offer enhanced functionality with moderate complexity. Suitable for business processes, advanced data manipulation, or multi-step workflows.
|
||||
|
||||
### POWERFUL Tier
|
||||
Advanced skills that provide comprehensive functionality with sophisticated implementation. Suitable for complex systems, enterprise-grade tools, or mission-critical applications.
|
||||
|
||||
## Requirements Matrix
|
||||
|
||||
| Component | BASIC | STANDARD | POWERFUL |
|
||||
|-----------|-------|----------|----------|
|
||||
| **SKILL.md Lines** | ≥100 | ≥200 | ≥300 |
|
||||
| **Scripts Count** | ≥1 | ≥1 | ≥2 |
|
||||
| **Script Size (LOC)** | 100-300 | 300-500 | 500-800 |
|
||||
| **Required Directories** | scripts | scripts, assets, references | scripts, assets, references, expected_outputs |
|
||||
| **Argparse Implementation** | Basic | Advanced | Complex with subcommands |
|
||||
| **Output Formats** | Human-readable | JSON + Human-readable | JSON + Human-readable + Custom |
|
||||
| **Error Handling** | Basic | Comprehensive | Advanced with recovery |
|
||||
| **Documentation Depth** | Functional | Comprehensive | Expert-level |
|
||||
| **Examples Provided** | ≥1 | ≥3 | ≥5 |
|
||||
| **Test Coverage** | Basic validation | Sample data testing | Comprehensive test suite |
|
||||
|
||||
## Detailed Requirements by Tier
|
||||
|
||||
### BASIC Tier Requirements
|
||||
|
||||
#### Documentation Requirements
|
||||
- **SKILL.md**: Minimum 100 lines of substantial content
|
||||
- **Required Sections**: Name, Description, Features, Usage, Examples
|
||||
- **README.md**: Basic usage instructions (200+ characters)
|
||||
- **Content Quality**: Clear and functional documentation
|
||||
- **Examples**: At least 1 practical usage example
|
||||
|
||||
#### Code Requirements
|
||||
- **Scripts**: Minimum 1 Python script (100-300 LOC)
|
||||
- **Argparse**: Basic command-line argument parsing
|
||||
- **Main Guard**: `if __name__ == "__main__":` protection
|
||||
- **Dependencies**: Python standard library only
|
||||
- **Output**: Human-readable format with clear messaging
|
||||
- **Error Handling**: Basic exception handling with user-friendly messages
|
||||
|
||||
#### Structure Requirements
|
||||
- **Mandatory Directories**: `scripts/`
|
||||
- **Recommended Directories**: `assets/`, `references/`
|
||||
- **File Organization**: Logical file naming and structure
|
||||
- **Assets**: Optional sample data files
|
||||
|
||||
#### Quality Standards
|
||||
- **Code Style**: Follows basic Python conventions
|
||||
- **Documentation**: Adequate coverage of functionality
|
||||
- **Usability**: Clear usage instructions and examples
|
||||
- **Completeness**: All essential components present
|
||||
|
||||
### STANDARD Tier Requirements
|
||||
|
||||
#### Documentation Requirements
|
||||
- **SKILL.md**: Minimum 200 lines with comprehensive coverage
|
||||
- **Required Sections**: All BASIC sections plus Architecture, Installation
|
||||
- **README.md**: Detailed usage instructions (500+ characters)
|
||||
- **References**: Technical documentation in `references/` directory
|
||||
- **Content Quality**: Professional-grade documentation with technical depth
|
||||
- **Examples**: At least 3 diverse usage examples
|
||||
|
||||
#### Code Requirements
|
||||
- **Scripts**: 1-2 Python scripts (300-500 LOC each)
|
||||
- **Argparse**: Advanced argument parsing with subcommands and validation
|
||||
- **Output Formats**: Both JSON and human-readable output support
|
||||
- **Error Handling**: Comprehensive exception handling with specific error types
|
||||
- **Code Structure**: Well-organized classes and functions
|
||||
- **Documentation**: Comprehensive docstrings for all functions
|
||||
|
||||
#### Structure Requirements
|
||||
- **Mandatory Directories**: `scripts/`, `assets/`, `references/`
|
||||
- **Recommended Directories**: `expected_outputs/`
|
||||
- **Assets**: Multiple sample files demonstrating different use cases
|
||||
- **References**: Technical specifications and API documentation
|
||||
- **Expected Outputs**: Sample results for validation
|
||||
|
||||
#### Quality Standards
|
||||
- **Code Quality**: Advanced Python patterns and best practices
|
||||
- **Documentation**: Expert-level technical documentation
|
||||
- **Testing**: Sample data processing with validation
|
||||
- **Integration**: Consideration for CI/CD and automation use
|
||||
|
||||
### POWERFUL Tier Requirements
|
||||
|
||||
#### Documentation Requirements
|
||||
- **SKILL.md**: Minimum 300 lines with expert-level comprehensiveness
|
||||
- **Required Sections**: All STANDARD sections plus Troubleshooting, Contributing, Advanced Usage
|
||||
- **README.md**: Comprehensive guide with installation and setup (1000+ characters)
|
||||
- **References**: Multiple technical documents with specifications
|
||||
- **Content Quality**: Publication-ready documentation with architectural details
|
||||
- **Examples**: At least 5 examples covering simple to complex scenarios
|
||||
|
||||
#### Code Requirements
|
||||
- **Scripts**: 2-3 Python scripts (500-800 LOC each)
|
||||
- **Argparse**: Complex argument parsing with multiple modes and configurations
|
||||
- **Output Formats**: JSON, human-readable, and custom format support
|
||||
- **Error Handling**: Advanced error handling with recovery mechanisms
|
||||
- **Code Architecture**: Sophisticated design patterns and modular structure
|
||||
- **Performance**: Optimized for efficiency and scalability
|
||||
|
||||
#### Structure Requirements
|
||||
- **Mandatory Directories**: `scripts/`, `assets/`, `references/`, `expected_outputs/`
|
||||
- **Optional Directories**: `tests/`, `examples/`, `docs/`
|
||||
- **Assets**: Comprehensive sample data covering edge cases
|
||||
- **References**: Complete technical specification suite
|
||||
- **Expected Outputs**: Full test result coverage including error cases
|
||||
- **Testing**: Comprehensive validation and test coverage
|
||||
|
||||
#### Quality Standards
|
||||
- **Enterprise Grade**: Production-ready code with enterprise patterns
|
||||
- **Documentation**: Comprehensive technical documentation suitable for technical teams
|
||||
- **Integration**: Full CI/CD integration capabilities
|
||||
- **Maintainability**: Designed for long-term maintenance and extension
|
||||
|
||||
## Tier Assessment Criteria
|
||||
|
||||
### Automatic Tier Classification
|
||||
Skills are automatically classified based on quantitative metrics:
|
||||
|
||||
```python
|
||||
def classify_tier(skill_metrics):
|
||||
if (skill_metrics['skill_md_lines'] >= 300 and
|
||||
skill_metrics['script_count'] >= 2 and
|
||||
skill_metrics['min_script_size'] >= 500 and
|
||||
all_required_dirs_present(['scripts', 'assets', 'references', 'expected_outputs'])):
|
||||
return 'POWERFUL'
|
||||
|
||||
elif (skill_metrics['skill_md_lines'] >= 200 and
|
||||
skill_metrics['script_count'] >= 1 and
|
||||
skill_metrics['min_script_size'] >= 300 and
|
||||
all_required_dirs_present(['scripts', 'assets', 'references'])):
|
||||
return 'STANDARD'
|
||||
|
||||
else:
|
||||
return 'BASIC'
|
||||
```
|
||||
|
||||
### Manual Tier Override
|
||||
Manual tier assignment may be considered when:
|
||||
- Skill provides exceptional value despite not meeting all quantitative requirements
|
||||
- Skill addresses critical infrastructure or security needs
|
||||
- Skill demonstrates innovative approaches or cutting-edge techniques
|
||||
- Skill provides essential integration or compatibility functions
|
||||
|
||||
### Tier Promotion Criteria
|
||||
Skills may be promoted to higher tiers when:
|
||||
- All quantitative requirements for higher tier are met
|
||||
- Quality assessment scores exceed tier thresholds
|
||||
- Community usage and feedback indicate higher value
|
||||
- Continuous integration and maintenance demonstrate reliability
|
||||
|
||||
### Tier Demotion Criteria
|
||||
Skills may be demoted to lower tiers when:
|
||||
- Quality degradation below tier standards
|
||||
- Lack of maintenance or updates
|
||||
- Compatibility issues or security vulnerabilities
|
||||
- Community feedback indicates reduced value
|
||||
|
||||
## Implementation Guidelines by Tier
|
||||
|
||||
### BASIC Tier Implementation
|
||||
```python
|
||||
# Example argparse implementation for BASIC tier
|
||||
parser = argparse.ArgumentParser(description="Basic skill functionality")
|
||||
parser.add_argument("input", help="Input file or parameter")
|
||||
parser.add_argument("--output", help="Output destination")
|
||||
parser.add_argument("--verbose", action="store_true", help="Verbose output")
|
||||
|
||||
# Basic error handling
|
||||
try:
|
||||
result = process_input(args.input)
|
||||
print(f"Processing completed: {result}")
|
||||
except FileNotFoundError:
|
||||
print("Error: Input file not found")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error: {str(e)}")
|
||||
sys.exit(1)
|
||||
```
|
||||
|
||||
### STANDARD Tier Implementation
|
||||
```python
|
||||
# Example argparse implementation for STANDARD tier
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Standard skill with advanced functionality",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="Examples:\n python script.py input.json --format json\n python script.py data/ --batch --output results/"
|
||||
)
|
||||
parser.add_argument("input", help="Input file or directory")
|
||||
parser.add_argument("--format", choices=["json", "text"], default="json", help="Output format")
|
||||
parser.add_argument("--batch", action="store_true", help="Process multiple files")
|
||||
parser.add_argument("--output", help="Output destination")
|
||||
|
||||
# Advanced error handling with specific exception types
|
||||
try:
|
||||
if args.batch:
|
||||
results = batch_process(args.input)
|
||||
else:
|
||||
results = single_process(args.input)
|
||||
|
||||
if args.format == "json":
|
||||
print(json.dumps(results, indent=2))
|
||||
else:
|
||||
print_human_readable(results)
|
||||
|
||||
except FileNotFoundError as e:
|
||||
logging.error(f"File not found: {e}")
|
||||
sys.exit(1)
|
||||
except ValueError as e:
|
||||
logging.error(f"Invalid input: {e}")
|
||||
sys.exit(2)
|
||||
except Exception as e:
|
||||
logging.error(f"Unexpected error: {e}")
|
||||
sys.exit(1)
|
||||
```
|
||||
|
||||
### POWERFUL Tier Implementation
|
||||
```python
|
||||
# Example argparse implementation for POWERFUL tier
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Powerful skill with comprehensive functionality",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
Basic usage:
|
||||
python script.py process input.json --output results/
|
||||
|
||||
Advanced batch processing:
|
||||
python script.py batch data/ --format json --parallel 4 --filter "*.csv"
|
||||
|
||||
Custom configuration:
|
||||
python script.py process input.json --config custom.yaml --dry-run
|
||||
"""
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(dest="command", help="Available commands")
|
||||
|
||||
# Process subcommand
|
||||
process_parser = subparsers.add_parser("process", help="Process single file")
|
||||
process_parser.add_argument("input", help="Input file path")
|
||||
process_parser.add_argument("--config", help="Configuration file")
|
||||
process_parser.add_argument("--dry-run", action="store_true", help="Show what would be done")
|
||||
|
||||
# Batch subcommand
|
||||
batch_parser = subparsers.add_parser("batch", help="Process multiple files")
|
||||
batch_parser.add_argument("directory", help="Input directory")
|
||||
batch_parser.add_argument("--parallel", type=int, default=1, help="Number of parallel processes")
|
||||
batch_parser.add_argument("--filter", help="File filter pattern")
|
||||
|
||||
# Comprehensive error handling with recovery
|
||||
try:
|
||||
if args.command == "process":
|
||||
result = process_with_recovery(args.input, args.config, args.dry_run)
|
||||
elif args.command == "batch":
|
||||
result = batch_process_with_monitoring(args.directory, args.parallel, args.filter)
|
||||
else:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
# Multiple output format support
|
||||
output_formatter = OutputFormatter(args.format)
|
||||
output_formatter.write(result, args.output)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logging.info("Processing interrupted by user")
|
||||
sys.exit(130)
|
||||
except ProcessingError as e:
|
||||
logging.error(f"Processing failed: {e}")
|
||||
if e.recoverable:
|
||||
logging.info("Attempting recovery...")
|
||||
# Recovery logic here
|
||||
sys.exit(1)
|
||||
except ValidationError as e:
|
||||
logging.error(f"Validation failed: {e}")
|
||||
logging.info("Check input format and try again")
|
||||
sys.exit(2)
|
||||
except Exception as e:
|
||||
logging.critical(f"Critical error: {e}")
|
||||
logging.info("Please report this issue")
|
||||
sys.exit(1)
|
||||
```
|
||||
|
||||
## Quality Scoring by Tier
|
||||
|
||||
### Scoring Thresholds
|
||||
- **POWERFUL Tier**: Overall score ≥80, all dimensions ≥75
|
||||
- **STANDARD Tier**: Overall score ≥70, 3+ dimensions ≥65
|
||||
- **BASIC Tier**: Overall score ≥60, meets minimum requirements
|
||||
|
||||
### Dimension Weights (All Tiers)
|
||||
- **Documentation**: 25%
|
||||
- **Code Quality**: 25%
|
||||
- **Completeness**: 25%
|
||||
- **Usability**: 25%
|
||||
|
||||
### Tier-Specific Quality Expectations
|
||||
|
||||
#### BASIC Tier Quality Profile
|
||||
- Documentation: Functional and clear (60+ points expected)
|
||||
- Code Quality: Clean and maintainable (60+ points expected)
|
||||
- Completeness: Essential components present (60+ points expected)
|
||||
- Usability: Easy to understand and use (60+ points expected)
|
||||
|
||||
#### STANDARD Tier Quality Profile
|
||||
- Documentation: Professional and comprehensive (70+ points expected)
|
||||
- Code Quality: Advanced patterns and best practices (70+ points expected)
|
||||
- Completeness: All recommended components (70+ points expected)
|
||||
- Usability: Well-designed user experience (70+ points expected)
|
||||
|
||||
#### POWERFUL Tier Quality Profile
|
||||
- Documentation: Expert-level and publication-ready (80+ points expected)
|
||||
- Code Quality: Enterprise-grade implementation (80+ points expected)
|
||||
- Completeness: Comprehensive test and validation coverage (80+ points expected)
|
||||
- Usability: Exceptional user experience with extensive help (80+ points expected)
|
||||
|
||||
## Tier Migration Process
|
||||
|
||||
### Promotion Process
|
||||
1. **Assessment**: Quality scorer evaluates skill against higher tier requirements
|
||||
2. **Review**: Engineering team reviews assessment and implementation
|
||||
3. **Testing**: Comprehensive testing against higher tier standards
|
||||
4. **Approval**: Team consensus on tier promotion
|
||||
5. **Update**: Skill metadata and documentation updated to reflect new tier
|
||||
|
||||
### Demotion Process
|
||||
1. **Issue Identification**: Quality degradation or standards violation identified
|
||||
2. **Assessment**: Current quality evaluated against tier requirements
|
||||
3. **Notice**: Skill maintainer notified of potential demotion
|
||||
4. **Grace Period**: 30-day period for remediation
|
||||
5. **Final Review**: Re-assessment after grace period
|
||||
6. **Action**: Tier adjustment or removal if standards not met
|
||||
|
||||
### Tier Change Communication
|
||||
- All tier changes logged in skill CHANGELOG.md
|
||||
- Repository-level tier change notifications
|
||||
- Integration with CI/CD systems for automated handling
|
||||
- Community notifications for significant changes
|
||||
|
||||
## Compliance Monitoring
|
||||
|
||||
### Automated Monitoring
|
||||
- Daily quality assessment scans
|
||||
- Tier compliance validation in CI/CD
|
||||
- Automated reporting of tier violations
|
||||
- Integration with code review processes
|
||||
|
||||
### Manual Review Process
|
||||
- Quarterly tier review cycles
|
||||
- Community feedback integration
|
||||
- Expert panel reviews for complex cases
|
||||
- Appeals process for tier disputes
|
||||
|
||||
### Enforcement Actions
|
||||
- **Warning**: First violation or minor issues
|
||||
- **Probation**: Repeated violations or moderate issues
|
||||
- **Demotion**: Serious violations or quality degradation
|
||||
- **Removal**: Critical violations or abandonment
|
||||
|
||||
This tier requirements matrix serves as the definitive guide for skill classification and quality standards within the claude-skills ecosystem. Regular updates ensure alignment with evolving best practices and community needs.
|
||||
1073
engineering/skill-tester/scripts/quality_scorer.py
Normal file
1073
engineering/skill-tester/scripts/quality_scorer.py
Normal file
File diff suppressed because it is too large
Load Diff
731
engineering/skill-tester/scripts/script_tester.py
Normal file
731
engineering/skill-tester/scripts/script_tester.py
Normal file
@@ -0,0 +1,731 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script Tester - Tests Python scripts in a skill directory
|
||||
|
||||
This script validates and tests Python scripts within a skill directory by checking
|
||||
syntax, imports, runtime execution, argparse functionality, and output formats.
|
||||
It ensures scripts meet quality standards and function correctly.
|
||||
|
||||
Usage:
|
||||
python script_tester.py <skill_path> [--timeout SECONDS] [--json] [--verbose]
|
||||
|
||||
Author: Claude Skills Engineering Team
|
||||
Version: 1.0.0
|
||||
Dependencies: Python Standard Library Only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional, Tuple, Union
|
||||
import threading
|
||||
|
||||
|
||||
class TestError(Exception):
|
||||
"""Custom exception for testing errors"""
|
||||
pass
|
||||
|
||||
|
||||
class ScriptTestResult:
|
||||
"""Container for individual script test results"""
|
||||
|
||||
def __init__(self, script_path: str):
|
||||
self.script_path = script_path
|
||||
self.script_name = Path(script_path).name
|
||||
self.timestamp = datetime.utcnow().isoformat() + "Z"
|
||||
self.tests = {}
|
||||
self.overall_status = "PENDING"
|
||||
self.execution_time = 0.0
|
||||
self.errors = []
|
||||
self.warnings = []
|
||||
|
||||
def add_test(self, test_name: str, passed: bool, message: str = "", details: Dict = None):
|
||||
"""Add a test result"""
|
||||
self.tests[test_name] = {
|
||||
"passed": passed,
|
||||
"message": message,
|
||||
"details": details or {}
|
||||
}
|
||||
|
||||
def add_error(self, error: str):
|
||||
"""Add an error message"""
|
||||
self.errors.append(error)
|
||||
|
||||
def add_warning(self, warning: str):
|
||||
"""Add a warning message"""
|
||||
self.warnings.append(warning)
|
||||
|
||||
def calculate_status(self):
|
||||
"""Calculate overall test status"""
|
||||
if not self.tests:
|
||||
self.overall_status = "NO_TESTS"
|
||||
return
|
||||
|
||||
failed_tests = [name for name, result in self.tests.items() if not result["passed"]]
|
||||
|
||||
if not failed_tests:
|
||||
self.overall_status = "PASS"
|
||||
elif len(failed_tests) <= len(self.tests) // 2:
|
||||
self.overall_status = "PARTIAL"
|
||||
else:
|
||||
self.overall_status = "FAIL"
|
||||
|
||||
|
||||
class TestSuite:
|
||||
"""Container for all test results"""
|
||||
|
||||
def __init__(self, skill_path: str):
|
||||
self.skill_path = skill_path
|
||||
self.timestamp = datetime.utcnow().isoformat() + "Z"
|
||||
self.script_results = {}
|
||||
self.summary = {}
|
||||
self.global_errors = []
|
||||
|
||||
def add_script_result(self, result: ScriptTestResult):
|
||||
"""Add a script test result"""
|
||||
self.script_results[result.script_name] = result
|
||||
|
||||
def add_global_error(self, error: str):
|
||||
"""Add a global error message"""
|
||||
self.global_errors.append(error)
|
||||
|
||||
def calculate_summary(self):
|
||||
"""Calculate summary statistics"""
|
||||
if not self.script_results:
|
||||
self.summary = {
|
||||
"total_scripts": 0,
|
||||
"passed": 0,
|
||||
"partial": 0,
|
||||
"failed": 0,
|
||||
"overall_status": "NO_SCRIPTS"
|
||||
}
|
||||
return
|
||||
|
||||
statuses = [result.overall_status for result in self.script_results.values()]
|
||||
|
||||
self.summary = {
|
||||
"total_scripts": len(self.script_results),
|
||||
"passed": statuses.count("PASS"),
|
||||
"partial": statuses.count("PARTIAL"),
|
||||
"failed": statuses.count("FAIL"),
|
||||
"no_tests": statuses.count("NO_TESTS")
|
||||
}
|
||||
|
||||
# Determine overall status
|
||||
if self.summary["failed"] == 0 and self.summary["no_tests"] == 0:
|
||||
self.summary["overall_status"] = "PASS"
|
||||
elif self.summary["passed"] > 0:
|
||||
self.summary["overall_status"] = "PARTIAL"
|
||||
else:
|
||||
self.summary["overall_status"] = "FAIL"
|
||||
|
||||
|
||||
class ScriptTester:
|
||||
"""Main script testing engine"""
|
||||
|
||||
def __init__(self, skill_path: str, timeout: int = 30, verbose: bool = False):
|
||||
self.skill_path = Path(skill_path).resolve()
|
||||
self.timeout = timeout
|
||||
self.verbose = verbose
|
||||
self.test_suite = TestSuite(str(self.skill_path))
|
||||
|
||||
def log_verbose(self, message: str):
|
||||
"""Log verbose message if verbose mode enabled"""
|
||||
if self.verbose:
|
||||
print(f"[VERBOSE] {message}", file=sys.stderr)
|
||||
|
||||
def test_all_scripts(self) -> TestSuite:
|
||||
"""Main entry point - test all scripts in the skill"""
|
||||
try:
|
||||
self.log_verbose(f"Starting script testing for {self.skill_path}")
|
||||
|
||||
# Check if skill path exists
|
||||
if not self.skill_path.exists():
|
||||
self.test_suite.add_global_error(f"Skill path does not exist: {self.skill_path}")
|
||||
return self.test_suite
|
||||
|
||||
scripts_dir = self.skill_path / "scripts"
|
||||
if not scripts_dir.exists():
|
||||
self.test_suite.add_global_error("No scripts directory found")
|
||||
return self.test_suite
|
||||
|
||||
# Find all Python scripts
|
||||
python_files = list(scripts_dir.glob("*.py"))
|
||||
if not python_files:
|
||||
self.test_suite.add_global_error("No Python scripts found in scripts directory")
|
||||
return self.test_suite
|
||||
|
||||
self.log_verbose(f"Found {len(python_files)} Python scripts to test")
|
||||
|
||||
# Test each script
|
||||
for script_path in python_files:
|
||||
try:
|
||||
result = self.test_single_script(script_path)
|
||||
self.test_suite.add_script_result(result)
|
||||
except Exception as e:
|
||||
# Create a failed result for the script
|
||||
result = ScriptTestResult(str(script_path))
|
||||
result.add_error(f"Failed to test script: {str(e)}")
|
||||
result.overall_status = "FAIL"
|
||||
self.test_suite.add_script_result(result)
|
||||
|
||||
# Calculate summary
|
||||
self.test_suite.calculate_summary()
|
||||
|
||||
except Exception as e:
|
||||
self.test_suite.add_global_error(f"Testing failed with exception: {str(e)}")
|
||||
|
||||
return self.test_suite
|
||||
|
||||
def test_single_script(self, script_path: Path) -> ScriptTestResult:
|
||||
"""Test a single Python script comprehensively"""
|
||||
result = ScriptTestResult(str(script_path))
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
self.log_verbose(f"Testing script: {script_path.name}")
|
||||
|
||||
# Read script content
|
||||
try:
|
||||
content = script_path.read_text(encoding='utf-8')
|
||||
except Exception as e:
|
||||
result.add_test("file_readable", False, f"Cannot read file: {str(e)}")
|
||||
result.add_error(f"Cannot read script file: {str(e)}")
|
||||
result.overall_status = "FAIL"
|
||||
return result
|
||||
|
||||
result.add_test("file_readable", True, "Script file is readable")
|
||||
|
||||
# Test 1: Syntax validation
|
||||
self._test_syntax(content, result)
|
||||
|
||||
# Test 2: Import validation
|
||||
self._test_imports(content, result)
|
||||
|
||||
# Test 3: Argparse validation
|
||||
self._test_argparse_implementation(content, result)
|
||||
|
||||
# Test 4: Main guard validation
|
||||
self._test_main_guard(content, result)
|
||||
|
||||
# Test 5: Runtime execution tests
|
||||
if result.tests.get("syntax_valid", {}).get("passed", False):
|
||||
self._test_script_execution(script_path, result)
|
||||
|
||||
# Test 6: Help functionality
|
||||
if result.tests.get("syntax_valid", {}).get("passed", False):
|
||||
self._test_help_functionality(script_path, result)
|
||||
|
||||
# Test 7: Sample data processing (if available)
|
||||
self._test_sample_data_processing(script_path, result)
|
||||
|
||||
# Test 8: Output format validation
|
||||
self._test_output_formats(script_path, result)
|
||||
|
||||
except Exception as e:
|
||||
result.add_error(f"Unexpected error during testing: {str(e)}")
|
||||
|
||||
finally:
|
||||
result.execution_time = time.time() - start_time
|
||||
result.calculate_status()
|
||||
|
||||
return result
|
||||
|
||||
def _test_syntax(self, content: str, result: ScriptTestResult):
|
||||
"""Test Python syntax validity"""
|
||||
self.log_verbose("Testing syntax...")
|
||||
|
||||
try:
|
||||
ast.parse(content)
|
||||
result.add_test("syntax_valid", True, "Python syntax is valid")
|
||||
except SyntaxError as e:
|
||||
result.add_test("syntax_valid", False, f"Syntax error: {str(e)}",
|
||||
{"error": str(e), "line": getattr(e, 'lineno', 'unknown')})
|
||||
result.add_error(f"Syntax error: {str(e)}")
|
||||
|
||||
def _test_imports(self, content: str, result: ScriptTestResult):
|
||||
"""Test import statements for external dependencies"""
|
||||
self.log_verbose("Testing imports...")
|
||||
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
external_imports = self._find_external_imports(tree)
|
||||
|
||||
if not external_imports:
|
||||
result.add_test("imports_valid", True, "Uses only standard library imports")
|
||||
else:
|
||||
result.add_test("imports_valid", False,
|
||||
f"Uses external imports: {', '.join(external_imports)}",
|
||||
{"external_imports": external_imports})
|
||||
result.add_error(f"External imports detected: {', '.join(external_imports)}")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("imports_valid", False, f"Error analyzing imports: {str(e)}")
|
||||
|
||||
def _find_external_imports(self, tree: ast.AST) -> List[str]:
|
||||
"""Find external (non-stdlib) imports"""
|
||||
# Comprehensive standard library module list
|
||||
stdlib_modules = {
|
||||
# Built-in modules
|
||||
'argparse', 'ast', 'json', 'os', 'sys', 'pathlib', 'datetime', 'typing',
|
||||
'collections', 're', 'math', 'random', 'itertools', 'functools', 'operator',
|
||||
'csv', 'sqlite3', 'urllib', 'http', 'html', 'xml', 'email', 'base64',
|
||||
'hashlib', 'hmac', 'secrets', 'tempfile', 'shutil', 'glob', 'fnmatch',
|
||||
'subprocess', 'threading', 'multiprocessing', 'queue', 'time', 'calendar',
|
||||
'locale', 'gettext', 'logging', 'warnings', 'unittest', 'doctest',
|
||||
'pickle', 'copy', 'pprint', 'reprlib', 'enum', 'dataclasses',
|
||||
'contextlib', 'abc', 'atexit', 'traceback', 'gc', 'weakref', 'types',
|
||||
'decimal', 'fractions', 'statistics', 'cmath', 'platform', 'errno',
|
||||
'io', 'codecs', 'unicodedata', 'stringprep', 'textwrap', 'string',
|
||||
'struct', 'difflib', 'heapq', 'bisect', 'array', 'uuid', 'mmap',
|
||||
'ctypes', 'winreg', 'msvcrt', 'winsound', 'posix', 'pwd', 'grp',
|
||||
'crypt', 'termios', 'tty', 'pty', 'fcntl', 'resource', 'nis',
|
||||
'syslog', 'signal', 'socket', 'ssl', 'select', 'selectors',
|
||||
'asyncio', 'asynchat', 'asyncore', 'netrc', 'xdrlib', 'plistlib',
|
||||
'mailbox', 'mimetypes', 'encodings', 'pkgutil', 'modulefinder',
|
||||
'runpy', 'importlib', 'imp', 'zipimport', 'zipfile', 'tarfile',
|
||||
'gzip', 'bz2', 'lzma', 'zlib', 'binascii', 'quopri', 'uu',
|
||||
'configparser', 'netrc', 'xdrlib', 'plistlib', 'token', 'tokenize',
|
||||
'keyword', 'heapq', 'bisect', 'array', 'weakref', 'types',
|
||||
'copyreg', 'shelve', 'marshal', 'dbm', 'sqlite3', 'zoneinfo'
|
||||
}
|
||||
|
||||
external_imports = []
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
module_name = alias.name.split('.')[0]
|
||||
if module_name not in stdlib_modules and not module_name.startswith('_'):
|
||||
external_imports.append(alias.name)
|
||||
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
module_name = node.module.split('.')[0]
|
||||
if module_name not in stdlib_modules and not module_name.startswith('_'):
|
||||
external_imports.append(node.module)
|
||||
|
||||
return list(set(external_imports))
|
||||
|
||||
def _test_argparse_implementation(self, content: str, result: ScriptTestResult):
|
||||
"""Test argparse implementation"""
|
||||
self.log_verbose("Testing argparse implementation...")
|
||||
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
|
||||
# Check for argparse import
|
||||
has_argparse_import = False
|
||||
has_parser_creation = False
|
||||
has_parse_args = False
|
||||
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, (ast.Import, ast.ImportFrom)):
|
||||
if (isinstance(node, ast.Import) and
|
||||
any(alias.name == 'argparse' for alias in node.names)):
|
||||
has_argparse_import = True
|
||||
elif (isinstance(node, ast.ImportFrom) and
|
||||
node.module == 'argparse'):
|
||||
has_argparse_import = True
|
||||
|
||||
elif isinstance(node, ast.Call):
|
||||
# Check for ArgumentParser creation
|
||||
if (isinstance(node.func, ast.Attribute) and
|
||||
isinstance(node.func.value, ast.Name) and
|
||||
node.func.value.id == 'argparse' and
|
||||
node.func.attr == 'ArgumentParser'):
|
||||
has_parser_creation = True
|
||||
|
||||
# Check for parse_args call
|
||||
if (isinstance(node.func, ast.Attribute) and
|
||||
node.func.attr == 'parse_args'):
|
||||
has_parse_args = True
|
||||
|
||||
argparse_score = sum([has_argparse_import, has_parser_creation, has_parse_args])
|
||||
|
||||
if argparse_score == 3:
|
||||
result.add_test("argparse_implementation", True, "Complete argparse implementation found")
|
||||
elif argparse_score > 0:
|
||||
result.add_test("argparse_implementation", False,
|
||||
"Partial argparse implementation",
|
||||
{"missing_components": [
|
||||
comp for comp, present in [
|
||||
("import", has_argparse_import),
|
||||
("parser_creation", has_parser_creation),
|
||||
("parse_args", has_parse_args)
|
||||
] if not present
|
||||
]})
|
||||
result.add_warning("Incomplete argparse implementation")
|
||||
else:
|
||||
result.add_test("argparse_implementation", False, "No argparse implementation found")
|
||||
result.add_error("Script should use argparse for command-line arguments")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("argparse_implementation", False, f"Error analyzing argparse: {str(e)}")
|
||||
|
||||
def _test_main_guard(self, content: str, result: ScriptTestResult):
|
||||
"""Test for if __name__ == '__main__' guard"""
|
||||
self.log_verbose("Testing main guard...")
|
||||
|
||||
has_main_guard = 'if __name__ == "__main__"' in content or "if __name__ == '__main__'" in content
|
||||
|
||||
if has_main_guard:
|
||||
result.add_test("main_guard", True, "Has proper main guard")
|
||||
else:
|
||||
result.add_test("main_guard", False, "Missing main guard")
|
||||
result.add_error("Script should have 'if __name__ == \"__main__\"' guard")
|
||||
|
||||
def _test_script_execution(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test basic script execution"""
|
||||
self.log_verbose("Testing script execution...")
|
||||
|
||||
try:
|
||||
# Try to run the script with no arguments (should not crash immediately)
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
|
||||
# Script might exit with error code if no args provided, but shouldn't crash
|
||||
if process.returncode in (0, 1, 2): # 0=success, 1=general error, 2=misuse
|
||||
result.add_test("basic_execution", True,
|
||||
f"Script runs without crashing (exit code: {process.returncode})")
|
||||
else:
|
||||
result.add_test("basic_execution", False,
|
||||
f"Script crashed with exit code {process.returncode}",
|
||||
{"stdout": process.stdout, "stderr": process.stderr})
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
result.add_test("basic_execution", False,
|
||||
f"Script execution timed out after {self.timeout} seconds")
|
||||
result.add_error(f"Script execution timeout ({self.timeout}s)")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("basic_execution", False, f"Execution error: {str(e)}")
|
||||
result.add_error(f"Script execution failed: {str(e)}")
|
||||
|
||||
def _test_help_functionality(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test --help functionality"""
|
||||
self.log_verbose("Testing help functionality...")
|
||||
|
||||
try:
|
||||
# Test --help flag
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path), '--help'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
|
||||
if process.returncode == 0:
|
||||
help_output = process.stdout
|
||||
|
||||
# Check for reasonable help content
|
||||
help_indicators = ['usage:', 'positional arguments:', 'optional arguments:',
|
||||
'options:', 'description:', 'help']
|
||||
has_help_content = any(indicator in help_output.lower() for indicator in help_indicators)
|
||||
|
||||
if has_help_content and len(help_output.strip()) > 50:
|
||||
result.add_test("help_functionality", True, "Provides comprehensive help text")
|
||||
else:
|
||||
result.add_test("help_functionality", False,
|
||||
"Help text is too brief or missing key sections",
|
||||
{"help_output": help_output})
|
||||
result.add_warning("Help text could be more comprehensive")
|
||||
|
||||
else:
|
||||
result.add_test("help_functionality", False,
|
||||
f"Help command failed with exit code {process.returncode}",
|
||||
{"stderr": process.stderr})
|
||||
result.add_error("--help flag does not work properly")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
result.add_test("help_functionality", False, "Help command timed out")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("help_functionality", False, f"Help test error: {str(e)}")
|
||||
|
||||
def _test_sample_data_processing(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test script against sample data if available"""
|
||||
self.log_verbose("Testing sample data processing...")
|
||||
|
||||
assets_dir = self.skill_path / "assets"
|
||||
if not assets_dir.exists():
|
||||
result.add_test("sample_data_processing", True, "No sample data to test (assets dir missing)")
|
||||
return
|
||||
|
||||
# Look for sample input files
|
||||
sample_files = list(assets_dir.rglob("*sample*")) + list(assets_dir.rglob("*test*"))
|
||||
sample_files = [f for f in sample_files if f.is_file() and not f.name.startswith('.')]
|
||||
|
||||
if not sample_files:
|
||||
result.add_test("sample_data_processing", True, "No sample data files found to test")
|
||||
return
|
||||
|
||||
tested_files = 0
|
||||
successful_tests = 0
|
||||
|
||||
for sample_file in sample_files[:3]: # Test up to 3 sample files
|
||||
try:
|
||||
self.log_verbose(f"Testing with sample file: {sample_file.name}")
|
||||
|
||||
# Try to run script with the sample file as input
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path), str(sample_file)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=self.timeout,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
|
||||
tested_files += 1
|
||||
|
||||
if process.returncode == 0:
|
||||
successful_tests += 1
|
||||
else:
|
||||
self.log_verbose(f"Sample test failed for {sample_file.name}: {process.stderr}")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
tested_files += 1
|
||||
result.add_warning(f"Sample data test timed out for {sample_file.name}")
|
||||
except Exception as e:
|
||||
tested_files += 1
|
||||
self.log_verbose(f"Sample test error for {sample_file.name}: {str(e)}")
|
||||
|
||||
if tested_files == 0:
|
||||
result.add_test("sample_data_processing", True, "No testable sample data found")
|
||||
elif successful_tests == tested_files:
|
||||
result.add_test("sample_data_processing", True,
|
||||
f"Successfully processed all {tested_files} sample files")
|
||||
elif successful_tests > 0:
|
||||
result.add_test("sample_data_processing", False,
|
||||
f"Processed {successful_tests}/{tested_files} sample files",
|
||||
{"success_rate": successful_tests / tested_files})
|
||||
result.add_warning("Some sample data processing failed")
|
||||
else:
|
||||
result.add_test("sample_data_processing", False,
|
||||
"Failed to process any sample data files")
|
||||
result.add_error("Script cannot process sample data")
|
||||
|
||||
def _test_output_formats(self, script_path: Path, result: ScriptTestResult):
|
||||
"""Test output format compliance"""
|
||||
self.log_verbose("Testing output formats...")
|
||||
|
||||
# Test if script supports JSON output
|
||||
json_support = False
|
||||
human_readable_support = False
|
||||
|
||||
try:
|
||||
# Read script content to check for output format indicators
|
||||
content = script_path.read_text(encoding='utf-8')
|
||||
|
||||
# Look for JSON-related code
|
||||
if any(indicator in content.lower() for indicator in ['json.dump', 'json.load', '"json"', '--json']):
|
||||
json_support = True
|
||||
|
||||
# Look for human-readable output indicators
|
||||
if any(indicator in content for indicator in ['print(', 'format(', 'f"', "f'"]):
|
||||
human_readable_support = True
|
||||
|
||||
# Try running with --json flag if it looks like it supports it
|
||||
if '--json' in content:
|
||||
try:
|
||||
process = subprocess.run(
|
||||
[sys.executable, str(script_path), '--json', '--help'],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
cwd=script_path.parent
|
||||
)
|
||||
if process.returncode == 0:
|
||||
json_support = True
|
||||
except:
|
||||
pass
|
||||
|
||||
# Evaluate dual output support
|
||||
if json_support and human_readable_support:
|
||||
result.add_test("output_formats", True, "Supports both JSON and human-readable output")
|
||||
elif json_support or human_readable_support:
|
||||
format_type = "JSON" if json_support else "human-readable"
|
||||
result.add_test("output_formats", False,
|
||||
f"Supports only {format_type} output",
|
||||
{"json_support": json_support, "human_readable_support": human_readable_support})
|
||||
result.add_warning("Consider adding dual output format support")
|
||||
else:
|
||||
result.add_test("output_formats", False, "No clear output format support detected")
|
||||
result.add_warning("Output format support is unclear")
|
||||
|
||||
except Exception as e:
|
||||
result.add_test("output_formats", False, f"Error testing output formats: {str(e)}")
|
||||
|
||||
|
||||
class TestReportFormatter:
|
||||
"""Formats test reports for output"""
|
||||
|
||||
@staticmethod
|
||||
def format_json(test_suite: TestSuite) -> str:
|
||||
"""Format test suite as JSON"""
|
||||
return json.dumps({
|
||||
"skill_path": test_suite.skill_path,
|
||||
"timestamp": test_suite.timestamp,
|
||||
"summary": test_suite.summary,
|
||||
"global_errors": test_suite.global_errors,
|
||||
"script_results": {
|
||||
name: {
|
||||
"script_path": result.script_path,
|
||||
"timestamp": result.timestamp,
|
||||
"overall_status": result.overall_status,
|
||||
"execution_time": round(result.execution_time, 2),
|
||||
"tests": result.tests,
|
||||
"errors": result.errors,
|
||||
"warnings": result.warnings
|
||||
}
|
||||
for name, result in test_suite.script_results.items()
|
||||
}
|
||||
}, indent=2)
|
||||
|
||||
@staticmethod
|
||||
def format_human_readable(test_suite: TestSuite) -> str:
|
||||
"""Format test suite as human-readable text"""
|
||||
lines = []
|
||||
lines.append("=" * 60)
|
||||
lines.append("SCRIPT TESTING REPORT")
|
||||
lines.append("=" * 60)
|
||||
lines.append(f"Skill: {test_suite.skill_path}")
|
||||
lines.append(f"Timestamp: {test_suite.timestamp}")
|
||||
lines.append("")
|
||||
|
||||
# Summary
|
||||
if test_suite.summary:
|
||||
lines.append("SUMMARY:")
|
||||
lines.append(f" Total Scripts: {test_suite.summary['total_scripts']}")
|
||||
lines.append(f" Passed: {test_suite.summary['passed']}")
|
||||
lines.append(f" Partial: {test_suite.summary['partial']}")
|
||||
lines.append(f" Failed: {test_suite.summary['failed']}")
|
||||
lines.append(f" Overall Status: {test_suite.summary['overall_status']}")
|
||||
lines.append("")
|
||||
|
||||
# Global errors
|
||||
if test_suite.global_errors:
|
||||
lines.append("GLOBAL ERRORS:")
|
||||
for error in test_suite.global_errors:
|
||||
lines.append(f" • {error}")
|
||||
lines.append("")
|
||||
|
||||
# Individual script results
|
||||
for script_name, result in test_suite.script_results.items():
|
||||
lines.append(f"SCRIPT: {script_name}")
|
||||
lines.append(f" Status: {result.overall_status}")
|
||||
lines.append(f" Execution Time: {result.execution_time:.2f}s")
|
||||
lines.append("")
|
||||
|
||||
# Tests
|
||||
if result.tests:
|
||||
lines.append(" TESTS:")
|
||||
for test_name, test_result in result.tests.items():
|
||||
status = "✓ PASS" if test_result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {test_result['message']}")
|
||||
lines.append("")
|
||||
|
||||
# Errors
|
||||
if result.errors:
|
||||
lines.append(" ERRORS:")
|
||||
for error in result.errors:
|
||||
lines.append(f" • {error}")
|
||||
lines.append("")
|
||||
|
||||
# Warnings
|
||||
if result.warnings:
|
||||
lines.append(" WARNINGS:")
|
||||
for warning in result.warnings:
|
||||
lines.append(f" • {warning}")
|
||||
lines.append("")
|
||||
|
||||
lines.append("-" * 40)
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Test Python scripts in a skill directory",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python script_tester.py engineering/my-skill
|
||||
python script_tester.py engineering/my-skill --timeout 60 --json
|
||||
python script_tester.py engineering/my-skill --verbose
|
||||
|
||||
Test Categories:
|
||||
- Syntax validation (AST parsing)
|
||||
- Import validation (stdlib only)
|
||||
- Argparse implementation
|
||||
- Main guard presence
|
||||
- Basic execution testing
|
||||
- Help functionality
|
||||
- Sample data processing
|
||||
- Output format compliance
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument("skill_path",
|
||||
help="Path to the skill directory containing scripts to test")
|
||||
parser.add_argument("--timeout",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Timeout for script execution tests in seconds (default: 30)")
|
||||
parser.add_argument("--json",
|
||||
action="store_true",
|
||||
help="Output results in JSON format")
|
||||
parser.add_argument("--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose logging")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Create tester and run tests
|
||||
tester = ScriptTester(args.skill_path, args.timeout, args.verbose)
|
||||
test_suite = tester.test_all_scripts()
|
||||
|
||||
# Format and output results
|
||||
if args.json:
|
||||
print(TestReportFormatter.format_json(test_suite))
|
||||
else:
|
||||
print(TestReportFormatter.format_human_readable(test_suite))
|
||||
|
||||
# Exit with appropriate code
|
||||
if test_suite.global_errors:
|
||||
sys.exit(1)
|
||||
elif test_suite.summary.get("overall_status") == "FAIL":
|
||||
sys.exit(1)
|
||||
elif test_suite.summary.get("overall_status") == "PARTIAL":
|
||||
sys.exit(2) # Partial success
|
||||
else:
|
||||
sys.exit(0) # Success
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nTesting interrupted by user", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
print(f"Testing failed: {str(e)}", file=sys.stderr)
|
||||
if args.verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
653
engineering/skill-tester/scripts/skill_validator.py
Normal file
653
engineering/skill-tester/scripts/skill_validator.py
Normal file
@@ -0,0 +1,653 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Skill Validator - Validates skill directories against quality standards
|
||||
|
||||
This script validates a skill directory structure, documentation, and Python scripts
|
||||
against the claude-skills ecosystem standards. It checks for required files, proper
|
||||
formatting, and compliance with tier-specific requirements.
|
||||
|
||||
Usage:
|
||||
python skill_validator.py <skill_path> [--tier TIER] [--json] [--verbose]
|
||||
|
||||
Author: Claude Skills Engineering Team
|
||||
Version: 1.0.0
|
||||
Dependencies: Python Standard Library Only
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import ast
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import yaml
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Any, Optional, Tuple
|
||||
|
||||
|
||||
class ValidationError(Exception):
|
||||
"""Custom exception for validation errors"""
|
||||
pass
|
||||
|
||||
|
||||
class ValidationReport:
|
||||
"""Container for validation results"""
|
||||
|
||||
def __init__(self, skill_path: str):
|
||||
self.skill_path = skill_path
|
||||
self.timestamp = datetime.utcnow().isoformat() + "Z"
|
||||
self.checks = {}
|
||||
self.warnings = []
|
||||
self.errors = []
|
||||
self.suggestions = []
|
||||
self.overall_score = 0.0
|
||||
self.compliance_level = "FAIL"
|
||||
|
||||
def add_check(self, check_name: str, passed: bool, message: str = "", score: float = 0.0):
|
||||
"""Add a validation check result"""
|
||||
self.checks[check_name] = {
|
||||
"passed": passed,
|
||||
"message": message,
|
||||
"score": score
|
||||
}
|
||||
|
||||
def add_warning(self, message: str):
|
||||
"""Add a warning message"""
|
||||
self.warnings.append(message)
|
||||
|
||||
def add_error(self, message: str):
|
||||
"""Add an error message"""
|
||||
self.errors.append(message)
|
||||
|
||||
def add_suggestion(self, message: str):
|
||||
"""Add an improvement suggestion"""
|
||||
self.suggestions.append(message)
|
||||
|
||||
def calculate_overall_score(self):
|
||||
"""Calculate overall compliance score"""
|
||||
if not self.checks:
|
||||
self.overall_score = 0.0
|
||||
return
|
||||
|
||||
total_score = sum(check["score"] for check in self.checks.values())
|
||||
max_score = len(self.checks) * 1.0
|
||||
self.overall_score = (total_score / max_score) * 100 if max_score > 0 else 0.0
|
||||
|
||||
# Determine compliance level
|
||||
if self.overall_score >= 90:
|
||||
self.compliance_level = "EXCELLENT"
|
||||
elif self.overall_score >= 75:
|
||||
self.compliance_level = "GOOD"
|
||||
elif self.overall_score >= 60:
|
||||
self.compliance_level = "ACCEPTABLE"
|
||||
elif self.overall_score >= 40:
|
||||
self.compliance_level = "NEEDS_IMPROVEMENT"
|
||||
else:
|
||||
self.compliance_level = "POOR"
|
||||
|
||||
|
||||
class SkillValidator:
|
||||
"""Main skill validation engine"""
|
||||
|
||||
# Tier requirements
|
||||
TIER_REQUIREMENTS = {
|
||||
"BASIC": {
|
||||
"min_skill_md_lines": 100,
|
||||
"min_scripts": 1,
|
||||
"script_size_range": (100, 300),
|
||||
"required_dirs": ["scripts"],
|
||||
"optional_dirs": ["assets", "references", "expected_outputs"],
|
||||
"features_required": ["argparse", "main_guard"]
|
||||
},
|
||||
"STANDARD": {
|
||||
"min_skill_md_lines": 200,
|
||||
"min_scripts": 1,
|
||||
"script_size_range": (300, 500),
|
||||
"required_dirs": ["scripts", "assets", "references"],
|
||||
"optional_dirs": ["expected_outputs"],
|
||||
"features_required": ["argparse", "main_guard", "json_output", "help_text"]
|
||||
},
|
||||
"POWERFUL": {
|
||||
"min_skill_md_lines": 300,
|
||||
"min_scripts": 2,
|
||||
"script_size_range": (500, 800),
|
||||
"required_dirs": ["scripts", "assets", "references", "expected_outputs"],
|
||||
"optional_dirs": [],
|
||||
"features_required": ["argparse", "main_guard", "json_output", "help_text", "error_handling"]
|
||||
}
|
||||
}
|
||||
|
||||
REQUIRED_SKILL_MD_SECTIONS = [
|
||||
"Name", "Description", "Features", "Usage", "Examples"
|
||||
]
|
||||
|
||||
FRONTMATTER_REQUIRED_FIELDS = [
|
||||
"Name", "Tier", "Category", "Dependencies", "Author", "Version"
|
||||
]
|
||||
|
||||
def __init__(self, skill_path: str, target_tier: Optional[str] = None, verbose: bool = False):
|
||||
self.skill_path = Path(skill_path).resolve()
|
||||
self.target_tier = target_tier
|
||||
self.verbose = verbose
|
||||
self.report = ValidationReport(str(self.skill_path))
|
||||
|
||||
def log_verbose(self, message: str):
|
||||
"""Log verbose message if verbose mode enabled"""
|
||||
if self.verbose:
|
||||
print(f"[VERBOSE] {message}", file=sys.stderr)
|
||||
|
||||
def validate_skill_structure(self) -> ValidationReport:
|
||||
"""Main validation entry point"""
|
||||
try:
|
||||
self.log_verbose(f"Starting validation of {self.skill_path}")
|
||||
|
||||
# Check if path exists
|
||||
if not self.skill_path.exists():
|
||||
self.report.add_error(f"Skill path does not exist: {self.skill_path}")
|
||||
return self.report
|
||||
|
||||
if not self.skill_path.is_dir():
|
||||
self.report.add_error(f"Skill path is not a directory: {self.skill_path}")
|
||||
return self.report
|
||||
|
||||
# Run all validation checks
|
||||
self._validate_required_files()
|
||||
self._validate_skill_md()
|
||||
self._validate_readme()
|
||||
self._validate_directory_structure()
|
||||
self._validate_python_scripts()
|
||||
self._validate_tier_compliance()
|
||||
|
||||
# Calculate overall score
|
||||
self.report.calculate_overall_score()
|
||||
|
||||
self.log_verbose(f"Validation completed. Score: {self.report.overall_score:.1f}")
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_error(f"Validation failed with exception: {str(e)}")
|
||||
|
||||
return self.report
|
||||
|
||||
def _validate_required_files(self):
|
||||
"""Validate presence of required files"""
|
||||
self.log_verbose("Checking required files...")
|
||||
|
||||
# Check SKILL.md
|
||||
skill_md_path = self.skill_path / "SKILL.md"
|
||||
if skill_md_path.exists():
|
||||
self.report.add_check("skill_md_exists", True, "SKILL.md found", 1.0)
|
||||
else:
|
||||
self.report.add_check("skill_md_exists", False, "SKILL.md missing", 0.0)
|
||||
self.report.add_error("SKILL.md is required but missing")
|
||||
|
||||
# Check README.md
|
||||
readme_path = self.skill_path / "README.md"
|
||||
if readme_path.exists():
|
||||
self.report.add_check("readme_exists", True, "README.md found", 1.0)
|
||||
else:
|
||||
self.report.add_check("readme_exists", False, "README.md missing", 0.0)
|
||||
self.report.add_warning("README.md is recommended but missing")
|
||||
self.report.add_suggestion("Add README.md with usage instructions and examples")
|
||||
|
||||
def _validate_skill_md(self):
|
||||
"""Validate SKILL.md content and format"""
|
||||
self.log_verbose("Validating SKILL.md...")
|
||||
|
||||
skill_md_path = self.skill_path / "SKILL.md"
|
||||
if not skill_md_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
content = skill_md_path.read_text(encoding='utf-8')
|
||||
lines = content.split('\n')
|
||||
line_count = len([line for line in lines if line.strip()])
|
||||
|
||||
# Check line count
|
||||
min_lines = self._get_tier_requirement("min_skill_md_lines", 100)
|
||||
if line_count >= min_lines:
|
||||
self.report.add_check("skill_md_length", True,
|
||||
f"SKILL.md has {line_count} lines (≥{min_lines})", 1.0)
|
||||
else:
|
||||
self.report.add_check("skill_md_length", False,
|
||||
f"SKILL.md has {line_count} lines (<{min_lines})", 0.0)
|
||||
self.report.add_error(f"SKILL.md too short: {line_count} lines, minimum {min_lines}")
|
||||
|
||||
# Validate frontmatter
|
||||
self._validate_frontmatter(content)
|
||||
|
||||
# Check required sections
|
||||
self._validate_required_sections(content)
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_check("skill_md_readable", False, f"Error reading SKILL.md: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Cannot read SKILL.md: {str(e)}")
|
||||
|
||||
def _validate_frontmatter(self, content: str):
|
||||
"""Validate SKILL.md frontmatter"""
|
||||
self.log_verbose("Validating frontmatter...")
|
||||
|
||||
# Extract frontmatter
|
||||
if content.startswith('---'):
|
||||
try:
|
||||
end_marker = content.find('---', 3)
|
||||
if end_marker == -1:
|
||||
self.report.add_check("frontmatter_format", False,
|
||||
"Frontmatter closing marker not found", 0.0)
|
||||
return
|
||||
|
||||
frontmatter_text = content[3:end_marker].strip()
|
||||
frontmatter = yaml.safe_load(frontmatter_text)
|
||||
|
||||
if not isinstance(frontmatter, dict):
|
||||
self.report.add_check("frontmatter_format", False,
|
||||
"Frontmatter is not a valid dictionary", 0.0)
|
||||
return
|
||||
|
||||
# Check required fields
|
||||
missing_fields = []
|
||||
for field in self.FRONTMATTER_REQUIRED_FIELDS:
|
||||
if field not in frontmatter:
|
||||
missing_fields.append(field)
|
||||
|
||||
if not missing_fields:
|
||||
self.report.add_check("frontmatter_complete", True,
|
||||
"All required frontmatter fields present", 1.0)
|
||||
else:
|
||||
self.report.add_check("frontmatter_complete", False,
|
||||
f"Missing fields: {', '.join(missing_fields)}", 0.0)
|
||||
self.report.add_error(f"Missing frontmatter fields: {', '.join(missing_fields)}")
|
||||
|
||||
except yaml.YAMLError as e:
|
||||
self.report.add_check("frontmatter_format", False,
|
||||
f"Invalid YAML frontmatter: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Invalid YAML frontmatter: {str(e)}")
|
||||
|
||||
else:
|
||||
self.report.add_check("frontmatter_exists", False,
|
||||
"No frontmatter found", 0.0)
|
||||
self.report.add_error("SKILL.md must start with YAML frontmatter")
|
||||
|
||||
def _validate_required_sections(self, content: str):
|
||||
"""Validate required sections in SKILL.md"""
|
||||
self.log_verbose("Checking required sections...")
|
||||
|
||||
missing_sections = []
|
||||
for section in self.REQUIRED_SKILL_MD_SECTIONS:
|
||||
pattern = rf'^#+\s*{re.escape(section)}\s*$'
|
||||
if not re.search(pattern, content, re.MULTILINE | re.IGNORECASE):
|
||||
missing_sections.append(section)
|
||||
|
||||
if not missing_sections:
|
||||
self.report.add_check("required_sections", True,
|
||||
"All required sections present", 1.0)
|
||||
else:
|
||||
self.report.add_check("required_sections", False,
|
||||
f"Missing sections: {', '.join(missing_sections)}", 0.0)
|
||||
self.report.add_error(f"Missing required sections: {', '.join(missing_sections)}")
|
||||
|
||||
def _validate_readme(self):
|
||||
"""Validate README.md content"""
|
||||
self.log_verbose("Validating README.md...")
|
||||
|
||||
readme_path = self.skill_path / "README.md"
|
||||
if not readme_path.exists():
|
||||
return
|
||||
|
||||
try:
|
||||
content = readme_path.read_text(encoding='utf-8')
|
||||
|
||||
# Check minimum content length
|
||||
if len(content.strip()) >= 200:
|
||||
self.report.add_check("readme_substantial", True,
|
||||
"README.md has substantial content", 1.0)
|
||||
else:
|
||||
self.report.add_check("readme_substantial", False,
|
||||
"README.md content is too brief", 0.5)
|
||||
self.report.add_suggestion("Expand README.md with more detailed usage instructions")
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_check("readme_readable", False,
|
||||
f"Error reading README.md: {str(e)}", 0.0)
|
||||
|
||||
def _validate_directory_structure(self):
|
||||
"""Validate directory structure against tier requirements"""
|
||||
self.log_verbose("Validating directory structure...")
|
||||
|
||||
required_dirs = self._get_tier_requirement("required_dirs", ["scripts"])
|
||||
optional_dirs = self._get_tier_requirement("optional_dirs", [])
|
||||
|
||||
# Check required directories
|
||||
missing_required = []
|
||||
for dir_name in required_dirs:
|
||||
dir_path = self.skill_path / dir_name
|
||||
if dir_path.exists() and dir_path.is_dir():
|
||||
self.report.add_check(f"dir_{dir_name}_exists", True,
|
||||
f"{dir_name}/ directory found", 1.0)
|
||||
else:
|
||||
missing_required.append(dir_name)
|
||||
self.report.add_check(f"dir_{dir_name}_exists", False,
|
||||
f"{dir_name}/ directory missing", 0.0)
|
||||
|
||||
if missing_required:
|
||||
self.report.add_error(f"Missing required directories: {', '.join(missing_required)}")
|
||||
|
||||
# Check optional directories and provide suggestions
|
||||
missing_optional = []
|
||||
for dir_name in optional_dirs:
|
||||
dir_path = self.skill_path / dir_name
|
||||
if not (dir_path.exists() and dir_path.is_dir()):
|
||||
missing_optional.append(dir_name)
|
||||
|
||||
if missing_optional:
|
||||
self.report.add_suggestion(f"Consider adding optional directories: {', '.join(missing_optional)}")
|
||||
|
||||
def _validate_python_scripts(self):
|
||||
"""Validate Python scripts in the scripts directory"""
|
||||
self.log_verbose("Validating Python scripts...")
|
||||
|
||||
scripts_dir = self.skill_path / "scripts"
|
||||
if not scripts_dir.exists():
|
||||
return
|
||||
|
||||
python_files = list(scripts_dir.glob("*.py"))
|
||||
min_scripts = self._get_tier_requirement("min_scripts", 1)
|
||||
|
||||
# Check minimum number of scripts
|
||||
if len(python_files) >= min_scripts:
|
||||
self.report.add_check("min_scripts_count", True,
|
||||
f"Found {len(python_files)} Python scripts (≥{min_scripts})", 1.0)
|
||||
else:
|
||||
self.report.add_check("min_scripts_count", False,
|
||||
f"Found {len(python_files)} Python scripts (<{min_scripts})", 0.0)
|
||||
self.report.add_error(f"Insufficient scripts: {len(python_files)}, minimum {min_scripts}")
|
||||
|
||||
# Validate each script
|
||||
for script_path in python_files:
|
||||
self._validate_single_script(script_path)
|
||||
|
||||
def _validate_single_script(self, script_path: Path):
|
||||
"""Validate a single Python script"""
|
||||
script_name = script_path.name
|
||||
self.log_verbose(f"Validating script: {script_name}")
|
||||
|
||||
try:
|
||||
content = script_path.read_text(encoding='utf-8')
|
||||
|
||||
# Count lines of code (excluding empty lines and comments)
|
||||
lines = content.split('\n')
|
||||
loc = len([line for line in lines if line.strip() and not line.strip().startswith('#')])
|
||||
|
||||
# Check script size against tier requirements
|
||||
size_range = self._get_tier_requirement("script_size_range", (100, 1000))
|
||||
min_size, max_size = size_range
|
||||
|
||||
if min_size <= loc <= max_size:
|
||||
self.report.add_check(f"script_size_{script_name}", True,
|
||||
f"{script_name} has {loc} LOC (within {min_size}-{max_size})", 1.0)
|
||||
else:
|
||||
self.report.add_check(f"script_size_{script_name}", False,
|
||||
f"{script_name} has {loc} LOC (outside {min_size}-{max_size})", 0.5)
|
||||
if loc < min_size:
|
||||
self.report.add_suggestion(f"Consider expanding {script_name} (currently {loc} LOC)")
|
||||
else:
|
||||
self.report.add_suggestion(f"Consider refactoring {script_name} (currently {loc} LOC)")
|
||||
|
||||
# Parse and validate Python syntax
|
||||
try:
|
||||
tree = ast.parse(content)
|
||||
self.report.add_check(f"script_syntax_{script_name}", True,
|
||||
f"{script_name} has valid Python syntax", 1.0)
|
||||
|
||||
# Check for required features
|
||||
self._validate_script_features(tree, script_name, content)
|
||||
|
||||
except SyntaxError as e:
|
||||
self.report.add_check(f"script_syntax_{script_name}", False,
|
||||
f"{script_name} has syntax error: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Syntax error in {script_name}: {str(e)}")
|
||||
|
||||
except Exception as e:
|
||||
self.report.add_check(f"script_readable_{script_name}", False,
|
||||
f"Cannot read {script_name}: {str(e)}", 0.0)
|
||||
self.report.add_error(f"Cannot read {script_name}: {str(e)}")
|
||||
|
||||
def _validate_script_features(self, tree: ast.AST, script_name: str, content: str):
|
||||
"""Validate required script features"""
|
||||
required_features = self._get_tier_requirement("features_required", ["argparse", "main_guard"])
|
||||
|
||||
# Check for argparse usage
|
||||
if "argparse" in required_features:
|
||||
has_argparse = self._check_argparse_usage(tree)
|
||||
self.report.add_check(f"script_argparse_{script_name}", has_argparse,
|
||||
f"{'Uses' if has_argparse else 'Missing'} argparse in {script_name}", 1.0 if has_argparse else 0.0)
|
||||
if not has_argparse:
|
||||
self.report.add_error(f"{script_name} must use argparse for command-line arguments")
|
||||
|
||||
# Check for main guard
|
||||
if "main_guard" in required_features:
|
||||
has_main_guard = 'if __name__ == "__main__"' in content
|
||||
self.report.add_check(f"script_main_guard_{script_name}", has_main_guard,
|
||||
f"{'Has' if has_main_guard else 'Missing'} main guard in {script_name}", 1.0 if has_main_guard else 0.0)
|
||||
if not has_main_guard:
|
||||
self.report.add_error(f"{script_name} must have 'if __name__ == \"__main__\"' guard")
|
||||
|
||||
# Check for external imports (should only use stdlib)
|
||||
external_imports = self._check_external_imports(tree)
|
||||
if not external_imports:
|
||||
self.report.add_check(f"script_imports_{script_name}", True,
|
||||
f"{script_name} uses only standard library", 1.0)
|
||||
else:
|
||||
self.report.add_check(f"script_imports_{script_name}", False,
|
||||
f"{script_name} uses external imports: {', '.join(external_imports)}", 0.0)
|
||||
self.report.add_error(f"{script_name} uses external imports: {', '.join(external_imports)}")
|
||||
|
||||
def _check_argparse_usage(self, tree: ast.AST) -> bool:
|
||||
"""Check if the script uses argparse"""
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name == 'argparse':
|
||||
return True
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module == 'argparse':
|
||||
return True
|
||||
return False
|
||||
|
||||
def _check_external_imports(self, tree: ast.AST) -> List[str]:
|
||||
"""Check for external (non-stdlib) imports"""
|
||||
# Simplified check - a more comprehensive solution would use a stdlib module list
|
||||
stdlib_modules = {
|
||||
'argparse', 'ast', 'json', 'os', 'sys', 'pathlib', 'datetime', 'typing',
|
||||
'collections', 're', 'math', 'random', 'itertools', 'functools', 'operator',
|
||||
'csv', 'sqlite3', 'urllib', 'http', 'html', 'xml', 'email', 'base64',
|
||||
'hashlib', 'hmac', 'secrets', 'tempfile', 'shutil', 'glob', 'fnmatch',
|
||||
'subprocess', 'threading', 'multiprocessing', 'queue', 'time', 'calendar',
|
||||
'zoneinfo', 'locale', 'gettext', 'logging', 'warnings', 'unittest',
|
||||
'doctest', 'pickle', 'copy', 'pprint', 'reprlib', 'enum', 'dataclasses',
|
||||
'contextlib', 'abc', 'atexit', 'traceback', 'gc', 'weakref', 'types',
|
||||
'copy', 'pprint', 'reprlib', 'enum', 'decimal', 'fractions', 'statistics',
|
||||
'cmath', 'platform', 'errno', 'io', 'codecs', 'unicodedata', 'stringprep',
|
||||
'textwrap', 'string', 'struct', 'difflib', 'heapq', 'bisect', 'array',
|
||||
'weakref', 'types', 'copyreg', 'uuid', 'mmap', 'ctypes'
|
||||
}
|
||||
|
||||
external_imports = []
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
module_name = alias.name.split('.')[0]
|
||||
if module_name not in stdlib_modules:
|
||||
external_imports.append(alias.name)
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
module_name = node.module.split('.')[0]
|
||||
if module_name not in stdlib_modules:
|
||||
external_imports.append(node.module)
|
||||
|
||||
return list(set(external_imports))
|
||||
|
||||
def _validate_tier_compliance(self):
|
||||
"""Validate overall tier compliance"""
|
||||
if not self.target_tier:
|
||||
return
|
||||
|
||||
self.log_verbose(f"Validating {self.target_tier} tier compliance...")
|
||||
|
||||
# This is a summary check - individual checks are done in other methods
|
||||
critical_checks = ["skill_md_exists", "min_scripts_count", "skill_md_length"]
|
||||
failed_critical = [check for check in critical_checks
|
||||
if check in self.report.checks and not self.report.checks[check]["passed"]]
|
||||
|
||||
if not failed_critical:
|
||||
self.report.add_check("tier_compliance", True,
|
||||
f"Meets {self.target_tier} tier requirements", 1.0)
|
||||
else:
|
||||
self.report.add_check("tier_compliance", False,
|
||||
f"Does not meet {self.target_tier} tier requirements", 0.0)
|
||||
self.report.add_error(f"Failed critical checks for {self.target_tier} tier: {', '.join(failed_critical)}")
|
||||
|
||||
def _get_tier_requirement(self, requirement: str, default: Any) -> Any:
|
||||
"""Get tier-specific requirement value"""
|
||||
if self.target_tier and self.target_tier in self.TIER_REQUIREMENTS:
|
||||
return self.TIER_REQUIREMENTS[self.target_tier].get(requirement, default)
|
||||
return default
|
||||
|
||||
|
||||
class ReportFormatter:
|
||||
"""Formats validation reports for output"""
|
||||
|
||||
@staticmethod
|
||||
def format_json(report: ValidationReport) -> str:
|
||||
"""Format report as JSON"""
|
||||
return json.dumps({
|
||||
"skill_path": report.skill_path,
|
||||
"timestamp": report.timestamp,
|
||||
"overall_score": round(report.overall_score, 1),
|
||||
"compliance_level": report.compliance_level,
|
||||
"checks": report.checks,
|
||||
"warnings": report.warnings,
|
||||
"errors": report.errors,
|
||||
"suggestions": report.suggestions
|
||||
}, indent=2)
|
||||
|
||||
@staticmethod
|
||||
def format_human_readable(report: ValidationReport) -> str:
|
||||
"""Format report as human-readable text"""
|
||||
lines = []
|
||||
lines.append("=" * 60)
|
||||
lines.append("SKILL VALIDATION REPORT")
|
||||
lines.append("=" * 60)
|
||||
lines.append(f"Skill: {report.skill_path}")
|
||||
lines.append(f"Timestamp: {report.timestamp}")
|
||||
lines.append(f"Overall Score: {report.overall_score:.1f}/100 ({report.compliance_level})")
|
||||
lines.append("")
|
||||
|
||||
# Group checks by category
|
||||
structure_checks = {k: v for k, v in report.checks.items() if k.startswith(('skill_md', 'readme', 'dir_'))}
|
||||
script_checks = {k: v for k, v in report.checks.items() if k.startswith('script_')}
|
||||
other_checks = {k: v for k, v in report.checks.items() if k not in structure_checks and k not in script_checks}
|
||||
|
||||
if structure_checks:
|
||||
lines.append("STRUCTURE VALIDATION:")
|
||||
for check_name, result in structure_checks.items():
|
||||
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {result['message']}")
|
||||
lines.append("")
|
||||
|
||||
if script_checks:
|
||||
lines.append("SCRIPT VALIDATION:")
|
||||
for check_name, result in script_checks.items():
|
||||
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {result['message']}")
|
||||
lines.append("")
|
||||
|
||||
if other_checks:
|
||||
lines.append("OTHER CHECKS:")
|
||||
for check_name, result in other_checks.items():
|
||||
status = "✓ PASS" if result["passed"] else "✗ FAIL"
|
||||
lines.append(f" {status}: {result['message']}")
|
||||
lines.append("")
|
||||
|
||||
if report.errors:
|
||||
lines.append("ERRORS:")
|
||||
for error in report.errors:
|
||||
lines.append(f" • {error}")
|
||||
lines.append("")
|
||||
|
||||
if report.warnings:
|
||||
lines.append("WARNINGS:")
|
||||
for warning in report.warnings:
|
||||
lines.append(f" • {warning}")
|
||||
lines.append("")
|
||||
|
||||
if report.suggestions:
|
||||
lines.append("SUGGESTIONS:")
|
||||
for suggestion in report.suggestions:
|
||||
lines.append(f" • {suggestion}")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate skill directories against quality standards",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
python skill_validator.py engineering/my-skill
|
||||
python skill_validator.py engineering/my-skill --tier POWERFUL --json
|
||||
python skill_validator.py engineering/my-skill --verbose
|
||||
|
||||
Tier Options:
|
||||
BASIC - Basic skill requirements (100+ lines SKILL.md, 1+ script)
|
||||
STANDARD - Standard skill requirements (200+ lines, advanced features)
|
||||
POWERFUL - Powerful skill requirements (300+ lines, comprehensive features)
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument("skill_path",
|
||||
help="Path to the skill directory to validate")
|
||||
parser.add_argument("--tier",
|
||||
choices=["BASIC", "STANDARD", "POWERFUL"],
|
||||
help="Target tier for validation (optional)")
|
||||
parser.add_argument("--json",
|
||||
action="store_true",
|
||||
help="Output results in JSON format")
|
||||
parser.add_argument("--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose logging")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
# Create validator and run validation
|
||||
validator = SkillValidator(args.skill_path, args.tier, args.verbose)
|
||||
report = validator.validate_skill_structure()
|
||||
|
||||
# Format and output report
|
||||
if args.json:
|
||||
print(ReportFormatter.format_json(report))
|
||||
else:
|
||||
print(ReportFormatter.format_human_readable(report))
|
||||
|
||||
# Exit with error code if validation failed
|
||||
if report.errors or report.overall_score < 60:
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\nValidation interrupted by user", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
except Exception as e:
|
||||
print(f"Validation failed: {str(e)}", file=sys.stderr)
|
||||
if args.verbose:
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user