9b724f33e3
## New Skill: video-comparer v1.0.0 - Compare original and compressed videos with interactive HTML reports - Calculate quality metrics (PSNR, SSIM) for compression analysis - Generate frame-by-frame visual comparisons (slider, side-by-side, grid) - Extract video metadata (codec, resolution, bitrate, duration) - Multi-platform FFmpeg support with security features ## transcript-fixer Enhancements - Add async AI processor for parallel processing - Add connection pool management for database operations - Add concurrency manager and rate limiter - Add audit log retention and database migrations - Add health check and metrics monitoring - Add comprehensive test suite (8 new test files) - Enhance security with domain and path validators ## Marketplace Updates - Update marketplace version from 1.8.0 to 1.9.0 - Update skills count from 15 to 16 - Update documentation (README.md, CLAUDE.md, CHANGELOG.md) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
567 lines
20 KiB
Python
567 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Database Migration Module - Production-Grade Migration Strategy
|
|
|
|
CRITICAL FIX (P1-6): Production database migration system
|
|
|
|
Features:
|
|
- Versioned migrations with forward and rollback capability
|
|
- Migration history tracking
|
|
- Atomic transactions with rollback support
|
|
- Dry-run mode for testing
|
|
- Migration validation and verification
|
|
- Backward compatibility checks
|
|
|
|
Migration Types:
|
|
- Forward: Apply new schema changes
|
|
- Rollback: Revert to previous version
|
|
- Validation: Check migration safety
|
|
- Dry-run: Test migrations without applying
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import sqlite3
|
|
from datetime import datetime
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple, Any, Callable
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, asdict
|
|
import hashlib
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class MigrationDirection(Enum):
|
|
"""Migration direction"""
|
|
FORWARD = "forward"
|
|
BACKWARD = "backward"
|
|
|
|
|
|
class MigrationStatus(Enum):
|
|
"""Migration execution status"""
|
|
PENDING = "pending"
|
|
RUNNING = "running"
|
|
COMPLETED = "completed"
|
|
FAILED = "failed"
|
|
ROLLED_BACK = "rolled_back"
|
|
|
|
|
|
@dataclass
|
|
class Migration:
|
|
"""Migration definition"""
|
|
version: str
|
|
name: str
|
|
description: str
|
|
forward_sql: str
|
|
backward_sql: Optional[str] = None # For rollback capability
|
|
dependencies: List[str] = None # List of required migration versions
|
|
check_function: Optional[Callable] = None # Validation function
|
|
is_breaking: bool = False # If True, requires explicit confirmation
|
|
|
|
def __post_init__(self):
|
|
if self.dependencies is None:
|
|
self.dependencies = []
|
|
|
|
def get_hash(self) -> str:
|
|
"""Get hash of migration content for integrity checking"""
|
|
content = f"{self.version}:{self.name}:{self.forward_sql}"
|
|
return hashlib.sha256(content.encode('utf-8')).hexdigest()
|
|
|
|
|
|
@dataclass
|
|
class MigrationRecord:
|
|
"""Migration execution record"""
|
|
id: int
|
|
version: str
|
|
name: str
|
|
status: MigrationStatus
|
|
direction: MigrationDirection
|
|
execution_time_ms: int
|
|
checksum: str
|
|
executed_at: str = ""
|
|
error_message: Optional[str] = None
|
|
details: Optional[Dict[str, Any]] = None
|
|
|
|
def to_dict(self) -> Dict[str, Any]:
|
|
"""Convert to dictionary for serialization"""
|
|
result = asdict(self)
|
|
result['status'] = self.status.value
|
|
result['direction'] = self.direction.value
|
|
return result
|
|
|
|
|
|
class DatabaseMigrationManager:
|
|
"""
|
|
Production-grade database migration manager
|
|
|
|
Handles versioned schema migrations with:
|
|
- Automatic rollback on failure
|
|
- Migration history tracking
|
|
- Dependency resolution
|
|
- Safety checks and validation
|
|
"""
|
|
|
|
def __init__(self, db_path: Path):
|
|
"""
|
|
Initialize migration manager
|
|
|
|
Args:
|
|
db_path: Path to SQLite database file
|
|
"""
|
|
self.db_path = Path(db_path)
|
|
self.migrations: Dict[str, Migration] = {}
|
|
self._ensure_migration_table()
|
|
|
|
def register_migration(self, migration: Migration) -> None:
|
|
"""
|
|
Register a migration definition
|
|
|
|
Args:
|
|
migration: Migration to register
|
|
"""
|
|
if migration.version in self.migrations:
|
|
raise ValueError(f"Migration version {migration.version} already registered")
|
|
|
|
# Validate dependencies exist
|
|
for dep_version in migration.dependencies:
|
|
if dep_version not in self.migrations:
|
|
raise ValueError(f"Dependency migration {dep_version} not found")
|
|
|
|
self.migrations[migration.version] = migration
|
|
logger.info(f"Registered migration {migration.version}: {migration.name}")
|
|
|
|
def _ensure_migration_table(self) -> None:
|
|
"""Create migration tracking table if not exists"""
|
|
with self._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
|
|
# Create migration history table
|
|
cursor.execute('''
|
|
CREATE TABLE IF NOT EXISTS schema_migrations (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
version TEXT NOT NULL UNIQUE,
|
|
name TEXT NOT NULL,
|
|
status TEXT NOT NULL CHECK(status IN ('pending', 'running', 'completed', 'failed', 'rolled_back')),
|
|
direction TEXT NOT NULL CHECK(direction IN ('forward', 'backward')),
|
|
execution_time_ms INTEGER NOT NULL CHECK(execution_time_ms >= 0),
|
|
checksum TEXT NOT NULL,
|
|
executed_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
error_message TEXT,
|
|
details TEXT
|
|
)
|
|
''')
|
|
|
|
# Create index for faster queries
|
|
cursor.execute('''
|
|
CREATE INDEX IF NOT EXISTS idx_migrations_version
|
|
ON schema_migrations(version)
|
|
''')
|
|
|
|
cursor.execute('''
|
|
CREATE INDEX IF NOT EXISTS idx_migrations_executed_at
|
|
ON schema_migrations(executed_at DESC)
|
|
''')
|
|
|
|
# Insert initial migration record if table is empty
|
|
cursor.execute('''
|
|
INSERT OR IGNORE INTO schema_migrations
|
|
(version, name, status, direction, execution_time_ms, checksum)
|
|
VALUES ('0.0', 'Initial empty schema', 'completed', 'forward', 0, 'empty')
|
|
''')
|
|
|
|
conn.commit()
|
|
|
|
@contextmanager
|
|
def _get_connection(self):
|
|
"""Get database connection with proper error handling"""
|
|
conn = sqlite3.connect(str(self.db_path))
|
|
conn.execute("PRAGMA foreign_keys = ON")
|
|
try:
|
|
yield conn
|
|
finally:
|
|
conn.close()
|
|
|
|
@contextmanager
|
|
def _transaction(self):
|
|
"""Context manager for database transactions"""
|
|
with self._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute("BEGIN")
|
|
try:
|
|
yield cursor
|
|
conn.commit()
|
|
except Exception:
|
|
conn.rollback()
|
|
raise
|
|
|
|
def get_current_version(self) -> str:
|
|
"""
|
|
Get current database schema version
|
|
|
|
Returns:
|
|
Current version string
|
|
"""
|
|
with self._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
SELECT version FROM schema_migrations
|
|
WHERE status = 'completed' AND direction = 'forward'
|
|
ORDER BY executed_at DESC LIMIT 1
|
|
''')
|
|
result = cursor.fetchone()
|
|
return result[0] if result else "0.0"
|
|
|
|
def get_migration_history(self) -> List[MigrationRecord]:
|
|
"""
|
|
Get migration execution history
|
|
|
|
Returns:
|
|
List of migration records, most recent first
|
|
"""
|
|
with self._get_connection() as conn:
|
|
cursor = conn.cursor()
|
|
cursor.execute('''
|
|
SELECT id, version, name, status, direction,
|
|
execution_time_ms, checksum, error_message,
|
|
executed_at, details
|
|
FROM schema_migrations
|
|
ORDER BY executed_at DESC
|
|
''')
|
|
|
|
records = []
|
|
for row in cursor.fetchall():
|
|
record = MigrationRecord(
|
|
id=row[0],
|
|
version=row[1],
|
|
name=row[2],
|
|
status=MigrationStatus(row[3]),
|
|
direction=MigrationDirection(row[4]),
|
|
execution_time_ms=row[5],
|
|
checksum=row[6],
|
|
error_message=row[7],
|
|
executed_at=row[8],
|
|
details=json.loads(row[9]) if row[9] else None
|
|
)
|
|
records.append(record)
|
|
|
|
return records
|
|
|
|
def _validate_migration(self, migration: Migration) -> Tuple[bool, List[str]]:
|
|
"""
|
|
Validate migration safety
|
|
|
|
Args:
|
|
migration: Migration to validate
|
|
|
|
Returns:
|
|
Tuple of (is_valid, error_messages)
|
|
"""
|
|
errors = []
|
|
|
|
# Check migration hash
|
|
if migration.get_hash() != migration.get_hash(): # Simple consistency check
|
|
errors.append("Migration content is inconsistent")
|
|
|
|
# Run custom validation function if provided
|
|
if migration.check_function:
|
|
try:
|
|
with self._get_connection() as conn:
|
|
is_valid, validation_error = migration.check_function(conn, migration)
|
|
if not is_valid:
|
|
errors.append(validation_error)
|
|
except Exception as e:
|
|
errors.append(f"Validation function failed: {e}")
|
|
|
|
return len(errors) == 0, errors
|
|
|
|
def _execute_migration_sql(self, cursor: sqlite3.Cursor, sql: str) -> None:
|
|
"""
|
|
Execute migration SQL safely
|
|
|
|
Args:
|
|
cursor: Database cursor
|
|
sql: SQL to execute
|
|
"""
|
|
# Split SQL into individual statements
|
|
statements = [s.strip() for s in sql.split(';') if s.strip()]
|
|
|
|
for statement in statements:
|
|
if statement:
|
|
cursor.execute(statement)
|
|
|
|
def _run_migration(self, migration: Migration, direction: MigrationDirection,
|
|
dry_run: bool = False) -> None:
|
|
"""
|
|
Run a single migration
|
|
|
|
Args:
|
|
migration: Migration to run
|
|
direction: Migration direction
|
|
dry_run: If True, only validate without executing
|
|
"""
|
|
start_time = datetime.now()
|
|
|
|
# Select appropriate SQL
|
|
if direction == MigrationDirection.FORWARD:
|
|
sql = migration.forward_sql
|
|
elif direction == MigrationDirection.BACKWARD:
|
|
if not migration.backward_sql:
|
|
raise ValueError(f"Migration {migration.version} cannot be rolled back")
|
|
sql = migration.backward_sql
|
|
else:
|
|
raise ValueError(f"Invalid migration direction: {direction}")
|
|
|
|
# Validate migration
|
|
is_valid, errors = self._validate_migration(migration)
|
|
if not is_valid:
|
|
raise ValueError(f"Migration validation failed: {'; '.join(errors)}")
|
|
|
|
if dry_run:
|
|
logger.info(f"[DRY RUN] Would apply {direction.value} migration {migration.version}")
|
|
return
|
|
|
|
# Record migration start
|
|
with self._transaction() as cursor:
|
|
# Insert running record
|
|
cursor.execute('''
|
|
INSERT INTO schema_migrations
|
|
(version, name, status, direction, execution_time_ms, checksum)
|
|
VALUES (?, ?, 'running', ?, 0, ?)
|
|
''', (migration.version, migration.name, direction.value, migration.get_hash()))
|
|
|
|
# Execute migration
|
|
try:
|
|
self._execute_migration_sql(cursor, sql)
|
|
|
|
# Calculate execution time
|
|
execution_time_ms = int((datetime.now() - start_time).total_seconds() * 1000)
|
|
|
|
# Update record as completed
|
|
cursor.execute('''
|
|
UPDATE schema_migrations
|
|
SET status = 'completed', execution_time_ms = ?
|
|
WHERE version = ? AND status = 'running' AND direction = ?
|
|
ORDER BY executed_at DESC LIMIT 1
|
|
''', (execution_time_ms, migration.version, direction.value))
|
|
|
|
logger.info(f"Successfully applied {direction.value} migration {migration.version} "
|
|
f"in {execution_time_ms}ms")
|
|
|
|
except Exception as e:
|
|
execution_time_ms = int((datetime.now() - start_time).total_seconds() * 1000)
|
|
|
|
# Update record as failed
|
|
cursor.execute('''
|
|
UPDATE schema_migrations
|
|
SET status = 'failed', error_message = ?
|
|
WHERE version = ? AND status = 'running' AND direction = ?
|
|
ORDER BY executed_at DESC LIMIT 1
|
|
''', (str(e), migration.version, direction.value))
|
|
|
|
logger.error(f"Migration {migration.version} failed: {e}")
|
|
raise RuntimeError(f"Migration {migration.version} failed: {e}")
|
|
|
|
def get_pending_migrations(self) -> List[Migration]:
|
|
"""
|
|
Get list of pending migrations
|
|
|
|
Returns:
|
|
List of migrations that need to be applied
|
|
"""
|
|
current_version = self.get_current_version()
|
|
pending = []
|
|
|
|
# Get all migration versions
|
|
all_versions = sorted(self.migrations.keys(), key=lambda v: tuple(map(int, v.split('.'))))
|
|
|
|
for version in all_versions:
|
|
if version > current_version:
|
|
migration = self.migrations[version]
|
|
pending.append(migration)
|
|
|
|
return pending
|
|
|
|
def migrate_to_version(self, target_version: str, dry_run: bool = False,
|
|
force: bool = False) -> None:
|
|
"""
|
|
Migrate database to target version
|
|
|
|
Args:
|
|
target_version: Target version to migrate to
|
|
dry_run: If True, only validate without executing
|
|
force: If True, skip breaking change confirmation
|
|
"""
|
|
current_version = self.get_current_version()
|
|
logger.info(f"Current version: {current_version}, Target version: {target_version}")
|
|
|
|
# Validate target version exists
|
|
if target_version != "latest" and target_version not in self.migrations:
|
|
raise ValueError(f"Target version {target_version} not found")
|
|
|
|
# Determine migration path
|
|
if target_version == "latest":
|
|
# Migrate forward to latest
|
|
target_migration = max(self.migrations.keys(), key=lambda v: tuple(map(int, v.split('.'))))
|
|
else:
|
|
target_migration = target_version
|
|
|
|
if target_migration > current_version:
|
|
# Forward migration
|
|
self._migrate_forward(current_version, target_migration, dry_run, force)
|
|
elif target_migration < current_version:
|
|
# Rollback
|
|
self._migrate_backward(current_version, target_migration, dry_run, force)
|
|
else:
|
|
logger.info("Database is already at target version")
|
|
|
|
def _migrate_forward(self, from_version: str, to_version: str,
|
|
dry_run: bool = False, force: bool = False) -> None:
|
|
"""Execute forward migrations"""
|
|
all_versions = sorted(self.migrations.keys(), key=lambda v: tuple(map(int, v.split('.'))))
|
|
|
|
for version in all_versions:
|
|
if version > from_version and version <= to_version:
|
|
migration = self.migrations[version]
|
|
|
|
# Check for breaking changes
|
|
if migration.is_breaking and not force:
|
|
raise RuntimeError(
|
|
f"Migration {migration.version} is a breaking change. "
|
|
f"Use --force to apply."
|
|
)
|
|
|
|
# Check dependencies
|
|
for dep in migration.dependencies:
|
|
if dep > from_version:
|
|
raise RuntimeError(
|
|
f"Migration {migration.version} requires dependency {dep} "
|
|
f"which is not yet applied"
|
|
)
|
|
|
|
self._run_migration(migration, MigrationDirection.FORWARD, dry_run)
|
|
|
|
def _migrate_backward(self, from_version: str, to_version: str,
|
|
dry_run: bool = False, force: bool = False) -> None:
|
|
"""Execute rollback migrations"""
|
|
all_versions = sorted(self.migrations.keys(), key=lambda v: tuple(map(int, v.split('.'))), reverse=True)
|
|
|
|
for version in all_versions:
|
|
if version <= from_version and version > to_version:
|
|
migration = self.migrations[version]
|
|
|
|
if not migration.backward_sql:
|
|
raise RuntimeError(f"Migration {migration.version} cannot be rolled back")
|
|
|
|
# Check if migration would break other migrations
|
|
dependent_migrations = [
|
|
v for v, m in self.migrations.items()
|
|
if version in m.dependencies and v <= from_version
|
|
]
|
|
if dependent_migrations and not force:
|
|
raise RuntimeError(
|
|
f"Cannot rollback {version} because it has dependencies: "
|
|
f"{', '.join(dependent_migrations)}"
|
|
)
|
|
|
|
self._run_migration(migration, MigrationDirection.BACKWARD, dry_run)
|
|
|
|
def rollback_migration(self, version: str, dry_run: bool = False,
|
|
force: bool = False) -> None:
|
|
"""
|
|
Rollback a specific migration
|
|
|
|
Args:
|
|
version: Migration version to rollback
|
|
dry_run: If True, only validate without executing
|
|
force: If True, skip safety checks
|
|
"""
|
|
if version not in self.migrations:
|
|
raise ValueError(f"Migration {version} not found")
|
|
|
|
migration = self.migrations[version]
|
|
if not migration.backward_sql:
|
|
raise ValueError(f"Migration {version} cannot be rolled back")
|
|
|
|
# Check if migration has been applied
|
|
history = self.get_migration_history()
|
|
applied_versions = [m.version for m in history if m.status == MigrationStatus.COMPLETED]
|
|
|
|
if version not in applied_versions:
|
|
raise ValueError(f"Migration {version} has not been applied")
|
|
|
|
# Check for dependent migrations
|
|
dependent_migrations = [
|
|
v for v, m in self.migrations.items()
|
|
if version in m.dependencies and v in applied_versions
|
|
]
|
|
if dependent_migrations and not force:
|
|
raise RuntimeError(
|
|
f"Cannot rollback {version} because it has dependencies: "
|
|
f"{', '.join(dependent_migrations)}"
|
|
)
|
|
|
|
logger.info(f"Rolling back migration {version}")
|
|
self._run_migration(migration, MigrationDirection.BACKWARD, dry_run)
|
|
|
|
def get_migration_plan(self, target_version: str = "latest") -> List[Dict[str, Any]]:
|
|
"""
|
|
Get migration execution plan
|
|
|
|
Args:
|
|
target_version: Target version to plan for
|
|
|
|
Returns:
|
|
List of migration steps with details
|
|
"""
|
|
current_version = self.get_current_version()
|
|
plan = []
|
|
|
|
if target_version == "latest":
|
|
target_version = max(self.migrations.keys(), key=lambda v: tuple(map(int, v.split('.'))))
|
|
|
|
all_versions = sorted(self.migrations.keys(), key=lambda v: tuple(map(int, v.split('.'))))
|
|
|
|
for version in all_versions:
|
|
if version > current_version and version <= target_version:
|
|
migration = self.migrations[version]
|
|
step = {
|
|
'version': version,
|
|
'name': migration.name,
|
|
'description': migration.description,
|
|
'is_breaking': migration.is_breaking,
|
|
'dependencies': migration.dependencies,
|
|
'has_rollback': migration.backward_sql is not None
|
|
}
|
|
plan.append(step)
|
|
|
|
return plan
|
|
|
|
def validate_migration_safety(self, target_version: str = "latest") -> Tuple[bool, List[str]]:
|
|
"""
|
|
Validate migration plan for safety issues
|
|
|
|
Args:
|
|
target_version: Target version to validate
|
|
|
|
Returns:
|
|
Tuple of (is_safe, safety_issues)
|
|
"""
|
|
plan = self.get_migration_plan(target_version)
|
|
issues = []
|
|
|
|
for step in plan:
|
|
migration = self.migrations[step['version']]
|
|
|
|
# Check breaking changes
|
|
if migration.is_breaking:
|
|
issues.append(f"Breaking change in {step['version']}: {step['name']}")
|
|
|
|
# Check rollback capability
|
|
if not migration.backward_sql:
|
|
issues.append(f"Migration {step['version']} cannot be rolled back")
|
|
|
|
return len(issues) == 0, issues |