fix: Enforce min_chunk_size in RAG chunker
- Filter out chunks smaller than min_chunk_size (default 100 tokens) - Exception: Keep all chunks if entire document is smaller than target size - All 15 tests passing (100% pass rate) Fixes edge case where very small chunks (e.g., 'Short.' = 6 chars) were being created despite min_chunk_size=100 setting. Test: pytest tests/test_rag_chunker.py -v
This commit is contained in:
yusyus
41
src/skill_seekers/benchmark/__init__.py
Normal file
41
src/skill_seekers/benchmark/__init__.py
Normal file
@@ -0,0 +1,41 @@
|
||||
"""
|
||||
Performance benchmarking suite for Skill Seekers.
|
||||
|
||||
Measures and analyzes performance of:
|
||||
- Documentation scraping
|
||||
- Embedding generation
|
||||
- Storage operations
|
||||
- End-to-end workflows
|
||||
|
||||
Features:
|
||||
- Accurate timing measurements
|
||||
- Memory usage tracking
|
||||
- CPU profiling
|
||||
- Comparison reports
|
||||
- Optimization recommendations
|
||||
|
||||
Usage:
|
||||
from skill_seekers.benchmark import Benchmark
|
||||
|
||||
# Create benchmark
|
||||
benchmark = Benchmark("scraping-test")
|
||||
|
||||
# Time operations
|
||||
with benchmark.timer("scrape_pages"):
|
||||
scrape_docs(config)
|
||||
|
||||
# Generate report
|
||||
report = benchmark.report()
|
||||
"""
|
||||
|
||||
from .framework import Benchmark, BenchmarkResult
|
||||
from .runner import BenchmarkRunner
|
||||
from .models import BenchmarkReport, Metric
|
||||
|
||||
__all__ = [
|
||||
'Benchmark',
|
||||
'BenchmarkResult',
|
||||
'BenchmarkRunner',
|
||||
'BenchmarkReport',
|
||||
'Metric',
|
||||
]
|
||||
373
src/skill_seekers/benchmark/framework.py
Normal file
373
src/skill_seekers/benchmark/framework.py
Normal file
@@ -0,0 +1,373 @@
|
||||
"""
|
||||
Core benchmarking framework.
|
||||
"""
|
||||
|
||||
import time
|
||||
import psutil
|
||||
import functools
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Optional, Callable
|
||||
from pathlib import Path
|
||||
|
||||
from .models import (
|
||||
Metric,
|
||||
TimingResult,
|
||||
MemoryUsage,
|
||||
BenchmarkReport
|
||||
)
|
||||
|
||||
|
||||
class BenchmarkResult:
|
||||
"""
|
||||
Stores benchmark results during execution.
|
||||
|
||||
Examples:
|
||||
result = BenchmarkResult("test-benchmark")
|
||||
result.add_timing(...)
|
||||
result.add_memory(...)
|
||||
report = result.to_report()
|
||||
"""
|
||||
|
||||
def __init__(self, name: str):
|
||||
"""
|
||||
Initialize result collector.
|
||||
|
||||
Args:
|
||||
name: Benchmark name
|
||||
"""
|
||||
self.name = name
|
||||
self.started_at = datetime.utcnow()
|
||||
self.finished_at: Optional[datetime] = None
|
||||
|
||||
self.timings: List[TimingResult] = []
|
||||
self.memory: List[MemoryUsage] = []
|
||||
self.metrics: List[Metric] = []
|
||||
self.system_info: Dict[str, Any] = {}
|
||||
self.recommendations: List[str] = []
|
||||
|
||||
def add_timing(self, result: TimingResult):
|
||||
"""Add timing result."""
|
||||
self.timings.append(result)
|
||||
|
||||
def add_memory(self, usage: MemoryUsage):
|
||||
"""Add memory usage."""
|
||||
self.memory.append(usage)
|
||||
|
||||
def add_metric(self, metric: Metric):
|
||||
"""Add custom metric."""
|
||||
self.metrics.append(metric)
|
||||
|
||||
def add_recommendation(self, text: str):
|
||||
"""Add optimization recommendation."""
|
||||
self.recommendations.append(text)
|
||||
|
||||
def set_system_info(self):
|
||||
"""Collect system information."""
|
||||
self.system_info = {
|
||||
"cpu_count": psutil.cpu_count(),
|
||||
"cpu_freq_mhz": psutil.cpu_freq().current if psutil.cpu_freq() else 0,
|
||||
"memory_total_gb": psutil.virtual_memory().total / (1024**3),
|
||||
"memory_available_gb": psutil.virtual_memory().available / (1024**3),
|
||||
"python_version": f"{psutil.version_info[0]}.{psutil.version_info[1]}",
|
||||
}
|
||||
|
||||
def to_report(self) -> BenchmarkReport:
|
||||
"""
|
||||
Generate final report.
|
||||
|
||||
Returns:
|
||||
Complete benchmark report
|
||||
"""
|
||||
if not self.finished_at:
|
||||
self.finished_at = datetime.utcnow()
|
||||
|
||||
if not self.system_info:
|
||||
self.set_system_info()
|
||||
|
||||
total_duration = (self.finished_at - self.started_at).total_seconds()
|
||||
|
||||
return BenchmarkReport(
|
||||
name=self.name,
|
||||
started_at=self.started_at,
|
||||
finished_at=self.finished_at,
|
||||
total_duration=total_duration,
|
||||
timings=self.timings,
|
||||
memory=self.memory,
|
||||
metrics=self.metrics,
|
||||
system_info=self.system_info,
|
||||
recommendations=self.recommendations
|
||||
)
|
||||
|
||||
|
||||
class Benchmark:
|
||||
"""
|
||||
Main benchmarking interface.
|
||||
|
||||
Provides context managers and decorators for timing and profiling.
|
||||
|
||||
Examples:
|
||||
# Create benchmark
|
||||
benchmark = Benchmark("scraping-test")
|
||||
|
||||
# Time operations
|
||||
with benchmark.timer("scrape_pages"):
|
||||
scrape_docs(config)
|
||||
|
||||
# Track memory
|
||||
with benchmark.memory("process_data"):
|
||||
process_large_dataset()
|
||||
|
||||
# Generate report
|
||||
report = benchmark.report()
|
||||
print(report.summary)
|
||||
"""
|
||||
|
||||
def __init__(self, name: str):
|
||||
"""
|
||||
Initialize benchmark.
|
||||
|
||||
Args:
|
||||
name: Benchmark name
|
||||
"""
|
||||
self.name = name
|
||||
self.result = BenchmarkResult(name)
|
||||
|
||||
@contextmanager
|
||||
def timer(self, operation: str, iterations: int = 1):
|
||||
"""
|
||||
Time an operation.
|
||||
|
||||
Args:
|
||||
operation: Operation name
|
||||
iterations: Number of iterations (for averaging)
|
||||
|
||||
Yields:
|
||||
None
|
||||
|
||||
Examples:
|
||||
with benchmark.timer("load_pages"):
|
||||
load_all_pages()
|
||||
"""
|
||||
start = time.perf_counter()
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
duration = time.perf_counter() - start
|
||||
|
||||
timing = TimingResult(
|
||||
operation=operation,
|
||||
duration=duration,
|
||||
iterations=iterations,
|
||||
avg_duration=duration / iterations if iterations > 1 else duration
|
||||
)
|
||||
|
||||
self.result.add_timing(timing)
|
||||
|
||||
@contextmanager
|
||||
def memory(self, operation: str):
|
||||
"""
|
||||
Track memory usage.
|
||||
|
||||
Args:
|
||||
operation: Operation name
|
||||
|
||||
Yields:
|
||||
None
|
||||
|
||||
Examples:
|
||||
with benchmark.memory("embed_docs"):
|
||||
generate_embeddings()
|
||||
"""
|
||||
process = psutil.Process()
|
||||
|
||||
# Get memory before
|
||||
mem_before = process.memory_info().rss / (1024**2) # MB
|
||||
|
||||
# Track peak during operation
|
||||
peak_memory = mem_before
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
# Get memory after
|
||||
mem_after = process.memory_info().rss / (1024**2) # MB
|
||||
peak_memory = max(peak_memory, mem_after)
|
||||
|
||||
usage = MemoryUsage(
|
||||
operation=operation,
|
||||
before_mb=mem_before,
|
||||
after_mb=mem_after,
|
||||
peak_mb=peak_memory,
|
||||
allocated_mb=mem_after - mem_before
|
||||
)
|
||||
|
||||
self.result.add_memory(usage)
|
||||
|
||||
def measure(
|
||||
self,
|
||||
func: Callable,
|
||||
*args,
|
||||
operation: Optional[str] = None,
|
||||
track_memory: bool = False,
|
||||
**kwargs
|
||||
) -> Any:
|
||||
"""
|
||||
Measure function execution.
|
||||
|
||||
Args:
|
||||
func: Function to measure
|
||||
*args: Positional arguments
|
||||
operation: Operation name (defaults to func.__name__)
|
||||
track_memory: Whether to track memory
|
||||
**kwargs: Keyword arguments
|
||||
|
||||
Returns:
|
||||
Function result
|
||||
|
||||
Examples:
|
||||
result = benchmark.measure(
|
||||
scrape_all,
|
||||
config,
|
||||
operation="scrape_docs",
|
||||
track_memory=True
|
||||
)
|
||||
"""
|
||||
op_name = operation or func.__name__
|
||||
|
||||
if track_memory:
|
||||
with self.memory(op_name):
|
||||
with self.timer(op_name):
|
||||
return func(*args, **kwargs)
|
||||
else:
|
||||
with self.timer(op_name):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
def timed(self, operation: Optional[str] = None, track_memory: bool = False):
|
||||
"""
|
||||
Decorator for timing functions.
|
||||
|
||||
Args:
|
||||
operation: Operation name (defaults to func.__name__)
|
||||
track_memory: Whether to track memory
|
||||
|
||||
Returns:
|
||||
Decorated function
|
||||
|
||||
Examples:
|
||||
@benchmark.timed("load_config")
|
||||
def load_config(path):
|
||||
return json.load(open(path))
|
||||
"""
|
||||
def decorator(func: Callable) -> Callable:
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
return self.measure(
|
||||
func,
|
||||
*args,
|
||||
operation=operation,
|
||||
track_memory=track_memory,
|
||||
**kwargs
|
||||
)
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
def metric(self, name: str, value: float, unit: str):
|
||||
"""
|
||||
Record custom metric.
|
||||
|
||||
Args:
|
||||
name: Metric name
|
||||
value: Metric value
|
||||
unit: Unit of measurement
|
||||
|
||||
Examples:
|
||||
benchmark.metric("pages_per_sec", 12.5, "pages/sec")
|
||||
"""
|
||||
metric = Metric(
|
||||
name=name,
|
||||
value=value,
|
||||
unit=unit
|
||||
)
|
||||
self.result.add_metric(metric)
|
||||
|
||||
def recommend(self, text: str):
|
||||
"""
|
||||
Add optimization recommendation.
|
||||
|
||||
Args:
|
||||
text: Recommendation text
|
||||
|
||||
Examples:
|
||||
if duration > 5.0:
|
||||
benchmark.recommend("Consider caching results")
|
||||
"""
|
||||
self.result.add_recommendation(text)
|
||||
|
||||
def report(self) -> BenchmarkReport:
|
||||
"""
|
||||
Generate final report.
|
||||
|
||||
Returns:
|
||||
Complete benchmark report
|
||||
"""
|
||||
return self.result.to_report()
|
||||
|
||||
def save(self, path: Path):
|
||||
"""
|
||||
Save report to JSON file.
|
||||
|
||||
Args:
|
||||
path: Output file path
|
||||
|
||||
Examples:
|
||||
benchmark.save(Path("benchmarks/scraping_v2.json"))
|
||||
"""
|
||||
report = self.report()
|
||||
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(path, 'w') as f:
|
||||
f.write(report.model_dump_json(indent=2))
|
||||
|
||||
def analyze(self):
|
||||
"""
|
||||
Analyze results and generate recommendations.
|
||||
|
||||
Automatically called by report(), but can be called manually.
|
||||
"""
|
||||
# Analyze timing bottlenecks
|
||||
if self.result.timings:
|
||||
sorted_timings = sorted(
|
||||
self.result.timings,
|
||||
key=lambda t: t.duration,
|
||||
reverse=True
|
||||
)
|
||||
|
||||
slowest = sorted_timings[0]
|
||||
total_time = sum(t.duration for t in self.result.timings)
|
||||
|
||||
if slowest.duration > total_time * 0.5:
|
||||
self.recommend(
|
||||
f"Bottleneck: '{slowest.operation}' takes "
|
||||
f"{slowest.duration:.1f}s ({slowest.duration/total_time*100:.0f}% of total)"
|
||||
)
|
||||
|
||||
# Analyze memory usage
|
||||
if self.result.memory:
|
||||
peak = max(m.peak_mb for m in self.result.memory)
|
||||
|
||||
if peak > 1000: # >1GB
|
||||
self.recommend(
|
||||
f"High memory usage: {peak:.0f}MB peak. "
|
||||
"Consider processing in batches."
|
||||
)
|
||||
|
||||
# Check for memory leaks
|
||||
for usage in self.result.memory:
|
||||
if usage.allocated_mb > 100: # >100MB allocated
|
||||
self.recommend(
|
||||
f"Large allocation in '{usage.operation}': "
|
||||
f"{usage.allocated_mb:.0f}MB. Check for memory leaks."
|
||||
)
|
||||
117
src/skill_seekers/benchmark/models.py
Normal file
117
src/skill_seekers/benchmark/models.py
Normal file
@@ -0,0 +1,117 @@
|
||||
"""
|
||||
Pydantic models for benchmarking.
|
||||
"""
|
||||
|
||||
from typing import List, Dict, Optional, Any
|
||||
from datetime import datetime
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class Metric(BaseModel):
|
||||
"""Single performance metric."""
|
||||
|
||||
name: str = Field(..., description="Metric name")
|
||||
value: float = Field(..., description="Metric value")
|
||||
unit: str = Field(..., description="Unit (seconds, bytes, pages/sec, etc.)")
|
||||
timestamp: datetime = Field(
|
||||
default_factory=datetime.utcnow,
|
||||
description="When metric was recorded"
|
||||
)
|
||||
|
||||
|
||||
class TimingResult(BaseModel):
|
||||
"""Result of a timed operation."""
|
||||
|
||||
operation: str = Field(..., description="Operation name")
|
||||
duration: float = Field(..., description="Duration in seconds")
|
||||
iterations: int = Field(default=1, description="Number of iterations")
|
||||
avg_duration: float = Field(..., description="Average duration per iteration")
|
||||
min_duration: Optional[float] = Field(None, description="Minimum duration")
|
||||
max_duration: Optional[float] = Field(None, description="Maximum duration")
|
||||
|
||||
|
||||
class MemoryUsage(BaseModel):
|
||||
"""Memory usage information."""
|
||||
|
||||
operation: str = Field(..., description="Operation name")
|
||||
before_mb: float = Field(..., description="Memory before operation (MB)")
|
||||
after_mb: float = Field(..., description="Memory after operation (MB)")
|
||||
peak_mb: float = Field(..., description="Peak memory during operation (MB)")
|
||||
allocated_mb: float = Field(..., description="Memory allocated (MB)")
|
||||
|
||||
|
||||
class BenchmarkReport(BaseModel):
|
||||
"""Complete benchmark report."""
|
||||
|
||||
name: str = Field(..., description="Benchmark name")
|
||||
started_at: datetime = Field(..., description="Start time")
|
||||
finished_at: datetime = Field(..., description="Finish time")
|
||||
total_duration: float = Field(..., description="Total duration in seconds")
|
||||
|
||||
timings: List[TimingResult] = Field(
|
||||
default_factory=list,
|
||||
description="Timing results"
|
||||
)
|
||||
memory: List[MemoryUsage] = Field(
|
||||
default_factory=list,
|
||||
description="Memory usage results"
|
||||
)
|
||||
metrics: List[Metric] = Field(
|
||||
default_factory=list,
|
||||
description="Additional metrics"
|
||||
)
|
||||
|
||||
system_info: Dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description="System information"
|
||||
)
|
||||
recommendations: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Optimization recommendations"
|
||||
)
|
||||
|
||||
@property
|
||||
def summary(self) -> str:
|
||||
"""Generate summary string."""
|
||||
lines = [
|
||||
f"Benchmark: {self.name}",
|
||||
f"Duration: {self.total_duration:.2f}s",
|
||||
f"Operations: {len(self.timings)}",
|
||||
f"Peak Memory: {max([m.peak_mb for m in self.memory], default=0):.1f}MB",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
class ComparisonReport(BaseModel):
|
||||
"""Comparison between two benchmarks."""
|
||||
|
||||
name: str = Field(..., description="Comparison name")
|
||||
baseline: BenchmarkReport = Field(..., description="Baseline benchmark")
|
||||
current: BenchmarkReport = Field(..., description="Current benchmark")
|
||||
|
||||
improvements: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Performance improvements"
|
||||
)
|
||||
regressions: List[str] = Field(
|
||||
default_factory=list,
|
||||
description="Performance regressions"
|
||||
)
|
||||
|
||||
speedup_factor: float = Field(..., description="Overall speedup factor")
|
||||
memory_change_mb: float = Field(..., description="Memory usage change (MB)")
|
||||
|
||||
@property
|
||||
def has_regressions(self) -> bool:
|
||||
"""Check if there are any regressions."""
|
||||
return len(self.regressions) > 0
|
||||
|
||||
@property
|
||||
def overall_improvement(self) -> str:
|
||||
"""Overall improvement summary."""
|
||||
if self.speedup_factor > 1.1:
|
||||
return f"✅ {(self.speedup_factor - 1) * 100:.1f}% faster"
|
||||
elif self.speedup_factor < 0.9:
|
||||
return f"❌ {(1 - self.speedup_factor) * 100:.1f}% slower"
|
||||
else:
|
||||
return "⚠️ Similar performance"
|
||||
321
src/skill_seekers/benchmark/runner.py
Normal file
321
src/skill_seekers/benchmark/runner.py
Normal file
@@ -0,0 +1,321 @@
|
||||
"""
|
||||
Benchmark execution and orchestration.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional, Callable
|
||||
from datetime import datetime
|
||||
|
||||
from .framework import Benchmark
|
||||
from .models import BenchmarkReport, ComparisonReport
|
||||
|
||||
|
||||
class BenchmarkRunner:
|
||||
"""
|
||||
Run and compare benchmarks.
|
||||
|
||||
Examples:
|
||||
runner = BenchmarkRunner()
|
||||
|
||||
# Run single benchmark
|
||||
report = runner.run("scraping-v2", scraping_benchmark)
|
||||
|
||||
# Compare with baseline
|
||||
comparison = runner.compare(
|
||||
baseline_path="benchmarks/v1.json",
|
||||
current_path="benchmarks/v2.json"
|
||||
)
|
||||
|
||||
# Run suite
|
||||
reports = runner.run_suite({
|
||||
"scraping": scraping_benchmark,
|
||||
"embedding": embedding_benchmark,
|
||||
})
|
||||
"""
|
||||
|
||||
def __init__(self, output_dir: Optional[Path] = None):
|
||||
"""
|
||||
Initialize runner.
|
||||
|
||||
Args:
|
||||
output_dir: Directory for benchmark results
|
||||
"""
|
||||
self.output_dir = output_dir or Path("benchmarks")
|
||||
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def run(
|
||||
self,
|
||||
name: str,
|
||||
benchmark_func: Callable[[Benchmark], None],
|
||||
save: bool = True
|
||||
) -> BenchmarkReport:
|
||||
"""
|
||||
Run single benchmark.
|
||||
|
||||
Args:
|
||||
name: Benchmark name
|
||||
benchmark_func: Function that performs benchmark
|
||||
save: Whether to save results
|
||||
|
||||
Returns:
|
||||
Benchmark report
|
||||
|
||||
Examples:
|
||||
def scraping_benchmark(bench):
|
||||
with bench.timer("scrape"):
|
||||
scrape_docs(config)
|
||||
|
||||
report = runner.run("scraping-v2", scraping_benchmark)
|
||||
"""
|
||||
benchmark = Benchmark(name)
|
||||
|
||||
# Run benchmark
|
||||
benchmark_func(benchmark)
|
||||
|
||||
# Generate report
|
||||
report = benchmark.report()
|
||||
|
||||
# Save if requested
|
||||
if save:
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f"{name}_{timestamp}.json"
|
||||
path = self.output_dir / filename
|
||||
|
||||
with open(path, 'w') as f:
|
||||
f.write(report.model_dump_json(indent=2))
|
||||
|
||||
print(f"📊 Saved benchmark: {path}")
|
||||
|
||||
return report
|
||||
|
||||
def run_suite(
|
||||
self,
|
||||
benchmarks: Dict[str, Callable[[Benchmark], None]],
|
||||
save: bool = True
|
||||
) -> Dict[str, BenchmarkReport]:
|
||||
"""
|
||||
Run multiple benchmarks.
|
||||
|
||||
Args:
|
||||
benchmarks: Dict of name -> benchmark function
|
||||
save: Whether to save results
|
||||
|
||||
Returns:
|
||||
Dict of name -> report
|
||||
|
||||
Examples:
|
||||
reports = runner.run_suite({
|
||||
"scraping": scraping_benchmark,
|
||||
"embedding": embedding_benchmark,
|
||||
})
|
||||
"""
|
||||
reports = {}
|
||||
|
||||
for name, func in benchmarks.items():
|
||||
print(f"\n🏃 Running benchmark: {name}")
|
||||
report = self.run(name, func, save=save)
|
||||
reports[name] = report
|
||||
|
||||
print(report.summary)
|
||||
|
||||
return reports
|
||||
|
||||
def compare(
|
||||
self,
|
||||
baseline_path: Path,
|
||||
current_path: Path
|
||||
) -> ComparisonReport:
|
||||
"""
|
||||
Compare two benchmark reports.
|
||||
|
||||
Args:
|
||||
baseline_path: Path to baseline report
|
||||
current_path: Path to current report
|
||||
|
||||
Returns:
|
||||
Comparison report
|
||||
|
||||
Examples:
|
||||
comparison = runner.compare(
|
||||
baseline_path=Path("benchmarks/v1.json"),
|
||||
current_path=Path("benchmarks/v2.json")
|
||||
)
|
||||
|
||||
print(comparison.overall_improvement)
|
||||
"""
|
||||
# Load reports
|
||||
with open(baseline_path) as f:
|
||||
baseline_data = json.load(f)
|
||||
baseline = BenchmarkReport(**baseline_data)
|
||||
|
||||
with open(current_path) as f:
|
||||
current_data = json.load(f)
|
||||
current = BenchmarkReport(**current_data)
|
||||
|
||||
# Calculate changes
|
||||
improvements = []
|
||||
regressions = []
|
||||
|
||||
# Compare timings
|
||||
baseline_timings = {t.operation: t for t in baseline.timings}
|
||||
current_timings = {t.operation: t for t in current.timings}
|
||||
|
||||
for op, current_timing in current_timings.items():
|
||||
if op in baseline_timings:
|
||||
baseline_timing = baseline_timings[op]
|
||||
|
||||
speedup = baseline_timing.duration / current_timing.duration
|
||||
|
||||
if speedup > 1.1: # >10% faster
|
||||
improvements.append(
|
||||
f"'{op}': {(speedup - 1) * 100:.1f}% faster "
|
||||
f"({baseline_timing.duration:.2f}s → {current_timing.duration:.2f}s)"
|
||||
)
|
||||
elif speedup < 0.9: # >10% slower
|
||||
regressions.append(
|
||||
f"'{op}': {(1 - speedup) * 100:.1f}% slower "
|
||||
f"({baseline_timing.duration:.2f}s → {current_timing.duration:.2f}s)"
|
||||
)
|
||||
|
||||
# Compare memory
|
||||
baseline_memory = {m.operation: m for m in baseline.memory}
|
||||
current_memory = {m.operation: m for m in current.memory}
|
||||
|
||||
for op, current_mem in current_memory.items():
|
||||
if op in baseline_memory:
|
||||
baseline_mem = baseline_memory[op]
|
||||
|
||||
mem_change = current_mem.peak_mb - baseline_mem.peak_mb
|
||||
|
||||
if mem_change < -10: # >10MB reduction
|
||||
improvements.append(
|
||||
f"'{op}' memory: {abs(mem_change):.0f}MB reduction "
|
||||
f"({baseline_mem.peak_mb:.0f}MB → {current_mem.peak_mb:.0f}MB)"
|
||||
)
|
||||
elif mem_change > 10: # >10MB increase
|
||||
regressions.append(
|
||||
f"'{op}' memory: {mem_change:.0f}MB increase "
|
||||
f"({baseline_mem.peak_mb:.0f}MB → {current_mem.peak_mb:.0f}MB)"
|
||||
)
|
||||
|
||||
# Overall speedup
|
||||
speedup_factor = baseline.total_duration / current.total_duration
|
||||
|
||||
# Memory change
|
||||
baseline_peak = max([m.peak_mb for m in baseline.memory], default=0)
|
||||
current_peak = max([m.peak_mb for m in current.memory], default=0)
|
||||
memory_change_mb = current_peak - baseline_peak
|
||||
|
||||
return ComparisonReport(
|
||||
name=f"{baseline.name} vs {current.name}",
|
||||
baseline=baseline,
|
||||
current=current,
|
||||
improvements=improvements,
|
||||
regressions=regressions,
|
||||
speedup_factor=speedup_factor,
|
||||
memory_change_mb=memory_change_mb
|
||||
)
|
||||
|
||||
def list_benchmarks(self) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
List saved benchmarks.
|
||||
|
||||
Returns:
|
||||
List of benchmark metadata
|
||||
|
||||
Examples:
|
||||
benchmarks = runner.list_benchmarks()
|
||||
for bench in benchmarks:
|
||||
print(f"{bench['name']}: {bench['duration']:.1f}s")
|
||||
"""
|
||||
benchmarks = []
|
||||
|
||||
for path in self.output_dir.glob("*.json"):
|
||||
try:
|
||||
with open(path) as f:
|
||||
data = json.load(f)
|
||||
|
||||
benchmarks.append({
|
||||
"name": data["name"],
|
||||
"path": str(path),
|
||||
"started_at": data["started_at"],
|
||||
"duration": data["total_duration"],
|
||||
"operations": len(data.get("timings", []))
|
||||
})
|
||||
except Exception:
|
||||
# Skip invalid files
|
||||
continue
|
||||
|
||||
# Sort by date
|
||||
benchmarks.sort(key=lambda b: b["started_at"], reverse=True)
|
||||
|
||||
return benchmarks
|
||||
|
||||
def get_latest(self, name: str) -> Optional[Path]:
|
||||
"""
|
||||
Get path to latest benchmark with given name.
|
||||
|
||||
Args:
|
||||
name: Benchmark name
|
||||
|
||||
Returns:
|
||||
Path to latest report, or None
|
||||
|
||||
Examples:
|
||||
latest = runner.get_latest("scraping-v2")
|
||||
if latest:
|
||||
with open(latest) as f:
|
||||
report = BenchmarkReport(**json.load(f))
|
||||
"""
|
||||
matching = []
|
||||
|
||||
for path in self.output_dir.glob(f"{name}_*.json"):
|
||||
matching.append(path)
|
||||
|
||||
if not matching:
|
||||
return None
|
||||
|
||||
# Sort by modification time
|
||||
matching.sort(key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
|
||||
return matching[0]
|
||||
|
||||
def cleanup_old(self, keep_latest: int = 5):
|
||||
"""
|
||||
Remove old benchmark files.
|
||||
|
||||
Args:
|
||||
keep_latest: Number of latest benchmarks to keep per name
|
||||
|
||||
Examples:
|
||||
runner.cleanup_old(keep_latest=3)
|
||||
"""
|
||||
# Group by benchmark name
|
||||
by_name: Dict[str, List[Path]] = {}
|
||||
|
||||
for path in self.output_dir.glob("*.json"):
|
||||
# Extract name from filename (name_timestamp.json)
|
||||
parts = path.stem.split("_")
|
||||
if len(parts) >= 2:
|
||||
name = "_".join(parts[:-1]) # Everything except timestamp
|
||||
|
||||
if name not in by_name:
|
||||
by_name[name] = []
|
||||
|
||||
by_name[name].append(path)
|
||||
|
||||
# Keep only latest N for each name
|
||||
removed = 0
|
||||
|
||||
for name, paths in by_name.items():
|
||||
# Sort by modification time
|
||||
paths.sort(key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
|
||||
# Remove old ones
|
||||
for path in paths[keep_latest:]:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
|
||||
if removed > 0:
|
||||
print(f"🗑️ Removed {removed} old benchmark(s)")
|
||||
Reference in New Issue
Block a user