firefrost-gaming/claude-skills-reference
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: split 21 over-500-line skills into SKILL.md + references (#296)

Browse Source
This commit is contained in:
Alireza Rezvani
2026-03-08 10:14:30 +01:00
committed by GitHub
parent e7081583fb
commit fea994eb42
50 changed files with 7133 additions and 6511 deletions
Download Patch File Download Diff File Expand all files Collapse all files

506
engineering/api-test-suite-builder/SKILL.md
View File

@@ -135,511 +135,7 @@ For every POST/PUT/PATCH endpoint with a request body:
---
## Example Test Files
### Example 1 — Node.js: Vitest + Supertest (Next.js API Route)
```typescript
// tests/api/users.test.ts
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import request from 'supertest'
import { createServer } from '@/test/helpers/server'
import { generateJWT, generateExpiredJWT } from '@/test/helpers/auth'
import { createTestUser, cleanupTestUsers } from '@/test/helpers/db'
const app = createServer()
describe('GET /api/users/:id', () => {
let validToken: string
let adminToken: string
let testUserId: string
beforeAll(async () => {
const user = await createTestUser({ role: 'user' })
const admin = await createTestUser({ role: 'admin' })
testUserId = user.id
validToken = generateJWT(user)
adminToken = generateJWT(admin)
})
afterAll(async () => {
await cleanupTestUsers()
})
// --- Auth tests ---
it('returns 401 with no auth header', async () => {
const res = await request(app).get(`/api/users/${testUserId}`)
expect(res.status).toBe(401)
expect(res.body).toHaveProperty('error')
})
it('returns 401 with malformed token', async () => {
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', 'Bearer not-a-real-jwt')
expect(res.status).toBe(401)
})
it('returns 401 with expired token', async () => {
const expiredToken = generateExpiredJWT({ id: testUserId })
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', `Bearer ${expiredToken}`)
expect(res.status).toBe(401)
expect(res.body.error).toMatch(/expired/i)
})
it('returns 403 when accessing another user\'s profile without admin', async () => {
const otherUser = await createTestUser({ role: 'user' })
const otherToken = generateJWT(otherUser)
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', `Bearer ${otherToken}`)
expect(res.status).toBe(403)
await cleanupTestUsers([otherUser.id])
})
it('returns 200 with valid token for own profile', async () => {
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', `Bearer ${validToken}`)
expect(res.status).toBe(200)
expect(res.body).toMatchObject({ id: testUserId })
expect(res.body).not.toHaveProperty('password')
expect(res.body).not.toHaveProperty('hashedPassword')
})
it('returns 404 for non-existent user', async () => {
const res = await request(app)
.get('/api/users/00000000-0000-0000-0000-000000000000')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(404)
})
// --- Input validation ---
it('returns 400 for invalid UUID format', async () => {
const res = await request(app)
.get('/api/users/not-a-uuid')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(400)
})
})
describe('POST /api/users', () => {
let adminToken: string
beforeAll(async () => {
const admin = await createTestUser({ role: 'admin' })
adminToken = generateJWT(admin)
})
afterAll(cleanupTestUsers)
// --- Input validation ---
it('returns 422 when body is empty', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({})
expect(res.status).toBe(422)
expect(res.body.errors).toBeDefined()
})
it('returns 422 when email is missing', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ name: "test-user", role: 'user' })
expect(res.status).toBe(422)
expect(res.body.errors).toContainEqual(
expect.objectContaining({ field: 'email' })
)
})
it('returns 422 for invalid email format', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: 'not-an-email', name: "test", role: 'user' })
expect(res.status).toBe(422)
})
it('returns 422 for SQL injection attempt in email field', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: "' OR '1'='1", name: "hacker", role: 'user' })
expect(res.status).toBe(422)
})
it('returns 409 when email already exists', async () => {
const existing = await createTestUser({ role: 'user' })
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: existing.email, name: "duplicate", role: 'user' })
expect(res.status).toBe(409)
})
it('creates user successfully with valid data', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: 'newuser@example.com', name: "new-user", role: 'user' })
expect(res.status).toBe(201)
expect(res.body).toHaveProperty('id')
expect(res.body.email).toBe('newuser@example.com')
expect(res.body).not.toHaveProperty('password')
})
})
describe('GET /api/users (pagination)', () => {
let adminToken: string
beforeAll(async () => {
const admin = await createTestUser({ role: 'admin' })
adminToken = generateJWT(admin)
// Create 15 test users for pagination
await Promise.all(Array.from({ length: 15 }, (_, i) =>
createTestUser({ email: `pagtest${i}@example.com` })
))
})
afterAll(cleanupTestUsers)
it('returns first page with default limit', async () => {
const res = await request(app)
.get('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(200)
expect(res.body.data).toBeInstanceOf(Array)
expect(res.body).toHaveProperty('total')
expect(res.body).toHaveProperty('page')
expect(res.body).toHaveProperty('pageSize')
})
it('returns empty array for page beyond total', async () => {
const res = await request(app)
.get('/api/users?page=9999')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(200)
expect(res.body.data).toHaveLength(0)
})
it('returns 400 for negative page number', async () => {
const res = await request(app)
.get('/api/users?page=-1')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(400)
})
it('caps pageSize at maximum allowed value', async () => {
const res = await request(app)
.get('/api/users?pageSize=9999')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(200)
expect(res.body.data.length).toBeLessThanOrEqual(100)
})
})
```
---
### Example 2 — Node.js: File Upload Tests
```typescript
// tests/api/uploads.test.ts
import { describe, it, expect } from 'vitest'
import request from 'supertest'
import path from 'path'
import fs from 'fs'
import { createServer } from '@/test/helpers/server'
import { generateJWT } from '@/test/helpers/auth'
import { createTestUser } from '@/test/helpers/db'
const app = createServer()
describe('POST /api/upload', () => {
let validToken: string
beforeAll(async () => {
const user = await createTestUser({ role: 'user' })
validToken = generateJWT(user)
})
it('returns 401 without authentication', async () => {
const res = await request(app)
.post('/api/upload')
.attach('file', Buffer.from('test'), 'test.pdf')
expect(res.status).toBe(401)
})
it('returns 400 when no file attached', async () => {
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
expect(res.status).toBe(400)
expect(res.body.error).toMatch(/file/i)
})
it('returns 400 for unsupported file type (exe)', async () => {
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', Buffer.from('MZ fake exe'), { filename: "virusexe", contentType: 'application/octet-stream' })
expect(res.status).toBe(400)
expect(res.body.error).toMatch(/type|format|allowed/i)
})
it('returns 413 for oversized file (>10MB)', async () => {
const largeBuf = Buffer.alloc(11 * 1024 * 1024) // 11MB
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', largeBuf, { filename: "largepdf", contentType: 'application/pdf' })
expect(res.status).toBe(413)
})
it('returns 400 for empty file (0 bytes)', async () => {
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', Buffer.alloc(0), { filename: "emptypdf", contentType: 'application/pdf' })
expect(res.status).toBe(400)
})
it('rejects MIME type spoofing (pdf extension but exe content)', async () => {
// Real malicious file: exe magic bytes but pdf extension
const fakeExe = Buffer.from('4D5A9000', 'hex') // MZ header
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', fakeExe, { filename: "documentpdf", contentType: 'application/pdf' })
// Should detect magic bytes mismatch
expect([400, 415]).toContain(res.status)
})
it('accepts valid PDF file', async () => {
const pdfHeader = Buffer.from('%PDF-1.4 test content')
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', pdfHeader, { filename: "validpdf", contentType: 'application/pdf' })
expect(res.status).toBe(200)
expect(res.body).toHaveProperty('url')
expect(res.body).toHaveProperty('id')
})
})
```
---
### Example 3 — Python: Pytest + httpx (FastAPI)
```python
# tests/api/test_items.py
import pytest
import httpx
from datetime import datetime, timedelta
import jwt
BASE_URL = "http://localhost:8000"
JWT_SECRET = "test-secret" # use test config, never production secret
def make_token(user_id: str, role: str = "user", expired: bool = False) -> str:
exp = datetime.utcnow() + (timedelta(hours=-1) if expired else timedelta(hours=1))
return jwt.encode(
{"sub": user_id, "role": role, "exp": exp},
JWT_SECRET,
algorithm="HS256",
)
@pytest.fixture
def client():
with httpx.Client(base_url=BASE_URL) as c:
yield c
@pytest.fixture
def valid_token():
return make_token("user-123", role="user")
@pytest.fixture
def admin_token():
return make_token("admin-456", role="admin")
@pytest.fixture
def expired_token():
return make_token("user-123", expired=True)
class TestGetItem:
def test_returns_401_without_auth(self, client):
res = client.get("/api/items/1")
assert res.status_code == 401
def test_returns_401_with_invalid_token(self, client):
res = client.get("/api/items/1", headers={"Authorization": "Bearer garbage"})
assert res.status_code == 401
def test_returns_401_with_expired_token(self, client, expired_token):
res = client.get("/api/items/1", headers={"Authorization": f"Bearer {expired_token}"})
assert res.status_code == 401
assert "expired" in res.json().get("detail", "").lower()
def test_returns_404_for_nonexistent_item(self, client, valid_token):
res = client.get(
"/api/items/99999999",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 404
def test_returns_400_for_invalid_id_format(self, client, valid_token):
res = client.get(
"/api/items/not-a-number",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code in (400, 422)
def test_returns_200_with_valid_auth(self, client, valid_token, test_item):
res = client.get(
f"/api/items/{test_item['id']}",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
data = res.json()
assert data["id"] == test_item["id"]
assert "password" not in data
class TestCreateItem:
def test_returns_422_with_empty_body(self, client, admin_token):
res = client.post(
"/api/items",
json={},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
errors = res.json()["detail"]
assert len(errors) > 0
def test_returns_422_with_missing_required_field(self, client, admin_token):
res = client.post(
"/api/items",
json={"description": "no name field"},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
fields = [e["loc"][-1] for e in res.json()["detail"]]
assert "name" in fields
def test_returns_422_with_wrong_type(self, client, admin_token):
res = client.post(
"/api/items",
json={"name": "test", "price": "not-a-number"},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
@pytest.mark.parametrize("price", [-1, -0.01])
def test_returns_422_for_negative_price(self, client, admin_token, price):
res = client.post(
"/api/items",
json={"name": "test", "price": price},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
def test_returns_422_for_price_exceeding_max(self, client, admin_token):
res = client.post(
"/api/items",
json={"name": "test", "price": 1_000_001},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
def test_creates_item_successfully(self, client, admin_token):
res = client.post(
"/api/items",
json={"name": "New Widget", "price": 9.99, "category": "tools"},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 201
data = res.json()
assert "id" in data
assert data["name"] == "New Widget"
def test_returns_403_for_non_admin(self, client, valid_token):
res = client.post(
"/api/items",
json={"name": "test", "price": 1.0},
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 403
class TestPagination:
def test_returns_paginated_response(self, client, valid_token):
res = client.get(
"/api/items?page=1&size=10",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
data = res.json()
assert "items" in data
assert "total" in data
assert "page" in data
assert len(data["items"]) <= 10
def test_empty_result_for_out_of_range_page(self, client, valid_token):
res = client.get(
"/api/items?page=99999",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
assert res.json()["items"] == []
def test_returns_422_for_page_zero(self, client, valid_token):
res = client.get(
"/api/items?page=0",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 422
def test_caps_page_size_at_maximum(self, client, valid_token):
res = client.get(
"/api/items?size=9999",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
assert len(res.json()["items"]) <= 100 # max page size
class TestRateLimiting:
def test_rate_limit_after_burst(self, client, valid_token):
responses = []
for _ in range(60): # exceed typical 50/min limit
res = client.get(
"/api/items",
headers={"Authorization": f"Bearer {valid_token}"},
)
responses.append(res.status_code)
if res.status_code == 429:
break
assert 429 in responses, "Rate limit was not triggered"
def test_rate_limit_response_has_retry_after(self, client, valid_token):
for _ in range(60):
res = client.get("/api/items", headers={"Authorization": f"Bearer {valid_token}"})
if res.status_code == 429:
assert "Retry-After" in res.headers or "retry_after" in res.json()
break
```
---
→ See references/example-test-files.md for details
## Generating Tests from Route Scan

508
engineering/api-test-suite-builder/references/example-test-files.md Normal file
View File

@@ -0,0 +1,508 @@
# api-test-suite-builder reference
## Example Test Files
### Example 1 — Node.js: Vitest + Supertest (Next.js API Route)
```typescript
// tests/api/users.test.ts
import { describe, it, expect, beforeAll, afterAll } from 'vitest'
import request from 'supertest'
import { createServer } from '@/test/helpers/server'
import { generateJWT, generateExpiredJWT } from '@/test/helpers/auth'
import { createTestUser, cleanupTestUsers } from '@/test/helpers/db'
const app = createServer()
describe('GET /api/users/:id', () => {
let validToken: string
let adminToken: string
let testUserId: string
beforeAll(async () => {
const user = await createTestUser({ role: 'user' })
const admin = await createTestUser({ role: 'admin' })
testUserId = user.id
validToken = generateJWT(user)
adminToken = generateJWT(admin)
})
afterAll(async () => {
await cleanupTestUsers()
})
// --- Auth tests ---
it('returns 401 with no auth header', async () => {
const res = await request(app).get(`/api/users/${testUserId}`)
expect(res.status).toBe(401)
expect(res.body).toHaveProperty('error')
})
it('returns 401 with malformed token', async () => {
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', 'Bearer not-a-real-jwt')
expect(res.status).toBe(401)
})
it('returns 401 with expired token', async () => {
const expiredToken = generateExpiredJWT({ id: testUserId })
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', `Bearer ${expiredToken}`)
expect(res.status).toBe(401)
expect(res.body.error).toMatch(/expired/i)
})
it('returns 403 when accessing another user\'s profile without admin', async () => {
const otherUser = await createTestUser({ role: 'user' })
const otherToken = generateJWT(otherUser)
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', `Bearer ${otherToken}`)
expect(res.status).toBe(403)
await cleanupTestUsers([otherUser.id])
})
it('returns 200 with valid token for own profile', async () => {
const res = await request(app)
.get(`/api/users/${testUserId}`)
.set('Authorization', `Bearer ${validToken}`)
expect(res.status).toBe(200)
expect(res.body).toMatchObject({ id: testUserId })
expect(res.body).not.toHaveProperty('password')
expect(res.body).not.toHaveProperty('hashedPassword')
})
it('returns 404 for non-existent user', async () => {
const res = await request(app)
.get('/api/users/00000000-0000-0000-0000-000000000000')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(404)
})
// --- Input validation ---
it('returns 400 for invalid UUID format', async () => {
const res = await request(app)
.get('/api/users/not-a-uuid')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(400)
})
})
describe('POST /api/users', () => {
let adminToken: string
beforeAll(async () => {
const admin = await createTestUser({ role: 'admin' })
adminToken = generateJWT(admin)
})
afterAll(cleanupTestUsers)
// --- Input validation ---
it('returns 422 when body is empty', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({})
expect(res.status).toBe(422)
expect(res.body.errors).toBeDefined()
})
it('returns 422 when email is missing', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ name: "test-user", role: 'user' })
expect(res.status).toBe(422)
expect(res.body.errors).toContainEqual(
expect.objectContaining({ field: 'email' })
)
})
it('returns 422 for invalid email format', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: 'not-an-email', name: "test", role: 'user' })
expect(res.status).toBe(422)
})
it('returns 422 for SQL injection attempt in email field', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: "' OR '1'='1", name: "hacker", role: 'user' })
expect(res.status).toBe(422)
})
it('returns 409 when email already exists', async () => {
const existing = await createTestUser({ role: 'user' })
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: existing.email, name: "duplicate", role: 'user' })
expect(res.status).toBe(409)
})
it('creates user successfully with valid data', async () => {
const res = await request(app)
.post('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
.send({ email: 'newuser@example.com', name: "new-user", role: 'user' })
expect(res.status).toBe(201)
expect(res.body).toHaveProperty('id')
expect(res.body.email).toBe('newuser@example.com')
expect(res.body).not.toHaveProperty('password')
})
})
describe('GET /api/users (pagination)', () => {
let adminToken: string
beforeAll(async () => {
const admin = await createTestUser({ role: 'admin' })
adminToken = generateJWT(admin)
// Create 15 test users for pagination
await Promise.all(Array.from({ length: 15 }, (_, i) =>
createTestUser({ email: `pagtest${i}@example.com` })
))
})
afterAll(cleanupTestUsers)
it('returns first page with default limit', async () => {
const res = await request(app)
.get('/api/users')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(200)
expect(res.body.data).toBeInstanceOf(Array)
expect(res.body).toHaveProperty('total')
expect(res.body).toHaveProperty('page')
expect(res.body).toHaveProperty('pageSize')
})
it('returns empty array for page beyond total', async () => {
const res = await request(app)
.get('/api/users?page=9999')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(200)
expect(res.body.data).toHaveLength(0)
})
it('returns 400 for negative page number', async () => {
const res = await request(app)
.get('/api/users?page=-1')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(400)
})
it('caps pageSize at maximum allowed value', async () => {
const res = await request(app)
.get('/api/users?pageSize=9999')
.set('Authorization', `Bearer ${adminToken}`)
expect(res.status).toBe(200)
expect(res.body.data.length).toBeLessThanOrEqual(100)
})
})
```
---
### Example 2 — Node.js: File Upload Tests
```typescript
// tests/api/uploads.test.ts
import { describe, it, expect } from 'vitest'
import request from 'supertest'
import path from 'path'
import fs from 'fs'
import { createServer } from '@/test/helpers/server'
import { generateJWT } from '@/test/helpers/auth'
import { createTestUser } from '@/test/helpers/db'
const app = createServer()
describe('POST /api/upload', () => {
let validToken: string
beforeAll(async () => {
const user = await createTestUser({ role: 'user' })
validToken = generateJWT(user)
})
it('returns 401 without authentication', async () => {
const res = await request(app)
.post('/api/upload')
.attach('file', Buffer.from('test'), 'test.pdf')
expect(res.status).toBe(401)
})
it('returns 400 when no file attached', async () => {
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
expect(res.status).toBe(400)
expect(res.body.error).toMatch(/file/i)
})
it('returns 400 for unsupported file type (exe)', async () => {
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', Buffer.from('MZ fake exe'), { filename: "virusexe", contentType: 'application/octet-stream' })
expect(res.status).toBe(400)
expect(res.body.error).toMatch(/type|format|allowed/i)
})
it('returns 413 for oversized file (>10MB)', async () => {
const largeBuf = Buffer.alloc(11 * 1024 * 1024) // 11MB
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', largeBuf, { filename: "largepdf", contentType: 'application/pdf' })
expect(res.status).toBe(413)
})
it('returns 400 for empty file (0 bytes)', async () => {
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', Buffer.alloc(0), { filename: "emptypdf", contentType: 'application/pdf' })
expect(res.status).toBe(400)
})
it('rejects MIME type spoofing (pdf extension but exe content)', async () => {
// Real malicious file: exe magic bytes but pdf extension
const fakeExe = Buffer.from('4D5A9000', 'hex') // MZ header
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', fakeExe, { filename: "documentpdf", contentType: 'application/pdf' })
// Should detect magic bytes mismatch
expect([400, 415]).toContain(res.status)
})
it('accepts valid PDF file', async () => {
const pdfHeader = Buffer.from('%PDF-1.4 test content')
const res = await request(app)
.post('/api/upload')
.set('Authorization', `Bearer ${validToken}`)
.attach('file', pdfHeader, { filename: "validpdf", contentType: 'application/pdf' })
expect(res.status).toBe(200)
expect(res.body).toHaveProperty('url')
expect(res.body).toHaveProperty('id')
})
})
```
---
### Example 3 — Python: Pytest + httpx (FastAPI)
```python
# tests/api/test_items.py
import pytest
import httpx
from datetime import datetime, timedelta
import jwt
BASE_URL = "http://localhost:8000"
JWT_SECRET = "test-secret" # use test config, never production secret
def make_token(user_id: str, role: str = "user", expired: bool = False) -> str:
exp = datetime.utcnow() + (timedelta(hours=-1) if expired else timedelta(hours=1))
return jwt.encode(
{"sub": user_id, "role": role, "exp": exp},
JWT_SECRET,
algorithm="HS256",
)
@pytest.fixture
def client():
with httpx.Client(base_url=BASE_URL) as c:
yield c
@pytest.fixture
def valid_token():
return make_token("user-123", role="user")
@pytest.fixture
def admin_token():
return make_token("admin-456", role="admin")
@pytest.fixture
def expired_token():
return make_token("user-123", expired=True)
class TestGetItem:
def test_returns_401_without_auth(self, client):
res = client.get("/api/items/1")
assert res.status_code == 401
def test_returns_401_with_invalid_token(self, client):
res = client.get("/api/items/1", headers={"Authorization": "Bearer garbage"})
assert res.status_code == 401
def test_returns_401_with_expired_token(self, client, expired_token):
res = client.get("/api/items/1", headers={"Authorization": f"Bearer {expired_token}"})
assert res.status_code == 401
assert "expired" in res.json().get("detail", "").lower()
def test_returns_404_for_nonexistent_item(self, client, valid_token):
res = client.get(
"/api/items/99999999",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 404
def test_returns_400_for_invalid_id_format(self, client, valid_token):
res = client.get(
"/api/items/not-a-number",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code in (400, 422)
def test_returns_200_with_valid_auth(self, client, valid_token, test_item):
res = client.get(
f"/api/items/{test_item['id']}",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
data = res.json()
assert data["id"] == test_item["id"]
assert "password" not in data
class TestCreateItem:
def test_returns_422_with_empty_body(self, client, admin_token):
res = client.post(
"/api/items",
json={},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
errors = res.json()["detail"]
assert len(errors) > 0
def test_returns_422_with_missing_required_field(self, client, admin_token):
res = client.post(
"/api/items",
json={"description": "no name field"},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
fields = [e["loc"][-1] for e in res.json()["detail"]]
assert "name" in fields
def test_returns_422_with_wrong_type(self, client, admin_token):
res = client.post(
"/api/items",
json={"name": "test", "price": "not-a-number"},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
@pytest.mark.parametrize("price", [-1, -0.01])
def test_returns_422_for_negative_price(self, client, admin_token, price):
res = client.post(
"/api/items",
json={"name": "test", "price": price},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
def test_returns_422_for_price_exceeding_max(self, client, admin_token):
res = client.post(
"/api/items",
json={"name": "test", "price": 1_000_001},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 422
def test_creates_item_successfully(self, client, admin_token):
res = client.post(
"/api/items",
json={"name": "New Widget", "price": 9.99, "category": "tools"},
headers={"Authorization": f"Bearer {admin_token}"},
)
assert res.status_code == 201
data = res.json()
assert "id" in data
assert data["name"] == "New Widget"
def test_returns_403_for_non_admin(self, client, valid_token):
res = client.post(
"/api/items",
json={"name": "test", "price": 1.0},
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 403
class TestPagination:
def test_returns_paginated_response(self, client, valid_token):
res = client.get(
"/api/items?page=1&size=10",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
data = res.json()
assert "items" in data
assert "total" in data
assert "page" in data
assert len(data["items"]) <= 10
def test_empty_result_for_out_of_range_page(self, client, valid_token):
res = client.get(
"/api/items?page=99999",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
assert res.json()["items"] == []
def test_returns_422_for_page_zero(self, client, valid_token):
res = client.get(
"/api/items?page=0",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 422
def test_caps_page_size_at_maximum(self, client, valid_token):
res = client.get(
"/api/items?size=9999",
headers={"Authorization": f"Bearer {valid_token}"},
)
assert res.status_code == 200
assert len(res.json()["items"]) <= 100 # max page size
class TestRateLimiting:
def test_rate_limit_after_burst(self, client, valid_token):
responses = []
for _ in range(60): # exceed typical 50/min limit
res = client.get(
"/api/items",
headers={"Authorization": f"Bearer {valid_token}"},
)
responses.append(res.status_code)
if res.status_code == 429:
break
assert 429 in responses, "Rate limit was not triggered"
def test_rate_limit_response_has_retry_after(self, client, valid_token):
for _ in range(60):
res = client.get("/api/items", headers={"Authorization": f"Bearer {valid_token}"})
if res.status_code == 429:
assert "Retry-After" in res.headers or "retry_after" in res.json()
break
```
---

39
engineering/codebase-onboarding/SKILL.md
View File

@@ -442,44 +442,7 @@ pnpm validate
---
## Output Formats
### Notion Export
```javascript
// Use Notion API to create onboarding page
const { Client } = require('@notionhq/client')
const notion = new Client({ auth: process.env.NOTION_TOKEN })
const blocks = markdownToNotionBlocks(onboardingMarkdown) // use notion-to-md
await notion.pages.create({
parent: { page_id: ONBOARDING_PARENT_PAGE_ID },
properties: { title: { title: [{ text: { content: 'Engineer Onboarding — MyApp' } }] } },
children: blocks,
})
```
### Confluence Export
```bash
# Using confluence-cli or REST API
curl -X POST \
-H "Content-Type: application/json" \
-u "user@example.com:$CONFLUENCE_TOKEN" \
"https://yourorg.atlassian.net/wiki/rest/api/content" \
-d '{
"type": "page",
"title": "Codebase Onboarding",
"space": {"key": "ENG"},
"body": {
"storage": {
"value": "<p>Generated content...</p>",
"representation": "storage"
}
}
}'
```
---
→ See references/output-format-templates.md for details
## Common Pitfalls

41
engineering/codebase-onboarding/references/output-format-templates.md Normal file
View File

@@ -0,0 +1,41 @@
# codebase-onboarding reference
## Output Formats
### Notion Export
```javascript
// Use Notion API to create onboarding page
const { Client } = require('@notionhq/client')
const notion = new Client({ auth: process.env.NOTION_TOKEN })
const blocks = markdownToNotionBlocks(onboardingMarkdown) // use notion-to-md
await notion.pages.create({
parent: { page_id: ONBOARDING_PARENT_PAGE_ID },
properties: { title: { title: [{ text: { content: 'Engineer Onboarding — MyApp' } }] } },
children: blocks,
})
```
### Confluence Export
```bash
# Using confluence-cli or REST API
curl -X POST \
-H "Content-Type: application/json" \
-u "user@example.com:$CONFLUENCE_TOKEN" \
"https://yourorg.atlassian.net/wiki/rest/api/content" \
-d '{
"type": "page",
"title": "Codebase Onboarding",
"space": {"key": "ENG"},
"body": {
"storage": {
"value": "<p>Generated content...</p>",
"representation": "storage"
}
}
}'
```
---

476
engineering/database-designer/SKILL.md
View File

@@ -34,479 +34,7 @@ A comprehensive database design skill that provides expert-level analysis, optim
- **Execution Planning**: Ordered migration steps with dependency resolution
## Database Design Principles
### Normalization Forms
#### First Normal Form (1NF)
- **Atomic Values**: Each column contains indivisible values
- **Unique Column Names**: No duplicate column names within a table
- **Uniform Data Types**: Each column contains the same type of data
- **Row Uniqueness**: No duplicate rows in the table
**Example Violation:**
```sql
-- BAD: Multiple phone numbers in one column
CREATE TABLE contacts (
id INT PRIMARY KEY,
name VARCHAR(100),
phones VARCHAR(200) -- "123-456-7890, 098-765-4321"
);
-- GOOD: Separate table for phone numbers
CREATE TABLE contacts (
id INT PRIMARY KEY,
name VARCHAR(100)
);
CREATE TABLE contact_phones (
id INT PRIMARY KEY,
contact_id INT REFERENCES contacts(id),
phone_number VARCHAR(20),
phone_type VARCHAR(10)
);
```
#### Second Normal Form (2NF)
- **1NF Compliance**: Must satisfy First Normal Form
- **Full Functional Dependency**: Non-key attributes depend on the entire primary key
- **Partial Dependency Elimination**: Remove attributes that depend on part of a composite key
**Example Violation:**
```sql
-- BAD: Student course table with partial dependencies
CREATE TABLE student_courses (
student_id INT,
course_id INT,
student_name VARCHAR(100), -- Depends only on student_id
course_name VARCHAR(100), -- Depends only on course_id
grade CHAR(1),
PRIMARY KEY (student_id, course_id)
);
-- GOOD: Separate tables eliminate partial dependencies
CREATE TABLE students (
id INT PRIMARY KEY,
name VARCHAR(100)
);
CREATE TABLE courses (
id INT PRIMARY KEY,
name VARCHAR(100)
);
CREATE TABLE enrollments (
student_id INT REFERENCES students(id),
course_id INT REFERENCES courses(id),
grade CHAR(1),
PRIMARY KEY (student_id, course_id)
);
```
#### Third Normal Form (3NF)
- **2NF Compliance**: Must satisfy Second Normal Form
- **Transitive Dependency Elimination**: Non-key attributes should not depend on other non-key attributes
- **Direct Dependency**: Non-key attributes depend directly on the primary key
**Example Violation:**
```sql
-- BAD: Employee table with transitive dependency
CREATE TABLE employees (
id INT PRIMARY KEY,
name VARCHAR(100),
department_id INT,
department_name VARCHAR(100), -- Depends on department_id, not employee id
department_budget DECIMAL(10,2) -- Transitive dependency
);
-- GOOD: Separate department information
CREATE TABLE departments (
id INT PRIMARY KEY,
name VARCHAR(100),
budget DECIMAL(10,2)
);
CREATE TABLE employees (
id INT PRIMARY KEY,
name VARCHAR(100),
department_id INT REFERENCES departments(id)
);
```
#### Boyce-Codd Normal Form (BCNF)
- **3NF Compliance**: Must satisfy Third Normal Form
- **Determinant Key Rule**: Every determinant must be a candidate key
- **Stricter 3NF**: Handles anomalies not covered by 3NF
### Denormalization Strategies
#### When to Denormalize
1. **Read-Heavy Workloads**: High query frequency with acceptable write trade-offs
2. **Performance Bottlenecks**: Join operations causing significant latency
3. **Aggregation Needs**: Frequent calculation of derived values
4. **Caching Requirements**: Pre-computed results for common queries
#### Common Denormalization Patterns
**Redundant Storage**
```sql
-- Store calculated values to avoid expensive joins
CREATE TABLE orders (
id INT PRIMARY KEY,
customer_id INT REFERENCES customers(id),
customer_name VARCHAR(100), -- Denormalized from customers table
order_total DECIMAL(10,2), -- Denormalized calculation
created_at TIMESTAMP
);
```
**Materialized Aggregates**
```sql
-- Pre-computed summary tables
CREATE TABLE customer_statistics (
customer_id INT PRIMARY KEY,
total_orders INT,
lifetime_value DECIMAL(12,2),
last_order_date DATE,
updated_at TIMESTAMP
);
```
## Index Optimization Strategies
### B-Tree Indexes
- **Default Choice**: Best for range queries, sorting, and equality matches
- **Column Order**: Most selective columns first for composite indexes
- **Prefix Matching**: Supports leading column subset queries
- **Maintenance Cost**: Balanced tree structure with logarithmic operations
### Hash Indexes
- **Equality Queries**: Optimal for exact match lookups
- **Memory Efficiency**: Constant-time access for single-value queries
- **Range Limitations**: Cannot support range or partial matches
- **Use Cases**: Primary keys, unique constraints, cache keys
### Composite Indexes
```sql
-- Query pattern determines optimal column order
-- Query: WHERE status = 'active' AND created_date > '2023-01-01' ORDER BY priority DESC
CREATE INDEX idx_task_status_date_priority
ON tasks (status, created_date, priority DESC);
-- Query: WHERE user_id = 123 AND category IN ('A', 'B') AND date_field BETWEEN '...' AND '...'
CREATE INDEX idx_user_category_date
ON user_activities (user_id, category, date_field);
```
### Covering Indexes
```sql
-- Include additional columns to avoid table lookups
CREATE INDEX idx_user_email_covering
ON users (email)
INCLUDE (first_name, last_name, status);
-- Query can be satisfied entirely from the index
-- SELECT first_name, last_name, status FROM users WHERE email = 'user@example.com';
```
### Partial Indexes
```sql
-- Index only relevant subset of data
CREATE INDEX idx_active_users_email
ON users (email)
WHERE status = 'active';
-- Index for recent orders only
CREATE INDEX idx_recent_orders_customer
ON orders (customer_id, created_at)
WHERE created_at > CURRENT_DATE - INTERVAL '30 days';
```
## Query Analysis & Optimization
### Query Patterns Recognition
1. **Equality Filters**: Single-column B-tree indexes
2. **Range Queries**: B-tree with proper column ordering
3. **Text Search**: Full-text indexes or trigram indexes
4. **Join Operations**: Foreign key indexes on both sides
5. **Sorting Requirements**: Indexes matching ORDER BY clauses
### Index Selection Algorithm
```
1. Identify WHERE clause columns
2. Determine most selective columns first
3. Consider JOIN conditions
4. Include ORDER BY columns if possible
5. Evaluate covering index opportunities
6. Check for existing overlapping indexes
```
## Data Modeling Patterns
### Star Schema (Data Warehousing)
```sql
-- Central fact table
CREATE TABLE sales_facts (
sale_id BIGINT PRIMARY KEY,
product_id INT REFERENCES products(id),
customer_id INT REFERENCES customers(id),
date_id INT REFERENCES date_dimension(id),
store_id INT REFERENCES stores(id),
quantity INT,
unit_price DECIMAL(8,2),
total_amount DECIMAL(10,2)
);
-- Dimension tables
CREATE TABLE date_dimension (
id INT PRIMARY KEY,
date_value DATE,
year INT,
quarter INT,
month INT,
day_of_week INT,
is_weekend BOOLEAN
);
```
### Snowflake Schema
```sql
-- Normalized dimension tables
CREATE TABLE products (
id INT PRIMARY KEY,
name VARCHAR(200),
category_id INT REFERENCES product_categories(id),
brand_id INT REFERENCES brands(id)
);
CREATE TABLE product_categories (
id INT PRIMARY KEY,
name VARCHAR(100),
parent_category_id INT REFERENCES product_categories(id)
);
```
### Document Model (JSON Storage)
```sql
-- Flexible document storage with indexing
CREATE TABLE documents (
id UUID PRIMARY KEY,
document_type VARCHAR(50),
data JSONB,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- Index on JSON properties
CREATE INDEX idx_documents_user_id
ON documents USING GIN ((data->>'user_id'));
CREATE INDEX idx_documents_status
ON documents ((data->>'status'))
WHERE document_type = 'order';
```
### Graph Data Patterns
```sql
-- Adjacency list for hierarchical data
CREATE TABLE categories (
id INT PRIMARY KEY,
name VARCHAR(100),
parent_id INT REFERENCES categories(id),
level INT,
path VARCHAR(500) -- Materialized path: "/1/5/12/"
);
-- Many-to-many relationships
CREATE TABLE relationships (
id UUID PRIMARY KEY,
from_entity_id UUID,
to_entity_id UUID,
relationship_type VARCHAR(50),
created_at TIMESTAMP,
INDEX (from_entity_id, relationship_type),
INDEX (to_entity_id, relationship_type)
);
```
## Migration Strategies
### Zero-Downtime Migration (Expand-Contract Pattern)
**Phase 1: Expand**
```sql
-- Add new column without constraints
ALTER TABLE users ADD COLUMN new_email VARCHAR(255);
-- Backfill data in batches
UPDATE users SET new_email = email WHERE id BETWEEN 1 AND 1000;
-- Continue in batches...
-- Add constraints after backfill
ALTER TABLE users ADD CONSTRAINT users_new_email_unique UNIQUE (new_email);
ALTER TABLE users ALTER COLUMN new_email SET NOT NULL;
```
**Phase 2: Contract**
```sql
-- Update application to use new column
-- Deploy application changes
-- Verify new column is being used
-- Remove old column
ALTER TABLE users DROP COLUMN email;
-- Rename new column
ALTER TABLE users RENAME COLUMN new_email TO email;
```
### Data Type Changes
```sql
-- Safe string to integer conversion
ALTER TABLE products ADD COLUMN sku_number INTEGER;
UPDATE products SET sku_number = CAST(sku AS INTEGER) WHERE sku ~ '^[0-9]+$';
-- Validate conversion success before dropping old column
```
## Partitioning Strategies
### Horizontal Partitioning (Sharding)
```sql
-- Range partitioning by date
CREATE TABLE sales_2023 PARTITION OF sales
FOR VALUES FROM ('2023-01-01') TO ('2024-01-01');
CREATE TABLE sales_2024 PARTITION OF sales
FOR VALUES FROM ('2024-01-01') TO ('2025-01-01');
-- Hash partitioning by user_id
CREATE TABLE user_data_0 PARTITION OF user_data
FOR VALUES WITH (MODULUS 4, REMAINDER 0);
CREATE TABLE user_data_1 PARTITION OF user_data
FOR VALUES WITH (MODULUS 4, REMAINDER 1);
```
### Vertical Partitioning
```sql
-- Separate frequently accessed columns
CREATE TABLE users_core (
id INT PRIMARY KEY,
email VARCHAR(255),
status VARCHAR(20),
created_at TIMESTAMP
);
-- Less frequently accessed profile data
CREATE TABLE users_profile (
user_id INT PRIMARY KEY REFERENCES users_core(id),
bio TEXT,
preferences JSONB,
last_login TIMESTAMP
);
```
## Connection Management
### Connection Pooling
- **Pool Size**: CPU cores × 2 + effective spindle count
- **Connection Lifetime**: Rotate connections to prevent resource leaks
- **Timeout Settings**: Connection, idle, and query timeouts
- **Health Checks**: Regular connection validation
### Read Replicas Strategy
```sql
-- Write queries to primary
INSERT INTO users (email, name) VALUES ('user@example.com', 'John Doe');
-- Read queries to replicas (with appropriate read preference)
SELECT * FROM users WHERE status = 'active'; -- Route to read replica
-- Consistent reads when required
SELECT * FROM users WHERE id = LAST_INSERT_ID(); -- Route to primary
```
## Caching Layers
### Cache-Aside Pattern
```python
def get_user(user_id):
# Try cache first
user = cache.get(f"user:{user_id}")
if user is None:
# Cache miss - query database
user = db.query("SELECT * FROM users WHERE id = %s", user_id)
# Store in cache
cache.set(f"user:{user_id}", user, ttl=3600)
return user
```
### Write-Through Cache
- **Consistency**: Always keep cache and database in sync
- **Write Latency**: Higher due to dual writes
- **Data Safety**: No data loss on cache failures
### Cache Invalidation Strategies
1. **TTL-Based**: Time-based expiration
2. **Event-Driven**: Invalidate on data changes
3. **Version-Based**: Use version numbers for consistency
4. **Tag-Based**: Group related cache entries
## Database Selection Guide
### SQL Databases
**PostgreSQL**
- **Strengths**: ACID compliance, complex queries, JSON support, extensibility
- **Use Cases**: OLTP applications, data warehousing, geospatial data
- **Scale**: Vertical scaling with read replicas
**MySQL**
- **Strengths**: Performance, replication, wide ecosystem support
- **Use Cases**: Web applications, content management, e-commerce
- **Scale**: Horizontal scaling through sharding
### NoSQL Databases
**Document Stores (MongoDB, CouchDB)**
- **Strengths**: Flexible schema, horizontal scaling, developer productivity
- **Use Cases**: Content management, catalogs, user profiles
- **Trade-offs**: Eventual consistency, complex queries limitations
**Key-Value Stores (Redis, DynamoDB)**
- **Strengths**: High performance, simple model, excellent caching
- **Use Cases**: Session storage, real-time analytics, gaming leaderboards
- **Trade-offs**: Limited query capabilities, data modeling constraints
**Column-Family (Cassandra, HBase)**
- **Strengths**: Write-heavy workloads, linear scalability, fault tolerance
- **Use Cases**: Time-series data, IoT applications, messaging systems
- **Trade-offs**: Query flexibility, consistency model complexity
**Graph Databases (Neo4j, Amazon Neptune)**
- **Strengths**: Relationship queries, pattern matching, recommendation engines
- **Use Cases**: Social networks, fraud detection, knowledge graphs
- **Trade-offs**: Specialized use cases, learning curve
### NewSQL Databases
**Distributed SQL (CockroachDB, TiDB, Spanner)**
- **Strengths**: SQL compatibility with horizontal scaling
- **Use Cases**: Global applications requiring ACID guarantees
- **Trade-offs**: Complexity, latency for distributed transactions
## Tools & Scripts
### Schema Analyzer
- **Input**: SQL DDL files, JSON schema definitions
- **Analysis**: Normalization compliance, constraint validation, naming conventions
- **Output**: Analysis report, Mermaid ERD, improvement recommendations
### Index Optimizer
- **Input**: Schema definition, query patterns
- **Analysis**: Missing indexes, redundancy detection, selectivity estimation
- **Output**: Index recommendations, CREATE INDEX statements, performance projections
### Migration Generator
- **Input**: Current and target schemas
- **Analysis**: Schema differences, dependency resolution, risk assessment
- **Output**: Migration scripts, rollback plans, validation queries
→ See references/database-design-reference.md for details
## Best Practices
@@ -535,4 +63,4 @@ def get_user(user_id):
Effective database design requires balancing multiple competing concerns: performance, scalability, maintainability, and business requirements. This skill provides the tools and knowledge to make informed decisions throughout the database lifecycle, from initial schema design through production optimization and evolution.
The included tools automate common analysis and optimization tasks, while the comprehensive guides provide the theoretical foundation for making sound architectural decisions. Whether building a new system or optimizing an existing one, these resources provide expert-level guidance for creating robust, scalable database solutions.
The included tools automate common analysis and optimization tasks, while the comprehensive guides provide the theoretical foundation for making sound architectural decisions. Whether building a new system or optimizing an existing one, these resources provide expert-level guidance for creating robust, scalable database solutions.

476
engineering/database-designer/references/database-design-reference.md Normal file
View File

@@ -0,0 +1,476 @@
# database-designer reference
## Database Design Principles
### Normalization Forms
#### First Normal Form (1NF)
- **Atomic Values**: Each column contains indivisible values
- **Unique Column Names**: No duplicate column names within a table
- **Uniform Data Types**: Each column contains the same type of data
- **Row Uniqueness**: No duplicate rows in the table
**Example Violation:**
```sql
-- BAD: Multiple phone numbers in one column
CREATE TABLE contacts (
id INT PRIMARY KEY,
name VARCHAR(100),
phones VARCHAR(200) -- "123-456-7890, 098-765-4321"
);
-- GOOD: Separate table for phone numbers
CREATE TABLE contacts (
id INT PRIMARY KEY,
name VARCHAR(100)
);
CREATE TABLE contact_phones (
id INT PRIMARY KEY,
contact_id INT REFERENCES contacts(id),
phone_number VARCHAR(20),
phone_type VARCHAR(10)
);
```
#### Second Normal Form (2NF)
- **1NF Compliance**: Must satisfy First Normal Form
- **Full Functional Dependency**: Non-key attributes depend on the entire primary key
- **Partial Dependency Elimination**: Remove attributes that depend on part of a composite key
**Example Violation:**
```sql
-- BAD: Student course table with partial dependencies
CREATE TABLE student_courses (
student_id INT,
course_id INT,
student_name VARCHAR(100), -- Depends only on student_id
course_name VARCHAR(100), -- Depends only on course_id
grade CHAR(1),
PRIMARY KEY (student_id, course_id)
);
-- GOOD: Separate tables eliminate partial dependencies
CREATE TABLE students (
id INT PRIMARY KEY,
name VARCHAR(100)
);
CREATE TABLE courses (
id INT PRIMARY KEY,
name VARCHAR(100)
);
CREATE TABLE enrollments (
student_id INT REFERENCES students(id),
course_id INT REFERENCES courses(id),
grade CHAR(1),
PRIMARY KEY (student_id, course_id)
);
```
#### Third Normal Form (3NF)
- **2NF Compliance**: Must satisfy Second Normal Form
- **Transitive Dependency Elimination**: Non-key attributes should not depend on other non-key attributes
- **Direct Dependency**: Non-key attributes depend directly on the primary key
**Example Violation:**
```sql
-- BAD: Employee table with transitive dependency
CREATE TABLE employees (
id INT PRIMARY KEY,
name VARCHAR(100),
department_id INT,
department_name VARCHAR(100), -- Depends on department_id, not employee id
department_budget DECIMAL(10,2) -- Transitive dependency
);
-- GOOD: Separate department information
CREATE TABLE departments (
id INT PRIMARY KEY,
name VARCHAR(100),
budget DECIMAL(10,2)
);
CREATE TABLE employees (
id INT PRIMARY KEY,
name VARCHAR(100),
department_id INT REFERENCES departments(id)
);
```
#### Boyce-Codd Normal Form (BCNF)
- **3NF Compliance**: Must satisfy Third Normal Form
- **Determinant Key Rule**: Every determinant must be a candidate key
- **Stricter 3NF**: Handles anomalies not covered by 3NF
### Denormalization Strategies
#### When to Denormalize
1. **Read-Heavy Workloads**: High query frequency with acceptable write trade-offs
2. **Performance Bottlenecks**: Join operations causing significant latency
3. **Aggregation Needs**: Frequent calculation of derived values
4. **Caching Requirements**: Pre-computed results for common queries
#### Common Denormalization Patterns
**Redundant Storage**
```sql
-- Store calculated values to avoid expensive joins
CREATE TABLE orders (
id INT PRIMARY KEY,
customer_id INT REFERENCES customers(id),
customer_name VARCHAR(100), -- Denormalized from customers table
order_total DECIMAL(10,2), -- Denormalized calculation
created_at TIMESTAMP
);
```
**Materialized Aggregates**
```sql
-- Pre-computed summary tables
CREATE TABLE customer_statistics (
customer_id INT PRIMARY KEY,
total_orders INT,
lifetime_value DECIMAL(12,2),
last_order_date DATE,
updated_at TIMESTAMP
);
```
## Index Optimization Strategies
### B-Tree Indexes
- **Default Choice**: Best for range queries, sorting, and equality matches
- **Column Order**: Most selective columns first for composite indexes
- **Prefix Matching**: Supports leading column subset queries
- **Maintenance Cost**: Balanced tree structure with logarithmic operations
### Hash Indexes
- **Equality Queries**: Optimal for exact match lookups
- **Memory Efficiency**: Constant-time access for single-value queries
- **Range Limitations**: Cannot support range or partial matches
- **Use Cases**: Primary keys, unique constraints, cache keys
### Composite Indexes
```sql
-- Query pattern determines optimal column order
-- Query: WHERE status = 'active' AND created_date > '2023-01-01' ORDER BY priority DESC
CREATE INDEX idx_task_status_date_priority
ON tasks (status, created_date, priority DESC);
-- Query: WHERE user_id = 123 AND category IN ('A', 'B') AND date_field BETWEEN '...' AND '...'
CREATE INDEX idx_user_category_date
ON user_activities (user_id, category, date_field);
```
### Covering Indexes
```sql
-- Include additional columns to avoid table lookups
CREATE INDEX idx_user_email_covering
ON users (email)
INCLUDE (first_name, last_name, status);
-- Query can be satisfied entirely from the index
-- SELECT first_name, last_name, status FROM users WHERE email = 'user@example.com';
```
### Partial Indexes
```sql
-- Index only relevant subset of data
CREATE INDEX idx_active_users_email
ON users (email)
WHERE status = 'active';
-- Index for recent orders only
CREATE INDEX idx_recent_orders_customer
ON orders (customer_id, created_at)
WHERE created_at > CURRENT_DATE - INTERVAL '30 days';
```
## Query Analysis & Optimization
### Query Patterns Recognition
1. **Equality Filters**: Single-column B-tree indexes
2. **Range Queries**: B-tree with proper column ordering
3. **Text Search**: Full-text indexes or trigram indexes
4. **Join Operations**: Foreign key indexes on both sides
5. **Sorting Requirements**: Indexes matching ORDER BY clauses
### Index Selection Algorithm
```
1. Identify WHERE clause columns
2. Determine most selective columns first
3. Consider JOIN conditions
4. Include ORDER BY columns if possible
5. Evaluate covering index opportunities
6. Check for existing overlapping indexes
```
## Data Modeling Patterns
### Star Schema (Data Warehousing)
```sql
-- Central fact table
CREATE TABLE sales_facts (
sale_id BIGINT PRIMARY KEY,
product_id INT REFERENCES products(id),
customer_id INT REFERENCES customers(id),
date_id INT REFERENCES date_dimension(id),
store_id INT REFERENCES stores(id),
quantity INT,
unit_price DECIMAL(8,2),
total_amount DECIMAL(10,2)
);
-- Dimension tables
CREATE TABLE date_dimension (
id INT PRIMARY KEY,
date_value DATE,
year INT,
quarter INT,
month INT,
day_of_week INT,
is_weekend BOOLEAN
);
```
### Snowflake Schema
```sql
-- Normalized dimension tables
CREATE TABLE products (
id INT PRIMARY KEY,
name VARCHAR(200),
category_id INT REFERENCES product_categories(id),
brand_id INT REFERENCES brands(id)
);
CREATE TABLE product_categories (
id INT PRIMARY KEY,
name VARCHAR(100),
parent_category_id INT REFERENCES product_categories(id)
);
```
### Document Model (JSON Storage)
```sql
-- Flexible document storage with indexing
CREATE TABLE documents (
id UUID PRIMARY KEY,
document_type VARCHAR(50),
data JSONB,
created_at TIMESTAMP DEFAULT NOW(),
updated_at TIMESTAMP DEFAULT NOW()
);
-- Index on JSON properties
CREATE INDEX idx_documents_user_id
ON documents USING GIN ((data->>'user_id'));
CREATE INDEX idx_documents_status
ON documents ((data->>'status'))
WHERE document_type = 'order';
```
### Graph Data Patterns
```sql
-- Adjacency list for hierarchical data
CREATE TABLE categories (
id INT PRIMARY KEY,
name VARCHAR(100),
parent_id INT REFERENCES categories(id),
level INT,
path VARCHAR(500) -- Materialized path: "/1/5/12/"
);
-- Many-to-many relationships
CREATE TABLE relationships (
id UUID PRIMARY KEY,
from_entity_id UUID,
to_entity_id UUID,
relationship_type VARCHAR(50),
created_at TIMESTAMP,
INDEX (from_entity_id, relationship_type),
INDEX (to_entity_id, relationship_type)
);
```
## Migration Strategies
### Zero-Downtime Migration (Expand-Contract Pattern)
**Phase 1: Expand**
```sql
-- Add new column without constraints
ALTER TABLE users ADD COLUMN new_email VARCHAR(255);
-- Backfill data in batches
UPDATE users SET new_email = email WHERE id BETWEEN 1 AND 1000;
-- Continue in batches...
-- Add constraints after backfill
ALTER TABLE users ADD CONSTRAINT users_new_email_unique UNIQUE (new_email);
ALTER TABLE users ALTER COLUMN new_email SET NOT NULL;
```
**Phase 2: Contract**
```sql
-- Update application to use new column
-- Deploy application changes
-- Verify new column is being used
-- Remove old column
ALTER TABLE users DROP COLUMN email;
-- Rename new column
ALTER TABLE users RENAME COLUMN new_email TO email;
```
### Data Type Changes
```sql
-- Safe string to integer conversion
ALTER TABLE products ADD COLUMN sku_number INTEGER;
UPDATE products SET sku_number = CAST(sku AS INTEGER) WHERE sku ~ '^[0-9]+$';
-- Validate conversion success before dropping old column
```
## Partitioning Strategies
### Horizontal Partitioning (Sharding)
```sql
-- Range partitioning by date
CREATE TABLE sales_2023 PARTITION OF sales
FOR VALUES FROM ('2023-01-01') TO ('2024-01-01');
CREATE TABLE sales_2024 PARTITION OF sales
FOR VALUES FROM ('2024-01-01') TO ('2025-01-01');
-- Hash partitioning by user_id
CREATE TABLE user_data_0 PARTITION OF user_data
FOR VALUES WITH (MODULUS 4, REMAINDER 0);
CREATE TABLE user_data_1 PARTITION OF user_data
FOR VALUES WITH (MODULUS 4, REMAINDER 1);
```
### Vertical Partitioning
```sql
-- Separate frequently accessed columns
CREATE TABLE users_core (
id INT PRIMARY KEY,
email VARCHAR(255),
status VARCHAR(20),
created_at TIMESTAMP
);
-- Less frequently accessed profile data
CREATE TABLE users_profile (
user_id INT PRIMARY KEY REFERENCES users_core(id),
bio TEXT,
preferences JSONB,
last_login TIMESTAMP
);
```
## Connection Management
### Connection Pooling
- **Pool Size**: CPU cores × 2 + effective spindle count
- **Connection Lifetime**: Rotate connections to prevent resource leaks
- **Timeout Settings**: Connection, idle, and query timeouts
- **Health Checks**: Regular connection validation
### Read Replicas Strategy
```sql
-- Write queries to primary
INSERT INTO users (email, name) VALUES ('user@example.com', 'John Doe');
-- Read queries to replicas (with appropriate read preference)
SELECT * FROM users WHERE status = 'active'; -- Route to read replica
-- Consistent reads when required
SELECT * FROM users WHERE id = LAST_INSERT_ID(); -- Route to primary
```
## Caching Layers
### Cache-Aside Pattern
```python
def get_user(user_id):
# Try cache first
user = cache.get(f"user:{user_id}")
if user is None:
# Cache miss - query database
user = db.query("SELECT * FROM users WHERE id = %s", user_id)
# Store in cache
cache.set(f"user:{user_id}", user, ttl=3600)
return user
```
### Write-Through Cache
- **Consistency**: Always keep cache and database in sync
- **Write Latency**: Higher due to dual writes
- **Data Safety**: No data loss on cache failures
### Cache Invalidation Strategies
1. **TTL-Based**: Time-based expiration
2. **Event-Driven**: Invalidate on data changes
3. **Version-Based**: Use version numbers for consistency
4. **Tag-Based**: Group related cache entries
## Database Selection Guide
### SQL Databases
**PostgreSQL**
- **Strengths**: ACID compliance, complex queries, JSON support, extensibility
- **Use Cases**: OLTP applications, data warehousing, geospatial data
- **Scale**: Vertical scaling with read replicas
**MySQL**
- **Strengths**: Performance, replication, wide ecosystem support
- **Use Cases**: Web applications, content management, e-commerce
- **Scale**: Horizontal scaling through sharding
### NoSQL Databases
**Document Stores (MongoDB, CouchDB)**
- **Strengths**: Flexible schema, horizontal scaling, developer productivity
- **Use Cases**: Content management, catalogs, user profiles
- **Trade-offs**: Eventual consistency, complex queries limitations
**Key-Value Stores (Redis, DynamoDB)**
- **Strengths**: High performance, simple model, excellent caching
- **Use Cases**: Session storage, real-time analytics, gaming leaderboards
- **Trade-offs**: Limited query capabilities, data modeling constraints
**Column-Family (Cassandra, HBase)**
- **Strengths**: Write-heavy workloads, linear scalability, fault tolerance
- **Use Cases**: Time-series data, IoT applications, messaging systems
- **Trade-offs**: Query flexibility, consistency model complexity
**Graph Databases (Neo4j, Amazon Neptune)**
- **Strengths**: Relationship queries, pattern matching, recommendation engines
- **Use Cases**: Social networks, fraud detection, knowledge graphs
- **Trade-offs**: Specialized use cases, learning curve
### NewSQL Databases
**Distributed SQL (CockroachDB, TiDB, Spanner)**
- **Strengths**: SQL compatibility with horizontal scaling
- **Use Cases**: Global applications requiring ACID guarantees
- **Trade-offs**: Complexity, latency for distributed transactions
## Tools & Scripts
### Schema Analyzer
- **Input**: SQL DDL files, JSON schema definitions
- **Analysis**: Normalization compliance, constraint validation, naming conventions
- **Output**: Analysis report, Mermaid ERD, improvement recommendations
### Index Optimizer
- **Input**: Schema definition, query patterns
- **Analysis**: Missing indexes, redundancy detection, selectivity estimation
- **Output**: Index recommendations, CREATE INDEX statements, performance projections
### Migration Generator
- **Input**: Current and target schemas
- **Analysis**: Schema differences, dependency resolution, risk assessment
- **Output**: Migration scripts, rollback plans, validation queries

281
engineering/database-schema-designer/SKILL.md
View File

@@ -69,286 +69,7 @@ User 1──* AuditLog
---
## Full Schema Example (Task Management SaaS)
### Prisma Schema
```prisma
// schema.prisma
generator client {
provider = "prisma-client-js"
}
datasource db {
provider = "postgresql"
url = env("DATABASE_URL")
}
// ── Multi-tenancy ─────────────────────────────────────────────────────────────
model Organization {
id String @id @default(cuid())
name String
slug String @unique
plan Plan @default(FREE)
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
users OrganizationMember[]
projects Project[]
auditLogs AuditLog[]
@@map("organizations")
}
model OrganizationMember {
id String @id @default(cuid())
organizationId String @map("organization_id")
userId String @map("user_id")
role OrgRole @default(MEMBER)
joinedAt DateTime @default(now()) @map("joined_at")
organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@unique([organizationId, userId])
@@index([userId])
@@map("organization_members")
}
model User {
id String @id @default(cuid())
email String @unique
name String?
avatarUrl String? @map("avatar_url")
passwordHash String? @map("password_hash")
emailVerifiedAt DateTime? @map("email_verified_at")
lastLoginAt DateTime? @map("last_login_at")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
memberships OrganizationMember[]
ownedProjects Project[] @relation("ProjectOwner")
assignedTasks TaskAssignment[]
comments Comment[]
auditLogs AuditLog[]
@@map("users")
}
// ── Core entities ─────────────────────────────────────────────────────────────
model Project {
id String @id @default(cuid())
organizationId String @map("organization_id")
ownerId String @map("owner_id")
name String
description String?
status ProjectStatus @default(ACTIVE)
settings Json @default("{}")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
organization Organization @relation(fields: [organizationId], references: [id])
owner User @relation("ProjectOwner", fields: [ownerId], references: [id])
tasks Task[]
labels Label[]
@@index([organizationId])
@@index([organizationId, status])
@@index([deletedAt])
@@map("projects")
}
model Task {
id String @id @default(cuid())
projectId String @map("project_id")
title String
description String?
status TaskStatus @default(TODO)
priority Priority @default(MEDIUM)
dueDate DateTime? @map("due_date")
position Float @default(0) // For drag-and-drop ordering
version Int @default(1) // Optimistic locking
createdById String @map("created_by_id")
updatedById String @map("updated_by_id")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
project Project @relation(fields: [projectId], references: [id])
assignments TaskAssignment[]
labels TaskLabel[]
comments Comment[]
attachments Attachment[]
@@index([projectId])
@@index([projectId, status])
@@index([projectId, deletedAt])
@@index([dueDate], where: { deletedAt: null }) // Partial index
@@map("tasks")
}
// ── Polymorphic attachments ───────────────────────────────────────────────────
model Attachment {
id String @id @default(cuid())
// Polymorphic association
entityType String @map("entity_type") // "task" | "comment"
entityId String @map("entity_id")
filename String
mimeType String @map("mime_type")
sizeBytes Int @map("size_bytes")
storageKey String @map("storage_key") // S3 key
uploadedById String @map("uploaded_by_id")
createdAt DateTime @default(now()) @map("created_at")
// Only one concrete relation (task) — polymorphic handled at app level
task Task? @relation(fields: [entityId], references: [id], map: "attachment_task_fk")
@@index([entityType, entityId])
@@map("attachments")
}
// ── Audit trail ───────────────────────────────────────────────────────────────
model AuditLog {
id String @id @default(cuid())
organizationId String @map("organization_id")
userId String? @map("user_id")
action String // "task.created", "task.status_changed"
entityType String @map("entity_type")
entityId String @map("entity_id")
before Json? // Previous state
after Json? // New state
ipAddress String? @map("ip_address")
userAgent String? @map("user_agent")
createdAt DateTime @default(now()) @map("created_at")
organization Organization @relation(fields: [organizationId], references: [id])
user User? @relation(fields: [userId], references: [id])
@@index([organizationId, createdAt(sort: Desc)])
@@index([entityType, entityId])
@@index([userId])
@@map("audit_logs")
}
enum Plan { FREE STARTER GROWTH ENTERPRISE }
enum OrgRole { OWNER ADMIN MEMBER VIEWER }
enum ProjectStatus { ACTIVE ARCHIVED }
enum TaskStatus { TODO IN_PROGRESS IN_REVIEW DONE CANCELLED }
enum Priority { LOW MEDIUM HIGH CRITICAL }
```
---
### Drizzle Schema (TypeScript)
```typescript
// db/schema.ts
import {
pgTable, text, timestamp, integer, boolean,
varchar, jsonb, real, pgEnum, uniqueIndex, index,
} from 'drizzle-orm/pg-core'
import { createId } from '@paralleldrive/cuid2'
export const taskStatusEnum = pgEnum('task_status', [
'todo', 'in_progress', 'in_review', 'done', 'cancelled'
])
export const priorityEnum = pgEnum('priority', ['low', 'medium', 'high', 'critical'])
export const tasks = pgTable('tasks', {
id: text('id').primaryKey().$defaultFn(() => createId()),
projectId: text('project_id').notNull().references(() => projects.id),
title: varchar('title', { length: 500 }).notNull(),
description: text('description'),
status: taskStatusEnum('status').notNull().default('todo'),
priority: priorityEnum('priority').notNull().default('medium'),
dueDate: timestamp('due_date', { withTimezone: true }),
position: real('position').notNull().default(0),
version: integer('version').notNull().default(1),
createdById: text('created_by_id').notNull().references(() => users.id),
updatedById: text('updated_by_id').notNull().references(() => users.id),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
deletedAt: timestamp('deleted_at', { withTimezone: true }),
}, (table) => ({
projectIdx: index('tasks_project_id_idx').on(table.projectId),
projectStatusIdx: index('tasks_project_status_idx').on(table.projectId, table.status),
}))
// Infer TypeScript types
export type Task = typeof tasks.$inferSelect
export type NewTask = typeof tasks.$inferInsert
```
---
### Alembic Migration (Python / SQLAlchemy)
```python
# alembic/versions/20260301_create_tasks.py
"""Create tasks table
Revision ID: a1b2c3d4e5f6
Revises: previous_revision
Create Date: 2026-03-01 12:00:00
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision = 'a1b2c3d4e5f6'
down_revision = 'previous_revision'
def upgrade() -> None:
# Create enums
task_status = postgresql.ENUM(
'todo', 'in_progress', 'in_review', 'done', 'cancelled',
name='task_status'
)
task_status.create(op.get_bind())
op.create_table(
'tasks',
sa.Column('id', sa.Text(), primary_key=True),
sa.Column('project_id', sa.Text(), sa.ForeignKey('projects.id'), nullable=False),
sa.Column('title', sa.VARCHAR(500), nullable=False),
sa.Column('description', sa.Text()),
sa.Column('status', postgresql.ENUM('todo', 'in_progress', 'in_review', 'done', 'cancelled', name='task_status', create_type=False), nullable=False, server_default='todo'),
sa.Column('priority', sa.Text(), nullable=False, server_default='medium'),
sa.Column('due_date', sa.TIMESTAMP(timezone=True)),
sa.Column('position', sa.Float(), nullable=False, server_default='0'),
sa.Column('version', sa.Integer(), nullable=False, server_default='1'),
sa.Column('created_by_id', sa.Text(), sa.ForeignKey('users.id'), nullable=False),
sa.Column('updated_by_id', sa.Text(), sa.ForeignKey('users.id'), nullable=False),
sa.Column('created_at', sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text('NOW()')),
sa.Column('updated_at', sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text('NOW()')),
sa.Column('deleted_at', sa.TIMESTAMP(timezone=True)),
)
# Indexes
op.create_index('tasks_project_id_idx', 'tasks', ['project_id'])
op.create_index('tasks_project_status_idx', 'tasks', ['project_id', 'status'])
# Partial index for active tasks only
op.create_index(
'tasks_due_date_active_idx',
'tasks', ['due_date'],
postgresql_where=sa.text('deleted_at IS NULL')
)
def downgrade() -> None:
op.drop_table('tasks')
op.execute("DROP TYPE IF EXISTS task_status")
```
---
→ See references/full-schema-examples.md for details
## Row-Level Security (RLS) Policies

283
engineering/database-schema-designer/references/full-schema-examples.md Normal file
View File

@@ -0,0 +1,283 @@
# database-schema-designer reference
## Full Schema Example (Task Management SaaS)
### Prisma Schema
```prisma
// schema.prisma
generator client {
provider = "prisma-client-js"
}
datasource db {
provider = "postgresql"
url = env("DATABASE_URL")
}
// ── Multi-tenancy ─────────────────────────────────────────────────────────────
model Organization {
id String @id @default(cuid())
name String
slug String @unique
plan Plan @default(FREE)
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
users OrganizationMember[]
projects Project[]
auditLogs AuditLog[]
@@map("organizations")
}
model OrganizationMember {
id String @id @default(cuid())
organizationId String @map("organization_id")
userId String @map("user_id")
role OrgRole @default(MEMBER)
joinedAt DateTime @default(now()) @map("joined_at")
organization Organization @relation(fields: [organizationId], references: [id], onDelete: Cascade)
user User @relation(fields: [userId], references: [id], onDelete: Cascade)
@@unique([organizationId, userId])
@@index([userId])
@@map("organization_members")
}
model User {
id String @id @default(cuid())
email String @unique
name String?
avatarUrl String? @map("avatar_url")
passwordHash String? @map("password_hash")
emailVerifiedAt DateTime? @map("email_verified_at")
lastLoginAt DateTime? @map("last_login_at")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
memberships OrganizationMember[]
ownedProjects Project[] @relation("ProjectOwner")
assignedTasks TaskAssignment[]
comments Comment[]
auditLogs AuditLog[]
@@map("users")
}
// ── Core entities ─────────────────────────────────────────────────────────────
model Project {
id String @id @default(cuid())
organizationId String @map("organization_id")
ownerId String @map("owner_id")
name String
description String?
status ProjectStatus @default(ACTIVE)
settings Json @default("{}")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
organization Organization @relation(fields: [organizationId], references: [id])
owner User @relation("ProjectOwner", fields: [ownerId], references: [id])
tasks Task[]
labels Label[]
@@index([organizationId])
@@index([organizationId, status])
@@index([deletedAt])
@@map("projects")
}
model Task {
id String @id @default(cuid())
projectId String @map("project_id")
title String
description String?
status TaskStatus @default(TODO)
priority Priority @default(MEDIUM)
dueDate DateTime? @map("due_date")
position Float @default(0) // For drag-and-drop ordering
version Int @default(1) // Optimistic locking
createdById String @map("created_by_id")
updatedById String @map("updated_by_id")
createdAt DateTime @default(now()) @map("created_at")
updatedAt DateTime @updatedAt @map("updated_at")
deletedAt DateTime? @map("deleted_at")
project Project @relation(fields: [projectId], references: [id])
assignments TaskAssignment[]
labels TaskLabel[]
comments Comment[]
attachments Attachment[]
@@index([projectId])
@@index([projectId, status])
@@index([projectId, deletedAt])
@@index([dueDate], where: { deletedAt: null }) // Partial index
@@map("tasks")
}
// ── Polymorphic attachments ───────────────────────────────────────────────────
model Attachment {
id String @id @default(cuid())
// Polymorphic association
entityType String @map("entity_type") // "task" | "comment"
entityId String @map("entity_id")
filename String
mimeType String @map("mime_type")
sizeBytes Int @map("size_bytes")
storageKey String @map("storage_key") // S3 key
uploadedById String @map("uploaded_by_id")
createdAt DateTime @default(now()) @map("created_at")
// Only one concrete relation (task) — polymorphic handled at app level
task Task? @relation(fields: [entityId], references: [id], map: "attachment_task_fk")
@@index([entityType, entityId])
@@map("attachments")
}
// ── Audit trail ───────────────────────────────────────────────────────────────
model AuditLog {
id String @id @default(cuid())
organizationId String @map("organization_id")
userId String? @map("user_id")
action String // "task.created", "task.status_changed"
entityType String @map("entity_type")
entityId String @map("entity_id")
before Json? // Previous state
after Json? // New state
ipAddress String? @map("ip_address")
userAgent String? @map("user_agent")
createdAt DateTime @default(now()) @map("created_at")
organization Organization @relation(fields: [organizationId], references: [id])
user User? @relation(fields: [userId], references: [id])
@@index([organizationId, createdAt(sort: Desc)])
@@index([entityType, entityId])
@@index([userId])
@@map("audit_logs")
}
enum Plan { FREE STARTER GROWTH ENTERPRISE }
enum OrgRole { OWNER ADMIN MEMBER VIEWER }
enum ProjectStatus { ACTIVE ARCHIVED }
enum TaskStatus { TODO IN_PROGRESS IN_REVIEW DONE CANCELLED }
enum Priority { LOW MEDIUM HIGH CRITICAL }
```
---
### Drizzle Schema (TypeScript)
```typescript
// db/schema.ts
import {
pgTable, text, timestamp, integer, boolean,
varchar, jsonb, real, pgEnum, uniqueIndex, index,
} from 'drizzle-orm/pg-core'
import { createId } from '@paralleldrive/cuid2'
export const taskStatusEnum = pgEnum('task_status', [
'todo', 'in_progress', 'in_review', 'done', 'cancelled'
])
export const priorityEnum = pgEnum('priority', ['low', 'medium', 'high', 'critical'])
export const tasks = pgTable('tasks', {
id: text('id').primaryKey().$defaultFn(() => createId()),
projectId: text('project_id').notNull().references(() => projects.id),
title: varchar('title', { length: 500 }).notNull(),
description: text('description'),
status: taskStatusEnum('status').notNull().default('todo'),
priority: priorityEnum('priority').notNull().default('medium'),
dueDate: timestamp('due_date', { withTimezone: true }),
position: real('position').notNull().default(0),
version: integer('version').notNull().default(1),
createdById: text('created_by_id').notNull().references(() => users.id),
updatedById: text('updated_by_id').notNull().references(() => users.id),
createdAt: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(),
updatedAt: timestamp('updated_at', { withTimezone: true }).notNull().defaultNow(),
deletedAt: timestamp('deleted_at', { withTimezone: true }),
}, (table) => ({
projectIdx: index('tasks_project_id_idx').on(table.projectId),
projectStatusIdx: index('tasks_project_status_idx').on(table.projectId, table.status),
}))
// Infer TypeScript types
export type Task = typeof tasks.$inferSelect
export type NewTask = typeof tasks.$inferInsert
```
---
### Alembic Migration (Python / SQLAlchemy)
```python
# alembic/versions/20260301_create_tasks.py
"""Create tasks table
Revision ID: a1b2c3d4e5f6
Revises: previous_revision
Create Date: 2026-03-01 12:00:00
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision = 'a1b2c3d4e5f6'
down_revision = 'previous_revision'
def upgrade() -> None:
# Create enums
task_status = postgresql.ENUM(
'todo', 'in_progress', 'in_review', 'done', 'cancelled',
name='task_status'
)
task_status.create(op.get_bind())
op.create_table(
'tasks',
sa.Column('id', sa.Text(), primary_key=True),
sa.Column('project_id', sa.Text(), sa.ForeignKey('projects.id'), nullable=False),
sa.Column('title', sa.VARCHAR(500), nullable=False),
sa.Column('description', sa.Text()),
sa.Column('status', postgresql.ENUM('todo', 'in_progress', 'in_review', 'done', 'cancelled', name='task_status', create_type=False), nullable=False, server_default='todo'),
sa.Column('priority', sa.Text(), nullable=False, server_default='medium'),
sa.Column('due_date', sa.TIMESTAMP(timezone=True)),
sa.Column('position', sa.Float(), nullable=False, server_default='0'),
sa.Column('version', sa.Integer(), nullable=False, server_default='1'),
sa.Column('created_by_id', sa.Text(), sa.ForeignKey('users.id'), nullable=False),
sa.Column('updated_by_id', sa.Text(), sa.ForeignKey('users.id'), nullable=False),
sa.Column('created_at', sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text('NOW()')),
sa.Column('updated_at', sa.TIMESTAMP(timezone=True), nullable=False, server_default=sa.text('NOW()')),
sa.Column('deleted_at', sa.TIMESTAMP(timezone=True)),
)
# Indexes
op.create_index('tasks_project_id_idx', 'tasks', ['project_id'])
op.create_index('tasks_project_status_idx', 'tasks', ['project_id', 'status'])
# Partial index for active tasks only
op.create_index(
'tasks_due_date_active_idx',
'tasks', ['due_date'],
postgresql_where=sa.text('deleted_at IS NULL')
)
def downgrade() -> None:
op.drop_table('tasks')
op.execute("DROP TYPE IF EXISTS task_status")
```
---

356
engineering/env-secrets-manager/SKILL.md
View File

@@ -196,361 +196,7 @@ git add .env.example
---
## Required Variable Validation Script
```bash
#!/bin/bash
# scripts/validate-env.sh
# Run at app startup or in CI before deploy
# Exit 1 if any required var is missing or empty
set -euo pipefail
MISSING=()
WARNINGS=()
# --- Define required vars by environment ---
ALWAYS_REQUIRED=(
APP_SECRET
APP_URL
DATABASE_URL
AUTH_JWT_SECRET
AUTH_REFRESH_SECRET
)
PROD_REQUIRED=(
STRIPE_SECRET_KEY
STRIPE_WEBHOOK_SECRET
SENTRY_DSN
)
# --- Check always-required vars ---
for var in "${ALWAYS_REQUIRED[@]}"; do
if [ -z "${!var:-}" ]; then
MISSING+=("$var")
fi
done
# --- Check prod-only vars ---
if [ "${APP_ENV:-}" = "production" ] || [ "${NODE_ENV:-}" = "production" ]; then
for var in "${PROD_REQUIRED[@]}"; do
if [ -z "${!var:-}" ]; then
MISSING+=("$var (required in production)")
fi
done
fi
# --- Validate format/length constraints ---
if [ -n "${AUTH_JWT_SECRET:-}" ] && [ ${#AUTH_JWT_SECRET} -lt 32 ]; then
WARNINGS+=("AUTH_JWT_SECRET is shorter than 32 chars — insecure")
fi
if [ -n "${DATABASE_URL:-}" ]; then
if ! echo "$DATABASE_URL" | grep -qE "^(postgres|postgresql|mysql|mongodb|redis)://"; then
WARNINGS+=("DATABASE_URL doesn't look like a valid connection string")
fi
fi
if [ -n "${APP_PORT:-}" ]; then
if ! [[ "$APP_PORT" =~ ^[0-9]+$ ]] || [ "$APP_PORT" -lt 1 ] || [ "$APP_PORT" -gt 65535 ]; then
WARNINGS+=("APP_PORT=$APP_PORT is not a valid port number")
fi
fi
# --- Report ---
if [ ${#WARNINGS[@]} -gt 0 ]; then
echo "WARNINGS:"
for w in "${WARNINGS[@]}"; do
echo " ⚠️ $w"
done
fi
if [ ${#MISSING[@]} -gt 0 ]; then
echo ""
echo "FATAL: Missing required environment variables:"
for var in "${MISSING[@]}"; do
echo " ❌ $var"
done
echo ""
echo "Copy .env.example to .env and fill in missing values."
exit 1
fi
echo "✅ All required environment variables are set"
```
Node.js equivalent:
```typescript
// src/config/validateEnv.ts
const required = [
'APP_SECRET', 'APP_URL', 'DATABASE_URL',
'AUTH_JWT_SECRET', 'AUTH_REFRESH_SECRET',
]
const missing = required.filter(key => !process.env[key])
if (missing.length > 0) {
console.error('FATAL: Missing required environment variables:', missing)
process.exit(1)
}
if (process.env.AUTH_JWT_SECRET && process.env.AUTH_JWT_SECRET.length < 32) {
console.error('FATAL: AUTH_JWT_SECRET must be at least 32 characters')
process.exit(1)
}
export const config = {
appSecret: process.env.APP_SECRET!,
appUrl: process.env.APP_URL!,
databaseUrl: process.env.DATABASE_URL!,
jwtSecret: process.env.AUTH_JWT_SECRET!,
refreshSecret: process.env.AUTH_REFRESH_SECRET!,
stripeKey: process.env.STRIPE_SECRET_KEY, // optional
port: parseInt(process.env.APP_PORT ?? '3000', 10),
} as const
```
---
## Secret Leak Detection
### Scan Working Tree
```bash
#!/bin/bash
# scripts/scan-secrets.sh
# Scan staged files and working tree for common secret patterns
FAIL=0
check() {
local label="$1"
local pattern="$2"
local matches
matches=$(git diff --cached -U0 2>/dev/null | grep "^+" | grep -vE "^(\+\+\+|#|\/\/)" | \
grep -E "$pattern" | grep -v ".env.example" | grep -v "test\|mock\|fixture\|fake" || true)
if [ -n "$matches" ]; then
echo "SECRET DETECTED [$label]:"
echo "$matches" | head -5
FAIL=1
fi
}
# AWS Access Keys
check "AWS Access Key" "AKIA[0-9A-Z]{16}"
check "AWS Secret Key" "aws_secret_access_key\s*=\s*['\"]?[A-Za-z0-9/+]{40}"
# Stripe
check "Stripe Live Key" "sk_live_[0-9a-zA-Z]{24,}"
check "Stripe Test Key" "sk_test_[0-9a-zA-Z]{24,}"
check "Stripe Webhook" "whsec_[0-9a-zA-Z]{32,}"
# JWT / Generic secrets
check "Hardcoded JWT" "eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}"
check "Generic Secret" "(secret|password|passwd|api_key|apikey|token)\s*[:=]\s*['\"][^'\"]{12,}['\"]"
# Private keys
check "Private Key Block" "-----BEGIN (RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----"
check "PEM Certificate" "-----BEGIN CERTIFICATE-----"
# Connection strings with credentials
check "DB Connection" "(postgres|mysql|mongodb)://[^:]+:[^@]+@"
check "Redis Auth" "redis://:[^@]+@\|rediss://:[^@]+@"
# Google
check "Google API Key" "AIza[0-9A-Za-z_-]{35}"
check "Google OAuth" "[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com"
# GitHub
check "GitHub Token" "gh[ps]_[A-Za-z0-9]{36,}"
check "GitHub Fine-grained" "github_pat_[A-Za-z0-9_]{82}"
# Slack
check "Slack Token" "xox[baprs]-[0-9A-Za-z]{10,}"
check "Slack Webhook" "https://hooks\.slack\.com/services/[A-Z0-9]{9,}/[A-Z0-9]{9,}/[A-Za-z0-9]{24,}"
# Twilio
check "Twilio SID" "AC[a-z0-9]{32}"
check "Twilio Token" "SK[a-z0-9]{32}"
if [ $FAIL -eq 1 ]; then
echo ""
echo "BLOCKED: Secrets detected in staged changes."
echo "Remove secrets before committing. Use environment variables instead."
echo "If this is a false positive, add it to .secretsignore or use:"
echo " git commit --no-verify (only if you're 100% certain it's safe)"
exit 1
fi
echo "No secrets detected in staged changes."
```
### Scan Git History (post-incident)
```bash
#!/bin/bash
# scripts/scan-history.sh — scan entire git history for leaked secrets
PATTERNS=(
"AKIA[0-9A-Z]{16}"
"sk_live_[0-9a-zA-Z]{24}"
"sk_test_[0-9a-zA-Z]{24}"
"-----BEGIN.*PRIVATE KEY-----"
"AIza[0-9A-Za-z_-]{35}"
"ghp_[A-Za-z0-9]{36}"
"xox[baprs]-[0-9A-Za-z]{10,}"
)
for pattern in "${PATTERNS[@]}"; do
echo "Scanning for: $pattern"
git log --all -p --no-color 2>/dev/null | \
grep -n "$pattern" | \
grep "^+" | \
grep -v "^+++" | \
head -10
done
# Alternative: use truffleHog or gitleaks for comprehensive scanning
# gitleaks detect --source . --log-opts="--all"
# trufflehog git file://. --only-verified
```
---
## Pre-commit Hook Installation
```bash
#!/bin/bash
# Install the pre-commit hook
HOOK_PATH=".git/hooks/pre-commit"
cat > "$HOOK_PATH" << 'HOOK'
#!/bin/bash
# Pre-commit: scan for secrets before every commit
SCRIPT="scripts/scan-secrets.sh"
if [ -f "$SCRIPT" ]; then
bash "$SCRIPT"
else
# Inline fallback if script not present
if git diff --cached -U0 | grep "^+" | grep -qE "AKIA[0-9A-Z]{16}|sk_live_|-----BEGIN.*PRIVATE KEY"; then
echo "BLOCKED: Possible secret detected in staged changes."
exit 1
fi
fi
HOOK
chmod +x "$HOOK_PATH"
echo "Pre-commit hook installed at $HOOK_PATH"
```
Using `pre-commit` framework (recommended for teams):
```yaml
# .pre-commit-config.yaml
repos:
- repo: https://github.com/gitleaks/gitleaks
rev: v8.18.0
hooks:
- id: gitleaks
- repo: local
hooks:
- id: validate-env-example
name: "check-envexample-is-up-to-date"
language: script
entry: bash scripts/check-env-example.sh
pass_filenames: false
```
---
## Credential Rotation Workflow
When a secret is leaked or compromised:
### Step 1 — Detect & Confirm
```bash
# Confirm which secret was exposed
git log --all -p --no-color | grep -A2 -B2 "AKIA\|sk_live_\|SECRET"
# Check if secret is in any open PRs
gh pr list --state open | while read pr; do
gh pr diff $(echo $pr | awk '{print $1}') | grep -E "AKIA|sk_live_" && echo "Found in PR: $pr"
done
```
### Step 2 — Identify Exposure Window
```bash
# Find first commit that introduced the secret
git log --all -p --no-color -- "*.env" "*.json" "*.yaml" "*.ts" "*.py" | \
grep -B 10 "THE_LEAKED_VALUE" | grep "^commit" | tail -1
# Get commit date
git show --format="%ci" COMMIT_HASH | head -1
# Check if secret appears in public repos (GitHub)
gh api search/code -X GET -f q="THE_LEAKED_VALUE" | jq '.total_count, .items[].html_url'
```
### Step 3 — Rotate Credential
Per service — rotate immediately:
- **AWS**: IAM console → delete access key → create new → update everywhere
- **Stripe**: Dashboard → Developers → API keys → Roll key
- **GitHub PAT**: Settings → Developer Settings → Personal access tokens → Revoke → Create new
- **DB password**: `ALTER USER app_user PASSWORD 'new-strong-password-here';`
- **JWT secret**: Rotate key (all existing sessions invalidated — users re-login)
### Step 4 — Update All Environments
```bash
# Update secret manager (source of truth)
# Then redeploy to pull new values
# Vault KV v2
vault kv put secret/myapp/prod \
STRIPE_SECRET_KEY="sk_live_NEW..." \
APP_SECRET="new-secret-here"
# AWS SSM
aws ssm put-parameter \
--name "/myapp/prod/STRIPE_SECRET_KEY" \
--value "sk_live_NEW..." \
--type "SecureString" \
--overwrite
# 1Password
op item edit "MyApp Prod" \
--field "STRIPE_SECRET_KEY[password]=sk_live_NEW..."
# Doppler
doppler secrets set STRIPE_SECRET_KEY="sk_live_NEW..." --project myapp --config prod
```
### Step 5 — Remove from Git History
```bash
# WARNING: rewrites history — coordinate with team first
git filter-repo --path-glob "*.env" --invert-paths
# Or remove specific string from all commits
git filter-repo --replace-text <(echo "LEAKED_VALUE==>REDACTED")
# Force push all branches (requires team coordination + force push permissions)
git push origin --force --all
# Notify all developers to re-clone
```
### Step 6 — Verify
```bash
# Confirm secret no longer in history
git log --all -p | grep "LEAKED_VALUE" | wc -l # should be 0
# Test new credentials work
curl -H "Authorization: Bearer $NEW_TOKEN" https://api.service.com/test
# Monitor for unauthorized usage of old credential (check service audit logs)
```
---
→ See references/validation-detection-rotation.md for details
## Secret Manager Integrations

358
engineering/env-secrets-manager/references/validation-detection-rotation.md Normal file
View File

@@ -0,0 +1,358 @@
# env-secrets-manager reference
## Required Variable Validation Script
```bash
#!/bin/bash
# scripts/validate-env.sh
# Run at app startup or in CI before deploy
# Exit 1 if any required var is missing or empty
set -euo pipefail
MISSING=()
WARNINGS=()
# --- Define required vars by environment ---
ALWAYS_REQUIRED=(
APP_SECRET
APP_URL
DATABASE_URL
AUTH_JWT_SECRET
AUTH_REFRESH_SECRET
)
PROD_REQUIRED=(
STRIPE_SECRET_KEY
STRIPE_WEBHOOK_SECRET
SENTRY_DSN
)
# --- Check always-required vars ---
for var in "${ALWAYS_REQUIRED[@]}"; do
if [ -z "${!var:-}" ]; then
MISSING+=("$var")
fi
done
# --- Check prod-only vars ---
if [ "${APP_ENV:-}" = "production" ] || [ "${NODE_ENV:-}" = "production" ]; then
for var in "${PROD_REQUIRED[@]}"; do
if [ -z "${!var:-}" ]; then
MISSING+=("$var (required in production)")
fi
done
fi
# --- Validate format/length constraints ---
if [ -n "${AUTH_JWT_SECRET:-}" ] && [ ${#AUTH_JWT_SECRET} -lt 32 ]; then
WARNINGS+=("AUTH_JWT_SECRET is shorter than 32 chars — insecure")
fi
if [ -n "${DATABASE_URL:-}" ]; then
if ! echo "$DATABASE_URL" | grep -qE "^(postgres|postgresql|mysql|mongodb|redis)://"; then
WARNINGS+=("DATABASE_URL doesn't look like a valid connection string")
fi
fi
if [ -n "${APP_PORT:-}" ]; then
if ! [[ "$APP_PORT" =~ ^[0-9]+$ ]] || [ "$APP_PORT" -lt 1 ] || [ "$APP_PORT" -gt 65535 ]; then
WARNINGS+=("APP_PORT=$APP_PORT is not a valid port number")
fi
fi
# --- Report ---
if [ ${#WARNINGS[@]} -gt 0 ]; then
echo "WARNINGS:"
for w in "${WARNINGS[@]}"; do
echo " ⚠️ $w"
done
fi
if [ ${#MISSING[@]} -gt 0 ]; then
echo ""
echo "FATAL: Missing required environment variables:"
for var in "${MISSING[@]}"; do
echo " ❌ $var"
done
echo ""
echo "Copy .env.example to .env and fill in missing values."
exit 1
fi
echo "✅ All required environment variables are set"
```
Node.js equivalent:
```typescript
// src/config/validateEnv.ts
const required = [
'APP_SECRET', 'APP_URL', 'DATABASE_URL',
'AUTH_JWT_SECRET', 'AUTH_REFRESH_SECRET',
]
const missing = required.filter(key => !process.env[key])
if (missing.length > 0) {
console.error('FATAL: Missing required environment variables:', missing)
process.exit(1)
}
if (process.env.AUTH_JWT_SECRET && process.env.AUTH_JWT_SECRET.length < 32) {
console.error('FATAL: AUTH_JWT_SECRET must be at least 32 characters')
process.exit(1)
}
export const config = {
appSecret: process.env.APP_SECRET!,
appUrl: process.env.APP_URL!,
databaseUrl: process.env.DATABASE_URL!,
jwtSecret: process.env.AUTH_JWT_SECRET!,
refreshSecret: process.env.AUTH_REFRESH_SECRET!,
stripeKey: process.env.STRIPE_SECRET_KEY, // optional
port: parseInt(process.env.APP_PORT ?? '3000', 10),
} as const
```
---
## Secret Leak Detection
### Scan Working Tree
```bash
#!/bin/bash
# scripts/scan-secrets.sh
# Scan staged files and working tree for common secret patterns
FAIL=0
check() {
local label="$1"
local pattern="$2"
local matches
matches=$(git diff --cached -U0 2>/dev/null | grep "^+" | grep -vE "^(\+\+\+|#|\/\/)" | \
grep -E "$pattern" | grep -v ".env.example" | grep -v "test\|mock\|fixture\|fake" || true)
if [ -n "$matches" ]; then
echo "SECRET DETECTED [$label]:"
echo "$matches" | head -5
FAIL=1
fi
}
# AWS Access Keys
check "AWS Access Key" "AKIA[0-9A-Z]{16}"
check "AWS Secret Key" "aws_secret_access_key\s*=\s*['\"]?[A-Za-z0-9/+]{40}"
# Stripe
check "Stripe Live Key" "sk_live_[0-9a-zA-Z]{24,}"
check "Stripe Test Key" "sk_test_[0-9a-zA-Z]{24,}"
check "Stripe Webhook" "whsec_[0-9a-zA-Z]{32,}"
# JWT / Generic secrets
check "Hardcoded JWT" "eyJ[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}"
check "Generic Secret" "(secret|password|passwd|api_key|apikey|token)\s*[:=]\s*['\"][^'\"]{12,}['\"]"
# Private keys
check "Private Key Block" "-----BEGIN (RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----"
check "PEM Certificate" "-----BEGIN CERTIFICATE-----"
# Connection strings with credentials
check "DB Connection" "(postgres|mysql|mongodb)://[^:]+:[^@]+@"
check "Redis Auth" "redis://:[^@]+@\|rediss://:[^@]+@"
# Google
check "Google API Key" "AIza[0-9A-Za-z_-]{35}"
check "Google OAuth" "[0-9]+-[0-9A-Za-z_]{32}\.apps\.googleusercontent\.com"
# GitHub
check "GitHub Token" "gh[ps]_[A-Za-z0-9]{36,}"
check "GitHub Fine-grained" "github_pat_[A-Za-z0-9_]{82}"
# Slack
check "Slack Token" "xox[baprs]-[0-9A-Za-z]{10,}"
check "Slack Webhook" "https://hooks\.slack\.com/services/[A-Z0-9]{9,}/[A-Z0-9]{9,}/[A-Za-z0-9]{24,}"
# Twilio
check "Twilio SID" "AC[a-z0-9]{32}"
check "Twilio Token" "SK[a-z0-9]{32}"
if [ $FAIL -eq 1 ]; then
echo ""
echo "BLOCKED: Secrets detected in staged changes."
echo "Remove secrets before committing. Use environment variables instead."
echo "If this is a false positive, add it to .secretsignore or use:"
echo " git commit --no-verify (only if you're 100% certain it's safe)"
exit 1
fi
echo "No secrets detected in staged changes."
```
### Scan Git History (post-incident)
```bash
#!/bin/bash
# scripts/scan-history.sh — scan entire git history for leaked secrets
PATTERNS=(
"AKIA[0-9A-Z]{16}"
"sk_live_[0-9a-zA-Z]{24}"
"sk_test_[0-9a-zA-Z]{24}"
"-----BEGIN.*PRIVATE KEY-----"
"AIza[0-9A-Za-z_-]{35}"
"ghp_[A-Za-z0-9]{36}"
"xox[baprs]-[0-9A-Za-z]{10,}"
)
for pattern in "${PATTERNS[@]}"; do
echo "Scanning for: $pattern"
git log --all -p --no-color 2>/dev/null | \
grep -n "$pattern" | \
grep "^+" | \
grep -v "^+++" | \
head -10
done
# Alternative: use truffleHog or gitleaks for comprehensive scanning
# gitleaks detect --source . --log-opts="--all"
# trufflehog git file://. --only-verified
```
---
## Pre-commit Hook Installation
```bash
#!/bin/bash
# Install the pre-commit hook
HOOK_PATH=".git/hooks/pre-commit"
cat > "$HOOK_PATH" << 'HOOK'
#!/bin/bash
# Pre-commit: scan for secrets before every commit
SCRIPT="scripts/scan-secrets.sh"
if [ -f "$SCRIPT" ]; then
bash "$SCRIPT"
else
# Inline fallback if script not present
if git diff --cached -U0 | grep "^+" | grep -qE "AKIA[0-9A-Z]{16}|sk_live_|-----BEGIN.*PRIVATE KEY"; then
echo "BLOCKED: Possible secret detected in staged changes."
exit 1
fi
fi
HOOK
chmod +x "$HOOK_PATH"
echo "Pre-commit hook installed at $HOOK_PATH"
```
Using `pre-commit` framework (recommended for teams):
```yaml
# .pre-commit-config.yaml
repos:
- repo: https://github.com/gitleaks/gitleaks
rev: v8.18.0
hooks:
- id: gitleaks
- repo: local
hooks:
- id: validate-env-example
name: "check-envexample-is-up-to-date"
language: script
entry: bash scripts/check-env-example.sh
pass_filenames: false
```
---
## Credential Rotation Workflow
When a secret is leaked or compromised:
### Step 1 — Detect & Confirm
```bash
# Confirm which secret was exposed
git log --all -p --no-color | grep -A2 -B2 "AKIA\|sk_live_\|SECRET"
# Check if secret is in any open PRs
gh pr list --state open | while read pr; do
gh pr diff $(echo $pr | awk '{print $1}') | grep -E "AKIA|sk_live_" && echo "Found in PR: $pr"
done
```
### Step 2 — Identify Exposure Window
```bash
# Find first commit that introduced the secret
git log --all -p --no-color -- "*.env" "*.json" "*.yaml" "*.ts" "*.py" | \
grep -B 10 "THE_LEAKED_VALUE" | grep "^commit" | tail -1
# Get commit date
git show --format="%ci" COMMIT_HASH | head -1
# Check if secret appears in public repos (GitHub)
gh api search/code -X GET -f q="THE_LEAKED_VALUE" | jq '.total_count, .items[].html_url'
```
### Step 3 — Rotate Credential
Per service — rotate immediately:
- **AWS**: IAM console → delete access key → create new → update everywhere
- **Stripe**: Dashboard → Developers → API keys → Roll key
- **GitHub PAT**: Settings → Developer Settings → Personal access tokens → Revoke → Create new
- **DB password**: `ALTER USER app_user PASSWORD 'new-strong-password-here';`
- **JWT secret**: Rotate key (all existing sessions invalidated — users re-login)
### Step 4 — Update All Environments
```bash
# Update secret manager (source of truth)
# Then redeploy to pull new values
# Vault KV v2
vault kv put secret/myapp/prod \
STRIPE_SECRET_KEY="sk_live_NEW..." \
APP_SECRET="new-secret-here"
# AWS SSM
aws ssm put-parameter \
--name "/myapp/prod/STRIPE_SECRET_KEY" \
--value "sk_live_NEW..." \
--type "SecureString" \
--overwrite
# 1Password
op item edit "MyApp Prod" \
--field "STRIPE_SECRET_KEY[password]=sk_live_NEW..."
# Doppler
doppler secrets set STRIPE_SECRET_KEY="sk_live_NEW..." --project myapp --config prod
```
### Step 5 — Remove from Git History
```bash
# WARNING: rewrites history — coordinate with team first
git filter-repo --path-glob "*.env" --invert-paths
# Or remove specific string from all commits
git filter-repo --replace-text <(echo "LEAKED_VALUE==>REDACTED")
# Force push all branches (requires team coordination + force push permissions)
git push origin --force --all
# Notify all developers to re-clone
```
### Step 6 — Verify
```bash
# Confirm secret no longer in history
git log --all -p | grep "LEAKED_VALUE" | wc -l # should be 0
# Test new credentials work
curl -H "Authorization: Bearer $NEW_TOKEN" https://api.service.com/test
# Monitor for unauthorized usage of old credential (check service audit logs)
```
---

514
engineering/monorepo-navigator/SKILL.md
View File

@@ -60,519 +60,7 @@ Most modern setups: **pnpm workspaces + Turborepo + Changesets**
---
## Turborepo
### turbo.json pipeline config
```json
{
"$schema": "https://turbo.build/schema.json",
"globalEnv": ["NODE_ENV", "DATABASE_URL"],
"pipeline": {
"build": {
"dependsOn": ["^build"], // build deps first (topological order)
"outputs": [".next/**", "dist/**", "build/**"],
"env": ["NEXT_PUBLIC_API_URL"]
},
"test": {
"dependsOn": ["^build"], // need built deps to test
"outputs": ["coverage/**"],
"cache": true
},
"lint": {
"outputs": [],
"cache": true
},
"dev": {
"cache": false, // never cache dev servers
"persistent": true // long-running process
},
"type-check": {
"dependsOn": ["^build"],
"outputs": []
}
}
}
```
### Key commands
```bash
# Build everything (respects dependency order)
turbo run build
# Build only affected packages (requires --filter)
turbo run build --filter=...[HEAD^1] # changed since last commit
turbo run build --filter=...[main] # changed vs main branch
# Test only affected
turbo run test --filter=...[HEAD^1]
# Run for a specific app and all its dependencies
turbo run build --filter=@myorg/web...
# Run for a specific package only (no dependencies)
turbo run build --filter=@myorg/ui
# Dry-run — see what would run without executing
turbo run build --dry-run
# Enable remote caching (Vercel Remote Cache)
turbo login
turbo link
```
### Remote caching setup
```bash
# .turbo/config.json (auto-created by turbo link)
{
"teamid": "team_xxxx",
"apiurl": "https://vercel.com"
}
# Self-hosted cache server (open-source alternative)
# Run ducktape/turborepo-remote-cache or Turborepo's official server
TURBO_API=http://your-cache-server.internal \
TURBO_TOKEN=your-token \
TURBO_TEAM=your-team \
turbo run build
```
---
## Nx
### Project graph and affected commands
```bash
# Install
npx create-nx-workspace@latest my-monorepo
# Visualize the project graph (opens browser)
nx graph
# Show affected packages for the current branch
nx affected:graph
# Run only affected tests
nx affected --target=test
# Run only affected builds
nx affected --target=build
# Run affected with base/head (for CI)
nx affected --target=test --base=main --head=HEAD
```
### nx.json configuration
```json
{
"$schema": "./node_modules/nx/schemas/nx-schema.json",
"targetDefaults": {
"build": {
"dependsOn": ["^build"],
"cache": true
},
"test": {
"cache": true,
"inputs": ["default", "^production"]
}
},
"namedInputs": {
"default": ["{projectRoot}/**/*", "sharedGlobals"],
"production": ["default", "!{projectRoot}/**/*.spec.ts", "!{projectRoot}/jest.config.*"],
"sharedGlobals": []
},
"parallel": 4,
"cacheDirectory": "/tmp/nx-cache"
}
```
---
## pnpm Workspaces
### pnpm-workspace.yaml
```yaml
packages:
- 'apps/*'
- 'packages/*'
- 'tools/*'
```
### workspace:* protocol for local packages
```json
// apps/web/package.json
{
"name": "@myorg/web",
"dependencies": {
"@myorg/ui": "workspace:*", // always use local version
"@myorg/utils": "workspace:^", // local, but respect semver on publish
"@myorg/types": "workspace:~"
}
}
```
### Useful pnpm workspace commands
```bash
# Install all packages across workspace
pnpm install
# Run script in a specific package
pnpm --filter @myorg/web dev
# Run script in all packages
pnpm --filter "*" build
# Run script in a package and all its dependencies
pnpm --filter @myorg/web... build
# Add a dependency to a specific package
pnpm --filter @myorg/web add react
# Add a shared dev dependency to root
pnpm add -D typescript -w
# List workspace packages
pnpm ls --depth -1 -r
```
---
## Cross-Package Impact Analysis
When a shared package changes, determine what's affected before you ship.
```bash
# Using Turborepo — show affected packages
turbo run build --filter=...[HEAD^1] --dry-run 2>&1 | grep "Tasks to run"
# Using Nx
nx affected:apps --base=main --head=HEAD # which apps are affected
nx affected:libs --base=main --head=HEAD # which libs are affected
# Manual analysis with pnpm
# Find all packages that depend on @myorg/utils:
grep -r '"@myorg/utils"' packages/*/package.json apps/*/package.json
# Using jq for structured output
for pkg in packages/*/package.json apps/*/package.json; do
name=$(jq -r '.name' "$pkg")
if jq -e '.dependencies["@myorg/utils"] // .devDependencies["@myorg/utils"]' "$pkg" > /dev/null 2>&1; then
echo "$name depends on @myorg/utils"
fi
done
```
---
## Dependency Graph Visualization
Generate a Mermaid diagram from your workspace:
```bash
# Generate dependency graph as Mermaid
cat > scripts/gen-dep-graph.js << 'EOF'
const { execSync } = require('child_process');
const fs = require('fs');
// Parse pnpm workspace packages
const packages = JSON.parse(
execSync('pnpm ls --depth -1 -r --json').toString()
);
let mermaid = 'graph TD\n';
packages.forEach(pkg => {
const deps = Object.keys(pkg.dependencies || {})
.filter(d => d.startsWith('@myorg/'));
deps.forEach(dep => {
const from = pkg.name.replace('@myorg/', '');
const to = dep.replace('@myorg/', '');
mermaid += ` ${from} --> ${to}\n`;
});
});
fs.writeFileSync('docs/dep-graph.md', '```mermaid\n' + mermaid + '```\n');
console.log('Written to docs/dep-graph.md');
EOF
node scripts/gen-dep-graph.js
```
**Example output:**
```mermaid
graph TD
web --> ui
web --> utils
web --> types
mobile --> ui
mobile --> utils
mobile --> types
admin --> ui
admin --> utils
api --> types
ui --> utils
```
---
## Claude Code Configuration (Workspace-Aware CLAUDE.md)
Place a root CLAUDE.md + per-package CLAUDE.md files:
```markdown
# /CLAUDE.md — Root (applies to all packages)
## Monorepo Structure
- apps/web — Next.js customer-facing app
- apps/admin — Next.js internal admin
- apps/api — Express REST API
- packages/ui — Shared React component library
- packages/utils — Shared utilities (pure functions only)
- packages/types — Shared TypeScript types (no runtime code)
## Build System
- pnpm workspaces + Turborepo
- Always use `pnpm --filter <package>` to scope commands
- Never run `npm install` or `yarn` — pnpm only
- Run `turbo run build --filter=...[HEAD^1]` before committing
## Task Scoping Rules
- When modifying packages/ui: also run tests for apps/web and apps/admin (they depend on it)
- When modifying packages/types: run type-check across ALL packages
- When modifying apps/api: only need to test apps/api
## Package Manager
pnpm — version pinned in packageManager field of root package.json
```
```markdown
# /packages/ui/CLAUDE.md — Package-specific
## This Package
Shared React component library. Zero business logic. Pure UI only.
## Rules
- All components must be exported from src/index.ts
- No direct API calls in components — accept data via props
- Every component needs a Storybook story in src/stories/
- Use Tailwind for styling — no CSS modules or styled-components
## Testing
- Component tests: `pnpm --filter @myorg/ui test`
- Visual regression: `pnpm --filter @myorg/ui test:storybook`
## Publishing
- Version bumps via changesets only — never edit package.json version manually
- Run `pnpm changeset` from repo root after changes
```
---
## Migration: Multi-Repo → Monorepo
```bash
# Step 1: Create monorepo scaffold
mkdir my-monorepo && cd my-monorepo
pnpm init
echo "packages:\n - 'apps/*'\n - 'packages/*'" > pnpm-workspace.yaml
# Step 2: Move repos with git history preserved
mkdir -p apps packages
# For each existing repo:
git clone https://github.com/myorg/web-app
cd web-app
git filter-repo --to-subdirectory-filter apps/web # rewrites history into subdir
cd ..
git remote add web-app ./web-app
git fetch web-app --tags
git merge web-app/main --allow-unrelated-histories
# Step 3: Update package names to scoped
# In each package.json, change "name": "web" to "name": "@myorg/web"
# Step 4: Replace cross-repo npm deps with workspace:*
# apps/web/package.json: "@myorg/ui": "1.2.3" → "@myorg/ui": "workspace:*"
# Step 5: Add shared configs to root
cp apps/web/.eslintrc.js .eslintrc.base.js
# Update each package's config to extend root:
# { "extends": ["../../.eslintrc.base.js"] }
# Step 6: Add Turborepo
pnpm add -D turbo -w
# Create turbo.json (see above)
# Step 7: Unified CI (see CI section below)
# Step 8: Test everything
turbo run build test lint
```
---
## CI Patterns
### GitHub Actions — Affected Only
```yaml
# .github/workflows/ci.yml
name: "ci"
on:
push:
branches: [main]
pull_request:
jobs:
affected:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # full history needed for affected detection
- uses: pnpm/action-setup@v3
with:
version: 9
- uses: actions/setup-node@v4
with:
node-version: 20
cache: pnpm
- run: pnpm install --frozen-lockfile
# Turborepo remote cache
- uses: actions/cache@v4
with:
path: .turbo
key: ${{ runner.os }}-turbo-${{ github.sha }}
restore-keys: ${{ runner.os }}-turbo-
# Only test/build affected packages
- name: "build-affected"
run: turbo run build --filter=...[origin/main]
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
- name: "test-affected"
run: turbo run test --filter=...[origin/main]
- name: "lint-affected"
run: turbo run lint --filter=...[origin/main]
```
### GitLab CI — Parallel Stages
```yaml
# .gitlab-ci.yml
stages: [install, build, test, publish]
variables:
PNPM_CACHE_FOLDER: .pnpm-store
cache:
key: pnpm-$CI_COMMIT_REF_SLUG
paths: [.pnpm-store/, .turbo/]
install:
stage: install
script:
- pnpm install --frozen-lockfile
artifacts:
paths: [node_modules/, packages/*/node_modules/, apps/*/node_modules/]
expire_in: 1h
build:affected:
stage: build
needs: [install]
script:
- turbo run build --filter=...[origin/main]
artifacts:
paths: [apps/*/dist/, apps/*/.next/, packages/*/dist/]
test:affected:
stage: test
needs: [build:affected]
script:
- turbo run test --filter=...[origin/main]
coverage: '/Statements\s*:\s*(\d+\.?\d*)%/'
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: "**/coverage/cobertura-coverage.xml"
```
---
## Publishing with Changesets
```bash
# Install changesets
pnpm add -D @changesets/cli -w
pnpm changeset init
# After making changes, create a changeset
pnpm changeset
# Interactive: select packages, choose semver bump, write changelog entry
# In CI — version packages + update changelogs
pnpm changeset version
# Publish all changed packages
pnpm changeset publish
# Pre-release channel (for alpha/beta)
pnpm changeset pre enter beta
pnpm changeset
pnpm changeset version # produces 1.2.0-beta.0
pnpm changeset publish --tag beta
pnpm changeset pre exit # back to stable releases
```
### Automated publish workflow (GitHub Actions)
```yaml
# .github/workflows/release.yml
name: "release"
on:
push:
branches: [main]
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
- uses: actions/setup-node@v4
with:
node-version: 20
registry-url: https://registry.npmjs.org
- run: pnpm install --frozen-lockfile
- name: "create-release-pr-or-publish"
uses: changesets/action@v1
with:
publish: pnpm changeset publish
version: pnpm changeset version
commit: "chore: release packages"
title: "chore: release packages"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
```
---
→ See references/monorepo-tooling-reference.md for details
## Common Pitfalls

516
engineering/monorepo-navigator/references/monorepo-tooling-reference.md Normal file
View File

@@ -0,0 +1,516 @@
# monorepo-navigator reference
## Turborepo
### turbo.json pipeline config
```json
{
"$schema": "https://turbo.build/schema.json",
"globalEnv": ["NODE_ENV", "DATABASE_URL"],
"pipeline": {
"build": {
"dependsOn": ["^build"], // build deps first (topological order)
"outputs": [".next/**", "dist/**", "build/**"],
"env": ["NEXT_PUBLIC_API_URL"]
},
"test": {
"dependsOn": ["^build"], // need built deps to test
"outputs": ["coverage/**"],
"cache": true
},
"lint": {
"outputs": [],
"cache": true
},
"dev": {
"cache": false, // never cache dev servers
"persistent": true // long-running process
},
"type-check": {
"dependsOn": ["^build"],
"outputs": []
}
}
}
```
### Key commands
```bash
# Build everything (respects dependency order)
turbo run build
# Build only affected packages (requires --filter)
turbo run build --filter=...[HEAD^1] # changed since last commit
turbo run build --filter=...[main] # changed vs main branch
# Test only affected
turbo run test --filter=...[HEAD^1]
# Run for a specific app and all its dependencies
turbo run build --filter=@myorg/web...
# Run for a specific package only (no dependencies)
turbo run build --filter=@myorg/ui
# Dry-run — see what would run without executing
turbo run build --dry-run
# Enable remote caching (Vercel Remote Cache)
turbo login
turbo link
```
### Remote caching setup
```bash
# .turbo/config.json (auto-created by turbo link)
{
"teamid": "team_xxxx",
"apiurl": "https://vercel.com"
}
# Self-hosted cache server (open-source alternative)
# Run ducktape/turborepo-remote-cache or Turborepo's official server
TURBO_API=http://your-cache-server.internal \
TURBO_TOKEN=your-token \
TURBO_TEAM=your-team \
turbo run build
```
---
## Nx
### Project graph and affected commands
```bash
# Install
npx create-nx-workspace@latest my-monorepo
# Visualize the project graph (opens browser)
nx graph
# Show affected packages for the current branch
nx affected:graph
# Run only affected tests
nx affected --target=test
# Run only affected builds
nx affected --target=build
# Run affected with base/head (for CI)
nx affected --target=test --base=main --head=HEAD
```
### nx.json configuration
```json
{
"$schema": "./node_modules/nx/schemas/nx-schema.json",
"targetDefaults": {
"build": {
"dependsOn": ["^build"],
"cache": true
},
"test": {
"cache": true,
"inputs": ["default", "^production"]
}
},
"namedInputs": {
"default": ["{projectRoot}/**/*", "sharedGlobals"],
"production": ["default", "!{projectRoot}/**/*.spec.ts", "!{projectRoot}/jest.config.*"],
"sharedGlobals": []
},
"parallel": 4,
"cacheDirectory": "/tmp/nx-cache"
}
```
---
## pnpm Workspaces
### pnpm-workspace.yaml
```yaml
packages:
- 'apps/*'
- 'packages/*'
- 'tools/*'
```
### workspace:* protocol for local packages
```json
// apps/web/package.json
{
"name": "@myorg/web",
"dependencies": {
"@myorg/ui": "workspace:*", // always use local version
"@myorg/utils": "workspace:^", // local, but respect semver on publish
"@myorg/types": "workspace:~"
}
}
```
### Useful pnpm workspace commands
```bash
# Install all packages across workspace
pnpm install
# Run script in a specific package
pnpm --filter @myorg/web dev
# Run script in all packages
pnpm --filter "*" build
# Run script in a package and all its dependencies
pnpm --filter @myorg/web... build
# Add a dependency to a specific package
pnpm --filter @myorg/web add react
# Add a shared dev dependency to root
pnpm add -D typescript -w
# List workspace packages
pnpm ls --depth -1 -r
```
---
## Cross-Package Impact Analysis
When a shared package changes, determine what's affected before you ship.
```bash
# Using Turborepo — show affected packages
turbo run build --filter=...[HEAD^1] --dry-run 2>&1 | grep "Tasks to run"
# Using Nx
nx affected:apps --base=main --head=HEAD # which apps are affected
nx affected:libs --base=main --head=HEAD # which libs are affected
# Manual analysis with pnpm
# Find all packages that depend on @myorg/utils:
grep -r '"@myorg/utils"' packages/*/package.json apps/*/package.json
# Using jq for structured output
for pkg in packages/*/package.json apps/*/package.json; do
name=$(jq -r '.name' "$pkg")
if jq -e '.dependencies["@myorg/utils"] // .devDependencies["@myorg/utils"]' "$pkg" > /dev/null 2>&1; then
echo "$name depends on @myorg/utils"
fi
done
```
---
## Dependency Graph Visualization
Generate a Mermaid diagram from your workspace:
```bash
# Generate dependency graph as Mermaid
cat > scripts/gen-dep-graph.js << 'EOF'
const { execSync } = require('child_process');
const fs = require('fs');
// Parse pnpm workspace packages
const packages = JSON.parse(
execSync('pnpm ls --depth -1 -r --json').toString()
);
let mermaid = 'graph TD\n';
packages.forEach(pkg => {
const deps = Object.keys(pkg.dependencies || {})
.filter(d => d.startsWith('@myorg/'));
deps.forEach(dep => {
const from = pkg.name.replace('@myorg/', '');
const to = dep.replace('@myorg/', '');
mermaid += ` ${from} --> ${to}\n`;
});
});
fs.writeFileSync('docs/dep-graph.md', '```mermaid\n' + mermaid + '```\n');
console.log('Written to docs/dep-graph.md');
EOF
node scripts/gen-dep-graph.js
```
**Example output:**
```mermaid
graph TD
web --> ui
web --> utils
web --> types
mobile --> ui
mobile --> utils
mobile --> types
admin --> ui
admin --> utils
api --> types
ui --> utils
```
---
## Claude Code Configuration (Workspace-Aware CLAUDE.md)
Place a root CLAUDE.md + per-package CLAUDE.md files:
```markdown
# /CLAUDE.md — Root (applies to all packages)
## Monorepo Structure
- apps/web — Next.js customer-facing app
- apps/admin — Next.js internal admin
- apps/api — Express REST API
- packages/ui — Shared React component library
- packages/utils — Shared utilities (pure functions only)
- packages/types — Shared TypeScript types (no runtime code)
## Build System
- pnpm workspaces + Turborepo
- Always use `pnpm --filter <package>` to scope commands
- Never run `npm install` or `yarn` — pnpm only
- Run `turbo run build --filter=...[HEAD^1]` before committing
## Task Scoping Rules
- When modifying packages/ui: also run tests for apps/web and apps/admin (they depend on it)
- When modifying packages/types: run type-check across ALL packages
- When modifying apps/api: only need to test apps/api
## Package Manager
pnpm — version pinned in packageManager field of root package.json
```
```markdown
# /packages/ui/CLAUDE.md — Package-specific
## This Package
Shared React component library. Zero business logic. Pure UI only.
## Rules
- All components must be exported from src/index.ts
- No direct API calls in components — accept data via props
- Every component needs a Storybook story in src/stories/
- Use Tailwind for styling — no CSS modules or styled-components
## Testing
- Component tests: `pnpm --filter @myorg/ui test`
- Visual regression: `pnpm --filter @myorg/ui test:storybook`
## Publishing
- Version bumps via changesets only — never edit package.json version manually
- Run `pnpm changeset` from repo root after changes
```
---
## Migration: Multi-Repo → Monorepo
```bash
# Step 1: Create monorepo scaffold
mkdir my-monorepo && cd my-monorepo
pnpm init
echo "packages:\n - 'apps/*'\n - 'packages/*'" > pnpm-workspace.yaml
# Step 2: Move repos with git history preserved
mkdir -p apps packages
# For each existing repo:
git clone https://github.com/myorg/web-app
cd web-app
git filter-repo --to-subdirectory-filter apps/web # rewrites history into subdir
cd ..
git remote add web-app ./web-app
git fetch web-app --tags
git merge web-app/main --allow-unrelated-histories
# Step 3: Update package names to scoped
# In each package.json, change "name": "web" to "name": "@myorg/web"
# Step 4: Replace cross-repo npm deps with workspace:*
# apps/web/package.json: "@myorg/ui": "1.2.3" → "@myorg/ui": "workspace:*"
# Step 5: Add shared configs to root
cp apps/web/.eslintrc.js .eslintrc.base.js
# Update each package's config to extend root:
# { "extends": ["../../.eslintrc.base.js"] }
# Step 6: Add Turborepo
pnpm add -D turbo -w
# Create turbo.json (see above)
# Step 7: Unified CI (see CI section below)
# Step 8: Test everything
turbo run build test lint
```
---
## CI Patterns
### GitHub Actions — Affected Only
```yaml
# .github/workflows/ci.yml
name: "ci"
on:
push:
branches: [main]
pull_request:
jobs:
affected:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # full history needed for affected detection
- uses: pnpm/action-setup@v3
with:
version: 9
- uses: actions/setup-node@v4
with:
node-version: 20
cache: pnpm
- run: pnpm install --frozen-lockfile
# Turborepo remote cache
- uses: actions/cache@v4
with:
path: .turbo
key: ${{ runner.os }}-turbo-${{ github.sha }}
restore-keys: ${{ runner.os }}-turbo-
# Only test/build affected packages
- name: "build-affected"
run: turbo run build --filter=...[origin/main]
env:
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_TEAM: ${{ vars.TURBO_TEAM }}
- name: "test-affected"
run: turbo run test --filter=...[origin/main]
- name: "lint-affected"
run: turbo run lint --filter=...[origin/main]
```
### GitLab CI — Parallel Stages
```yaml
# .gitlab-ci.yml
stages: [install, build, test, publish]
variables:
PNPM_CACHE_FOLDER: .pnpm-store
cache:
key: pnpm-$CI_COMMIT_REF_SLUG
paths: [.pnpm-store/, .turbo/]
install:
stage: install
script:
- pnpm install --frozen-lockfile
artifacts:
paths: [node_modules/, packages/*/node_modules/, apps/*/node_modules/]
expire_in: 1h
build:affected:
stage: build
needs: [install]
script:
- turbo run build --filter=...[origin/main]
artifacts:
paths: [apps/*/dist/, apps/*/.next/, packages/*/dist/]
test:affected:
stage: test
needs: [build:affected]
script:
- turbo run test --filter=...[origin/main]
coverage: '/Statements\s*:\s*(\d+\.?\d*)%/'
artifacts:
reports:
coverage_report:
coverage_format: cobertura
path: "**/coverage/cobertura-coverage.xml"
```
---
## Publishing with Changesets
```bash
# Install changesets
pnpm add -D @changesets/cli -w
pnpm changeset init
# After making changes, create a changeset
pnpm changeset
# Interactive: select packages, choose semver bump, write changelog entry
# In CI — version packages + update changelogs
pnpm changeset version
# Publish all changed packages
pnpm changeset publish
# Pre-release channel (for alpha/beta)
pnpm changeset pre enter beta
pnpm changeset
pnpm changeset version # produces 1.2.0-beta.0
pnpm changeset publish --tag beta
pnpm changeset pre exit # back to stable releases
```
### Automated publish workflow (GitHub Actions)
```yaml
# .github/workflows/release.yml
name: "release"
on:
push:
branches: [main]
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
- uses: actions/setup-node@v4
with:
node-version: 20
registry-url: https://registry.npmjs.org
- run: pnpm install --frozen-lockfile
- name: "create-release-pr-or-publish"
uses: changesets/action@v1
with:
publish: pnpm changeset publish
version: pnpm changeset version
commit: "chore: release packages"
title: "chore: release packages"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
```
---

473
engineering/performance-profiler/SKILL.md
View File

@@ -50,478 +50,7 @@ Systematic performance profiling for Node.js, Python, and Go applications. Ident
---
## Node.js Profiling
### CPU Flamegraph
```bash
# Method 1: clinic.js (best for development)
npm install -g clinic
# CPU flamegraph
clinic flame -- node dist/server.js
# Heap profiler
clinic heapprofiler -- node dist/server.js
# Bubble chart (event loop blocking)
clinic bubbles -- node dist/server.js
# Load with autocannon while profiling
autocannon -c 50 -d 30 http://localhost:3000/api/tasks &
clinic flame -- node dist/server.js
```
```bash
# Method 2: Node.js built-in profiler
node --prof dist/server.js
# After running some load:
node --prof-process isolate-*.log | head -100
```
```bash
# Method 3: V8 CPU profiler via inspector
node --inspect dist/server.js
# Open Chrome DevTools → Performance → Record
```
### Heap Snapshot / Memory Leak Detection
```javascript
// Add to your server for on-demand heap snapshots
import v8 from 'v8'
import fs from 'fs'
// Endpoint: POST /debug/heap-snapshot (protect with auth!)
app.post('/debug/heap-snapshot', (req, res) => {
const filename = `heap-${Date.now()}.heapsnapshot`
const snapshot = v8.writeHeapSnapshot(filename)
res.json({ snapshot })
})
```
```bash
# Take snapshots over time and compare in Chrome DevTools
curl -X POST http://localhost:3000/debug/heap-snapshot
# Wait 5 minutes of load
curl -X POST http://localhost:3000/debug/heap-snapshot
# Open both snapshots in Chrome → Memory → Compare
```
### Detect Event Loop Blocking
```javascript
// Add blocked-at to detect synchronous blocking
import blocked from 'blocked-at'
blocked((time, stack) => {
console.warn(`Event loop blocked for ${time}ms`)
console.warn(stack.join('\n'))
}, { threshold: 100 }) // Alert if blocked > 100ms
```
### Node.js Memory Profiling Script
```javascript
// scripts/memory-profile.mjs
// Run: node --experimental-vm-modules scripts/memory-profile.mjs
import { createRequire } from 'module'
const require = createRequire(import.meta.url)
function formatBytes(bytes) {
return (bytes / 1024 / 1024).toFixed(2) + ' MB'
}
function measureMemory(label) {
const mem = process.memoryUsage()
console.log(`\n[${label}]`)
console.log(` RSS: ${formatBytes(mem.rss)}`)
console.log(` Heap Used: ${formatBytes(mem.heapUsed)}`)
console.log(` Heap Total:${formatBytes(mem.heapTotal)}`)
console.log(` External: ${formatBytes(mem.external)}`)
return mem
}
const baseline = measureMemory('Baseline')
// Simulate your operation
for (let i = 0; i < 1000; i++) {
// Replace with your actual operation
const result = await someOperation()
}
const after = measureMemory('After 1000 operations')
console.log(`\n[Delta]`)
console.log(` Heap Used: +${formatBytes(after.heapUsed - baseline.heapUsed)}`)
// If heap keeps growing across GC cycles, you have a leak
global.gc?.() // Run with --expose-gc flag
const afterGC = measureMemory('After GC')
if (afterGC.heapUsed > baseline.heapUsed * 1.1) {
console.warn('⚠️ Possible memory leak detected (>10% growth after GC)')
}
```
---
## Python Profiling
### CPU Profiling with py-spy
```bash
# Install
pip install py-spy
# Profile a running process (no code changes needed)
py-spy top --pid $(pgrep -f "uvicorn")
# Generate flamegraph SVG
py-spy record -o flamegraph.svg --pid $(pgrep -f "uvicorn") --duration 30
# Profile from the start
py-spy record -o flamegraph.svg -- python -m uvicorn app.main:app
# Open flamegraph.svg in browser — look for wide bars = hot code paths
```
### cProfile for function-level profiling
```python
# scripts/profile_endpoint.py
import cProfile
import pstats
import io
from app.services.task_service import TaskService
def run():
service = TaskService()
for _ in range(100):
service.list_tasks(user_id="user_1", page=1, limit=20)
profiler = cProfile.Profile()
profiler.enable()
run()
profiler.disable()
# Print top 20 functions by cumulative time
stream = io.StringIO()
stats = pstats.Stats(profiler, stream=stream)
stats.sort_stats('cumulative')
stats.print_stats(20)
print(stream.getvalue())
```
### Memory profiling with memory_profiler
```python
# pip install memory-profiler
from memory_profiler import profile
@profile
def my_function():
# Function to profile
data = load_large_dataset()
result = process(data)
return result
```
```bash
# Run with line-by-line memory tracking
python -m memory_profiler scripts/profile_function.py
# Output:
# Line # Mem usage Increment Line Contents
# ================================================
# 10 45.3 MiB 45.3 MiB def my_function():
# 11 78.1 MiB 32.8 MiB data = load_large_dataset()
# 12 156.2 MiB 78.1 MiB result = process(data)
```
---
## Go Profiling with pprof
```go
// main.go — add pprof endpoints
import _ "net/http/pprof"
import "net/http"
func main() {
// pprof endpoints at /debug/pprof/
go func() {
log.Println(http.ListenAndServe(":6060", nil))
}()
// ... rest of your app
}
```
```bash
# CPU profile (30s)
go tool pprof -http=:8080 http://localhost:6060/debug/pprof/profile?seconds=30
# Memory profile
go tool pprof -http=:8080 http://localhost:6060/debug/pprof/heap
# Goroutine leak detection
curl http://localhost:6060/debug/pprof/goroutine?debug=1
# In pprof UI: "Flame Graph" view → find the tallest bars
```
---
## Bundle Size Analysis
### Next.js Bundle Analyzer
```bash
# Install
pnpm add -D @next/bundle-analyzer
# next.config.js
const withBundleAnalyzer = require('@next/bundle-analyzer')({
enabled: process.env.ANALYZE === 'true',
})
module.exports = withBundleAnalyzer({})
# Run analyzer
ANALYZE=true pnpm build
# Opens browser with treemap of bundle
```
### What to look for
```bash
# Find the largest chunks
pnpm build 2>&1 | grep -E "^\s+(λ|○|●)" | sort -k4 -rh | head -20
# Check if a specific package is too large
# Visit: https://bundlephobia.com/package/moment@2.29.4
# moment: 67.9kB gzipped → replace with date-fns (13.8kB) or dayjs (6.9kB)
# Find duplicate packages
pnpm dedupe --check
# Visualize what's in a chunk
npx source-map-explorer .next/static/chunks/*.js
```
### Common bundle wins
```typescript
// Before: import entire lodash
import _ from 'lodash' // 71kB
// After: import only what you need
import debounce from 'lodash/debounce' // 2kB
// Before: moment.js
import moment from 'moment' // 67kB
// After: dayjs
import dayjs from 'dayjs' // 7kB
// Before: static import (always in bundle)
import HeavyChart from '@/components/HeavyChart'
// After: dynamic import (loaded on demand)
const HeavyChart = dynamic(() => import('@/components/HeavyChart'), {
loading: () => <Skeleton />,
})
```
---
## Database Query Optimization
### Find slow queries
```sql
-- PostgreSQL: enable pg_stat_statements
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-- Top 20 slowest queries
SELECT
round(mean_exec_time::numeric, 2) AS mean_ms,
calls,
round(total_exec_time::numeric, 2) AS total_ms,
round(stddev_exec_time::numeric, 2) AS stddev_ms,
left(query, 80) AS query
FROM pg_stat_statements
WHERE calls > 10
ORDER BY mean_exec_time DESC
LIMIT 20;
-- Reset stats
SELECT pg_stat_statements_reset();
```
```bash
# MySQL slow query log
mysql -e "SET GLOBAL slow_query_log = 'ON'; SET GLOBAL long_query_time = 0.1;"
tail -f /var/log/mysql/slow-query.log
```
### EXPLAIN ANALYZE
```sql
-- Always use EXPLAIN (ANALYZE, BUFFERS) for real timing
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
SELECT t.*, u.name as assignee_name
FROM tasks t
LEFT JOIN users u ON u.id = t.assignee_id
WHERE t.project_id = 'proj_123'
AND t.deleted_at IS NULL
ORDER BY t.created_at DESC
LIMIT 20;
-- Look for:
-- Seq Scan on large table → needs index
-- Nested Loop with high rows → N+1, consider JOIN or batch
-- Sort → can index handle the sort?
-- Hash Join → fine for moderate sizes
```
### Detect N+1 Queries
```typescript
// Add query logging in dev
import { db } from './client'
// Drizzle: enable logging
const db = drizzle(pool, { logger: true })
// Or use a query counter middleware
let queryCount = 0
db.$on('query', () => queryCount++)
// In tests:
queryCount = 0
const tasks = await getTasksWithAssignees(projectId)
expect(queryCount).toBe(1) // Fail if it's 21 (1 + 20 N+1s)
```
```python
# Django: detect N+1 with django-silk or nplusone
from nplusone.ext.django.middleware import NPlusOneMiddleware
MIDDLEWARE = ['nplusone.ext.django.middleware.NPlusOneMiddleware']
NPLUSONE_RAISE = True # Raise exception on N+1 in tests
```
### Fix N+1 — Before/After
```typescript
// Before: N+1 (1 query for tasks + N queries for assignees)
const tasks = await db.select().from(tasksTable)
for (const task of tasks) {
task.assignee = await db.select().from(usersTable)
.where(eq(usersTable.id, task.assigneeId))
.then(r => r[0])
}
// After: 1 query with JOIN
const tasks = await db
.select({
id: tasksTable.id,
title: tasksTable.title,
assigneeName: usersTable.name,
assigneeEmail: usersTable.email,
})
.from(tasksTable)
.leftJoin(usersTable, eq(usersTable.id, tasksTable.assigneeId))
.where(eq(tasksTable.projectId, projectId))
```
---
## Load Testing with k6
```javascript
// tests/load/api-load-test.js
import http from 'k6/http'
import { check, sleep } from 'k6'
import { Rate, Trend } from 'k6/metrics'
const errorRate = new Rate('errors')
const taskListDuration = new Trend('task_list_duration')
export const options = {
stages: [
{ duration: '30s', target: 10 }, // Ramp up to 10 VUs
{ duration: '1m', target: 50 }, // Ramp to 50 VUs
{ duration: '2m', target: 50 }, // Sustain 50 VUs
{ duration: '30s', target: 100 }, // Spike to 100 VUs
{ duration: '1m', target: 50 }, // Back to 50
{ duration: '30s', target: 0 }, // Ramp down
],
thresholds: {
http_req_duration: ['p(95)<500'], // 95% of requests < 500ms
http_req_duration: ['p(99)<1000'], // 99% < 1s
errors: ['rate<0.01'], // Error rate < 1%
task_list_duration: ['p(95)<200'], // Task list specifically < 200ms
},
}
const BASE_URL = __ENV.BASE_URL || 'http://localhost:3000'
export function setup() {
// Get auth token once
const loginRes = http.post(`${BASE_URL}/api/auth/login`, JSON.stringify({
email: 'loadtest@example.com',
password: 'loadtest123',
}), { headers: { 'Content-Type': 'application/json' } })
return { token: loginRes.json('token') }
}
export default function(data) {
const headers = {
'Authorization': `Bearer ${data.token}`,
'Content-Type': 'application/json',
}
// Scenario 1: List tasks
const start = Date.now()
const listRes = http.get(`${BASE_URL}/api/tasks?limit=20`, { headers })
taskListDuration.add(Date.now() - start)
check(listRes, {
'list tasks: status 200': (r) => r.status === 200,
'list tasks: has items': (r) => r.json('items') !== undefined,
}) || errorRate.add(1)
sleep(0.5)
// Scenario 2: Create task
const createRes = http.post(
`${BASE_URL}/api/tasks`,
JSON.stringify({ title: `Load test task ${Date.now()}`, priority: 'medium' }),
{ headers }
)
check(createRes, {
'create task: status 201': (r) => r.status === 201,
}) || errorRate.add(1)
sleep(1)
}
export function teardown(data) {
// Cleanup: delete load test tasks
}
```
```bash
# Run load test
k6 run tests/load/api-load-test.js \
--env BASE_URL=https://staging.myapp.com
# With Grafana output
k6 run --out influxdb=http://localhost:8086/k6 tests/load/api-load-test.js
```
---
→ See references/profiling-recipes.md for details
## Before/After Measurement Template

475
engineering/performance-profiler/references/profiling-recipes.md Normal file
View File

@@ -0,0 +1,475 @@
# performance-profiler reference
## Node.js Profiling
### CPU Flamegraph
```bash
# Method 1: clinic.js (best for development)
npm install -g clinic
# CPU flamegraph
clinic flame -- node dist/server.js
# Heap profiler
clinic heapprofiler -- node dist/server.js
# Bubble chart (event loop blocking)
clinic bubbles -- node dist/server.js
# Load with autocannon while profiling
autocannon -c 50 -d 30 http://localhost:3000/api/tasks &
clinic flame -- node dist/server.js
```
```bash
# Method 2: Node.js built-in profiler
node --prof dist/server.js
# After running some load:
node --prof-process isolate-*.log | head -100
```
```bash
# Method 3: V8 CPU profiler via inspector
node --inspect dist/server.js
# Open Chrome DevTools → Performance → Record
```
### Heap Snapshot / Memory Leak Detection
```javascript
// Add to your server for on-demand heap snapshots
import v8 from 'v8'
import fs from 'fs'
// Endpoint: POST /debug/heap-snapshot (protect with auth!)
app.post('/debug/heap-snapshot', (req, res) => {
const filename = `heap-${Date.now()}.heapsnapshot`
const snapshot = v8.writeHeapSnapshot(filename)
res.json({ snapshot })
})
```
```bash
# Take snapshots over time and compare in Chrome DevTools
curl -X POST http://localhost:3000/debug/heap-snapshot
# Wait 5 minutes of load
curl -X POST http://localhost:3000/debug/heap-snapshot
# Open both snapshots in Chrome → Memory → Compare
```
### Detect Event Loop Blocking
```javascript
// Add blocked-at to detect synchronous blocking
import blocked from 'blocked-at'
blocked((time, stack) => {
console.warn(`Event loop blocked for ${time}ms`)
console.warn(stack.join('\n'))
}, { threshold: 100 }) // Alert if blocked > 100ms
```
### Node.js Memory Profiling Script
```javascript
// scripts/memory-profile.mjs
// Run: node --experimental-vm-modules scripts/memory-profile.mjs
import { createRequire } from 'module'
const require = createRequire(import.meta.url)
function formatBytes(bytes) {
return (bytes / 1024 / 1024).toFixed(2) + ' MB'
}
function measureMemory(label) {
const mem = process.memoryUsage()
console.log(`\n[${label}]`)
console.log(` RSS: ${formatBytes(mem.rss)}`)
console.log(` Heap Used: ${formatBytes(mem.heapUsed)}`)
console.log(` Heap Total:${formatBytes(mem.heapTotal)}`)
console.log(` External: ${formatBytes(mem.external)}`)
return mem
}
const baseline = measureMemory('Baseline')
// Simulate your operation
for (let i = 0; i < 1000; i++) {
// Replace with your actual operation
const result = await someOperation()
}
const after = measureMemory('After 1000 operations')
console.log(`\n[Delta]`)
console.log(` Heap Used: +${formatBytes(after.heapUsed - baseline.heapUsed)}`)
// If heap keeps growing across GC cycles, you have a leak
global.gc?.() // Run with --expose-gc flag
const afterGC = measureMemory('After GC')
if (afterGC.heapUsed > baseline.heapUsed * 1.1) {
console.warn('⚠️ Possible memory leak detected (>10% growth after GC)')
}
```
---
## Python Profiling
### CPU Profiling with py-spy
```bash
# Install
pip install py-spy
# Profile a running process (no code changes needed)
py-spy top --pid $(pgrep -f "uvicorn")
# Generate flamegraph SVG
py-spy record -o flamegraph.svg --pid $(pgrep -f "uvicorn") --duration 30
# Profile from the start
py-spy record -o flamegraph.svg -- python -m uvicorn app.main:app
# Open flamegraph.svg in browser — look for wide bars = hot code paths
```
### cProfile for function-level profiling
```python
# scripts/profile_endpoint.py
import cProfile
import pstats
import io
from app.services.task_service import TaskService
def run():
service = TaskService()
for _ in range(100):
service.list_tasks(user_id="user_1", page=1, limit=20)
profiler = cProfile.Profile()
profiler.enable()
run()
profiler.disable()
# Print top 20 functions by cumulative time
stream = io.StringIO()
stats = pstats.Stats(profiler, stream=stream)
stats.sort_stats('cumulative')
stats.print_stats(20)
print(stream.getvalue())
```
### Memory profiling with memory_profiler
```python
# pip install memory-profiler
from memory_profiler import profile
@profile
def my_function():
# Function to profile
data = load_large_dataset()
result = process(data)
return result
```
```bash
# Run with line-by-line memory tracking
python -m memory_profiler scripts/profile_function.py
# Output:
# Line # Mem usage Increment Line Contents
# ================================================
# 10 45.3 MiB 45.3 MiB def my_function():
# 11 78.1 MiB 32.8 MiB data = load_large_dataset()
# 12 156.2 MiB 78.1 MiB result = process(data)
```
---
## Go Profiling with pprof
```go
// main.go — add pprof endpoints
import _ "net/http/pprof"
import "net/http"
func main() {
// pprof endpoints at /debug/pprof/
go func() {
log.Println(http.ListenAndServe(":6060", nil))
}()
// ... rest of your app
}
```
```bash
# CPU profile (30s)
go tool pprof -http=:8080 http://localhost:6060/debug/pprof/profile?seconds=30
# Memory profile
go tool pprof -http=:8080 http://localhost:6060/debug/pprof/heap
# Goroutine leak detection
curl http://localhost:6060/debug/pprof/goroutine?debug=1
# In pprof UI: "Flame Graph" view → find the tallest bars
```
---
## Bundle Size Analysis
### Next.js Bundle Analyzer
```bash
# Install
pnpm add -D @next/bundle-analyzer
# next.config.js
const withBundleAnalyzer = require('@next/bundle-analyzer')({
enabled: process.env.ANALYZE === 'true',
})
module.exports = withBundleAnalyzer({})
# Run analyzer
ANALYZE=true pnpm build
# Opens browser with treemap of bundle
```
### What to look for
```bash
# Find the largest chunks
pnpm build 2>&1 | grep -E "^\s+(λ|○|●)" | sort -k4 -rh | head -20
# Check if a specific package is too large
# Visit: https://bundlephobia.com/package/moment@2.29.4
# moment: 67.9kB gzipped → replace with date-fns (13.8kB) or dayjs (6.9kB)
# Find duplicate packages
pnpm dedupe --check
# Visualize what's in a chunk
npx source-map-explorer .next/static/chunks/*.js
```
### Common bundle wins
```typescript
// Before: import entire lodash
import _ from 'lodash' // 71kB
// After: import only what you need
import debounce from 'lodash/debounce' // 2kB
// Before: moment.js
import moment from 'moment' // 67kB
// After: dayjs
import dayjs from 'dayjs' // 7kB
// Before: static import (always in bundle)
import HeavyChart from '@/components/HeavyChart'
// After: dynamic import (loaded on demand)
const HeavyChart = dynamic(() => import('@/components/HeavyChart'), {
loading: () => <Skeleton />,
})
```
---
## Database Query Optimization
### Find slow queries
```sql
-- PostgreSQL: enable pg_stat_statements
CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-- Top 20 slowest queries
SELECT
round(mean_exec_time::numeric, 2) AS mean_ms,
calls,
round(total_exec_time::numeric, 2) AS total_ms,
round(stddev_exec_time::numeric, 2) AS stddev_ms,
left(query, 80) AS query
FROM pg_stat_statements
WHERE calls > 10
ORDER BY mean_exec_time DESC
LIMIT 20;
-- Reset stats
SELECT pg_stat_statements_reset();
```
```bash
# MySQL slow query log
mysql -e "SET GLOBAL slow_query_log = 'ON'; SET GLOBAL long_query_time = 0.1;"
tail -f /var/log/mysql/slow-query.log
```
### EXPLAIN ANALYZE
```sql
-- Always use EXPLAIN (ANALYZE, BUFFERS) for real timing
EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT)
SELECT t.*, u.name as assignee_name
FROM tasks t
LEFT JOIN users u ON u.id = t.assignee_id
WHERE t.project_id = 'proj_123'
AND t.deleted_at IS NULL
ORDER BY t.created_at DESC
LIMIT 20;
-- Look for:
-- Seq Scan on large table → needs index
-- Nested Loop with high rows → N+1, consider JOIN or batch
-- Sort → can index handle the sort?
-- Hash Join → fine for moderate sizes
```
### Detect N+1 Queries
```typescript
// Add query logging in dev
import { db } from './client'
// Drizzle: enable logging
const db = drizzle(pool, { logger: true })
// Or use a query counter middleware
let queryCount = 0
db.$on('query', () => queryCount++)
// In tests:
queryCount = 0
const tasks = await getTasksWithAssignees(projectId)
expect(queryCount).toBe(1) // Fail if it's 21 (1 + 20 N+1s)
```
```python
# Django: detect N+1 with django-silk or nplusone
from nplusone.ext.django.middleware import NPlusOneMiddleware
MIDDLEWARE = ['nplusone.ext.django.middleware.NPlusOneMiddleware']
NPLUSONE_RAISE = True # Raise exception on N+1 in tests
```
### Fix N+1 — Before/After
```typescript
// Before: N+1 (1 query for tasks + N queries for assignees)
const tasks = await db.select().from(tasksTable)
for (const task of tasks) {
task.assignee = await db.select().from(usersTable)
.where(eq(usersTable.id, task.assigneeId))
.then(r => r[0])
}
// After: 1 query with JOIN
const tasks = await db
.select({
id: tasksTable.id,
title: tasksTable.title,
assigneeName: usersTable.name,
assigneeEmail: usersTable.email,
})
.from(tasksTable)
.leftJoin(usersTable, eq(usersTable.id, tasksTable.assigneeId))
.where(eq(tasksTable.projectId, projectId))
```
---
## Load Testing with k6
```javascript
// tests/load/api-load-test.js
import http from 'k6/http'
import { check, sleep } from 'k6'
import { Rate, Trend } from 'k6/metrics'
const errorRate = new Rate('errors')
const taskListDuration = new Trend('task_list_duration')
export const options = {
stages: [
{ duration: '30s', target: 10 }, // Ramp up to 10 VUs
{ duration: '1m', target: 50 }, // Ramp to 50 VUs
{ duration: '2m', target: 50 }, // Sustain 50 VUs
{ duration: '30s', target: 100 }, // Spike to 100 VUs
{ duration: '1m', target: 50 }, // Back to 50
{ duration: '30s', target: 0 }, // Ramp down
],
thresholds: {
http_req_duration: ['p(95)<500'], // 95% of requests < 500ms
http_req_duration: ['p(99)<1000'], // 99% < 1s
errors: ['rate<0.01'], // Error rate < 1%
task_list_duration: ['p(95)<200'], // Task list specifically < 200ms
},
}
const BASE_URL = __ENV.BASE_URL || 'http://localhost:3000'
export function setup() {
// Get auth token once
const loginRes = http.post(`${BASE_URL}/api/auth/login`, JSON.stringify({
email: 'loadtest@example.com',
password: 'loadtest123',
}), { headers: { 'Content-Type': 'application/json' } })
return { token: loginRes.json('token') }
}
export default function(data) {
const headers = {
'Authorization': `Bearer ${data.token}`,
'Content-Type': 'application/json',
}
// Scenario 1: List tasks
const start = Date.now()
const listRes = http.get(`${BASE_URL}/api/tasks?limit=20`, { headers })
taskListDuration.add(Date.now() - start)
check(listRes, {
'list tasks: status 200': (r) => r.status === 200,
'list tasks: has items': (r) => r.json('items') !== undefined,
}) || errorRate.add(1)
sleep(0.5)
// Scenario 2: Create task
const createRes = http.post(
`${BASE_URL}/api/tasks`,
JSON.stringify({ title: `Load test task ${Date.now()}`, priority: 'medium' }),
{ headers }
)
check(createRes, {
'create task: status 201': (r) => r.status === 201,
}) || errorRate.add(1)
sleep(1)
}
export function teardown(data) {
// Cleanup: delete load test tasks
}
```
```bash
# Run load test
k6 run tests/load/api-load-test.js \
--env BASE_URL=https://staging.myapp.com
# With Grafana output
k6 run --out influxdb=http://localhost:8086/k6 tests/load/api-load-test.js
```
---

486
engineering/tech-debt-tracker/SKILL.md
View File

@@ -26,489 +26,7 @@ This skill offers three interconnected tools that form a complete tech debt mana
Together, these tools enable engineering teams to make data-driven decisions about tech debt, balancing new feature development with maintenance work.
## Technical Debt Classification Framework
### 1. Code Debt
Code-level issues that make the codebase harder to understand, modify, and maintain.
**Indicators:**
- Long functions (>50 lines for complex logic, >20 for simple operations)
- Deep nesting (>4 levels of indentation)
- High cyclomatic complexity (>10)
- Duplicate code patterns (>3 similar blocks)
- Missing or inadequate error handling
- Poor variable/function naming
- Magic numbers and hardcoded values
- Commented-out code blocks
**Impact:**
- Increased debugging time
- Higher defect rates
- Slower feature development
- Knowledge silos (only original author understands the code)
**Detection Methods:**
- AST parsing for structural analysis
- Pattern matching for common anti-patterns
- Complexity metrics calculation
- Duplicate code detection algorithms
### 2. Architecture Debt
High-level design decisions that seemed reasonable at the time but now limit scalability or maintainability.
**Indicators:**
- Monolithic components that should be modular
- Circular dependencies between modules
- Violation of separation of concerns
- Inconsistent data flow patterns
- Over-engineering or under-engineering for current scale
- Tightly coupled components
- Missing abstraction layers
**Impact:**
- Difficult to scale individual components
- Cascading changes required for simple modifications
- Testing becomes complex and brittle
- Onboarding new team members takes longer
**Detection Methods:**
- Dependency analysis
- Module coupling metrics
- Component size analysis
- Interface consistency checks
### 3. Test Debt
Inadequate or missing test coverage, poor test quality, and testing infrastructure issues.
**Indicators:**
- Low test coverage (<80% for critical paths)
- Missing unit tests for complex logic
- No integration tests for key workflows
- Flaky tests that pass/fail intermittently
- Slow test execution (>10 minutes for unit tests)
- Tests that don't test meaningful behavior
- Missing test data management strategy
**Impact:**
- Fear of refactoring ("don't touch it, it works")
- Regression bugs in production
- Slow feedback cycles during development
- Difficulty validating complex business logic
**Detection Methods:**
- Coverage report analysis
- Test execution time monitoring
- Test failure pattern analysis
- Test code quality assessment
### 4. Documentation Debt
Missing, outdated, or poor-quality documentation that makes the system harder to understand and maintain.
**Indicators:**
- Missing API documentation
- Outdated README files
- No architectural decision records (ADRs)
- Missing code comments for complex algorithms
- No onboarding documentation for new team members
- Inconsistent documentation formats
- Documentation that contradicts actual implementation
**Impact:**
- Increased onboarding time for new team members
- Knowledge loss when team members leave
- Miscommunication between teams
- Repeated questions in team channels
**Detection Methods:**
- Documentation coverage analysis
- Freshness checking (last modified dates)
- Link validation
- Comment density analysis
### 5. Dependency Debt
Issues related to external libraries, frameworks, and system dependencies.
**Indicators:**
- Outdated packages with known security vulnerabilities
- Dependencies with incompatible licenses
- Unused dependencies bloating the build
- Version conflicts between packages
- Deprecated APIs still in use
- Heavy dependencies for simple tasks
- Missing dependency pinning
**Impact:**
- Security vulnerabilities
- Build instability
- Longer build times
- Legal compliance issues
- Difficulty upgrading core frameworks
**Detection Methods:**
- Vulnerability scanning
- License compliance checking
- Usage analysis
- Version compatibility checking
### 6. Infrastructure Debt
Operations and deployment-related technical debt.
**Indicators:**
- Manual deployment processes
- Missing monitoring and alerting
- Inadequate logging
- No disaster recovery plan
- Inconsistent environments (dev/staging/prod)
- Missing CI/CD pipelines
- Infrastructure as code gaps
**Impact:**
- Deployment risks and downtime
- Difficult troubleshooting
- Inconsistent behavior across environments
- Manual work that should be automated
**Detection Methods:**
- Infrastructure audit checklists
- Configuration drift detection
- Monitoring coverage analysis
- Deployment process documentation review
## Severity Scoring Framework
Each piece of tech debt is scored on multiple dimensions to determine overall severity:
### Impact Assessment (1-10 scale)
**Development Velocity Impact**
- 1-2: Negligible impact on development speed
- 3-4: Minor slowdown, workarounds available
- 5-6: Moderate impact, affects some features
- 7-8: Significant slowdown, affects most work
- 9-10: Critical blocker, prevents new development
**Quality Impact**
- 1-2: No impact on defect rates
- 3-4: Minor increase in minor bugs
- 5-6: Moderate increase in defects
- 7-8: Regular production issues
- 9-10: Critical reliability problems
**Team Productivity Impact**
- 1-2: No impact on team morale or efficiency
- 3-4: Occasional frustration
- 5-6: Regular complaints from developers
- 7-8: Team actively avoiding the area
- 9-10: Causing developer turnover
**Business Impact**
- 1-2: No customer-facing impact
- 3-4: Minor UX degradation
- 5-6: Moderate performance impact
- 7-8: Customer complaints or churn
- 9-10: Revenue-impacting issues
### Effort Assessment
**Size (Story Points or Hours)**
- XS (1-4 hours): Simple refactor or documentation update
- S (1-2 days): Minor architectural change
- M (3-5 days): Moderate refactoring effort
- L (1-2 weeks): Major component restructuring
- XL (3+ weeks): System-wide architectural changes
**Risk Level**
- Low: Well-understood change with clear scope
- Medium: Some unknowns but manageable
- High: Significant unknowns, potential for scope creep
**Skill Requirements**
- Junior: Can be handled by any team member
- Mid: Requires experienced developer
- Senior: Needs architectural expertise
- Expert: Requires deep system knowledge
## Interest Rate Calculation
Technical debt accrues "interest" - the additional cost of leaving it unfixed. This interest rate helps prioritize which debt to pay down first.
### Interest Rate Formula
```
Interest Rate = (Impact Score × Frequency of Encounter) / Time Period
```
Where:
- **Impact Score**: Average severity score (1-10)
- **Frequency of Encounter**: How often developers interact with this code
- **Time Period**: Usually measured per sprint or month
### Cost of Delay Calculation
```
Cost of Delay = Interest Rate × Time Until Fix × Team Size Multiplier
```
### Example Calculation
**Scenario**: Legacy authentication module with poor error handling
- Impact Score: 7 (causes regular production issues)
- Frequency: 15 encounters per sprint (3 developers × 5 times each)
- Team Size: 8 developers
- Current sprint: 1, planned fix: sprint 4
```
Interest Rate = 7 × 15 = 105 points per sprint
Cost of Delay = 105 × 3 × 1.2 = 378 total cost points
```
This debt item should be prioritized over lower-cost items.
## Debt Inventory Management
### Data Structure
Each debt item is tracked with the following attributes:
```json
{
"id": "DEBT-2024-001",
"title": "Legacy user authentication module",
"category": "code",
"subcategory": "error_handling",
"location": "src/auth/legacy_auth.py:45-120",
"description": "Authentication error handling uses generic exceptions",
"impact": {
"velocity": 7,
"quality": 8,
"productivity": 6,
"business": 5
},
"effort": {
"size": "M",
"risk": "medium",
"skill_required": "mid"
},
"interest_rate": 105,
"cost_of_delay": 378,
"priority": "high",
"created_date": "2024-01-15",
"last_updated": "2024-01-20",
"assigned_to": null,
"status": "identified",
"tags": ["security", "user-experience", "maintainability"]
}
```
### Status Lifecycle
1. **Identified** - Debt detected but not yet analyzed
2. **Analyzed** - Impact and effort assessed
3. **Prioritized** - Added to backlog with priority
4. **Planned** - Assigned to specific sprint/release
5. **In Progress** - Actively being worked on
6. **Review** - Implementation complete, under review
7. **Done** - Debt resolved and verified
8. **Won't Fix** - Consciously decided not to address
## Prioritization Frameworks
### 1. Cost-of-Delay vs Effort Matrix
Plot debt items on a 2D matrix:
- X-axis: Effort (XS to XL)
- Y-axis: Cost of Delay (calculated value)
**Priority Quadrants:**
- High Cost, Low Effort: **Immediate** (quick wins)
- High Cost, High Effort: **Planned** (major initiatives)
- Low Cost, Low Effort: **Opportunistic** (during related work)
- Low Cost, High Effort: **Backlog** (consider for future)
### 2. Weighted Shortest Job First (WSJF)
```
WSJF Score = (Business Value + Time Criticality + Risk Reduction) / Effort
```
Where each component is scored 1-10:
- **Business Value**: Direct impact on customer value
- **Time Criticality**: How much value decreases over time
- **Risk Reduction**: How much risk is mitigated by fixing this debt
### 3. Technical Debt Quadrant
Based on Martin Fowler's framework:
**Quadrant 1: Reckless & Deliberate**
- "We don't have time for design"
- Highest priority for remediation
**Quadrant 2: Prudent & Deliberate**
- "We must ship now and deal with consequences"
- Schedule for near-term resolution
**Quadrant 3: Reckless & Inadvertent**
- "What's layering?"
- Focus on education and process improvement
**Quadrant 4: Prudent & Inadvertent**
- "Now we know how we should have done it"
- Normal part of learning, lowest priority
## Refactoring Strategies
### 1. Strangler Fig Pattern
Gradually replace old system by building new functionality around it.
**When to use:**
- Large, monolithic systems
- High-risk changes to critical paths
- Long-term architectural migrations
**Implementation:**
1. Identify boundaries for extraction
2. Create abstraction layer
3. Route new features to new implementation
4. Gradually migrate existing features
5. Remove old implementation
### 2. Branch by Abstraction
Create abstraction layer to allow parallel implementations.
**When to use:**
- Need to support old and new systems simultaneously
- High-risk changes with rollback requirements
- A/B testing infrastructure changes
**Implementation:**
1. Create abstraction interface
2. Implement abstraction for current system
3. Replace direct calls with abstraction calls
4. Implement new version behind same abstraction
5. Switch implementations via configuration
6. Remove old implementation
### 3. Feature Toggles
Use configuration flags to control code execution.
**When to use:**
- Gradual rollout of refactored components
- Risk mitigation during large changes
- Experimental refactoring approaches
**Implementation:**
1. Identify decision points in code
2. Add toggle checks at decision points
3. Implement both old and new paths
4. Test both paths thoroughly
5. Gradually move toggle to new implementation
6. Remove old path and toggle
### 4. Parallel Run
Run old and new implementations simultaneously to verify correctness.
**When to use:**
- Critical business logic changes
- Data processing pipeline changes
- Algorithm improvements
**Implementation:**
1. Implement new version alongside old
2. Run both versions with same inputs
3. Compare outputs and log discrepancies
4. Investigate and fix discrepancies
5. Build confidence through parallel execution
6. Switch to new implementation
7. Remove old implementation
## Sprint Allocation Recommendations
### Debt-to-Feature Ratio
Maintain healthy balance between new features and debt reduction:
**Team Velocity < 70% of capacity:**
- 60% tech debt, 40% features
- Focus on removing major blockers
**Team Velocity 70-85% of capacity:**
- 30% tech debt, 70% features
- Balanced maintenance approach
**Team Velocity > 85% of capacity:**
- 15% tech debt, 85% features
- Opportunistic debt reduction only
### Sprint Planning Integration
**Story Point Allocation:**
- Reserve 20% of sprint capacity for tech debt
- Prioritize debt items with highest interest rates
- Include "debt tax" in feature estimates when working in high-debt areas
**Debt Budget Tracking:**
- Track debt points completed per sprint
- Monitor debt interest rate trend
- Alert when debt accumulation exceeds team's paydown rate
### Quarterly Planning
**Debt Initiatives:**
- Identify 1-2 major debt themes per quarter
- Allocate dedicated sprints for large-scale refactoring
- Plan debt work around major feature releases
**Success Metrics:**
- Debt interest rate reduction
- Developer velocity improvements
- Defect rate reduction
- Code review cycle time improvement
## Stakeholder Reporting
### Executive Dashboard
**Key Metrics:**
- Overall tech debt health score (0-100)
- Debt trend direction (improving/declining)
- Cost of delayed fixes (in development days)
- High-risk debt items count
**Monthly Report Structure:**
1. **Executive Summary** (3 bullet points)
2. **Health Score Trend** (6-month view)
3. **Top 3 Risk Items** (business impact focus)
4. **Investment Recommendation** (resource allocation)
5. **Success Stories** (debt reduced last month)
### Engineering Team Dashboard
**Daily Metrics:**
- New debt items identified
- Debt items resolved
- Interest rate by team/component
- Debt hotspots (most problematic areas)
**Sprint Reviews:**
- Debt points completed vs. planned
- Velocity impact from debt work
- Newly discovered debt during feature work
- Team sentiment on code quality
### Product Manager Reports
**Feature Impact Analysis:**
- How debt affects feature development time
- Quality risk assessment for upcoming features
- Debt that blocks planned features
- Recommendations for feature sequence planning
**Customer Impact Translation:**
- Debt that affects performance
- Debt that increases bug rates
- Debt that limits feature flexibility
- Investment required to maintain current quality
→ See references/debt-frameworks.md for details
## Implementation Roadmap
@@ -572,4 +90,4 @@ Maintain healthy balance between new features and debt reduction:
**Problem**: Building complex debt management systems that nobody uses.
**Solution**: Start simple, iterate based on actual usage patterns.
Technical debt management is not just about writing better code - it's about creating sustainable development practices that balance short-term delivery pressure with long-term system health. Use these tools and frameworks to make informed decisions about when and how to invest in debt reduction.
Technical debt management is not just about writing better code - it's about creating sustainable development practices that balance short-term delivery pressure with long-term system health. Use these tools and frameworks to make informed decisions about when and how to invest in debt reduction.

486
engineering/tech-debt-tracker/references/debt-frameworks.md Normal file
View File

@@ -0,0 +1,486 @@
# tech-debt-tracker reference
## Technical Debt Classification Framework
### 1. Code Debt
Code-level issues that make the codebase harder to understand, modify, and maintain.
**Indicators:**
- Long functions (>50 lines for complex logic, >20 for simple operations)
- Deep nesting (>4 levels of indentation)
- High cyclomatic complexity (>10)
- Duplicate code patterns (>3 similar blocks)
- Missing or inadequate error handling
- Poor variable/function naming
- Magic numbers and hardcoded values
- Commented-out code blocks
**Impact:**
- Increased debugging time
- Higher defect rates
- Slower feature development
- Knowledge silos (only original author understands the code)
**Detection Methods:**
- AST parsing for structural analysis
- Pattern matching for common anti-patterns
- Complexity metrics calculation
- Duplicate code detection algorithms
### 2. Architecture Debt
High-level design decisions that seemed reasonable at the time but now limit scalability or maintainability.
**Indicators:**
- Monolithic components that should be modular
- Circular dependencies between modules
- Violation of separation of concerns
- Inconsistent data flow patterns
- Over-engineering or under-engineering for current scale
- Tightly coupled components
- Missing abstraction layers
**Impact:**
- Difficult to scale individual components
- Cascading changes required for simple modifications
- Testing becomes complex and brittle
- Onboarding new team members takes longer
**Detection Methods:**
- Dependency analysis
- Module coupling metrics
- Component size analysis
- Interface consistency checks
### 3. Test Debt
Inadequate or missing test coverage, poor test quality, and testing infrastructure issues.
**Indicators:**
- Low test coverage (<80% for critical paths)
- Missing unit tests for complex logic
- No integration tests for key workflows
- Flaky tests that pass/fail intermittently
- Slow test execution (>10 minutes for unit tests)
- Tests that don't test meaningful behavior
- Missing test data management strategy
**Impact:**
- Fear of refactoring ("don't touch it, it works")
- Regression bugs in production
- Slow feedback cycles during development
- Difficulty validating complex business logic
**Detection Methods:**
- Coverage report analysis
- Test execution time monitoring
- Test failure pattern analysis
- Test code quality assessment
### 4. Documentation Debt
Missing, outdated, or poor-quality documentation that makes the system harder to understand and maintain.
**Indicators:**
- Missing API documentation
- Outdated README files
- No architectural decision records (ADRs)
- Missing code comments for complex algorithms
- No onboarding documentation for new team members
- Inconsistent documentation formats
- Documentation that contradicts actual implementation
**Impact:**
- Increased onboarding time for new team members
- Knowledge loss when team members leave
- Miscommunication between teams
- Repeated questions in team channels
**Detection Methods:**
- Documentation coverage analysis
- Freshness checking (last modified dates)
- Link validation
- Comment density analysis
### 5. Dependency Debt
Issues related to external libraries, frameworks, and system dependencies.
**Indicators:**
- Outdated packages with known security vulnerabilities
- Dependencies with incompatible licenses
- Unused dependencies bloating the build
- Version conflicts between packages
- Deprecated APIs still in use
- Heavy dependencies for simple tasks
- Missing dependency pinning
**Impact:**
- Security vulnerabilities
- Build instability
- Longer build times
- Legal compliance issues
- Difficulty upgrading core frameworks
**Detection Methods:**
- Vulnerability scanning
- License compliance checking
- Usage analysis
- Version compatibility checking
### 6. Infrastructure Debt
Operations and deployment-related technical debt.
**Indicators:**
- Manual deployment processes
- Missing monitoring and alerting
- Inadequate logging
- No disaster recovery plan
- Inconsistent environments (dev/staging/prod)
- Missing CI/CD pipelines
- Infrastructure as code gaps
**Impact:**
- Deployment risks and downtime
- Difficult troubleshooting
- Inconsistent behavior across environments
- Manual work that should be automated
**Detection Methods:**
- Infrastructure audit checklists
- Configuration drift detection
- Monitoring coverage analysis
- Deployment process documentation review
## Severity Scoring Framework
Each piece of tech debt is scored on multiple dimensions to determine overall severity:
### Impact Assessment (1-10 scale)
**Development Velocity Impact**
- 1-2: Negligible impact on development speed
- 3-4: Minor slowdown, workarounds available
- 5-6: Moderate impact, affects some features
- 7-8: Significant slowdown, affects most work
- 9-10: Critical blocker, prevents new development
**Quality Impact**
- 1-2: No impact on defect rates
- 3-4: Minor increase in minor bugs
- 5-6: Moderate increase in defects
- 7-8: Regular production issues
- 9-10: Critical reliability problems
**Team Productivity Impact**
- 1-2: No impact on team morale or efficiency
- 3-4: Occasional frustration
- 5-6: Regular complaints from developers
- 7-8: Team actively avoiding the area
- 9-10: Causing developer turnover
**Business Impact**
- 1-2: No customer-facing impact
- 3-4: Minor UX degradation
- 5-6: Moderate performance impact
- 7-8: Customer complaints or churn
- 9-10: Revenue-impacting issues
### Effort Assessment
**Size (Story Points or Hours)**
- XS (1-4 hours): Simple refactor or documentation update
- S (1-2 days): Minor architectural change
- M (3-5 days): Moderate refactoring effort
- L (1-2 weeks): Major component restructuring
- XL (3+ weeks): System-wide architectural changes
**Risk Level**
- Low: Well-understood change with clear scope
- Medium: Some unknowns but manageable
- High: Significant unknowns, potential for scope creep
**Skill Requirements**
- Junior: Can be handled by any team member
- Mid: Requires experienced developer
- Senior: Needs architectural expertise
- Expert: Requires deep system knowledge
## Interest Rate Calculation
Technical debt accrues "interest" - the additional cost of leaving it unfixed. This interest rate helps prioritize which debt to pay down first.
### Interest Rate Formula
```
Interest Rate = (Impact Score × Frequency of Encounter) / Time Period
```
Where:
- **Impact Score**: Average severity score (1-10)
- **Frequency of Encounter**: How often developers interact with this code
- **Time Period**: Usually measured per sprint or month
### Cost of Delay Calculation
```
Cost of Delay = Interest Rate × Time Until Fix × Team Size Multiplier
```
### Example Calculation
**Scenario**: Legacy authentication module with poor error handling
- Impact Score: 7 (causes regular production issues)
- Frequency: 15 encounters per sprint (3 developers × 5 times each)
- Team Size: 8 developers
- Current sprint: 1, planned fix: sprint 4
```
Interest Rate = 7 × 15 = 105 points per sprint
Cost of Delay = 105 × 3 × 1.2 = 378 total cost points
```
This debt item should be prioritized over lower-cost items.
## Debt Inventory Management
### Data Structure
Each debt item is tracked with the following attributes:
```json
{
"id": "DEBT-2024-001",
"title": "Legacy user authentication module",
"category": "code",
"subcategory": "error_handling",
"location": "src/auth/legacy_auth.py:45-120",
"description": "Authentication error handling uses generic exceptions",
"impact": {
"velocity": 7,
"quality": 8,
"productivity": 6,
"business": 5
},
"effort": {
"size": "M",
"risk": "medium",
"skill_required": "mid"
},
"interest_rate": 105,
"cost_of_delay": 378,
"priority": "high",
"created_date": "2024-01-15",
"last_updated": "2024-01-20",
"assigned_to": null,
"status": "identified",
"tags": ["security", "user-experience", "maintainability"]
}
```
### Status Lifecycle
1. **Identified** - Debt detected but not yet analyzed
2. **Analyzed** - Impact and effort assessed
3. **Prioritized** - Added to backlog with priority
4. **Planned** - Assigned to specific sprint/release
5. **In Progress** - Actively being worked on
6. **Review** - Implementation complete, under review
7. **Done** - Debt resolved and verified
8. **Won't Fix** - Consciously decided not to address
## Prioritization Frameworks
### 1. Cost-of-Delay vs Effort Matrix
Plot debt items on a 2D matrix:
- X-axis: Effort (XS to XL)
- Y-axis: Cost of Delay (calculated value)
**Priority Quadrants:**
- High Cost, Low Effort: **Immediate** (quick wins)
- High Cost, High Effort: **Planned** (major initiatives)
- Low Cost, Low Effort: **Opportunistic** (during related work)
- Low Cost, High Effort: **Backlog** (consider for future)
### 2. Weighted Shortest Job First (WSJF)
```
WSJF Score = (Business Value + Time Criticality + Risk Reduction) / Effort
```
Where each component is scored 1-10:
- **Business Value**: Direct impact on customer value
- **Time Criticality**: How much value decreases over time
- **Risk Reduction**: How much risk is mitigated by fixing this debt
### 3. Technical Debt Quadrant
Based on Martin Fowler's framework:
**Quadrant 1: Reckless & Deliberate**
- "We don't have time for design"
- Highest priority for remediation
**Quadrant 2: Prudent & Deliberate**
- "We must ship now and deal with consequences"
- Schedule for near-term resolution
**Quadrant 3: Reckless & Inadvertent**
- "What's layering?"
- Focus on education and process improvement
**Quadrant 4: Prudent & Inadvertent**
- "Now we know how we should have done it"
- Normal part of learning, lowest priority
## Refactoring Strategies
### 1. Strangler Fig Pattern
Gradually replace old system by building new functionality around it.
**When to use:**
- Large, monolithic systems
- High-risk changes to critical paths
- Long-term architectural migrations
**Implementation:**
1. Identify boundaries for extraction
2. Create abstraction layer
3. Route new features to new implementation
4. Gradually migrate existing features
5. Remove old implementation
### 2. Branch by Abstraction
Create abstraction layer to allow parallel implementations.
**When to use:**
- Need to support old and new systems simultaneously
- High-risk changes with rollback requirements
- A/B testing infrastructure changes
**Implementation:**
1. Create abstraction interface
2. Implement abstraction for current system
3. Replace direct calls with abstraction calls
4. Implement new version behind same abstraction
5. Switch implementations via configuration
6. Remove old implementation
### 3. Feature Toggles
Use configuration flags to control code execution.
**When to use:**
- Gradual rollout of refactored components
- Risk mitigation during large changes
- Experimental refactoring approaches
**Implementation:**
1. Identify decision points in code
2. Add toggle checks at decision points
3. Implement both old and new paths
4. Test both paths thoroughly
5. Gradually move toggle to new implementation
6. Remove old path and toggle
### 4. Parallel Run
Run old and new implementations simultaneously to verify correctness.
**When to use:**
- Critical business logic changes
- Data processing pipeline changes
- Algorithm improvements
**Implementation:**
1. Implement new version alongside old
2. Run both versions with same inputs
3. Compare outputs and log discrepancies
4. Investigate and fix discrepancies
5. Build confidence through parallel execution
6. Switch to new implementation
7. Remove old implementation
## Sprint Allocation Recommendations
### Debt-to-Feature Ratio
Maintain healthy balance between new features and debt reduction:
**Team Velocity < 70% of capacity:**
- 60% tech debt, 40% features
- Focus on removing major blockers
**Team Velocity 70-85% of capacity:**
- 30% tech debt, 70% features
- Balanced maintenance approach
**Team Velocity > 85% of capacity:**
- 15% tech debt, 85% features
- Opportunistic debt reduction only
### Sprint Planning Integration
**Story Point Allocation:**
- Reserve 20% of sprint capacity for tech debt
- Prioritize debt items with highest interest rates
- Include "debt tax" in feature estimates when working in high-debt areas
**Debt Budget Tracking:**
- Track debt points completed per sprint
- Monitor debt interest rate trend
- Alert when debt accumulation exceeds team's paydown rate
### Quarterly Planning
**Debt Initiatives:**
- Identify 1-2 major debt themes per quarter
- Allocate dedicated sprints for large-scale refactoring
- Plan debt work around major feature releases
**Success Metrics:**
- Debt interest rate reduction
- Developer velocity improvements
- Defect rate reduction
- Code review cycle time improvement
## Stakeholder Reporting
### Executive Dashboard
**Key Metrics:**
- Overall tech debt health score (0-100)
- Debt trend direction (improving/declining)
- Cost of delayed fixes (in development days)
- High-risk debt items count
**Monthly Report Structure:**
1. **Executive Summary** (3 bullet points)
2. **Health Score Trend** (6-month view)
3. **Top 3 Risk Items** (business impact focus)
4. **Investment Recommendation** (resource allocation)
5. **Success Stories** (debt reduced last month)
### Engineering Team Dashboard
**Daily Metrics:**
- New debt items identified
- Debt items resolved
- Interest rate by team/component
- Debt hotspots (most problematic areas)
**Sprint Reviews:**
- Debt points completed vs. planned
- Velocity impact from debt work
- Newly discovered debt during feature work
- Team sentiment on code quality
### Product Manager Reports
**Feature Impact Analysis:**
- How debt affects feature development time
- Quality risk assessment for upcoming features
- Debt that blocks planned features
- Recommendations for feature sequence planning
**Customer Impact Translation:**
- Debt that affects performance
- Debt that increases bug rates
- Debt that limits feature flexibility
- Investment required to maintain current quality