feat(repo): Add warning budget and maintainer audit

Freeze the accepted validation warning count at 135 so repo-state and release-state checks fail if the warning baseline grows silently while legacy warnings remain intentionally preserved.

Add a read-only maintainer audit command plus regression tests so maintainers can inspect repo health quickly without mutating files.
This commit is contained in:
sickn33
2026-03-21 11:08:57 +01:00
parent 2463affbac
commit fc5b383f34
12 changed files with 408 additions and 9 deletions

View File

@@ -0,0 +1,3 @@
{
"maxWarnings": 135
}

View File

@@ -0,0 +1,59 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from _project_paths import find_repo_root
from update_readme import configure_utf8_output
from validate_skills import collect_validation_results
def load_warning_budget(base_dir: str | Path) -> int:
root = Path(base_dir)
budget_path = root / "tools" / "config" / "validation-budget.json"
payload = json.loads(budget_path.read_text(encoding="utf-8"))
max_warnings = payload.get("maxWarnings")
if not isinstance(max_warnings, int) or max_warnings < 0:
raise ValueError("tools/config/validation-budget.json must define a non-negative integer maxWarnings")
return max_warnings
def check_warning_budget(base_dir: str | Path) -> dict[str, int | bool]:
root = Path(base_dir)
skills_dir = root / "skills"
actual = len(collect_validation_results(str(skills_dir))["warnings"])
maximum = load_warning_budget(root)
return {
"actual": actual,
"max": maximum,
"within_budget": actual <= maximum,
}
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Fail if validation warnings exceed the repository budget.")
parser.add_argument("--json", action="store_true", help="Print the budget summary as JSON.")
return parser.parse_args()
def main() -> int:
args = parse_args()
root = find_repo_root(__file__)
summary = check_warning_budget(root)
if args.json:
print(json.dumps(summary, indent=2))
elif summary["within_budget"]:
print(f"✅ Validation warnings within budget: {summary['actual']}/{summary['max']}")
else:
print(f"❌ Validation warnings exceed budget: {summary['actual']}/{summary['max']}")
return 0 if summary["within_budget"] else 1
if __name__ == "__main__":
configure_utf8_output()
sys.exit(main())

View File

@@ -0,0 +1,103 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from pathlib import Path
from _project_paths import find_repo_root
from audit_consistency import find_local_consistency_issues
from check_validation_warning_budget import check_warning_budget
from update_readme import configure_utf8_output, load_metadata
def get_git_status(base_dir: str | Path) -> list[str]:
result = subprocess.run(
["git", "status", "--short"],
cwd=str(base_dir),
check=True,
capture_output=True,
text=True,
)
return [line for line in result.stdout.splitlines() if line.strip()]
def build_audit_summary(
base_dir: str | Path,
warning_budget_checker=check_warning_budget,
consistency_finder=find_local_consistency_issues,
git_status_resolver=get_git_status,
) -> dict:
root = Path(base_dir)
metadata = load_metadata(str(root))
consistency_issues = consistency_finder(root)
git_status = git_status_resolver(root)
return {
"repo": metadata["repo"],
"version": metadata["version"],
"total_skills": metadata["total_skills"],
"total_skills_label": metadata["total_skills_label"],
"warning_budget": warning_budget_checker(root),
"consistency_issues": consistency_issues,
"git": {
"clean": len(git_status) == 0,
"changed_files": git_status,
},
}
def print_human_summary(summary: dict) -> None:
warning_budget = summary["warning_budget"]
warning_status = "within budget" if warning_budget["within_budget"] else "over budget"
consistency_status = "clean" if not summary["consistency_issues"] else f"{len(summary['consistency_issues'])} issue(s)"
git_status = "clean" if summary["git"]["clean"] else f"{len(summary['git']['changed_files'])} changed file(s)"
print(f"Repository: {summary['repo']}")
print(f"Version: {summary['version']}")
print(f"Skills: {summary['total_skills_label']}")
print(f"Warning budget: {warning_status} ({warning_budget['actual']}/{warning_budget['max']})")
print(f"Consistency: {consistency_status}")
print(f"Git working tree: {git_status}")
if summary["consistency_issues"]:
print("\nConsistency issues:")
for issue in summary["consistency_issues"]:
print(f"- {issue}")
if summary["git"]["changed_files"]:
print("\nChanged files:")
for line in summary["git"]["changed_files"]:
print(f"- {line}")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Print a maintainer-friendly repository health summary.")
parser.add_argument("--json", action="store_true", help="Print the full audit summary as JSON.")
return parser.parse_args()
def main() -> int:
args = parse_args()
root = find_repo_root(__file__)
summary = build_audit_summary(root)
if args.json:
print(json.dumps(summary, indent=2))
else:
print_human_summary(summary)
if not summary["warning_budget"]["within_budget"]:
return 1
if summary["consistency_issues"]:
return 1
if not summary["git"]["clean"]:
return 1
return 0
if __name__ == "__main__":
configure_utf8_output()
sys.exit(main())

View File

@@ -18,6 +18,14 @@ assert.ok(
packageJson.scripts["sync:release-state"],
"package.json should expose a deterministic release-state sync command",
);
assert.ok(
packageJson.scripts["check:warning-budget"],
"package.json should expose a warning-budget guardrail command",
);
assert.ok(
packageJson.scripts["audit:maintainer"],
"package.json should expose a maintainer audit command",
);
assert.ok(
packageJson.scripts["sync:web-assets"],
"package.json should expose a web-asset sync command for tracked web artifacts",
@@ -27,11 +35,21 @@ assert.match(
/sync:web-assets/,
"sync:release-state should refresh tracked web assets before auditing release drift",
);
assert.match(
packageJson.scripts["sync:release-state"],
/check:warning-budget/,
"sync:release-state should enforce the frozen validation warning budget",
);
assert.match(
packageJson.scripts["sync:repo-state"],
/sync:web-assets/,
"sync:repo-state should refresh tracked web assets before maintainer audits",
);
assert.match(
packageJson.scripts["sync:repo-state"],
/check:warning-budget/,
"sync:repo-state should enforce the frozen validation warning budget",
);
for (const filePath of [
"apps/web-app/public/sitemap.xml",

View File

@@ -31,6 +31,8 @@ const LOCAL_TEST_COMMANDS = [
[path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_sync_microsoft_skills_security.py")],
[path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_sync_repo_metadata.py")],
[path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_sync_contributors.py")],
[path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_validation_warning_budget.py")],
[path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_maintainer_audit.py")],
[path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_validate_skills_headings.py")],
];
const NETWORK_TEST_COMMANDS = [

View File

@@ -0,0 +1,101 @@
import importlib.util
import json
import sys
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
TOOLS_SCRIPTS_DIR = REPO_ROOT / "tools" / "scripts"
if str(TOOLS_SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(TOOLS_SCRIPTS_DIR))
def load_module(relative_path: str, module_name: str):
module_path = REPO_ROOT / relative_path
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
maintainer_audit = load_module(
"tools/scripts/maintainer_audit.py",
"maintainer_audit_test",
)
class MaintainerAuditTests(unittest.TestCase):
def test_build_audit_summary_reports_clean_state(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
(root / "README.md").write_text(
"""<!-- registry-sync: version=8.4.0; skills=1; stars=26132; updated_at=2026-03-21T00:00:00+00:00 -->
# Test Repo
""",
encoding="utf-8",
)
(root / "package.json").write_text(
json.dumps(
{
"name": "antigravity-awesome-skills",
"version": "8.4.0",
"description": "1+ agentic skills for Claude Code, Gemini CLI, Cursor, Antigravity & more. Installer CLI.",
}
),
encoding="utf-8",
)
(root / "skills_index.json").write_text(json.dumps([{}]), encoding="utf-8")
summary = maintainer_audit.build_audit_summary(
root,
warning_budget_checker=lambda _base_dir: {"actual": 135, "max": 135, "within_budget": True},
consistency_finder=lambda _base_dir: [],
git_status_resolver=lambda _base_dir: [],
)
self.assertEqual(summary["version"], "8.4.0")
self.assertEqual(summary["total_skills"], 1)
self.assertTrue(summary["warning_budget"]["within_budget"])
self.assertEqual(summary["consistency_issues"], [])
self.assertTrue(summary["git"]["clean"])
def test_build_audit_summary_reports_drift(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
(root / "README.md").write_text(
"""<!-- registry-sync: version=8.4.0; skills=1; stars=26132; updated_at=2026-03-21T00:00:00+00:00 -->
# Test Repo
""",
encoding="utf-8",
)
(root / "package.json").write_text(
json.dumps(
{
"name": "antigravity-awesome-skills",
"version": "8.4.0",
"description": "1+ agentic skills for Claude Code, Gemini CLI, Cursor, Antigravity & more. Installer CLI.",
}
),
encoding="utf-8",
)
(root / "skills_index.json").write_text(json.dumps([{}]), encoding="utf-8")
summary = maintainer_audit.build_audit_summary(
root,
warning_budget_checker=lambda _base_dir: {"actual": 140, "max": 135, "within_budget": False},
consistency_finder=lambda _base_dir: ["README drift"],
git_status_resolver=lambda _base_dir: [" M README.md"],
)
self.assertFalse(summary["warning_budget"]["within_budget"])
self.assertEqual(summary["consistency_issues"], ["README drift"])
self.assertFalse(summary["git"]["clean"])
self.assertEqual(summary["git"]["changed_files"], [" M README.md"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,90 @@
import importlib.util
import json
import sys
import tempfile
import unittest
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[3]
TOOLS_SCRIPTS_DIR = REPO_ROOT / "tools" / "scripts"
if str(TOOLS_SCRIPTS_DIR) not in sys.path:
sys.path.insert(0, str(TOOLS_SCRIPTS_DIR))
def load_module(relative_path: str, module_name: str):
module_path = REPO_ROOT / relative_path
spec = importlib.util.spec_from_file_location(module_name, module_path)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
warning_budget = load_module(
"tools/scripts/check_validation_warning_budget.py",
"check_validation_warning_budget_test",
)
class ValidationWarningBudgetTests(unittest.TestCase):
def test_warning_budget_passes_when_actual_matches_budget(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
(root / "tools" / "config").mkdir(parents=True)
(root / "skills" / "example-skill").mkdir(parents=True)
(root / "tools" / "config" / "validation-budget.json").write_text(
json.dumps({"maxWarnings": 1}),
encoding="utf-8",
)
(root / "skills" / "example-skill" / "SKILL.md").write_text(
"""---
name: example-skill
description: Example skill
risk: safe
source: community
---
# Example Skill
""",
encoding="utf-8",
)
summary = warning_budget.check_warning_budget(root)
self.assertEqual(summary["actual"], 1)
self.assertEqual(summary["max"], 1)
self.assertTrue(summary["within_budget"])
def test_warning_budget_fails_when_actual_exceeds_budget(self):
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
(root / "tools" / "config").mkdir(parents=True)
(root / "skills" / "example-skill").mkdir(parents=True)
(root / "tools" / "config" / "validation-budget.json").write_text(
json.dumps({"maxWarnings": 0}),
encoding="utf-8",
)
(root / "skills" / "example-skill" / "SKILL.md").write_text(
"""---
name: example-skill
description: Example skill
risk: safe
source: community
---
# Example Skill
""",
encoding="utf-8",
)
summary = warning_budget.check_warning_budget(root)
self.assertEqual(summary["actual"], 1)
self.assertEqual(summary["max"], 0)
self.assertFalse(summary["within_budget"])
if __name__ == "__main__":
unittest.main()

View File

@@ -77,16 +77,11 @@ def parse_frontmatter(content, rel_path=None):
except yaml.YAMLError as e:
return None, [f"YAML Syntax Error: {e}"]
def validate_skills(skills_dir, strict_mode=False):
configure_utf8_output()
print(f"🔍 Validating skills in: {skills_dir}")
print(f"⚙️ Mode: {'STRICT (CI)' if strict_mode else 'Standard (Dev)'}")
def collect_validation_results(skills_dir, strict_mode=False):
errors = []
warnings = []
skill_count = 0
# Pre-compiled regex
security_disclaimer_pattern = re.compile(r"AUTHORIZED USE ONLY", re.IGNORECASE)
@@ -188,6 +183,25 @@ def validate_skills(skills_dir, strict_mode=False):
if not os.path.exists(target_path):
errors.append(f"{rel_path}: Dangling link detected. Path '{link_clean}' (from '...({link})') does not exist locally.")
return {
"skill_count": skill_count,
"warnings": warnings,
"errors": errors,
"strict_mode": strict_mode,
}
def validate_skills(skills_dir, strict_mode=False):
configure_utf8_output()
print(f"🔍 Validating skills in: {skills_dir}")
print(f"⚙️ Mode: {'STRICT (CI)' if strict_mode else 'Standard (Dev)'}")
results = collect_validation_results(skills_dir, strict_mode=strict_mode)
warnings = results["warnings"]
errors = results["errors"]
skill_count = results["skill_count"]
# Reporting
print(f"\n📊 Checked {skill_count} skills.")