From fc5b383f34cbe061d0bdd461d5e3e9c29be4660f Mon Sep 17 00:00:00 2001 From: sickn33 Date: Sat, 21 Mar 2026 11:08:57 +0100 Subject: [PATCH] feat(repo): Add warning budget and maintainer audit Freeze the accepted validation warning count at 135 so repo-state and release-state checks fail if the warning baseline grows silently while legacy warnings remain intentionally preserved. Add a read-only maintainer audit command plus regression tests so maintainers can inspect repo health quickly without mutating files. --- .github/MAINTENANCE.md | 5 + CHANGELOG.md | 1 + package.json | 6 +- tools/config/validation-budget.json | 3 + .../check_validation_warning_budget.py | 59 ++++++++++ tools/scripts/maintainer_audit.py | 103 ++++++++++++++++++ .../tests/automation_workflows.test.js | 18 +++ tools/scripts/tests/run-test-suite.js | 2 + tools/scripts/tests/test_maintainer_audit.py | 101 +++++++++++++++++ .../tests/test_validation_warning_budget.py | 90 +++++++++++++++ tools/scripts/validate_skills.py | 28 +++-- walkthrough.md | 1 + 12 files changed, 408 insertions(+), 9 deletions(-) create mode 100644 tools/config/validation-budget.json create mode 100644 tools/scripts/check_validation_warning_budget.py create mode 100644 tools/scripts/maintainer_audit.py create mode 100644 tools/scripts/tests/test_maintainer_audit.py create mode 100644 tools/scripts/tests/test_validation_warning_budget.py diff --git a/.github/MAINTENANCE.md b/.github/MAINTENANCE.md index aaccf57f..aed8e81d 100644 --- a/.github/MAINTENANCE.md +++ b/.github/MAINTENANCE.md @@ -86,6 +86,7 @@ Before ANY commit that adds/modifies skills, run the chain: ``` This wraps `chain + catalog + sync:web-assets + sync:contributors + audit:consistency` for a full local repo-state refresh. The scheduled GitHub Actions workflow `Repo Hygiene` runs this same sweep weekly to catch slow drift on `main`. + It also enforces the frozen validation warning budget, so new warnings do not creep in silently while the legacy `135` known warnings remain accepted. When you need the live GitHub repo metadata updated too, run: @@ -93,6 +94,10 @@ Before ANY commit that adds/modifies skills, run the chain: npm run sync:github-about npm run audit:consistency:github ``` + For a read-only summary of current repo health, run: + ```bash + npm run audit:maintainer + ``` 4. **COMMIT GENERATED FILES**: ```bash diff --git a/CHANGELOG.md b/CHANGELOG.md index ded68d91..763dfef4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Automated metadata propagation for curated docs and package copy so `npm run chain` now keeps README, package description, and the main count-sensitive user/maintainer docs aligned when the skill catalog changes. - Added an explicit `sync:github-about` automation path so GitHub About description, homepage, and topics can be refreshed from the same metadata source instead of being updated manually. - Added contributor sync plus repo-state audit automation: `sync:contributors`, `sync:web-assets`, `check:stale-claims`, `audit:consistency`, `sync:release-state`, and `sync:repo-state` now cover contributor acknowledgements, tracked web artifacts, stale count/version drift, deterministic release-state verification, and end-to-end maintainer sanity checks. Main CI, the weekly `Repo Hygiene` workflow, and the npm publish workflow now reuse those paths instead of maintaining separate ad hoc sync steps. +- Added a frozen validation warning budget (`135`) plus a read-only maintainer audit command so the accepted legacy warnings stay stable while maintainers can get a one-command repo health summary without mutating files. ## [8.4.0] - 2026-03-20 - "Discovery, Metadata, and Release Hardening" diff --git a/package.json b/package.json index f48af0ff..42ad3237 100644 --- a/package.json +++ b/package.json @@ -21,14 +21,16 @@ "sync:web-assets": "npm run app:setup && cd apps/web-app && npm run generate:sitemap", "chain": "npm run validate && npm run index && npm run sync:metadata", "sync:all": "npm run chain", - "sync:release-state": "npm run chain && npm run catalog && npm run sync:web-assets && npm run audit:consistency", - "sync:repo-state": "npm run chain && npm run catalog && npm run sync:web-assets && npm run sync:contributors && npm run audit:consistency", + "sync:release-state": "npm run chain && npm run catalog && npm run sync:web-assets && npm run audit:consistency && npm run check:warning-budget", + "sync:repo-state": "npm run chain && npm run catalog && npm run sync:web-assets && npm run sync:contributors && npm run audit:consistency && npm run check:warning-budget", "sync:repo-state:full": "npm run sync:repo-state && npm run sync:github-about && npm run audit:consistency:github", "catalog": "node tools/scripts/build-catalog.js", "build": "npm run chain && npm run catalog", "check:stale-claims": "node tools/scripts/run-python.js tools/scripts/check_stale_claims.py", + "check:warning-budget": "node tools/scripts/run-python.js tools/scripts/check_validation_warning_budget.py", "audit:consistency": "node tools/scripts/run-python.js tools/scripts/audit_consistency.py", "audit:consistency:github": "node tools/scripts/run-python.js tools/scripts/audit_consistency.py --check-github-about", + "audit:maintainer": "node tools/scripts/run-python.js tools/scripts/maintainer_audit.py", "security:docs": "node tools/scripts/tests/docs_security_content.test.js", "pr:preflight": "node tools/scripts/pr_preflight.js", "release:preflight": "node tools/scripts/release_workflow.js preflight", diff --git a/tools/config/validation-budget.json b/tools/config/validation-budget.json new file mode 100644 index 00000000..68d85a91 --- /dev/null +++ b/tools/config/validation-budget.json @@ -0,0 +1,3 @@ +{ + "maxWarnings": 135 +} diff --git a/tools/scripts/check_validation_warning_budget.py b/tools/scripts/check_validation_warning_budget.py new file mode 100644 index 00000000..8f299bd6 --- /dev/null +++ b/tools/scripts/check_validation_warning_budget.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from _project_paths import find_repo_root +from update_readme import configure_utf8_output +from validate_skills import collect_validation_results + + +def load_warning_budget(base_dir: str | Path) -> int: + root = Path(base_dir) + budget_path = root / "tools" / "config" / "validation-budget.json" + payload = json.loads(budget_path.read_text(encoding="utf-8")) + max_warnings = payload.get("maxWarnings") + if not isinstance(max_warnings, int) or max_warnings < 0: + raise ValueError("tools/config/validation-budget.json must define a non-negative integer maxWarnings") + return max_warnings + + +def check_warning_budget(base_dir: str | Path) -> dict[str, int | bool]: + root = Path(base_dir) + skills_dir = root / "skills" + actual = len(collect_validation_results(str(skills_dir))["warnings"]) + maximum = load_warning_budget(root) + return { + "actual": actual, + "max": maximum, + "within_budget": actual <= maximum, + } + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Fail if validation warnings exceed the repository budget.") + parser.add_argument("--json", action="store_true", help="Print the budget summary as JSON.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + root = find_repo_root(__file__) + summary = check_warning_budget(root) + + if args.json: + print(json.dumps(summary, indent=2)) + elif summary["within_budget"]: + print(f"āœ… Validation warnings within budget: {summary['actual']}/{summary['max']}") + else: + print(f"āŒ Validation warnings exceed budget: {summary['actual']}/{summary['max']}") + + return 0 if summary["within_budget"] else 1 + + +if __name__ == "__main__": + configure_utf8_output() + sys.exit(main()) diff --git a/tools/scripts/maintainer_audit.py b/tools/scripts/maintainer_audit.py new file mode 100644 index 00000000..591267ce --- /dev/null +++ b/tools/scripts/maintainer_audit.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +from _project_paths import find_repo_root +from audit_consistency import find_local_consistency_issues +from check_validation_warning_budget import check_warning_budget +from update_readme import configure_utf8_output, load_metadata + + +def get_git_status(base_dir: str | Path) -> list[str]: + result = subprocess.run( + ["git", "status", "--short"], + cwd=str(base_dir), + check=True, + capture_output=True, + text=True, + ) + return [line for line in result.stdout.splitlines() if line.strip()] + + +def build_audit_summary( + base_dir: str | Path, + warning_budget_checker=check_warning_budget, + consistency_finder=find_local_consistency_issues, + git_status_resolver=get_git_status, +) -> dict: + root = Path(base_dir) + metadata = load_metadata(str(root)) + consistency_issues = consistency_finder(root) + git_status = git_status_resolver(root) + + return { + "repo": metadata["repo"], + "version": metadata["version"], + "total_skills": metadata["total_skills"], + "total_skills_label": metadata["total_skills_label"], + "warning_budget": warning_budget_checker(root), + "consistency_issues": consistency_issues, + "git": { + "clean": len(git_status) == 0, + "changed_files": git_status, + }, + } + + +def print_human_summary(summary: dict) -> None: + warning_budget = summary["warning_budget"] + warning_status = "within budget" if warning_budget["within_budget"] else "over budget" + consistency_status = "clean" if not summary["consistency_issues"] else f"{len(summary['consistency_issues'])} issue(s)" + git_status = "clean" if summary["git"]["clean"] else f"{len(summary['git']['changed_files'])} changed file(s)" + + print(f"Repository: {summary['repo']}") + print(f"Version: {summary['version']}") + print(f"Skills: {summary['total_skills_label']}") + print(f"Warning budget: {warning_status} ({warning_budget['actual']}/{warning_budget['max']})") + print(f"Consistency: {consistency_status}") + print(f"Git working tree: {git_status}") + + if summary["consistency_issues"]: + print("\nConsistency issues:") + for issue in summary["consistency_issues"]: + print(f"- {issue}") + + if summary["git"]["changed_files"]: + print("\nChanged files:") + for line in summary["git"]["changed_files"]: + print(f"- {line}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Print a maintainer-friendly repository health summary.") + parser.add_argument("--json", action="store_true", help="Print the full audit summary as JSON.") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + root = find_repo_root(__file__) + summary = build_audit_summary(root) + + if args.json: + print(json.dumps(summary, indent=2)) + else: + print_human_summary(summary) + + if not summary["warning_budget"]["within_budget"]: + return 1 + if summary["consistency_issues"]: + return 1 + if not summary["git"]["clean"]: + return 1 + return 0 + + +if __name__ == "__main__": + configure_utf8_output() + sys.exit(main()) diff --git a/tools/scripts/tests/automation_workflows.test.js b/tools/scripts/tests/automation_workflows.test.js index ea5b2013..77f2015b 100644 --- a/tools/scripts/tests/automation_workflows.test.js +++ b/tools/scripts/tests/automation_workflows.test.js @@ -18,6 +18,14 @@ assert.ok( packageJson.scripts["sync:release-state"], "package.json should expose a deterministic release-state sync command", ); +assert.ok( + packageJson.scripts["check:warning-budget"], + "package.json should expose a warning-budget guardrail command", +); +assert.ok( + packageJson.scripts["audit:maintainer"], + "package.json should expose a maintainer audit command", +); assert.ok( packageJson.scripts["sync:web-assets"], "package.json should expose a web-asset sync command for tracked web artifacts", @@ -27,11 +35,21 @@ assert.match( /sync:web-assets/, "sync:release-state should refresh tracked web assets before auditing release drift", ); +assert.match( + packageJson.scripts["sync:release-state"], + /check:warning-budget/, + "sync:release-state should enforce the frozen validation warning budget", +); assert.match( packageJson.scripts["sync:repo-state"], /sync:web-assets/, "sync:repo-state should refresh tracked web assets before maintainer audits", ); +assert.match( + packageJson.scripts["sync:repo-state"], + /check:warning-budget/, + "sync:repo-state should enforce the frozen validation warning budget", +); for (const filePath of [ "apps/web-app/public/sitemap.xml", diff --git a/tools/scripts/tests/run-test-suite.js b/tools/scripts/tests/run-test-suite.js index cff5e0e9..84003f74 100644 --- a/tools/scripts/tests/run-test-suite.js +++ b/tools/scripts/tests/run-test-suite.js @@ -31,6 +31,8 @@ const LOCAL_TEST_COMMANDS = [ [path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_sync_microsoft_skills_security.py")], [path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_sync_repo_metadata.py")], [path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_sync_contributors.py")], + [path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_validation_warning_budget.py")], + [path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_maintainer_audit.py")], [path.join(TOOL_SCRIPTS, "run-python.js"), path.join(TOOL_TESTS, "test_validate_skills_headings.py")], ]; const NETWORK_TEST_COMMANDS = [ diff --git a/tools/scripts/tests/test_maintainer_audit.py b/tools/scripts/tests/test_maintainer_audit.py new file mode 100644 index 00000000..6f8a3e6a --- /dev/null +++ b/tools/scripts/tests/test_maintainer_audit.py @@ -0,0 +1,101 @@ +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[3] +TOOLS_SCRIPTS_DIR = REPO_ROOT / "tools" / "scripts" +if str(TOOLS_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_SCRIPTS_DIR)) + + +def load_module(relative_path: str, module_name: str): + module_path = REPO_ROOT / relative_path + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +maintainer_audit = load_module( + "tools/scripts/maintainer_audit.py", + "maintainer_audit_test", +) + + +class MaintainerAuditTests(unittest.TestCase): + def test_build_audit_summary_reports_clean_state(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "README.md").write_text( + """ +# Test Repo +""", + encoding="utf-8", + ) + (root / "package.json").write_text( + json.dumps( + { + "name": "antigravity-awesome-skills", + "version": "8.4.0", + "description": "1+ agentic skills for Claude Code, Gemini CLI, Cursor, Antigravity & more. Installer CLI.", + } + ), + encoding="utf-8", + ) + (root / "skills_index.json").write_text(json.dumps([{}]), encoding="utf-8") + + summary = maintainer_audit.build_audit_summary( + root, + warning_budget_checker=lambda _base_dir: {"actual": 135, "max": 135, "within_budget": True}, + consistency_finder=lambda _base_dir: [], + git_status_resolver=lambda _base_dir: [], + ) + + self.assertEqual(summary["version"], "8.4.0") + self.assertEqual(summary["total_skills"], 1) + self.assertTrue(summary["warning_budget"]["within_budget"]) + self.assertEqual(summary["consistency_issues"], []) + self.assertTrue(summary["git"]["clean"]) + + def test_build_audit_summary_reports_drift(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "README.md").write_text( + """ +# Test Repo +""", + encoding="utf-8", + ) + (root / "package.json").write_text( + json.dumps( + { + "name": "antigravity-awesome-skills", + "version": "8.4.0", + "description": "1+ agentic skills for Claude Code, Gemini CLI, Cursor, Antigravity & more. Installer CLI.", + } + ), + encoding="utf-8", + ) + (root / "skills_index.json").write_text(json.dumps([{}]), encoding="utf-8") + + summary = maintainer_audit.build_audit_summary( + root, + warning_budget_checker=lambda _base_dir: {"actual": 140, "max": 135, "within_budget": False}, + consistency_finder=lambda _base_dir: ["README drift"], + git_status_resolver=lambda _base_dir: [" M README.md"], + ) + + self.assertFalse(summary["warning_budget"]["within_budget"]) + self.assertEqual(summary["consistency_issues"], ["README drift"]) + self.assertFalse(summary["git"]["clean"]) + self.assertEqual(summary["git"]["changed_files"], [" M README.md"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/scripts/tests/test_validation_warning_budget.py b/tools/scripts/tests/test_validation_warning_budget.py new file mode 100644 index 00000000..24caf61d --- /dev/null +++ b/tools/scripts/tests/test_validation_warning_budget.py @@ -0,0 +1,90 @@ +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[3] +TOOLS_SCRIPTS_DIR = REPO_ROOT / "tools" / "scripts" +if str(TOOLS_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(TOOLS_SCRIPTS_DIR)) + + +def load_module(relative_path: str, module_name: str): + module_path = REPO_ROOT / relative_path + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +warning_budget = load_module( + "tools/scripts/check_validation_warning_budget.py", + "check_validation_warning_budget_test", +) + + +class ValidationWarningBudgetTests(unittest.TestCase): + def test_warning_budget_passes_when_actual_matches_budget(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "tools" / "config").mkdir(parents=True) + (root / "skills" / "example-skill").mkdir(parents=True) + (root / "tools" / "config" / "validation-budget.json").write_text( + json.dumps({"maxWarnings": 1}), + encoding="utf-8", + ) + (root / "skills" / "example-skill" / "SKILL.md").write_text( + """--- +name: example-skill +description: Example skill +risk: safe +source: community +--- + +# Example Skill +""", + encoding="utf-8", + ) + + summary = warning_budget.check_warning_budget(root) + + self.assertEqual(summary["actual"], 1) + self.assertEqual(summary["max"], 1) + self.assertTrue(summary["within_budget"]) + + def test_warning_budget_fails_when_actual_exceeds_budget(self): + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + (root / "tools" / "config").mkdir(parents=True) + (root / "skills" / "example-skill").mkdir(parents=True) + (root / "tools" / "config" / "validation-budget.json").write_text( + json.dumps({"maxWarnings": 0}), + encoding="utf-8", + ) + (root / "skills" / "example-skill" / "SKILL.md").write_text( + """--- +name: example-skill +description: Example skill +risk: safe +source: community +--- + +# Example Skill +""", + encoding="utf-8", + ) + + summary = warning_budget.check_warning_budget(root) + + self.assertEqual(summary["actual"], 1) + self.assertEqual(summary["max"], 0) + self.assertFalse(summary["within_budget"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/scripts/validate_skills.py b/tools/scripts/validate_skills.py index 69f91555..e4304d4e 100644 --- a/tools/scripts/validate_skills.py +++ b/tools/scripts/validate_skills.py @@ -77,16 +77,11 @@ def parse_frontmatter(content, rel_path=None): except yaml.YAMLError as e: return None, [f"YAML Syntax Error: {e}"] -def validate_skills(skills_dir, strict_mode=False): - configure_utf8_output() - - print(f"šŸ” Validating skills in: {skills_dir}") - print(f"āš™ļø Mode: {'STRICT (CI)' if strict_mode else 'Standard (Dev)'}") - +def collect_validation_results(skills_dir, strict_mode=False): errors = [] warnings = [] skill_count = 0 - + # Pre-compiled regex security_disclaimer_pattern = re.compile(r"AUTHORIZED USE ONLY", re.IGNORECASE) @@ -188,6 +183,25 @@ def validate_skills(skills_dir, strict_mode=False): if not os.path.exists(target_path): errors.append(f"āŒ {rel_path}: Dangling link detected. Path '{link_clean}' (from '...({link})') does not exist locally.") + return { + "skill_count": skill_count, + "warnings": warnings, + "errors": errors, + "strict_mode": strict_mode, + } + + +def validate_skills(skills_dir, strict_mode=False): + configure_utf8_output() + + print(f"šŸ” Validating skills in: {skills_dir}") + print(f"āš™ļø Mode: {'STRICT (CI)' if strict_mode else 'Standard (Dev)'}") + + results = collect_validation_results(skills_dir, strict_mode=strict_mode) + warnings = results["warnings"] + errors = results["errors"] + skill_count = results["skill_count"] + # Reporting print(f"\nšŸ“Š Checked {skill_count} skills.") diff --git a/walkthrough.md b/walkthrough.md index 289f6816..180db275 100644 --- a/walkthrough.md +++ b/walkthrough.md @@ -129,3 +129,4 @@ - Added a remote GitHub About sync path (`npm run sync:github-about`) backed by `gh repo edit` + `gh api .../topics` so the public repository metadata can be refreshed from the same source of truth on demand. - Added maintainer automation for repo-state hygiene: `sync:contributors` updates the README contributor list from GitHub contributors, `check:stale-claims`/`audit:consistency` catch drift in count-sensitive docs, and `sync:repo-state` now chains the local maintainer sweep into a single command. - Hardened automation surfaces beyond the local CLI: `main` CI now runs the unified repo-state sync, tracked web artifacts are refreshed through `sync:web-assets`, release verification now uses a deterministic `sync:release-state` path plus `npm pack --dry-run`, the npm publish workflow reruns those checks before publishing, and a weekly `Repo Hygiene` GitHub Actions workflow now sweeps slow drift on `main`. +- Added two maintainer niceties on top of the hardening work: `check:warning-budget` freezes the accepted `135` validation warnings so they cannot silently grow, and `audit:maintainer` prints a read-only health snapshot of warning budget, consistency drift, and git cleanliness.