feat: add prompt injection check workflow for content security (#324)
New bundled workflow `prompt-injection-check` scans scraped content for prompt injection patterns (role assumption, instruction overrides, delimiter injection, hidden instructions, encoded payloads) using AI. Flags suspicious content without removing it — preserves documentation accuracy while warning about adversarial content. Added as first stage in both `default` and `security-focus` workflows so it runs automatically with --enhance-level >= 1. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
94
tests/test_workflow_prompt_injection.py
Normal file
94
tests/test_workflow_prompt_injection.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""Tests for prompt injection check workflow (#324).
|
||||
|
||||
Validates that:
|
||||
- prompt-injection-check.yaml is a valid bundled workflow
|
||||
- default.yaml includes injection_scan as its first stage
|
||||
- security-focus.yaml includes injection_scan as its first stage
|
||||
- The workflow YAML is structurally correct
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
def _load_bundled_yaml(name: str) -> dict:
|
||||
"""Load a bundled workflow YAML by name."""
|
||||
from importlib.resources import files as importlib_files
|
||||
|
||||
for suffix in (".yaml", ".yml"):
|
||||
try:
|
||||
ref = importlib_files("skill_seekers.workflows").joinpath(name + suffix)
|
||||
return yaml.safe_load(ref.read_text(encoding="utf-8"))
|
||||
except (FileNotFoundError, TypeError, ModuleNotFoundError):
|
||||
continue
|
||||
raise FileNotFoundError(f"Bundled workflow '{name}' not found")
|
||||
|
||||
|
||||
class TestPromptInjectionCheckWorkflow:
|
||||
"""Validate the standalone prompt-injection-check workflow."""
|
||||
|
||||
def test_workflow_loads(self):
|
||||
data = _load_bundled_yaml("prompt-injection-check")
|
||||
assert data["name"] == "prompt-injection-check"
|
||||
|
||||
def test_has_stages(self):
|
||||
data = _load_bundled_yaml("prompt-injection-check")
|
||||
assert "stages" in data
|
||||
assert len(data["stages"]) >= 1
|
||||
|
||||
def test_injection_scan_stage_present(self):
|
||||
data = _load_bundled_yaml("prompt-injection-check")
|
||||
stage_names = [s["name"] for s in data["stages"]]
|
||||
assert "injection_scan" in stage_names
|
||||
|
||||
def test_injection_scan_has_prompt(self):
|
||||
data = _load_bundled_yaml("prompt-injection-check")
|
||||
scan_stage = next(s for s in data["stages"] if s["name"] == "injection_scan")
|
||||
assert scan_stage.get("prompt")
|
||||
assert "prompt injection" in scan_stage["prompt"].lower()
|
||||
|
||||
def test_injection_scan_targets_all(self):
|
||||
data = _load_bundled_yaml("prompt-injection-check")
|
||||
scan_stage = next(s for s in data["stages"] if s["name"] == "injection_scan")
|
||||
assert scan_stage["target"] == "all"
|
||||
|
||||
def test_applies_to_all_source_types(self):
|
||||
data = _load_bundled_yaml("prompt-injection-check")
|
||||
applies = data.get("applies_to", [])
|
||||
assert "doc_scraping" in applies
|
||||
assert "github_analysis" in applies
|
||||
assert "codebase_analysis" in applies
|
||||
|
||||
def test_post_process_metadata(self):
|
||||
data = _load_bundled_yaml("prompt-injection-check")
|
||||
meta = data.get("post_process", {}).get("add_metadata", {})
|
||||
assert meta.get("security_scanned") is True
|
||||
|
||||
|
||||
class TestDefaultWorkflowHasInjectionScan:
|
||||
"""Validate that default.yaml runs injection_scan first."""
|
||||
|
||||
def test_injection_scan_is_first_stage(self):
|
||||
data = _load_bundled_yaml("default")
|
||||
assert data["stages"][0]["name"] == "injection_scan"
|
||||
|
||||
def test_injection_scan_has_prompt(self):
|
||||
data = _load_bundled_yaml("default")
|
||||
scan_stage = data["stages"][0]
|
||||
assert scan_stage.get("prompt")
|
||||
assert "injection" in scan_stage["prompt"].lower()
|
||||
|
||||
|
||||
class TestSecurityFocusHasInjectionScan:
|
||||
"""Validate that security-focus.yaml runs injection_scan first."""
|
||||
|
||||
def test_injection_scan_is_first_stage(self):
|
||||
data = _load_bundled_yaml("security-focus")
|
||||
assert data["stages"][0]["name"] == "injection_scan"
|
||||
|
||||
def test_injection_scan_has_prompt(self):
|
||||
data = _load_bundled_yaml("security-focus")
|
||||
scan_stage = data["stages"][0]
|
||||
assert scan_stage.get("prompt")
|
||||
assert "injection" in scan_stage["prompt"].lower()
|
||||
Reference in New Issue
Block a user