generated from coulomb/repo-seed
Harden WSJF triage report recovery
This commit is contained in:
@@ -4,7 +4,8 @@ Covers:
|
||||
- UntrustedFieldError raised when prompt references untrusted field
|
||||
- Object-type attribute rejected even when listed in trusted_fields
|
||||
- Injection fixture: untrusted field raises UntrustedFieldError before rendering
|
||||
- Schema validation: NullLLM returning invalid JSON → retry → second invalid → []
|
||||
- Schema validation: invalid JSON retries once; report-sink instructions preserve
|
||||
a validation-failure artifact after the second invalid output.
|
||||
- review_required flag: present on InstructionDef model
|
||||
"""
|
||||
|
||||
@@ -98,6 +99,7 @@ def _instr(
|
||||
max_tokens: int | None = None,
|
||||
max_depth: int | None = None,
|
||||
model_params: dict[str, Any] | None = None,
|
||||
report_sinks: list[dict[str, Any]] | None = None,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id=id,
|
||||
@@ -111,6 +113,7 @@ def _instr(
|
||||
model_params=model_params or {},
|
||||
output_schema=output_schema,
|
||||
review_required=review_required,
|
||||
report_sinks=report_sinks or [],
|
||||
)
|
||||
|
||||
|
||||
@@ -353,6 +356,58 @@ def test_execute_instruction_with_audit_rejects_invalid_report_schema():
|
||||
assert llm.call_count == 2
|
||||
|
||||
|
||||
def test_execute_instruction_with_audit_preserves_invalid_report_with_sinks(
|
||||
tmp_path,
|
||||
monkeypatch,
|
||||
):
|
||||
schema_dir = tmp_path / "schemas"
|
||||
schema_dir.mkdir()
|
||||
schema_path = schema_dir / "daily-triage-report.json"
|
||||
schema_path.write_text(
|
||||
json.dumps({
|
||||
"type": "object",
|
||||
"required": ["summary", "recommendations"],
|
||||
"properties": {
|
||||
"summary": {"type": "string"},
|
||||
"recommendations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"required": ["action"],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
report_data = {
|
||||
"summary": "Generated partial triage.",
|
||||
"recommendations": [{"rank": 1, "candidate": "CUST-WP-0045"}],
|
||||
}
|
||||
llm = _CountingLLM([json.dumps(report_data), json.dumps(report_data)])
|
||||
instr = _instr(
|
||||
id="daily-triage-report",
|
||||
prompt="Report.",
|
||||
trusted_fields=[],
|
||||
output_schema="schemas/daily-triage-report.json",
|
||||
report_sinks=[{"type": "working-memory", "path": "/tmp"}],
|
||||
)
|
||||
|
||||
result = execute_instruction_with_audit(instr, _Event(), {}, llm)
|
||||
|
||||
assert result.tasks == []
|
||||
assert result.output_validated is False
|
||||
assert result.review_required is True
|
||||
assert result.validation_error == "$.recommendations[0]: missing required property 'action'"
|
||||
assert result.report is not None
|
||||
assert result.report["status"] == "validation_failed"
|
||||
assert result.report["partial_summary"] == "Generated partial triage."
|
||||
assert result.report["partial_report"] == report_data
|
||||
assert llm.call_count == 2
|
||||
|
||||
|
||||
def test_execute_instruction_with_audit_accepts_report_and_tasks_envelope():
|
||||
envelope = {
|
||||
"report": {"summary": "Review needed."},
|
||||
|
||||
Reference in New Issue
Block a user