Add quality gate override flow

This commit is contained in:
2026-05-15 16:30:58 +02:00
parent d4f363af72
commit effea4d0d6
8 changed files with 278 additions and 1 deletions

View File

@@ -126,6 +126,8 @@ def _split_audit_list(value: str) -> list[str]:
def _reviewer_type(action: str) -> str: def _reviewer_type(action: str) -> str:
if action == "quality_gate_override":
return "human"
if action.startswith("agentic_"): if action.startswith("agentic_"):
return "agent" return "agent"
if action == "trusted_auto_approve_candidate_graph": if action == "trusted_auto_approve_candidate_graph":
@@ -152,6 +154,8 @@ def _decision_kind(action: str) -> str:
return "downgraded" return "downgraded"
if "request_human_review" in action: if "request_human_review" in action:
return "needs_human" return "needs_human"
if "override" in action:
return "override"
if "propose_edit" in action: if "propose_edit" in action:
return "proposed_edit" return "proposed_edit"
if "relink" in action: if "relink" in action:

View File

@@ -313,6 +313,37 @@ class RegistryService:
"review without approving registry truth." "review without approving registry truth."
) )
def record_quality_gate_override(
self,
repository_id: int,
analysis_run_id: int,
*,
criterion_id: str,
element_type: str,
element_id: int,
reason: str,
notes: str = "",
) -> ReviewDecision:
if not reason.strip():
raise ValueError("quality gate override requires a reason")
decision_id = self.store.create_review_decision(
repository_id,
analysis_run_id,
action="quality_gate_override",
notes=(
f"criteria_version={active_quality_criteria_version()}; "
f"criteria={criterion_id}; "
f"target={element_type}:{element_id}; "
f"rationale={reason.strip()}; "
f"notes={notes.strip()}"
).strip(),
)
return next(
decision
for decision in self.list_review_decisions(repository_id, analysis_run_id)
if decision.id == decision_id
)
def _generate_candidates( def _generate_candidates(
self, self,
repository: Repository, repository: Repository,

View File

@@ -65,6 +65,7 @@ from repo_registry.web_api.schemas import (
IdResponse, IdResponse,
ObservedFactResponse, ObservedFactResponse,
QualityCriteriaRegistryResponse, QualityCriteriaRegistryResponse,
QualityGateOverrideCreate,
RepositoryAbilityMapResponse, RepositoryAbilityMapResponse,
RepositoryComparisonResponse, RepositoryComparisonResponse,
RepositoryCreate, RepositoryCreate,
@@ -471,6 +472,29 @@ def list_analysis_run_review_decisions(
raise HTTPException(status_code=404, detail=str(exc)) from exc raise HTTPException(status_code=404, detail=str(exc)) from exc
@app.post(
"/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides",
tags=["review"],
response_model=ReviewDecisionResponse,
)
def create_quality_gate_override(
repository_id: int,
analysis_run_id: int,
payload: QualityGateOverrideCreate,
service: RegistryService = Depends(get_service),
) -> dict[str, object]:
try:
return asdict(
service.record_quality_gate_override(
repository_id,
analysis_run_id,
**payload.model_dump(),
)
)
except (NotFoundError, ValueError) as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
@app.get( @app.get(
"/repos/{repository_id}/observed-facts", "/repos/{repository_id}/observed-facts",
tags=["analysis"], tags=["analysis"],

View File

@@ -566,6 +566,14 @@ class QualityCriteriaRegistryResponse(BaseModel):
criteria: list[QualityCriterionResponse] criteria: list[QualityCriterionResponse]
class QualityGateOverrideCreate(BaseModel):
criterion_id: str
element_type: str
element_id: int
reason: str
notes: str = ""
class ObservedFactResponse(BaseModel): class ObservedFactResponse(BaseModel):
id: int id: int
repository_id: int repository_id: int

View File

@@ -9,6 +9,10 @@ from urllib.parse import quote_plus, urlparse
from fastapi import APIRouter, Depends, Form, HTTPException, Query from fastapi import APIRouter, Depends, Form, HTTPException, Query
from fastapi.responses import HTMLResponse, PlainTextResponse, RedirectResponse from fastapi.responses import HTMLResponse, PlainTextResponse, RedirectResponse
from repo_registry.acceptance import (
evaluate_candidate_graph_quality,
quality_gate_outcome_dicts,
)
from repo_registry.core.service import RegistryService from repo_registry.core.service import RegistryService
from repo_registry.self_scoping.comparison import compare_assessment_to_golden from repo_registry.self_scoping.comparison import compare_assessment_to_golden
from repo_registry.self_scoping.review_store import ( from repo_registry.self_scoping.review_store import (
@@ -2018,6 +2022,10 @@ def analysis_run_detail(
display_name = repository_display_name(repository) display_name = repository_display_name(repository)
candidate_graph = service.candidate_graph(repository_id, analysis_run_id) candidate_graph = service.candidate_graph(repository_id, analysis_run_id)
candidate_graph_data = asdict(candidate_graph) candidate_graph_data = asdict(candidate_graph)
quality_gate_outcomes = quality_gate_outcome_dicts(
evaluate_candidate_graph_quality(candidate_graph)
)
candidate_graph_data["quality_gate_outcomes"] = quality_gate_outcomes
facts = service.list_observed_facts(repository_id, analysis_run_id) facts = service.list_observed_facts(repository_id, analysis_run_id)
chunks = service.list_content_chunks(repository_id, analysis_run_id) chunks = service.list_content_chunks(repository_id, analysis_run_id)
decisions = service.list_review_decisions(repository_id, analysis_run_id) decisions = service.list_review_decisions(repository_id, analysis_run_id)
@@ -2070,6 +2078,12 @@ def analysis_run_detail(
</form> </form>
</div> </div>
{render_candidate_graph(candidate_graph_data, repository_id, analysis_run_id)} {render_candidate_graph(candidate_graph_data, repository_id, analysis_run_id)}
<h2>Quality Gate Outcomes</h2>
{render_quality_gate_outcomes(
quality_gate_outcomes,
repository_id=repository_id,
analysis_run_id=analysis_run_id,
)}
</section> </section>
<section class="panel"> <section class="panel">
<div class="actions"> <div class="actions">
@@ -2139,6 +2153,32 @@ def create_expectation_gap_from_form(
) )
@router.post("/ui/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides")
def create_quality_gate_override_from_form(
repository_id: int,
analysis_run_id: int,
criterion_id: str = Form(...),
element_type: str = Form(...),
element_id: int = Form(...),
reason: str = Form(...),
notes: str = Form(""),
service: RegistryService = Depends(get_service),
) -> RedirectResponse:
service.record_quality_gate_override(
repository_id,
analysis_run_id,
criterion_id=criterion_id,
element_type=element_type,
element_id=element_id,
reason=reason,
notes=notes,
)
return RedirectResponse(
f"/ui/repos/{repository_id}/analysis-runs/{analysis_run_id}",
status_code=303,
)
@router.post("/ui/repos/{repository_id}/expectation-gaps") @router.post("/ui/repos/{repository_id}/expectation-gaps")
def create_repository_expectation_gap_from_form( def create_repository_expectation_gap_from_form(
repository_id: int, repository_id: int,
@@ -4731,6 +4771,45 @@ def render_review_decisions(decisions: list) -> str:
""" """
def render_quality_gate_outcomes(
outcomes: list[dict],
*,
repository_id: int,
analysis_run_id: int,
) -> str:
if not outcomes:
return '<p class="muted">No quality gates fired for this run.</p>'
rows = "\n".join(
f"""
<tr>
<td><span class="pill">{escape(outcome["criterion_id"])}</span></td>
<td>{escape(outcome["outcome"])}</td>
<td>{escape(outcome["element_type"])} #{outcome["element_id"]}</td>
<td>{escape(outcome["reason"])}</td>
<td>
<form class="stack" method="post" action="/ui/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides">
<input type="hidden" name="criterion_id" value="{escape(outcome["criterion_id"])}">
<input type="hidden" name="element_type" value="{escape(outcome["element_type"])}">
<input type="hidden" name="element_id" value="{outcome["element_id"]}">
<input name="reason" required placeholder="Override reason">
<input name="notes" placeholder="Optional notes">
<button class="secondary" type="submit">Override</button>
</form>
</td>
</tr>
"""
for outcome in outcomes
)
return f"""
<table>
<thead>
<tr><th>Criterion</th><th>Outcome</th><th>Element</th><th>Reason</th><th>Override</th></tr>
</thead>
<tbody>{rows}</tbody>
</table>
"""
def render_expectation_gap_form( def render_expectation_gap_form(
*, *,
action: str, action: str,

View File

@@ -164,3 +164,18 @@ def test_analysis_records_deterministic_gate_review_decision(tmp_path):
assert gate_decision.criteria_version == "repo-scoping-quality-criteria/v1" assert gate_decision.criteria_version == "repo-scoping-quality-criteria/v1"
assert "without approving registry truth" in gate_decision.rationale assert "without approving registry truth" in gate_decision.rationale
assert service.ability_map(repository.id).abilities == [] assert service.ability_map(repository.id).abilities == []
override = service.record_quality_gate_override(
repository.id,
summary.analysis_run.id,
criterion_id="RREG-QC-002",
element_type="capability",
element_id=10,
reason="Curator confirmed this repo now owns provider routing.",
notes="Future criteria update may be needed.",
)
assert override.action == "quality_gate_override"
assert override.reviewer_type == "human"
assert override.decision_kind == "override"
assert override.criterion_ids == ["RREG-QC-002"]
assert override.rationale == "Curator confirmed this repo now owns provider routing."

View File

@@ -135,6 +135,62 @@ def test_quality_criteria_api_lists_active_registry():
) )
def test_quality_gate_override_api_records_auditable_override(tmp_path):
source = tmp_path / "provider-repo"
source.mkdir()
(source / "README.md").write_text("# Provider Repo\n", encoding="utf-8")
(source / "providers.py").write_text(
"provider_registry = {'openrouter': OpenRouterAdapter}\n",
encoding="utf-8",
)
database_path = str(tmp_path / "quality-gate.sqlite3")
def override_settings():
return Settings(
database_path=database_path,
checkout_root=str(tmp_path / "checkouts"),
)
app.dependency_overrides[get_settings] = override_settings
client = TestClient(app)
try:
repository_response = client.post(
"/repos",
json={"name": "Provider Repo", "url": str(source)},
)
repository_id = repository_response.json()["id"]
run_response = client.post(
f"/repos/{repository_id}/analysis-runs",
json={"use_llm_assistance": False},
)
run_id = run_response.json()["analysis_run"]["id"]
graph_response = client.get(
f"/repos/{repository_id}/analysis-runs/{run_id}/candidate-graph"
)
outcome = graph_response.json()["quality_gate_outcomes"][0]
override_response = client.post(
f"/repos/{repository_id}/analysis-runs/{run_id}/quality-gate-overrides",
json={
"criterion_id": outcome["criterion_id"],
"element_type": outcome["element_type"],
"element_id": outcome["element_id"],
"reason": "Curator inspected source and accepts this exception.",
"notes": "Track for criteria refinement.",
},
)
assert override_response.status_code == 200
override = override_response.json()
assert override["action"] == "quality_gate_override"
assert override["reviewer_type"] == "human"
assert override["decision_kind"] == "override"
assert override["criterion_ids"] == [outcome["criterion_id"]]
assert override["rationale"].startswith("Curator inspected source")
finally:
app.dependency_overrides.clear()
def test_openapi_contract_snapshot_for_stable_agent_paths(): def test_openapi_contract_snapshot_for_stable_agent_paths():
client = TestClient(app) client = TestClient(app)
@@ -276,6 +332,9 @@ def test_openapi_contract_snapshot_for_stable_agent_paths():
"/repos/{repository_id}/analysis-runs/{analysis_run_id}/review-decisions": { "/repos/{repository_id}/analysis-runs/{analysis_run_id}/review-decisions": {
"get": {"tags": ["review"], "success_schema": "list[ReviewDecisionResponse]"} "get": {"tags": ["review"], "success_schema": "list[ReviewDecisionResponse]"}
}, },
"/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides": {
"post": {"tags": ["review"], "success_schema": "ReviewDecisionResponse"}
},
"/repos/{repository_id}/analysis-runs/{base_analysis_run_id}/diff/{target_analysis_run_id}": { "/repos/{repository_id}/analysis-runs/{base_analysis_run_id}/diff/{target_analysis_run_id}": {
"get": {"tags": ["review"], "success_schema": "AnalysisRunDiffResponse"} "get": {"tags": ["review"], "success_schema": "AnalysisRunDiffResponse"}
}, },
@@ -2107,6 +2166,57 @@ def test_ui_register_and_explore_lands_on_analysis_result(tmp_path):
app.dependency_overrides.clear() app.dependency_overrides.clear()
def test_ui_quality_gate_outcomes_can_be_overridden(tmp_path):
source = tmp_path / "provider-ui"
source.mkdir()
(source / "README.md").write_text("# Provider UI\n", encoding="utf-8")
(source / "providers.py").write_text(
"provider_registry = {'openrouter': OpenRouterAdapter}\n",
encoding="utf-8",
)
def override_settings():
return Settings(
database_path=str(tmp_path / "ui-quality-gates.sqlite3"),
checkout_root=str(tmp_path / "ui-quality-gates-checkouts"),
)
app.dependency_overrides[get_settings] = override_settings
client = TestClient(app)
try:
repository = client.post(
"/repos",
json={"name": "Provider UI", "url": str(source)},
).json()
run = client.post(
f"/ui/repos/{repository['id']}/analysis-runs",
data={"source_path": "", "use_llm_assistance": ""},
follow_redirects=False,
)
detail = client.get(run.headers["location"])
assert detail.status_code == 200
assert "Quality Gate Outcomes" in detail.text
assert "RREG-QC-002" in detail.text
override = client.post(
f"/ui/repos/{repository['id']}/analysis-runs/1/quality-gate-overrides",
data={
"criterion_id": "RREG-QC-002",
"element_type": "capability",
"element_id": "1",
"reason": "Curator accepts this exception.",
"notes": "Review criteria later.",
},
follow_redirects=False,
)
assert override.status_code == 303
updated = client.get(override.headers["location"])
assert "quality_gate_override" in updated.text
assert "Curator accepts this exception." in updated.text
finally:
app.dependency_overrides.clear()
def test_rebuild_characteristics_endpoint_dry_run_and_confirm(tmp_path): def test_rebuild_characteristics_endpoint_dry_run_and_confirm(tmp_path):
source = tmp_path / "rebuild-api" source = tmp_path / "rebuild-api"
source.mkdir() source.mkdir()

View File

@@ -205,7 +205,7 @@ outcome counts, criteria version, and a rationale that no approval was applied.
```task ```task
id: RREG-WP-0014-T06 id: RREG-WP-0014-T06
status: todo status: done
priority: medium priority: medium
state_hub_task_id: "bcba3237-fb87-4a38-8e96-12b872d5e6a9" state_hub_task_id: "bcba3237-fb87-4a38-8e96-12b872d5e6a9"
``` ```
@@ -220,6 +220,12 @@ Acceptance criteria:
- The UI makes it clear when a candidate is blocked by formal criteria versus - The UI makes it clear when a candidate is blocked by formal criteria versus
merely awaiting judgement. merely awaiting judgement.
Implementation note 2026-05-15: added quality-gate outcome rendering to the
analysis-run UI, a reviewer override form with required rationale, a
`quality_gate_override` review decision path in service/API/UI, and audit fields
that make overrides searchable through review decisions. Criteria changes remain
versioned in `docs/quality-criteria` and linked through workplan notes.
## T07: Regression Coverage For Acceptance Boundary ## T07: Regression Coverage For Acceptance Boundary
```task ```task