generated from coulomb/repo-seed
Add quality gate override flow
This commit is contained in:
@@ -126,6 +126,8 @@ def _split_audit_list(value: str) -> list[str]:
|
|||||||
|
|
||||||
|
|
||||||
def _reviewer_type(action: str) -> str:
|
def _reviewer_type(action: str) -> str:
|
||||||
|
if action == "quality_gate_override":
|
||||||
|
return "human"
|
||||||
if action.startswith("agentic_"):
|
if action.startswith("agentic_"):
|
||||||
return "agent"
|
return "agent"
|
||||||
if action == "trusted_auto_approve_candidate_graph":
|
if action == "trusted_auto_approve_candidate_graph":
|
||||||
@@ -152,6 +154,8 @@ def _decision_kind(action: str) -> str:
|
|||||||
return "downgraded"
|
return "downgraded"
|
||||||
if "request_human_review" in action:
|
if "request_human_review" in action:
|
||||||
return "needs_human"
|
return "needs_human"
|
||||||
|
if "override" in action:
|
||||||
|
return "override"
|
||||||
if "propose_edit" in action:
|
if "propose_edit" in action:
|
||||||
return "proposed_edit"
|
return "proposed_edit"
|
||||||
if "relink" in action:
|
if "relink" in action:
|
||||||
|
|||||||
@@ -313,6 +313,37 @@ class RegistryService:
|
|||||||
"review without approving registry truth."
|
"review without approving registry truth."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def record_quality_gate_override(
|
||||||
|
self,
|
||||||
|
repository_id: int,
|
||||||
|
analysis_run_id: int,
|
||||||
|
*,
|
||||||
|
criterion_id: str,
|
||||||
|
element_type: str,
|
||||||
|
element_id: int,
|
||||||
|
reason: str,
|
||||||
|
notes: str = "",
|
||||||
|
) -> ReviewDecision:
|
||||||
|
if not reason.strip():
|
||||||
|
raise ValueError("quality gate override requires a reason")
|
||||||
|
decision_id = self.store.create_review_decision(
|
||||||
|
repository_id,
|
||||||
|
analysis_run_id,
|
||||||
|
action="quality_gate_override",
|
||||||
|
notes=(
|
||||||
|
f"criteria_version={active_quality_criteria_version()}; "
|
||||||
|
f"criteria={criterion_id}; "
|
||||||
|
f"target={element_type}:{element_id}; "
|
||||||
|
f"rationale={reason.strip()}; "
|
||||||
|
f"notes={notes.strip()}"
|
||||||
|
).strip(),
|
||||||
|
)
|
||||||
|
return next(
|
||||||
|
decision
|
||||||
|
for decision in self.list_review_decisions(repository_id, analysis_run_id)
|
||||||
|
if decision.id == decision_id
|
||||||
|
)
|
||||||
|
|
||||||
def _generate_candidates(
|
def _generate_candidates(
|
||||||
self,
|
self,
|
||||||
repository: Repository,
|
repository: Repository,
|
||||||
|
|||||||
@@ -65,6 +65,7 @@ from repo_registry.web_api.schemas import (
|
|||||||
IdResponse,
|
IdResponse,
|
||||||
ObservedFactResponse,
|
ObservedFactResponse,
|
||||||
QualityCriteriaRegistryResponse,
|
QualityCriteriaRegistryResponse,
|
||||||
|
QualityGateOverrideCreate,
|
||||||
RepositoryAbilityMapResponse,
|
RepositoryAbilityMapResponse,
|
||||||
RepositoryComparisonResponse,
|
RepositoryComparisonResponse,
|
||||||
RepositoryCreate,
|
RepositoryCreate,
|
||||||
@@ -471,6 +472,29 @@ def list_analysis_run_review_decisions(
|
|||||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@app.post(
|
||||||
|
"/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides",
|
||||||
|
tags=["review"],
|
||||||
|
response_model=ReviewDecisionResponse,
|
||||||
|
)
|
||||||
|
def create_quality_gate_override(
|
||||||
|
repository_id: int,
|
||||||
|
analysis_run_id: int,
|
||||||
|
payload: QualityGateOverrideCreate,
|
||||||
|
service: RegistryService = Depends(get_service),
|
||||||
|
) -> dict[str, object]:
|
||||||
|
try:
|
||||||
|
return asdict(
|
||||||
|
service.record_quality_gate_override(
|
||||||
|
repository_id,
|
||||||
|
analysis_run_id,
|
||||||
|
**payload.model_dump(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except (NotFoundError, ValueError) as exc:
|
||||||
|
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
@app.get(
|
@app.get(
|
||||||
"/repos/{repository_id}/observed-facts",
|
"/repos/{repository_id}/observed-facts",
|
||||||
tags=["analysis"],
|
tags=["analysis"],
|
||||||
|
|||||||
@@ -566,6 +566,14 @@ class QualityCriteriaRegistryResponse(BaseModel):
|
|||||||
criteria: list[QualityCriterionResponse]
|
criteria: list[QualityCriterionResponse]
|
||||||
|
|
||||||
|
|
||||||
|
class QualityGateOverrideCreate(BaseModel):
|
||||||
|
criterion_id: str
|
||||||
|
element_type: str
|
||||||
|
element_id: int
|
||||||
|
reason: str
|
||||||
|
notes: str = ""
|
||||||
|
|
||||||
|
|
||||||
class ObservedFactResponse(BaseModel):
|
class ObservedFactResponse(BaseModel):
|
||||||
id: int
|
id: int
|
||||||
repository_id: int
|
repository_id: int
|
||||||
|
|||||||
@@ -9,6 +9,10 @@ from urllib.parse import quote_plus, urlparse
|
|||||||
from fastapi import APIRouter, Depends, Form, HTTPException, Query
|
from fastapi import APIRouter, Depends, Form, HTTPException, Query
|
||||||
from fastapi.responses import HTMLResponse, PlainTextResponse, RedirectResponse
|
from fastapi.responses import HTMLResponse, PlainTextResponse, RedirectResponse
|
||||||
|
|
||||||
|
from repo_registry.acceptance import (
|
||||||
|
evaluate_candidate_graph_quality,
|
||||||
|
quality_gate_outcome_dicts,
|
||||||
|
)
|
||||||
from repo_registry.core.service import RegistryService
|
from repo_registry.core.service import RegistryService
|
||||||
from repo_registry.self_scoping.comparison import compare_assessment_to_golden
|
from repo_registry.self_scoping.comparison import compare_assessment_to_golden
|
||||||
from repo_registry.self_scoping.review_store import (
|
from repo_registry.self_scoping.review_store import (
|
||||||
@@ -2018,6 +2022,10 @@ def analysis_run_detail(
|
|||||||
display_name = repository_display_name(repository)
|
display_name = repository_display_name(repository)
|
||||||
candidate_graph = service.candidate_graph(repository_id, analysis_run_id)
|
candidate_graph = service.candidate_graph(repository_id, analysis_run_id)
|
||||||
candidate_graph_data = asdict(candidate_graph)
|
candidate_graph_data = asdict(candidate_graph)
|
||||||
|
quality_gate_outcomes = quality_gate_outcome_dicts(
|
||||||
|
evaluate_candidate_graph_quality(candidate_graph)
|
||||||
|
)
|
||||||
|
candidate_graph_data["quality_gate_outcomes"] = quality_gate_outcomes
|
||||||
facts = service.list_observed_facts(repository_id, analysis_run_id)
|
facts = service.list_observed_facts(repository_id, analysis_run_id)
|
||||||
chunks = service.list_content_chunks(repository_id, analysis_run_id)
|
chunks = service.list_content_chunks(repository_id, analysis_run_id)
|
||||||
decisions = service.list_review_decisions(repository_id, analysis_run_id)
|
decisions = service.list_review_decisions(repository_id, analysis_run_id)
|
||||||
@@ -2070,6 +2078,12 @@ def analysis_run_detail(
|
|||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
{render_candidate_graph(candidate_graph_data, repository_id, analysis_run_id)}
|
{render_candidate_graph(candidate_graph_data, repository_id, analysis_run_id)}
|
||||||
|
<h2>Quality Gate Outcomes</h2>
|
||||||
|
{render_quality_gate_outcomes(
|
||||||
|
quality_gate_outcomes,
|
||||||
|
repository_id=repository_id,
|
||||||
|
analysis_run_id=analysis_run_id,
|
||||||
|
)}
|
||||||
</section>
|
</section>
|
||||||
<section class="panel">
|
<section class="panel">
|
||||||
<div class="actions">
|
<div class="actions">
|
||||||
@@ -2139,6 +2153,32 @@ def create_expectation_gap_from_form(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/ui/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides")
|
||||||
|
def create_quality_gate_override_from_form(
|
||||||
|
repository_id: int,
|
||||||
|
analysis_run_id: int,
|
||||||
|
criterion_id: str = Form(...),
|
||||||
|
element_type: str = Form(...),
|
||||||
|
element_id: int = Form(...),
|
||||||
|
reason: str = Form(...),
|
||||||
|
notes: str = Form(""),
|
||||||
|
service: RegistryService = Depends(get_service),
|
||||||
|
) -> RedirectResponse:
|
||||||
|
service.record_quality_gate_override(
|
||||||
|
repository_id,
|
||||||
|
analysis_run_id,
|
||||||
|
criterion_id=criterion_id,
|
||||||
|
element_type=element_type,
|
||||||
|
element_id=element_id,
|
||||||
|
reason=reason,
|
||||||
|
notes=notes,
|
||||||
|
)
|
||||||
|
return RedirectResponse(
|
||||||
|
f"/ui/repos/{repository_id}/analysis-runs/{analysis_run_id}",
|
||||||
|
status_code=303,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@router.post("/ui/repos/{repository_id}/expectation-gaps")
|
@router.post("/ui/repos/{repository_id}/expectation-gaps")
|
||||||
def create_repository_expectation_gap_from_form(
|
def create_repository_expectation_gap_from_form(
|
||||||
repository_id: int,
|
repository_id: int,
|
||||||
@@ -4731,6 +4771,45 @@ def render_review_decisions(decisions: list) -> str:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def render_quality_gate_outcomes(
|
||||||
|
outcomes: list[dict],
|
||||||
|
*,
|
||||||
|
repository_id: int,
|
||||||
|
analysis_run_id: int,
|
||||||
|
) -> str:
|
||||||
|
if not outcomes:
|
||||||
|
return '<p class="muted">No quality gates fired for this run.</p>'
|
||||||
|
rows = "\n".join(
|
||||||
|
f"""
|
||||||
|
<tr>
|
||||||
|
<td><span class="pill">{escape(outcome["criterion_id"])}</span></td>
|
||||||
|
<td>{escape(outcome["outcome"])}</td>
|
||||||
|
<td>{escape(outcome["element_type"])} #{outcome["element_id"]}</td>
|
||||||
|
<td>{escape(outcome["reason"])}</td>
|
||||||
|
<td>
|
||||||
|
<form class="stack" method="post" action="/ui/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides">
|
||||||
|
<input type="hidden" name="criterion_id" value="{escape(outcome["criterion_id"])}">
|
||||||
|
<input type="hidden" name="element_type" value="{escape(outcome["element_type"])}">
|
||||||
|
<input type="hidden" name="element_id" value="{outcome["element_id"]}">
|
||||||
|
<input name="reason" required placeholder="Override reason">
|
||||||
|
<input name="notes" placeholder="Optional notes">
|
||||||
|
<button class="secondary" type="submit">Override</button>
|
||||||
|
</form>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
"""
|
||||||
|
for outcome in outcomes
|
||||||
|
)
|
||||||
|
return f"""
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr><th>Criterion</th><th>Outcome</th><th>Element</th><th>Reason</th><th>Override</th></tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>{rows}</tbody>
|
||||||
|
</table>
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
def render_expectation_gap_form(
|
def render_expectation_gap_form(
|
||||||
*,
|
*,
|
||||||
action: str,
|
action: str,
|
||||||
|
|||||||
@@ -164,3 +164,18 @@ def test_analysis_records_deterministic_gate_review_decision(tmp_path):
|
|||||||
assert gate_decision.criteria_version == "repo-scoping-quality-criteria/v1"
|
assert gate_decision.criteria_version == "repo-scoping-quality-criteria/v1"
|
||||||
assert "without approving registry truth" in gate_decision.rationale
|
assert "without approving registry truth" in gate_decision.rationale
|
||||||
assert service.ability_map(repository.id).abilities == []
|
assert service.ability_map(repository.id).abilities == []
|
||||||
|
|
||||||
|
override = service.record_quality_gate_override(
|
||||||
|
repository.id,
|
||||||
|
summary.analysis_run.id,
|
||||||
|
criterion_id="RREG-QC-002",
|
||||||
|
element_type="capability",
|
||||||
|
element_id=10,
|
||||||
|
reason="Curator confirmed this repo now owns provider routing.",
|
||||||
|
notes="Future criteria update may be needed.",
|
||||||
|
)
|
||||||
|
assert override.action == "quality_gate_override"
|
||||||
|
assert override.reviewer_type == "human"
|
||||||
|
assert override.decision_kind == "override"
|
||||||
|
assert override.criterion_ids == ["RREG-QC-002"]
|
||||||
|
assert override.rationale == "Curator confirmed this repo now owns provider routing."
|
||||||
|
|||||||
@@ -135,6 +135,62 @@ def test_quality_criteria_api_lists_active_registry():
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_quality_gate_override_api_records_auditable_override(tmp_path):
|
||||||
|
source = tmp_path / "provider-repo"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "README.md").write_text("# Provider Repo\n", encoding="utf-8")
|
||||||
|
(source / "providers.py").write_text(
|
||||||
|
"provider_registry = {'openrouter': OpenRouterAdapter}\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
database_path = str(tmp_path / "quality-gate.sqlite3")
|
||||||
|
|
||||||
|
def override_settings():
|
||||||
|
return Settings(
|
||||||
|
database_path=database_path,
|
||||||
|
checkout_root=str(tmp_path / "checkouts"),
|
||||||
|
)
|
||||||
|
|
||||||
|
app.dependency_overrides[get_settings] = override_settings
|
||||||
|
client = TestClient(app)
|
||||||
|
try:
|
||||||
|
repository_response = client.post(
|
||||||
|
"/repos",
|
||||||
|
json={"name": "Provider Repo", "url": str(source)},
|
||||||
|
)
|
||||||
|
repository_id = repository_response.json()["id"]
|
||||||
|
run_response = client.post(
|
||||||
|
f"/repos/{repository_id}/analysis-runs",
|
||||||
|
json={"use_llm_assistance": False},
|
||||||
|
)
|
||||||
|
run_id = run_response.json()["analysis_run"]["id"]
|
||||||
|
graph_response = client.get(
|
||||||
|
f"/repos/{repository_id}/analysis-runs/{run_id}/candidate-graph"
|
||||||
|
)
|
||||||
|
outcome = graph_response.json()["quality_gate_outcomes"][0]
|
||||||
|
|
||||||
|
override_response = client.post(
|
||||||
|
f"/repos/{repository_id}/analysis-runs/{run_id}/quality-gate-overrides",
|
||||||
|
json={
|
||||||
|
"criterion_id": outcome["criterion_id"],
|
||||||
|
"element_type": outcome["element_type"],
|
||||||
|
"element_id": outcome["element_id"],
|
||||||
|
"reason": "Curator inspected source and accepts this exception.",
|
||||||
|
"notes": "Track for criteria refinement.",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert override_response.status_code == 200
|
||||||
|
override = override_response.json()
|
||||||
|
assert override["action"] == "quality_gate_override"
|
||||||
|
assert override["reviewer_type"] == "human"
|
||||||
|
assert override["decision_kind"] == "override"
|
||||||
|
assert override["criterion_ids"] == [outcome["criterion_id"]]
|
||||||
|
assert override["rationale"].startswith("Curator inspected source")
|
||||||
|
finally:
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
def test_openapi_contract_snapshot_for_stable_agent_paths():
|
def test_openapi_contract_snapshot_for_stable_agent_paths():
|
||||||
client = TestClient(app)
|
client = TestClient(app)
|
||||||
|
|
||||||
@@ -276,6 +332,9 @@ def test_openapi_contract_snapshot_for_stable_agent_paths():
|
|||||||
"/repos/{repository_id}/analysis-runs/{analysis_run_id}/review-decisions": {
|
"/repos/{repository_id}/analysis-runs/{analysis_run_id}/review-decisions": {
|
||||||
"get": {"tags": ["review"], "success_schema": "list[ReviewDecisionResponse]"}
|
"get": {"tags": ["review"], "success_schema": "list[ReviewDecisionResponse]"}
|
||||||
},
|
},
|
||||||
|
"/repos/{repository_id}/analysis-runs/{analysis_run_id}/quality-gate-overrides": {
|
||||||
|
"post": {"tags": ["review"], "success_schema": "ReviewDecisionResponse"}
|
||||||
|
},
|
||||||
"/repos/{repository_id}/analysis-runs/{base_analysis_run_id}/diff/{target_analysis_run_id}": {
|
"/repos/{repository_id}/analysis-runs/{base_analysis_run_id}/diff/{target_analysis_run_id}": {
|
||||||
"get": {"tags": ["review"], "success_schema": "AnalysisRunDiffResponse"}
|
"get": {"tags": ["review"], "success_schema": "AnalysisRunDiffResponse"}
|
||||||
},
|
},
|
||||||
@@ -2107,6 +2166,57 @@ def test_ui_register_and_explore_lands_on_analysis_result(tmp_path):
|
|||||||
app.dependency_overrides.clear()
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
|
def test_ui_quality_gate_outcomes_can_be_overridden(tmp_path):
|
||||||
|
source = tmp_path / "provider-ui"
|
||||||
|
source.mkdir()
|
||||||
|
(source / "README.md").write_text("# Provider UI\n", encoding="utf-8")
|
||||||
|
(source / "providers.py").write_text(
|
||||||
|
"provider_registry = {'openrouter': OpenRouterAdapter}\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
def override_settings():
|
||||||
|
return Settings(
|
||||||
|
database_path=str(tmp_path / "ui-quality-gates.sqlite3"),
|
||||||
|
checkout_root=str(tmp_path / "ui-quality-gates-checkouts"),
|
||||||
|
)
|
||||||
|
|
||||||
|
app.dependency_overrides[get_settings] = override_settings
|
||||||
|
client = TestClient(app)
|
||||||
|
try:
|
||||||
|
repository = client.post(
|
||||||
|
"/repos",
|
||||||
|
json={"name": "Provider UI", "url": str(source)},
|
||||||
|
).json()
|
||||||
|
run = client.post(
|
||||||
|
f"/ui/repos/{repository['id']}/analysis-runs",
|
||||||
|
data={"source_path": "", "use_llm_assistance": ""},
|
||||||
|
follow_redirects=False,
|
||||||
|
)
|
||||||
|
detail = client.get(run.headers["location"])
|
||||||
|
assert detail.status_code == 200
|
||||||
|
assert "Quality Gate Outcomes" in detail.text
|
||||||
|
assert "RREG-QC-002" in detail.text
|
||||||
|
|
||||||
|
override = client.post(
|
||||||
|
f"/ui/repos/{repository['id']}/analysis-runs/1/quality-gate-overrides",
|
||||||
|
data={
|
||||||
|
"criterion_id": "RREG-QC-002",
|
||||||
|
"element_type": "capability",
|
||||||
|
"element_id": "1",
|
||||||
|
"reason": "Curator accepts this exception.",
|
||||||
|
"notes": "Review criteria later.",
|
||||||
|
},
|
||||||
|
follow_redirects=False,
|
||||||
|
)
|
||||||
|
assert override.status_code == 303
|
||||||
|
updated = client.get(override.headers["location"])
|
||||||
|
assert "quality_gate_override" in updated.text
|
||||||
|
assert "Curator accepts this exception." in updated.text
|
||||||
|
finally:
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
|
||||||
def test_rebuild_characteristics_endpoint_dry_run_and_confirm(tmp_path):
|
def test_rebuild_characteristics_endpoint_dry_run_and_confirm(tmp_path):
|
||||||
source = tmp_path / "rebuild-api"
|
source = tmp_path / "rebuild-api"
|
||||||
source.mkdir()
|
source.mkdir()
|
||||||
|
|||||||
@@ -205,7 +205,7 @@ outcome counts, criteria version, and a rationale that no approval was applied.
|
|||||||
|
|
||||||
```task
|
```task
|
||||||
id: RREG-WP-0014-T06
|
id: RREG-WP-0014-T06
|
||||||
status: todo
|
status: done
|
||||||
priority: medium
|
priority: medium
|
||||||
state_hub_task_id: "bcba3237-fb87-4a38-8e96-12b872d5e6a9"
|
state_hub_task_id: "bcba3237-fb87-4a38-8e96-12b872d5e6a9"
|
||||||
```
|
```
|
||||||
@@ -220,6 +220,12 @@ Acceptance criteria:
|
|||||||
- The UI makes it clear when a candidate is blocked by formal criteria versus
|
- The UI makes it clear when a candidate is blocked by formal criteria versus
|
||||||
merely awaiting judgement.
|
merely awaiting judgement.
|
||||||
|
|
||||||
|
Implementation note 2026-05-15: added quality-gate outcome rendering to the
|
||||||
|
analysis-run UI, a reviewer override form with required rationale, a
|
||||||
|
`quality_gate_override` review decision path in service/API/UI, and audit fields
|
||||||
|
that make overrides searchable through review decisions. Criteria changes remain
|
||||||
|
versioned in `docs/quality-criteria` and linked through workplan notes.
|
||||||
|
|
||||||
## T07: Regression Coverage For Acceptance Boundary
|
## T07: Regression Coverage For Acceptance Boundary
|
||||||
|
|
||||||
```task
|
```task
|
||||||
|
|||||||
Reference in New Issue
Block a user