generated from coulomb/repo-seed
218 lines
6.8 KiB
Python
218 lines
6.8 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from dataclasses import dataclass
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
|
|
SELF_SCOPING_ROOT_ENV = "REPO_SCOPING_SELF_SCOPING_ROOT"
|
|
OUTCOME_SCHEMA_VERSION = "self-scoping-review-outcome/v1"
|
|
ALLOWED_OUTCOMES = {
|
|
"prefer_golden",
|
|
"prefer_assessment",
|
|
"prefer_baseline",
|
|
"prefer_challenger",
|
|
"tie",
|
|
"needs_human",
|
|
"reject_assessment",
|
|
"reject_challenger",
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ReviewArtifact:
|
|
path: str
|
|
artifact_id: str
|
|
title: str
|
|
updated_at: str
|
|
|
|
|
|
def self_scoping_root(root: str | Path | None = None) -> Path:
|
|
configured = root or os.environ.get(SELF_SCOPING_ROOT_ENV) or "docs/self-scoping"
|
|
return Path(configured).resolve()
|
|
|
|
|
|
def list_golden_profiles(root: str | Path | None = None) -> list[ReviewArtifact]:
|
|
return _list_artifacts("golden", root=root)
|
|
|
|
|
|
def list_assessment_artifacts(root: str | Path | None = None) -> list[ReviewArtifact]:
|
|
return _list_artifacts("assessments", root=root)
|
|
|
|
|
|
def load_json_artifact(
|
|
relative_path: str,
|
|
root: str | Path | None = None,
|
|
) -> dict[str, Any]:
|
|
artifact_path = _safe_artifact_path(relative_path, root=root)
|
|
return json.loads(artifact_path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def list_outcome_records(root: str | Path | None = None) -> list[dict[str, Any]]:
|
|
outcomes_dir = self_scoping_root(root) / "outcomes"
|
|
if not outcomes_dir.exists():
|
|
return []
|
|
records: list[dict[str, Any]] = []
|
|
for path in sorted(outcomes_dir.glob("*.json"), reverse=True):
|
|
try:
|
|
records.append(json.loads(path.read_text(encoding="utf-8")))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
return records
|
|
|
|
|
|
def record_assessment_outcome(
|
|
*,
|
|
golden_path: str,
|
|
assessment_path: str,
|
|
outcome: str,
|
|
reviewer: str,
|
|
notes: str,
|
|
comparison_status: str,
|
|
root: str | Path | None = None,
|
|
) -> dict[str, Any]:
|
|
if outcome not in ALLOWED_OUTCOMES:
|
|
raise ValueError(f"unsupported review outcome: {outcome}")
|
|
|
|
base = self_scoping_root(root)
|
|
golden = load_json_artifact(golden_path, root=base)
|
|
assessment = load_json_artifact(assessment_path, root=base)
|
|
created_at = _created_at()
|
|
outcome_id = _outcome_id(created_at, assessment_path, outcome)
|
|
record = {
|
|
"schema_version": OUTCOME_SCHEMA_VERSION,
|
|
"outcome_id": outcome_id,
|
|
"created_at": created_at,
|
|
"reviewer": reviewer.strip() or "codex",
|
|
"outcome": outcome,
|
|
"notes": notes.strip(),
|
|
"comparison_status": comparison_status,
|
|
"golden_profile_path": golden_path,
|
|
"golden_profile_id": golden.get("profile_id", ""),
|
|
"assessment_artifact_path": assessment_path,
|
|
"assessment_artifact_id": assessment.get("artifact_id", ""),
|
|
"engine_identity": assessment.get("engine_identity", {}),
|
|
"decision_scope": "baseline-comparison",
|
|
}
|
|
|
|
_write_outcome(record, base)
|
|
return record
|
|
|
|
|
|
def record_assessment_pair_outcome(
|
|
*,
|
|
baseline_path: str,
|
|
challenger_path: str,
|
|
outcome: str,
|
|
reviewer: str,
|
|
notes: str,
|
|
comparison_status: str,
|
|
root: str | Path | None = None,
|
|
) -> dict[str, Any]:
|
|
if outcome not in ALLOWED_OUTCOMES:
|
|
raise ValueError(f"unsupported review outcome: {outcome}")
|
|
|
|
base = self_scoping_root(root)
|
|
baseline = load_json_artifact(baseline_path, root=base)
|
|
challenger = load_json_artifact(challenger_path, root=base)
|
|
created_at = _created_at()
|
|
outcome_id = _outcome_id(
|
|
created_at,
|
|
f"{Path(baseline_path).stem}__{Path(challenger_path).stem}",
|
|
outcome,
|
|
)
|
|
record = {
|
|
"schema_version": OUTCOME_SCHEMA_VERSION,
|
|
"outcome_id": outcome_id,
|
|
"created_at": created_at,
|
|
"reviewer": reviewer.strip() or "codex",
|
|
"outcome": outcome,
|
|
"notes": notes.strip(),
|
|
"comparison_status": comparison_status,
|
|
"baseline_assessment_path": baseline_path,
|
|
"baseline_assessment_artifact_id": baseline.get("artifact_id", ""),
|
|
"baseline_engine_identity": baseline.get("engine_identity", {}),
|
|
"challenger_assessment_path": challenger_path,
|
|
"challenger_assessment_artifact_id": challenger.get("artifact_id", ""),
|
|
"challenger_engine_identity": challenger.get("engine_identity", {}),
|
|
"decision_scope": "assessment-pair-comparison",
|
|
}
|
|
_write_outcome(record, base)
|
|
return record
|
|
|
|
|
|
def _created_at() -> str:
|
|
return (
|
|
datetime.now(UTC)
|
|
.replace(microsecond=0)
|
|
.isoformat()
|
|
.replace("+00:00", "Z")
|
|
)
|
|
|
|
|
|
def _write_outcome(record: dict[str, Any], base: Path) -> None:
|
|
outcomes_dir = base / "outcomes"
|
|
outcomes_dir.mkdir(parents=True, exist_ok=True)
|
|
output_path = outcomes_dir / f"{record['outcome_id']}.json"
|
|
output_path.write_text(
|
|
json.dumps(record, indent=2, sort_keys=True) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def _list_artifacts(kind: str, root: str | Path | None = None) -> list[ReviewArtifact]:
|
|
base = self_scoping_root(root)
|
|
artifacts: list[ReviewArtifact] = []
|
|
for path in sorted((base / kind).glob("*.json")):
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
artifacts.append(
|
|
ReviewArtifact(
|
|
path=path.relative_to(base).as_posix(),
|
|
artifact_id=str(
|
|
payload.get("artifact_id") or payload.get("profile_id") or path.stem
|
|
),
|
|
title=str(
|
|
payload.get("title")
|
|
or payload.get("assessment", {}).get("summary")
|
|
or payload.get("artifact_type")
|
|
or path.stem
|
|
),
|
|
updated_at=str(
|
|
payload.get("updated_at") or payload.get("created_at") or ""
|
|
),
|
|
)
|
|
)
|
|
return artifacts
|
|
|
|
|
|
def _safe_artifact_path(relative_path: str, root: str | Path | None = None) -> Path:
|
|
base = self_scoping_root(root)
|
|
artifact_path = (base / relative_path).resolve()
|
|
try:
|
|
artifact_path.relative_to(base)
|
|
except ValueError as exc:
|
|
raise ValueError(f"artifact path escapes self-scoping root: {relative_path}") from exc
|
|
if artifact_path.suffix != ".json":
|
|
raise ValueError(f"artifact path is not JSON: {relative_path}")
|
|
if not artifact_path.exists():
|
|
raise FileNotFoundError(relative_path)
|
|
return artifact_path
|
|
|
|
|
|
def _outcome_id(created_at: str, assessment_path: str, outcome: str) -> str:
|
|
timestamp = (
|
|
created_at.replace("-", "")
|
|
.replace(":", "")
|
|
.replace("T", "-")
|
|
.replace("Z", "")
|
|
)
|
|
assessment_stem = Path(assessment_path).stem.replace(".", "-")
|
|
return f"{timestamp}__{assessment_stem}__{outcome}__{uuid4().hex[:8]}"
|