coulomb/info-tech-canon
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add user-engine evaluation readiness pack

Browse Source
This commit is contained in:
Bernd Worsch
2026-05-23 05:18:54 +02:00
parent cab14fdd7e
commit 4e4cec6555
34 changed files with 1597 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

197
src/info_tech_canon/validation.py
View File

@@ -54,12 +54,17 @@ REQUIRED_SCHEMAS = (
RETRIEVAL_BRIEF_KINDS = {
"concept-catalog",
"consumer-workplan-brief",
"evaluation-pack",
"evaluation-question-set",
"example",
"interface-card-expectation",
"kernel",
"mapping",
"model",
"model-extension",
"pattern",
"profile-alignment",
"profile",
"standard",
}
@@ -90,6 +95,50 @@ PURPOSE_REQUIRED_CONSUMERS = {
"repo-scoping",
}
USER_ENGINE_EVALUATION_ARTIFACT_IDS = {
"evaluation/user-engine",
"evaluation/user-engine/consumer-workplan-brief",
"evaluation/user-engine/interface-card-expectations",
"evaluation/user-engine/questions",
"evaluation/user-engine/small-saas-alignment",
}
USER_ENGINE_QUESTION_DOMAINS = {
"access-control",
"data",
"governance",
"organization",
"purposes",
"security",
"task",
}
USER_ENGINE_REQUIRED_ENTITY_IDS = {
"access-role",
"account",
"control",
"evidence",
"organization-role",
"policy",
"principal",
"subject",
"team",
"tenant",
"user",
}
USER_ENGINE_REQUIRED_EDGE_TYPES = {
"assigned_role",
"authenticates_as",
"belongs_to_tenant",
"evidenced_by",
"evaluated_as",
"governed_by",
"implemented_by",
"member_of",
"scoped_to",
}
def structural_checks(context: Any) -> dict[str, list[dict[str, Any]]]:
errors: list[dict[str, Any]] = []
@@ -102,6 +151,11 @@ def structural_checks(context: Any) -> dict[str, list[dict[str, Any]]]:
_check_artifact_index(context.repo_root, context.infospace_root, errors)
_check_agent_assets(context.infospace_root, context.infospace.artifacts, errors)
_check_purpose_demand_assets(context.infospace_root, context.infospace.artifacts, errors)
_check_user_engine_evaluation_assets(
context.infospace_root,
context.infospace.artifacts,
errors,
)
_check_optional_assets(context.infospace_root, warnings)
return {"errors": errors, "warnings": warnings}
@@ -522,6 +576,149 @@ def _check_purpose_demand_assets(
)
def _check_user_engine_evaluation_assets(
infospace_root: Path,
artifacts: list[Any],
errors: list[dict[str, Any]],
) -> None:
artifact_ids = {artifact.id for artifact in artifacts}
for artifact_id in sorted(USER_ENGINE_EVALUATION_ARTIFACT_IDS - artifact_ids):
errors.append(
{
"code": "missing_user_engine_evaluation_artifact",
"artifact_id": artifact_id,
}
)
pack = _read_yaml(
infospace_root / "evaluations" / "user-engine" / "evaluation-pack.yaml",
errors,
)
if isinstance(pack, dict):
components = pack.get("pack_components") or {}
if not isinstance(components, dict):
errors.append(
{
"code": "invalid_user_engine_pack_components",
"path": "infospace/evaluations/user-engine/evaluation-pack.yaml",
}
)
else:
for component in (
"questions",
"interface_card_expectations",
"small_saas_alignment",
"consumer_workplan_brief",
):
if not components.get(component):
errors.append(
{
"code": "missing_user_engine_pack_component",
"component": component,
}
)
questions = _read_yaml(
infospace_root / "evaluations" / "user-engine" / "questions.yaml",
errors,
)
if isinstance(questions, dict):
domains = questions.get("question_domains") or []
domain_ids = {
str(domain.get("id"))
for domain in domains
if isinstance(domain, dict) and domain.get("id")
}
for domain_id in sorted(USER_ENGINE_QUESTION_DOMAINS - domain_ids):
errors.append(
{
"code": "missing_user_engine_question_domain",
"domain": domain_id,
}
)
for domain in domains:
if isinstance(domain, dict) and not domain.get("questions"):
errors.append(
{
"code": "empty_user_engine_question_domain",
"domain": domain.get("id"),
}
)
expectations = _read_yaml(
infospace_root
/ "evaluations"
/ "user-engine"
/ "interface-card-expectations.yaml",
errors,
)
if isinstance(expectations, dict):
entity_ids = {
str(entity.get("id"))
for entity in expectations.get("expected_entities") or []
if isinstance(entity, dict) and entity.get("id")
}
for entity_id in sorted(USER_ENGINE_REQUIRED_ENTITY_IDS - entity_ids):
errors.append(
{
"code": "missing_user_engine_expected_entity",
"entity": entity_id,
}
)
edge_types = {
str(edge.get("type"))
for edge in expectations.get("expected_edges") or []
if isinstance(edge, dict) and edge.get("type")
}
for edge_type in sorted(USER_ENGINE_REQUIRED_EDGE_TYPES - edge_types):
errors.append(
{
"code": "missing_user_engine_expected_edge",
"edge": edge_type,
}
)
evidence = expectations.get("evidence_required") or []
if not isinstance(evidence, list) or not evidence:
errors.append(
{
"code": "missing_user_engine_evidence_expectations",
"path": "infospace/evaluations/user-engine/interface-card-expectations.yaml",
}
)
alignment = _read_yaml(
infospace_root
/ "evaluations"
/ "user-engine"
/ "small-saas-alignment.yaml",
errors,
)
if isinstance(alignment, dict):
if alignment.get("profile") != "profile/small-saas":
errors.append(
{
"code": "invalid_user_engine_alignment_profile",
"value": alignment.get("profile"),
}
)
if not alignment.get("profile_requirements"):
errors.append(
{
"code": "missing_user_engine_profile_requirements",
"path": "infospace/evaluations/user-engine/small-saas-alignment.yaml",
}
)
brief_path = infospace_root / "evaluations" / "user-engine" / "consumer-workplan-brief.md"
if not brief_path.is_file():
errors.append(
{
"code": "missing_user_engine_consumer_workplan_brief",
"path": "infospace/evaluations/user-engine/consumer-workplan-brief.md",
}
)
def _artifact_paths_by_path(
infospace_root: Path,
errors: list[dict[str, Any]],