generated from coulomb/repo-seed
context loading, path resolution, form state, dynamic rules, and provider-neutral assessment runner/cache boundary
This commit is contained in:
388
tests/test_runtime_context_forms_assessment.py
Normal file
388
tests/test_runtime_context_forms_assessment.py
Normal file
@@ -0,0 +1,388 @@
|
||||
from pathlib import Path
|
||||
|
||||
from click.testing import CliRunner
|
||||
|
||||
from markitect_tool.cli import main
|
||||
from markitect_tool.contract import check_markdown_file, load_contract_file
|
||||
from markitect_tool.core import parse_markdown, parse_markdown_file
|
||||
from markitect_tool.runtime import (
|
||||
AssessmentResult,
|
||||
AssessmentRunner,
|
||||
MemoryAssessmentCache,
|
||||
assessment_requests_for_contract,
|
||||
evaluate_form_state,
|
||||
load_runtime_context_file,
|
||||
)
|
||||
from markitect_tool.workflow import WorkflowRunner, load_workflow_file
|
||||
|
||||
|
||||
LETTER_CONTRACT = """# Letter Contract
|
||||
|
||||
```yaml contract
|
||||
id: letter-runtime-v1
|
||||
document:
|
||||
type: business-letter
|
||||
fields:
|
||||
recipient_name:
|
||||
type: string
|
||||
required: true
|
||||
source: context.recipient.name
|
||||
sender_name:
|
||||
type: string
|
||||
required: true
|
||||
source: context.sender.name
|
||||
sender_email:
|
||||
type: string
|
||||
required: true
|
||||
source: context.sender.email
|
||||
pattern: "@example\\\\.com$"
|
||||
delivery_channel:
|
||||
type: string
|
||||
required: true
|
||||
default: email
|
||||
enum: [email, print]
|
||||
postal_address:
|
||||
type: string
|
||||
contact_label:
|
||||
type: string
|
||||
rules:
|
||||
- id: postal-address-for-print
|
||||
if:
|
||||
path: fields.delivery_channel.value
|
||||
equals: print
|
||||
then:
|
||||
required: [postal_address]
|
||||
visible:
|
||||
postal_address: true
|
||||
else:
|
||||
hidden: [postal_address]
|
||||
- id: calculate-contact-label
|
||||
then:
|
||||
set:
|
||||
contact_label: "${fields.sender_name.value} <${fields.sender_email.value}>"
|
||||
- id: sender-email-domain
|
||||
assert:
|
||||
path: context.sender.email
|
||||
matches: "@example\\\\.com$"
|
||||
message: Sender email must come from example.com.
|
||||
severity: warning
|
||||
sections:
|
||||
- id: greeting
|
||||
title: Greeting
|
||||
presence: required
|
||||
level: 2
|
||||
- id: body
|
||||
title: Body
|
||||
presence: required
|
||||
level: 2
|
||||
- id: closing
|
||||
title: Closing
|
||||
presence: required
|
||||
level: 2
|
||||
rubrics:
|
||||
- id: tone-fit
|
||||
scope: section.body
|
||||
criteria: The body should match the recipient relationship.
|
||||
```
|
||||
"""
|
||||
|
||||
|
||||
LETTER_DOC = """---
|
||||
document_type: business-letter
|
||||
---
|
||||
|
||||
# Follow Up
|
||||
|
||||
## Greeting
|
||||
|
||||
Dear Ada,
|
||||
|
||||
## Body
|
||||
|
||||
Thank you for the productive discussion. We will follow up with a concise
|
||||
proposal and next steps for the Markdown workflow.
|
||||
|
||||
## Closing
|
||||
|
||||
Kind regards
|
||||
"""
|
||||
|
||||
|
||||
LETTER_CONTEXT = """metadata:
|
||||
case_id: case-42
|
||||
schema:
|
||||
type: object
|
||||
required: [recipient, sender]
|
||||
properties:
|
||||
recipient:
|
||||
type: object
|
||||
required: [name]
|
||||
sender:
|
||||
type: object
|
||||
required: [name, email]
|
||||
context:
|
||||
recipient:
|
||||
name: Ada Lovelace
|
||||
sender:
|
||||
name: Markitect Team
|
||||
email: hello@example.com
|
||||
"""
|
||||
|
||||
|
||||
def test_runtime_context_loads_yaml_and_validates_schema(tmp_path: Path):
|
||||
context_file = tmp_path / "context.yaml"
|
||||
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
||||
|
||||
context = load_runtime_context_file(context_file)
|
||||
|
||||
assert context.valid is True
|
||||
assert context.data["recipient"]["name"] == "Ada Lovelace"
|
||||
assert context.metadata["case_id"] == "case-42"
|
||||
|
||||
|
||||
def test_runtime_context_reports_schema_failure(tmp_path: Path):
|
||||
context_file = tmp_path / "context.yaml"
|
||||
context_file.write_text(
|
||||
"schema:\n type: object\n required: [recipient]\ncontext: {}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
context = load_runtime_context_file(context_file)
|
||||
|
||||
assert context.valid is False
|
||||
assert context.diagnostics[0].code == "runtime.context.schema"
|
||||
|
||||
|
||||
def test_form_state_prefills_defaults_hides_fields_and_calculates_values(tmp_path: Path):
|
||||
contract_file = tmp_path / "letter.contract.md"
|
||||
context_file = tmp_path / "context.yaml"
|
||||
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
||||
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
||||
document = parse_markdown(LETTER_DOC, source_path="letter.md")
|
||||
|
||||
form_state = evaluate_form_state(
|
||||
document,
|
||||
load_contract_file(contract_file),
|
||||
load_runtime_context_file(context_file),
|
||||
)
|
||||
fields = {field.id: field for field in form_state.fields}
|
||||
|
||||
assert form_state.valid is True
|
||||
assert fields["recipient_name"].origin == "prefilled"
|
||||
assert fields["delivery_channel"].origin == "defaulted"
|
||||
assert fields["postal_address"].visible is False
|
||||
assert fields["contact_label"].origin == "calculated"
|
||||
assert fields["contact_label"].value == "Markitect Team <hello@example.com>"
|
||||
|
||||
|
||||
def test_context_conflict_keeps_manual_document_value_as_warning(tmp_path: Path):
|
||||
contract_file = tmp_path / "letter.contract.md"
|
||||
context_file = tmp_path / "context.yaml"
|
||||
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
||||
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
||||
document = parse_markdown(
|
||||
LETTER_DOC.replace(
|
||||
"document_type: business-letter",
|
||||
"document_type: business-letter\nrecipient_name: Grace Hopper",
|
||||
),
|
||||
source_path="letter.md",
|
||||
)
|
||||
|
||||
form_state = evaluate_form_state(
|
||||
document,
|
||||
load_contract_file(contract_file),
|
||||
load_runtime_context_file(context_file),
|
||||
)
|
||||
recipient = next(field for field in form_state.fields if field.id == "recipient_name")
|
||||
|
||||
assert form_state.valid is True
|
||||
assert recipient.value == "Grace Hopper"
|
||||
assert "runtime.field.conflict" in {diagnostic.code for diagnostic in form_state.diagnostics}
|
||||
|
||||
|
||||
def test_dynamic_rule_requires_print_address(tmp_path: Path):
|
||||
contract_file = tmp_path / "letter.contract.md"
|
||||
context_file = tmp_path / "context.yaml"
|
||||
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
||||
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
||||
document = parse_markdown(
|
||||
LETTER_DOC.replace(
|
||||
"document_type: business-letter",
|
||||
"document_type: business-letter\ndelivery_channel: print",
|
||||
),
|
||||
source_path="letter.md",
|
||||
)
|
||||
|
||||
form_state = evaluate_form_state(
|
||||
document,
|
||||
load_contract_file(contract_file),
|
||||
load_runtime_context_file(context_file),
|
||||
)
|
||||
|
||||
assert form_state.valid is False
|
||||
assert "contract.field.missing" in {diagnostic.code for diagnostic in form_state.diagnostics}
|
||||
|
||||
|
||||
def test_dynamic_section_rule_can_require_section(tmp_path: Path):
|
||||
contract_file = tmp_path / "workplan.contract.md"
|
||||
contract_file.write_text(
|
||||
"""# Workplan Contract
|
||||
|
||||
```yaml contract
|
||||
id: dynamic-workplan-v1
|
||||
document:
|
||||
type: workplan
|
||||
fields:
|
||||
status:
|
||||
type: string
|
||||
required: true
|
||||
sections:
|
||||
- id: purpose
|
||||
title: Purpose
|
||||
presence: required
|
||||
- id: decision-point
|
||||
title: Decision Point
|
||||
presence: optional
|
||||
rules:
|
||||
- id: require-decision-when-done
|
||||
if:
|
||||
path: fields.status.value
|
||||
equals: done
|
||||
then:
|
||||
sections:
|
||||
decision-point:
|
||||
presence: required
|
||||
```
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
document = parse_markdown(
|
||||
"---\ndocument_type: workplan\nstatus: done\n---\n# WP\n\n## Purpose\n\nDone.\n",
|
||||
source_path="workplan.md",
|
||||
)
|
||||
|
||||
form_state = evaluate_form_state(document, load_contract_file(contract_file))
|
||||
|
||||
assert form_state.valid is False
|
||||
assert "runtime.section.missing" in {diagnostic.code for diagnostic in form_state.diagnostics}
|
||||
|
||||
|
||||
def test_contract_check_uses_runtime_context(tmp_path: Path):
|
||||
contract_file = tmp_path / "letter.contract.md"
|
||||
document_file = tmp_path / "letter.md"
|
||||
context_file = tmp_path / "context.yaml"
|
||||
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
||||
document_file.write_text(LETTER_DOC, encoding="utf-8")
|
||||
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
||||
|
||||
result = check_markdown_file(document_file, contract_file, context_path=context_file)
|
||||
|
||||
assert result.valid is True
|
||||
assert result.runtime["form_state"]["field_values"]["recipient_name"] == "Ada Lovelace"
|
||||
|
||||
|
||||
def test_contract_cli_accepts_context_and_reports_form_state(tmp_path: Path):
|
||||
contract_file = tmp_path / "letter.contract.md"
|
||||
document_file = tmp_path / "letter.md"
|
||||
context_file = tmp_path / "context.yaml"
|
||||
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
||||
document_file.write_text(LETTER_DOC, encoding="utf-8")
|
||||
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
||||
|
||||
check = CliRunner().invoke(
|
||||
main,
|
||||
[
|
||||
"contract",
|
||||
"check",
|
||||
str(document_file),
|
||||
"--contract",
|
||||
str(contract_file),
|
||||
"--context",
|
||||
str(context_file),
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
)
|
||||
form_state = CliRunner().invoke(
|
||||
main,
|
||||
[
|
||||
"contract",
|
||||
"form-state",
|
||||
str(document_file),
|
||||
"--contract",
|
||||
str(contract_file),
|
||||
"--context",
|
||||
str(context_file),
|
||||
"--format",
|
||||
"text",
|
||||
],
|
||||
)
|
||||
|
||||
assert check.exit_code == 0
|
||||
assert '"runtime"' in check.output
|
||||
assert form_state.exit_code == 0
|
||||
assert "recipient_name: Ada Lovelace [prefilled]" in form_state.output
|
||||
|
||||
|
||||
def test_assessment_runner_normalizes_cache_and_failure_diagnostics(tmp_path: Path):
|
||||
contract_file = tmp_path / "letter.contract.md"
|
||||
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
||||
document = parse_markdown_file(_write_file(tmp_path / "letter.md", LETTER_DOC))
|
||||
requests = assessment_requests_for_contract(document, load_contract_file(contract_file))
|
||||
|
||||
class Adapter:
|
||||
calls = 0
|
||||
|
||||
def assess(self, request):
|
||||
self.calls += 1
|
||||
return AssessmentResult(
|
||||
rule_id=request.rule_id,
|
||||
passed=False,
|
||||
score=0.2,
|
||||
reason="Tone is too vague.",
|
||||
provider="mock",
|
||||
model="mock-grader",
|
||||
)
|
||||
|
||||
adapter = Adapter()
|
||||
runner = AssessmentRunner(adapter, cache=MemoryAssessmentCache())
|
||||
first = runner.assess(requests[0])
|
||||
second = runner.assess(requests[0])
|
||||
run = runner.assess_all(requests)
|
||||
|
||||
assert first.cached is False
|
||||
assert second.cached is True
|
||||
assert adapter.calls == 1
|
||||
assert "runtime.assessment.failed" in {diagnostic.code for diagnostic in run.diagnostics}
|
||||
|
||||
|
||||
def test_workflow_form_state_step_uses_context(tmp_path: Path):
|
||||
contract_file = tmp_path / "letter.contract.md"
|
||||
document_file = tmp_path / "letter.md"
|
||||
context_file = tmp_path / "context.yaml"
|
||||
workflow_file = tmp_path / "workflow.yaml"
|
||||
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
||||
document_file.write_text(LETTER_DOC, encoding="utf-8")
|
||||
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
||||
workflow_file.write_text(
|
||||
"""metadata:
|
||||
id: runtime-workflow
|
||||
steps:
|
||||
- id: form
|
||||
kind: form_state
|
||||
document: letter.md
|
||||
contract: letter.contract.md
|
||||
context: context.yaml
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = WorkflowRunner(load_workflow_file(workflow_file)).run()
|
||||
|
||||
assert result.valid is True
|
||||
assert result.steps["form"]["field_values"]["recipient_name"] == "Ada Lovelace"
|
||||
|
||||
|
||||
def _write_file(path: Path, text: str) -> Path:
|
||||
path.write_text(text, encoding="utf-8")
|
||||
return path
|
||||
Reference in New Issue
Block a user