generated from coulomb/repo-seed
389 lines
11 KiB
Python
389 lines
11 KiB
Python
from pathlib import Path
|
|
|
|
from click.testing import CliRunner
|
|
|
|
from markitect_tool.cli import main
|
|
from markitect_tool.contract import check_markdown_file, load_contract_file
|
|
from markitect_tool.core import parse_markdown, parse_markdown_file
|
|
from markitect_tool.runtime import (
|
|
AssessmentResult,
|
|
AssessmentRunner,
|
|
MemoryAssessmentCache,
|
|
assessment_requests_for_contract,
|
|
evaluate_form_state,
|
|
load_runtime_context_file,
|
|
)
|
|
from markitect_tool.workflow import WorkflowRunner, load_workflow_file
|
|
|
|
|
|
LETTER_CONTRACT = """# Letter Contract
|
|
|
|
```yaml contract
|
|
id: letter-runtime-v1
|
|
document:
|
|
type: business-letter
|
|
fields:
|
|
recipient_name:
|
|
type: string
|
|
required: true
|
|
source: context.recipient.name
|
|
sender_name:
|
|
type: string
|
|
required: true
|
|
source: context.sender.name
|
|
sender_email:
|
|
type: string
|
|
required: true
|
|
source: context.sender.email
|
|
pattern: "@example\\\\.com$"
|
|
delivery_channel:
|
|
type: string
|
|
required: true
|
|
default: email
|
|
enum: [email, print]
|
|
postal_address:
|
|
type: string
|
|
contact_label:
|
|
type: string
|
|
rules:
|
|
- id: postal-address-for-print
|
|
if:
|
|
path: fields.delivery_channel.value
|
|
equals: print
|
|
then:
|
|
required: [postal_address]
|
|
visible:
|
|
postal_address: true
|
|
else:
|
|
hidden: [postal_address]
|
|
- id: calculate-contact-label
|
|
then:
|
|
set:
|
|
contact_label: "${fields.sender_name.value} <${fields.sender_email.value}>"
|
|
- id: sender-email-domain
|
|
assert:
|
|
path: context.sender.email
|
|
matches: "@example\\\\.com$"
|
|
message: Sender email must come from example.com.
|
|
severity: warning
|
|
sections:
|
|
- id: greeting
|
|
title: Greeting
|
|
presence: required
|
|
level: 2
|
|
- id: body
|
|
title: Body
|
|
presence: required
|
|
level: 2
|
|
- id: closing
|
|
title: Closing
|
|
presence: required
|
|
level: 2
|
|
rubrics:
|
|
- id: tone-fit
|
|
scope: section.body
|
|
criteria: The body should match the recipient relationship.
|
|
```
|
|
"""
|
|
|
|
|
|
LETTER_DOC = """---
|
|
document_type: business-letter
|
|
---
|
|
|
|
# Follow Up
|
|
|
|
## Greeting
|
|
|
|
Dear Ada,
|
|
|
|
## Body
|
|
|
|
Thank you for the productive discussion. We will follow up with a concise
|
|
proposal and next steps for the Markdown workflow.
|
|
|
|
## Closing
|
|
|
|
Kind regards
|
|
"""
|
|
|
|
|
|
LETTER_CONTEXT = """metadata:
|
|
case_id: case-42
|
|
schema:
|
|
type: object
|
|
required: [recipient, sender]
|
|
properties:
|
|
recipient:
|
|
type: object
|
|
required: [name]
|
|
sender:
|
|
type: object
|
|
required: [name, email]
|
|
context:
|
|
recipient:
|
|
name: Ada Lovelace
|
|
sender:
|
|
name: Markitect Team
|
|
email: hello@example.com
|
|
"""
|
|
|
|
|
|
def test_runtime_context_loads_yaml_and_validates_schema(tmp_path: Path):
|
|
context_file = tmp_path / "context.yaml"
|
|
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
|
|
|
context = load_runtime_context_file(context_file)
|
|
|
|
assert context.valid is True
|
|
assert context.data["recipient"]["name"] == "Ada Lovelace"
|
|
assert context.metadata["case_id"] == "case-42"
|
|
|
|
|
|
def test_runtime_context_reports_schema_failure(tmp_path: Path):
|
|
context_file = tmp_path / "context.yaml"
|
|
context_file.write_text(
|
|
"schema:\n type: object\n required: [recipient]\ncontext: {}\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
context = load_runtime_context_file(context_file)
|
|
|
|
assert context.valid is False
|
|
assert context.diagnostics[0].code == "runtime.context.schema"
|
|
|
|
|
|
def test_form_state_prefills_defaults_hides_fields_and_calculates_values(tmp_path: Path):
|
|
contract_file = tmp_path / "letter.contract.md"
|
|
context_file = tmp_path / "context.yaml"
|
|
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
|
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
|
document = parse_markdown(LETTER_DOC, source_path="letter.md")
|
|
|
|
form_state = evaluate_form_state(
|
|
document,
|
|
load_contract_file(contract_file),
|
|
load_runtime_context_file(context_file),
|
|
)
|
|
fields = {field.id: field for field in form_state.fields}
|
|
|
|
assert form_state.valid is True
|
|
assert fields["recipient_name"].origin == "prefilled"
|
|
assert fields["delivery_channel"].origin == "defaulted"
|
|
assert fields["postal_address"].visible is False
|
|
assert fields["contact_label"].origin == "calculated"
|
|
assert fields["contact_label"].value == "Markitect Team <hello@example.com>"
|
|
|
|
|
|
def test_context_conflict_keeps_manual_document_value_as_warning(tmp_path: Path):
|
|
contract_file = tmp_path / "letter.contract.md"
|
|
context_file = tmp_path / "context.yaml"
|
|
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
|
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
|
document = parse_markdown(
|
|
LETTER_DOC.replace(
|
|
"document_type: business-letter",
|
|
"document_type: business-letter\nrecipient_name: Grace Hopper",
|
|
),
|
|
source_path="letter.md",
|
|
)
|
|
|
|
form_state = evaluate_form_state(
|
|
document,
|
|
load_contract_file(contract_file),
|
|
load_runtime_context_file(context_file),
|
|
)
|
|
recipient = next(field for field in form_state.fields if field.id == "recipient_name")
|
|
|
|
assert form_state.valid is True
|
|
assert recipient.value == "Grace Hopper"
|
|
assert "runtime.field.conflict" in {diagnostic.code for diagnostic in form_state.diagnostics}
|
|
|
|
|
|
def test_dynamic_rule_requires_print_address(tmp_path: Path):
|
|
contract_file = tmp_path / "letter.contract.md"
|
|
context_file = tmp_path / "context.yaml"
|
|
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
|
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
|
document = parse_markdown(
|
|
LETTER_DOC.replace(
|
|
"document_type: business-letter",
|
|
"document_type: business-letter\ndelivery_channel: print",
|
|
),
|
|
source_path="letter.md",
|
|
)
|
|
|
|
form_state = evaluate_form_state(
|
|
document,
|
|
load_contract_file(contract_file),
|
|
load_runtime_context_file(context_file),
|
|
)
|
|
|
|
assert form_state.valid is False
|
|
assert "contract.field.missing" in {diagnostic.code for diagnostic in form_state.diagnostics}
|
|
|
|
|
|
def test_dynamic_section_rule_can_require_section(tmp_path: Path):
|
|
contract_file = tmp_path / "workplan.contract.md"
|
|
contract_file.write_text(
|
|
"""# Workplan Contract
|
|
|
|
```yaml contract
|
|
id: dynamic-workplan-v1
|
|
document:
|
|
type: workplan
|
|
fields:
|
|
status:
|
|
type: string
|
|
required: true
|
|
sections:
|
|
- id: purpose
|
|
title: Purpose
|
|
presence: required
|
|
- id: decision-point
|
|
title: Decision Point
|
|
presence: optional
|
|
rules:
|
|
- id: require-decision-when-done
|
|
if:
|
|
path: fields.status.value
|
|
equals: done
|
|
then:
|
|
sections:
|
|
decision-point:
|
|
presence: required
|
|
```
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
document = parse_markdown(
|
|
"---\ndocument_type: workplan\nstatus: done\n---\n# WP\n\n## Purpose\n\nDone.\n",
|
|
source_path="workplan.md",
|
|
)
|
|
|
|
form_state = evaluate_form_state(document, load_contract_file(contract_file))
|
|
|
|
assert form_state.valid is False
|
|
assert "runtime.section.missing" in {diagnostic.code for diagnostic in form_state.diagnostics}
|
|
|
|
|
|
def test_contract_check_uses_runtime_context(tmp_path: Path):
|
|
contract_file = tmp_path / "letter.contract.md"
|
|
document_file = tmp_path / "letter.md"
|
|
context_file = tmp_path / "context.yaml"
|
|
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
|
document_file.write_text(LETTER_DOC, encoding="utf-8")
|
|
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
|
|
|
result = check_markdown_file(document_file, contract_file, context_path=context_file)
|
|
|
|
assert result.valid is True
|
|
assert result.runtime["form_state"]["field_values"]["recipient_name"] == "Ada Lovelace"
|
|
|
|
|
|
def test_contract_cli_accepts_context_and_reports_form_state(tmp_path: Path):
|
|
contract_file = tmp_path / "letter.contract.md"
|
|
document_file = tmp_path / "letter.md"
|
|
context_file = tmp_path / "context.yaml"
|
|
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
|
document_file.write_text(LETTER_DOC, encoding="utf-8")
|
|
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
|
|
|
check = CliRunner().invoke(
|
|
main,
|
|
[
|
|
"contract",
|
|
"check",
|
|
str(document_file),
|
|
"--contract",
|
|
str(contract_file),
|
|
"--context",
|
|
str(context_file),
|
|
"--format",
|
|
"json",
|
|
],
|
|
)
|
|
form_state = CliRunner().invoke(
|
|
main,
|
|
[
|
|
"contract",
|
|
"form-state",
|
|
str(document_file),
|
|
"--contract",
|
|
str(contract_file),
|
|
"--context",
|
|
str(context_file),
|
|
"--format",
|
|
"text",
|
|
],
|
|
)
|
|
|
|
assert check.exit_code == 0
|
|
assert '"runtime"' in check.output
|
|
assert form_state.exit_code == 0
|
|
assert "recipient_name: Ada Lovelace [prefilled]" in form_state.output
|
|
|
|
|
|
def test_assessment_runner_normalizes_cache_and_failure_diagnostics(tmp_path: Path):
|
|
contract_file = tmp_path / "letter.contract.md"
|
|
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
|
document = parse_markdown_file(_write_file(tmp_path / "letter.md", LETTER_DOC))
|
|
requests = assessment_requests_for_contract(document, load_contract_file(contract_file))
|
|
|
|
class Adapter:
|
|
calls = 0
|
|
|
|
def assess(self, request):
|
|
self.calls += 1
|
|
return AssessmentResult(
|
|
rule_id=request.rule_id,
|
|
passed=False,
|
|
score=0.2,
|
|
reason="Tone is too vague.",
|
|
provider="mock",
|
|
model="mock-grader",
|
|
)
|
|
|
|
adapter = Adapter()
|
|
runner = AssessmentRunner(adapter, cache=MemoryAssessmentCache())
|
|
first = runner.assess(requests[0])
|
|
second = runner.assess(requests[0])
|
|
run = runner.assess_all(requests)
|
|
|
|
assert first.cached is False
|
|
assert second.cached is True
|
|
assert adapter.calls == 1
|
|
assert "runtime.assessment.failed" in {diagnostic.code for diagnostic in run.diagnostics}
|
|
|
|
|
|
def test_workflow_form_state_step_uses_context(tmp_path: Path):
|
|
contract_file = tmp_path / "letter.contract.md"
|
|
document_file = tmp_path / "letter.md"
|
|
context_file = tmp_path / "context.yaml"
|
|
workflow_file = tmp_path / "workflow.yaml"
|
|
contract_file.write_text(LETTER_CONTRACT, encoding="utf-8")
|
|
document_file.write_text(LETTER_DOC, encoding="utf-8")
|
|
context_file.write_text(LETTER_CONTEXT, encoding="utf-8")
|
|
workflow_file.write_text(
|
|
"""metadata:
|
|
id: runtime-workflow
|
|
steps:
|
|
- id: form
|
|
kind: form_state
|
|
document: letter.md
|
|
contract: letter.contract.md
|
|
context: context.yaml
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
result = WorkflowRunner(load_workflow_file(workflow_file)).run()
|
|
|
|
assert result.valid is True
|
|
assert result.steps["form"]["field_values"]["recipient_name"] == "Ada Lovelace"
|
|
|
|
|
|
def _write_file(path: Path, text: str) -> Path:
|
|
path.write_text(text, encoding="utf-8")
|
|
return path
|