tegwick
3f2449aea1
Fix failing tests that expected content to start with heading but now include schema reference comments. Also fix validate command syntax in test (positional to --schema flag). Fixes: - test_generate_stub_with_explicit_associated_path - test_generate_stub_interactive_mode_defaults_to_associated_path - test_generate_stub_validates_generated_draft_against_schema These tests were failing due to changes from Issue #55 schema reference metadata feature and validate command syntax updates. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude <noreply@anthropic.com>
632 lines
24 KiB
Python
632 lines
24 KiB
Python
"""
|
|
Tests for Issue #55: Schema-based draft generation
|
|
|
|
This test module implements comprehensive tests for enhanced schema-based draft
|
|
generation that utilizes content field instructions and schema metadata.
|
|
|
|
Following TDD8 methodology - these tests are written before implementation.
|
|
"""
|
|
|
|
import json
|
|
import pytest
|
|
from pathlib import Path
|
|
from tempfile import NamedTemporaryFile
|
|
from click.testing import CliRunner
|
|
|
|
from markitect.cli import cli
|
|
from markitect.stub_generator import StubGenerator
|
|
|
|
|
|
class TestIssue55SchemaBasedDraftGeneration:
|
|
"""Test suite for enhanced schema-based draft generation functionality."""
|
|
|
|
def setup_method(self):
|
|
"""Set up test fixtures."""
|
|
self.stub_generator = StubGenerator()
|
|
self.runner = CliRunner()
|
|
|
|
def test_generate_stub_uses_content_instructions_from_schema(self):
|
|
"""Test that generate-stub uses content instructions instead of generic placeholders."""
|
|
# Arrange
|
|
schema_with_instructions = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "API Documentation",
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"level": {"type": "integer"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Write the main API documentation title"
|
|
},
|
|
"x-markitect-instruction-type": {
|
|
"type": "string",
|
|
"enum": ["description"]
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
},
|
|
"level_2": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"level": {"type": "integer"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Describe each API endpoint section"
|
|
},
|
|
"x-markitect-instruction-type": {
|
|
"type": "string",
|
|
"enum": ["description"]
|
|
}
|
|
}
|
|
},
|
|
"minItems": 2,
|
|
"maxItems": 2
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema_with_instructions, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file)
|
|
])
|
|
|
|
# Assert
|
|
assert result.exit_code == 0
|
|
output = result.output
|
|
|
|
# Should use specific content instructions, not generic placeholders
|
|
assert "Write the main API documentation title" in output
|
|
assert "Describe each API endpoint section" in output
|
|
|
|
# Should NOT contain generic placeholder text
|
|
assert "TODO: Add content for" not in output
|
|
assert "section_level_2 section" not in output
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
|
|
def test_generate_stub_includes_schema_reference_metadata(self):
|
|
"""Test that generated drafts include reference to their source schema."""
|
|
# Arrange
|
|
schema = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Requirements Document",
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Provide the document title"
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file)
|
|
])
|
|
|
|
# Assert
|
|
assert result.exit_code == 0
|
|
output = result.output
|
|
|
|
# Should include schema reference metadata
|
|
assert "<!-- Generated from schema:" in output or "Source schema:" in output
|
|
assert str(schema_file.name) in output or schema_file.name in output
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
|
|
def test_generate_stub_supports_different_instruction_types(self):
|
|
"""Test that generate-stub handles different instruction types appropriately."""
|
|
# Arrange
|
|
schema_with_example_instructions = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Tutorial Guide",
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Example: Getting Started with Our API"
|
|
},
|
|
"x-markitect-instruction-type": {
|
|
"type": "string",
|
|
"enum": ["example"]
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema_with_example_instructions, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file)
|
|
])
|
|
|
|
# Assert
|
|
assert result.exit_code == 0
|
|
output = result.output
|
|
|
|
# Should use the example-type instruction
|
|
assert "Example: Getting Started with Our API" in output
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
|
|
def test_generate_stub_handles_schemas_without_content_instructions(self):
|
|
"""Test that generate-stub gracefully handles schemas without content instructions."""
|
|
# Arrange - Schema without content instructions (backward compatibility)
|
|
schema_without_instructions = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Basic Document",
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"level": {"type": "integer"}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema_without_instructions, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file)
|
|
])
|
|
|
|
# Assert - Should still work with generic placeholders
|
|
assert result.exit_code == 0
|
|
output = result.output
|
|
|
|
# Should fall back to generic placeholder behavior
|
|
assert "Basic Document" in output # Should use schema title
|
|
assert len(output) > 0 # Should generate some content
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
|
|
def test_generate_stub_supports_output_file_with_schema_reference(self):
|
|
"""Test that generate-stub writes to output file with schema reference."""
|
|
# Arrange
|
|
schema = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Project Plan",
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Provide the project name and overview"
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
|
|
output_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file),
|
|
'--output', str(output_file)
|
|
])
|
|
|
|
# Assert
|
|
assert result.exit_code == 0
|
|
assert output_file.exists()
|
|
|
|
content = output_file.read_text()
|
|
assert "Provide the project name and overview" in content
|
|
assert "Project Plan" in content
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
if output_file.exists():
|
|
output_file.unlink()
|
|
|
|
def test_generate_stub_validates_generated_draft_against_schema(self):
|
|
"""Test that generated drafts can be validated against their source schema."""
|
|
# Arrange
|
|
schema = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Meeting Notes",
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Meeting title and date"
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.md', delete=False) as f:
|
|
draft_file = Path(f.name)
|
|
|
|
try:
|
|
# Act - Generate draft
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file),
|
|
'--output', str(draft_file)
|
|
])
|
|
|
|
assert result.exit_code == 0
|
|
|
|
# Act - Validate generated draft against schema
|
|
validate_result = self.runner.invoke(cli, [
|
|
'validate',
|
|
str(draft_file),
|
|
'--schema', str(schema_file)
|
|
])
|
|
|
|
# Assert - Generated draft should be valid against its source schema
|
|
assert validate_result.exit_code == 0
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
if draft_file.exists():
|
|
draft_file.unlink()
|
|
|
|
def test_stub_generator_class_supports_content_instructions_directly(self):
|
|
"""Test that StubGenerator class can be used directly with content instruction schemas."""
|
|
# Arrange
|
|
schema = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Architecture Document",
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "System architecture overview title"
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Act
|
|
stub_content = self.stub_generator.generate_stub_from_schema(schema)
|
|
|
|
# Assert
|
|
assert "System architecture overview title" in stub_content
|
|
assert "Architecture Document" in stub_content
|
|
|
|
def test_generate_stub_with_outline_mode_schema_integration(self):
|
|
"""Test that generate-stub works with schemas created by outline mode + content instructions."""
|
|
# Arrange - Schema that would be generated by outline mode with content instructions
|
|
outline_schema_with_instructions = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Schema from user_guide.md",
|
|
"x-markitect-outline-mode": True,
|
|
"x-markitect-outline-depth": 2,
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Provide content for the 'level 1 heading' section"
|
|
},
|
|
"x-markitect-instruction-type": {
|
|
"type": "string",
|
|
"enum": ["description"]
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
},
|
|
"level_2": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Provide content for the 'level 2 heading' section"
|
|
},
|
|
"x-markitect-instruction-type": {
|
|
"type": "string",
|
|
"enum": ["description"]
|
|
}
|
|
}
|
|
},
|
|
"minItems": 3,
|
|
"maxItems": 3
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(outline_schema_with_instructions, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file)
|
|
])
|
|
|
|
# Assert
|
|
assert result.exit_code == 0
|
|
output = result.output
|
|
|
|
# Should use content instructions from outline mode schema
|
|
assert "Provide content for the 'level 1 heading' section" in output
|
|
assert "Provide content for the 'level 2 heading' section" in output
|
|
|
|
# Should respect outline mode structure (depth limited to 2)
|
|
assert output.count('##') >= 3 # Should have multiple level 2 headings
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
|
|
def test_generate_stub_with_heading_text_capture_schema_integration(self):
|
|
"""Test that generate-stub works with schemas that have heading text capture."""
|
|
# Arrange - Schema with both heading text capture and content instructions
|
|
schema_with_heading_capture = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Schema from api_docs.md",
|
|
"x-markitect-heading-text-capture": True,
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {
|
|
"type": "string",
|
|
"enum": ["API Reference"] # From heading text capture
|
|
},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Complete API reference documentation"
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema_with_heading_capture, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file)
|
|
])
|
|
|
|
# Assert
|
|
assert result.exit_code == 0
|
|
output = result.output
|
|
|
|
# Should use the specific heading text from enum constraint
|
|
assert "# API Reference" in output
|
|
|
|
# Should use content instructions
|
|
assert "Complete API reference documentation" in output
|
|
|
|
finally:
|
|
schema_file.unlink()
|
|
|
|
def test_generate_stub_preserves_schema_metadata_in_output(self):
|
|
"""Test that important schema metadata is preserved in the generated draft."""
|
|
# Arrange
|
|
schema_with_metadata = {
|
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
"type": "object",
|
|
"title": "Design Document",
|
|
"description": "Template for system design documents",
|
|
"x-markitect-content-instructions-enabled": True,
|
|
"x-markitect-outline-mode": True,
|
|
"properties": {
|
|
"headings": {
|
|
"type": "object",
|
|
"properties": {
|
|
"level_1": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"content": {"type": "string"},
|
|
"x-markitect-content-instructions": {
|
|
"type": "string",
|
|
"const": "Design document title and scope"
|
|
}
|
|
}
|
|
},
|
|
"minItems": 1,
|
|
"maxItems": 1
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
with NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
|
|
json.dump(schema_with_metadata, f, indent=2)
|
|
schema_file = Path(f.name)
|
|
|
|
try:
|
|
# Act
|
|
result = self.runner.invoke(cli, [
|
|
'generate-stub',
|
|
str(schema_file)
|
|
])
|
|
|
|
# Assert
|
|
assert result.exit_code == 0
|
|
output = result.output
|
|
|
|
# Should include schema metadata information
|
|
assert any(marker in output.lower() for marker in [
|
|
"generated from", "source schema", "template for", "schema:", "outline mode"
|
|
])
|
|
|
|
finally:
|
|
schema_file.unlink() |