tegwick
9b12875681
Phase 0 - Project Organization: - Create docs/PROJECT_STRUCTURE.md documenting codebase layout - Create markitect/core/ with parser, serializer, document_manager, workspace - Create markitect/schema/ consolidating 6 schema_*.py modules - Create markitect/storage/ with database module - Maintain backward compatibility via re-exports from original locations - Add docs/roadmap/information-space-service/ with README and WORKPLAN Phase 1 - Foundation (Weeks 1-3): - Week 1: Core domain models (InformationSpace, SpaceDocument, SpaceConfig, SpaceMetadata, SpaceVariable, TransclusionReference, SpaceStatus) - Week 2: Repository layer with interfaces (ISpaceRepository, IDocumentAssociationRepository, IVariableRepository, IReferenceRepository) and SQLite implementations with foreign key cascade deletes - Week 3: SpaceService orchestration layer with full CRUD, document, variable, and reference tracking operations Test coverage: 124 tests (25 model + 63 repository + 36 integration) Capabilities delivered: - CAP-001: InformationSpace entity with lifecycle management - CAP-002: SpaceRepository CRUD with SQLite backing - CAP-003: Document-Space associations with path-based organization - CAP-004: Space metadata and configuration schemas - CAP-005: Database schema with migrations Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
360 lines
12 KiB
Python
360 lines
12 KiB
Python
"""
|
|
AST to Markdown Serialization - Issue #2 Completion
|
|
|
|
This module provides functionality to serialize markdown-it AST tokens back into
|
|
markdown format, enabling roundtrip validation and document manipulation.
|
|
|
|
Key Features:
|
|
- Convert AST tokens back to markdown text
|
|
- Preserve front matter during serialization
|
|
- Support for content manipulation operations
|
|
- Roundtrip integrity validation
|
|
"""
|
|
|
|
from typing import List, Dict, Any, Optional
|
|
import yaml
|
|
|
|
|
|
class ASTSerializer:
|
|
"""
|
|
Serializes markdown-it AST tokens back to markdown format.
|
|
|
|
Provides roundtrip capability: markdown -> AST -> markdown
|
|
Supports front matter preservation and content manipulation.
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""Initialize the AST serializer."""
|
|
pass
|
|
|
|
def serialize_to_markdown(self, ast: List[Dict[str, Any]], front_matter: Optional[Dict[str, Any]] = None) -> str:
|
|
"""
|
|
Convert AST tokens back to markdown format.
|
|
|
|
Args:
|
|
ast: List of markdown-it AST tokens
|
|
front_matter: Optional YAML front matter dictionary
|
|
|
|
Returns:
|
|
Markdown text with optional front matter
|
|
|
|
Example:
|
|
serializer = ASTSerializer()
|
|
markdown = serializer.serialize_to_markdown(ast, front_matter)
|
|
"""
|
|
markdown_parts = []
|
|
|
|
# Add front matter if present
|
|
if front_matter and isinstance(front_matter, dict) and front_matter:
|
|
yaml_content = yaml.dump(front_matter, default_flow_style=False).strip()
|
|
markdown_parts.append(f"---\n{yaml_content}\n---\n\n")
|
|
|
|
# Process AST tokens
|
|
markdown_content = self._process_tokens(ast)
|
|
markdown_parts.append(markdown_content)
|
|
|
|
return ''.join(markdown_parts)
|
|
|
|
def _process_tokens(self, tokens: List[Dict[str, Any]]) -> str:
|
|
"""
|
|
Process a list of AST tokens into markdown text.
|
|
|
|
Args:
|
|
tokens: List of markdown-it tokens
|
|
|
|
Returns:
|
|
Markdown text representation
|
|
"""
|
|
markdown_lines = []
|
|
current_line = ""
|
|
list_level = 0
|
|
|
|
for token in tokens:
|
|
token_type = token.get('type', '')
|
|
content = token.get('content', '')
|
|
markup = token.get('markup', '')
|
|
tag = token.get('tag', '')
|
|
nesting = token.get('nesting', 0)
|
|
level = token.get('level', 0)
|
|
|
|
# Handle different token types
|
|
if token_type == 'heading_open':
|
|
heading_level = int(tag[1]) if tag.startswith('h') else 1
|
|
current_line = '#' * heading_level + ' '
|
|
elif token_type == 'heading_close':
|
|
if current_line:
|
|
markdown_lines.append(current_line.rstrip())
|
|
current_line = ""
|
|
markdown_lines.append("") # Empty line after heading
|
|
|
|
elif token_type == 'paragraph_open':
|
|
pass # Start of paragraph
|
|
elif token_type == 'paragraph_close':
|
|
if current_line:
|
|
markdown_lines.append(current_line.rstrip())
|
|
current_line = ""
|
|
markdown_lines.append("") # Empty line after paragraph
|
|
|
|
elif token_type == 'inline':
|
|
# Process inline content and children
|
|
if content:
|
|
current_line += content
|
|
elif 'children' in token:
|
|
current_line += self._process_inline_children(token['children'])
|
|
|
|
elif token_type == 'list_item_open':
|
|
# Handle list items
|
|
indent = ' ' * (level // 2)
|
|
if markup in ('-', '*'):
|
|
current_line = indent + '- '
|
|
elif markup.isdigit():
|
|
current_line = indent + '1. '
|
|
elif token_type == 'list_item_close':
|
|
if current_line:
|
|
markdown_lines.append(current_line.rstrip())
|
|
current_line = ""
|
|
|
|
elif token_type in ('bullet_list_open', 'ordered_list_open'):
|
|
list_level += 1
|
|
elif token_type in ('bullet_list_close', 'ordered_list_close'):
|
|
list_level -= 1
|
|
if list_level == 0:
|
|
markdown_lines.append("") # Empty line after list
|
|
|
|
elif token_type == 'blockquote_open':
|
|
pass
|
|
elif token_type == 'blockquote_close':
|
|
markdown_lines.append("")
|
|
|
|
elif token_type == 'code_block':
|
|
markdown_lines.append(f"```{token.get('info', '')}")
|
|
markdown_lines.append(content.rstrip())
|
|
markdown_lines.append("```")
|
|
markdown_lines.append("")
|
|
|
|
elif token_type == 'fence':
|
|
if nesting == 1: # Opening fence
|
|
markdown_lines.append(f"```{token.get('info', '')}")
|
|
else: # Closing fence
|
|
markdown_lines.append("```")
|
|
markdown_lines.append("")
|
|
|
|
elif token_type == 'hr':
|
|
markdown_lines.append("---")
|
|
markdown_lines.append("")
|
|
|
|
elif token_type == 'text':
|
|
current_line += content
|
|
|
|
# Add any remaining content
|
|
if current_line:
|
|
markdown_lines.append(current_line.rstrip())
|
|
|
|
# Clean up extra empty lines at the end
|
|
while markdown_lines and markdown_lines[-1] == "":
|
|
markdown_lines.pop()
|
|
|
|
return '\n'.join(markdown_lines)
|
|
|
|
def _process_inline_children(self, children: List[Dict[str, Any]]) -> str:
|
|
"""
|
|
Process inline children tokens (emphasis, strong, links, etc.).
|
|
|
|
Args:
|
|
children: List of inline token children
|
|
|
|
Returns:
|
|
Processed inline markdown text
|
|
"""
|
|
result = ""
|
|
|
|
for child in children:
|
|
token_type = child.get('type', '')
|
|
content = child.get('content', '')
|
|
markup = child.get('markup', '')
|
|
|
|
if token_type == 'text':
|
|
result += content
|
|
elif token_type == 'code_inline':
|
|
result += f"`{content}`"
|
|
elif token_type == 'em_open':
|
|
result += markup or '*'
|
|
elif token_type == 'em_close':
|
|
result += markup or '*'
|
|
elif token_type == 'strong_open':
|
|
result += markup or '**'
|
|
elif token_type == 'strong_close':
|
|
result += markup or '**'
|
|
elif token_type == 'link_open':
|
|
# Extract href from attrs
|
|
href = ""
|
|
if 'attrs' in child and child['attrs']:
|
|
for attr in child['attrs']:
|
|
if attr[0] == 'href':
|
|
href = attr[1]
|
|
break
|
|
result += "["
|
|
elif token_type == 'link_close':
|
|
# This is tricky - we need to get the href from the opening token
|
|
# For now, we'll use a placeholder approach
|
|
result += "](#)"
|
|
elif token_type == 'softbreak':
|
|
result += '\n'
|
|
elif token_type == 'hardbreak':
|
|
result += ' \n'
|
|
|
|
return result
|
|
|
|
def modify_ast_content(self, ast: List[Dict[str, Any]], modifications: Dict[str, Any]) -> List[Dict[str, Any]]:
|
|
"""
|
|
Modify AST content based on provided modifications.
|
|
|
|
Args:
|
|
ast: Original AST tokens
|
|
modifications: Dictionary of modifications to apply
|
|
|
|
Returns:
|
|
Modified AST tokens
|
|
|
|
Supported modifications:
|
|
- add_section: Add a new section with title and content
|
|
- update_front_matter: Update front matter values
|
|
"""
|
|
modified_ast = ast.copy()
|
|
|
|
# Handle adding sections
|
|
if 'add_section' in modifications:
|
|
section_data = modifications['add_section']
|
|
title = section_data.get('title', 'New Section')
|
|
content = section_data.get('content', '')
|
|
level = section_data.get('level', 2)
|
|
|
|
# Create new section tokens
|
|
new_tokens = [
|
|
{
|
|
"type": "heading_open",
|
|
"tag": f"h{level}",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": 1,
|
|
"level": 0,
|
|
"content": "",
|
|
"markup": "#" * level,
|
|
"info": "",
|
|
"meta": {},
|
|
"block": True,
|
|
"hidden": False
|
|
},
|
|
{
|
|
"type": "inline",
|
|
"tag": "",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": 0,
|
|
"level": 1,
|
|
"children": [
|
|
{
|
|
"type": "text",
|
|
"tag": "",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": 0,
|
|
"level": 0,
|
|
"content": title,
|
|
"markup": "",
|
|
"info": "",
|
|
"meta": {},
|
|
"block": False,
|
|
"hidden": False
|
|
}
|
|
],
|
|
"content": title,
|
|
"markup": "",
|
|
"info": "",
|
|
"meta": {},
|
|
"block": True,
|
|
"hidden": False
|
|
},
|
|
{
|
|
"type": "heading_close",
|
|
"tag": f"h{level}",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": -1,
|
|
"level": 0,
|
|
"content": "",
|
|
"markup": "#" * level,
|
|
"info": "",
|
|
"meta": {},
|
|
"block": True,
|
|
"hidden": False
|
|
}
|
|
]
|
|
|
|
if content:
|
|
new_tokens.extend([
|
|
{
|
|
"type": "paragraph_open",
|
|
"tag": "p",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": 1,
|
|
"level": 0,
|
|
"content": "",
|
|
"markup": "",
|
|
"info": "",
|
|
"meta": {},
|
|
"block": True,
|
|
"hidden": False
|
|
},
|
|
{
|
|
"type": "inline",
|
|
"tag": "",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": 0,
|
|
"level": 1,
|
|
"children": [
|
|
{
|
|
"type": "text",
|
|
"tag": "",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": 0,
|
|
"level": 0,
|
|
"content": content,
|
|
"markup": "",
|
|
"info": "",
|
|
"meta": {},
|
|
"block": False,
|
|
"hidden": False
|
|
}
|
|
],
|
|
"content": content,
|
|
"markup": "",
|
|
"info": "",
|
|
"meta": {},
|
|
"block": True,
|
|
"hidden": False
|
|
},
|
|
{
|
|
"type": "paragraph_close",
|
|
"tag": "p",
|
|
"attrs": {},
|
|
"map": None,
|
|
"nesting": -1,
|
|
"level": 0,
|
|
"content": "",
|
|
"markup": "",
|
|
"info": "",
|
|
"meta": {},
|
|
"block": True,
|
|
"hidden": False
|
|
}
|
|
])
|
|
|
|
# Add to end of AST
|
|
modified_ast.extend(new_tokens)
|
|
|
|
return modified_ast
|