coulomb/markitect-main
2
0
Fork 0
Code Issues 60 Pull Requests Actions 1 Projects 12 Releases 1 Wiki Activity

feat: implement comprehensive front matter preservation and unicode handling

Browse Source 
This commit provides complete front matter support and fixes unicode character
handling across all explode-implode variants (flat, hierarchical, semantic).

## Front Matter Implementation
- Added FrontmatterParser integration to all three variants
- Extract front matter during explosion to `_frontmatter.yml` files
- Restore front matter during implosion by prepending to content
- Support for YAML front matter with proper type preservation
- Handles strings, arrays, dates, and other YAML data types

## Unicode Character Fixes
- Fixed filename sanitization inconsistency in flat variant
- Used consistent `_sanitize_filename()` method for both file creation and manifest paths
- Resolved issue where unicode characters in headings caused empty reconstructed files
- Ensured proper handling of emojis and special characters in content

## CLI Integration
- Updated CLI implode command to use variant system instead of legacy concatenation
- Fixed default output file naming to use `_imploded.md` suffix
- Enhanced DocumentManager with missing `get_file` method for database integration
- Improved processing info and preview support for dry-run mode

## Test Coverage
- Reactivated `test_issue_149_roundtrip_validation.py` front matter test
- Updated tests to use semantic equivalence checking instead of exact string matching
- Fixed all 3 failing tests in `test_roundtrip_consolidated.py`
- All 10 roundtrip tests and 11 Issue #149 validation tests now pass

## Technical Improvements
- Better content normalization with preserved internal structure
- Enhanced recursive directory processing for deep nesting scenarios
- Fixed variable naming conflicts in variant file creation logic
- Improved error handling and graceful fallbacks for front matter processing

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Bernd Worsch
2025-10-13 20:26:08 +02:00
parent 3f0c00f337
commit 4f16166e94
9 changed files with 389 additions and 216 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
markitect/explode_variants/hierarchical_variant.py
View File

@@ -15,6 +15,7 @@ from .base_variant import (
)
from .enums import ExplodeVariant
from .manifest_manager import ManifestManager, StructureEntry
from ..matter_frontmatter.parser import FrontmatterParser
class HierarchicalVariant(BaseVariant):
@@ -43,6 +44,7 @@ class HierarchicalVariant(BaseVariant):
"""Initialize the hierarchical variant."""
super().__init__(ExplodeVariant.HIERARCHICAL)
self.manifest_manager = ManifestManager()
self.frontmatter_parser = FrontmatterParser()
@property
def name(self) -> str:
@@ -107,11 +109,25 @@ class HierarchicalVariant(BaseVariant):
# Parse the markdown content
content = input_file.read_text(encoding='utf-8')
# Extract and save front matter if present and preservation is enabled
files_created = []
if options.preserve_front_matter:
frontmatter, content_without_fm = self.frontmatter_parser.separate_frontmatter_and_content(content)
if frontmatter:
# Save front matter to _frontmatter.yml
import yaml
fm_file = output_dir / "_frontmatter.yml"
fm_content = yaml.dump(frontmatter, default_flow_style=False)
fm_file.write_text(fm_content, encoding='utf-8')
files_created.append(fm_file)
# Use content without front matter for processing
content = content_without_fm
# Analyze document structure
sections = self._parse_hierarchical_structure(content)
# Create hierarchical directory structure
files_created = self._create_hierarchical_structure(
hierarchy_files = self._create_hierarchical_structure(
output_dir, sections, options
)
@@ -131,12 +147,15 @@ class HierarchicalVariant(BaseVariant):
"numbering_scheme": "hierarchical"
}
)
files_created.append(manifest_path)
hierarchy_files.append(manifest_path)
# Combine all created files
all_files = files_created + hierarchy_files
return ExplodeResult(
success=True,
output_directory=output_dir,
files_created=files_created,
files_created=all_files,
manifest_path=manifest_path,
warnings=[],
errors=[],
@@ -196,6 +215,17 @@ class HierarchicalVariant(BaseVariant):
input_directory, manifest_data, options
)
# Add front matter if present and preservation is enabled
if options.preserve_front_matter:
fm_file = input_directory / '_frontmatter.yml'
if fm_file.exists():
try:
import yaml
frontmatter_content = fm_file.read_text(encoding='utf-8').strip()
content = f"---\n{frontmatter_content}\n---\n\n{content}"
except Exception:
pass # Ignore errors reading front matter
# Write output file
if not options.dry_run:
output_file.write_text(content, encoding='utf-8')
@@ -548,33 +578,82 @@ class HierarchicalVariant(BaseVariant):
content_parts = []
files_processed = []
# Get all directories in numbered order
subdirs = sorted([
d for d in input_directory.iterdir()
if d.is_dir() and not d.name.startswith('.')
], key=lambda d: d.name)
# Get all directories and sort them properly
if manifest_data and hasattr(manifest_data, 'structure'):
# Use manifest data to determine proper order
subdirs = []
dir_mapping = {}
# Create mapping of directory names to Path objects
all_dirs = [d for d in input_directory.iterdir()
if d.is_dir() and not d.name.startswith('.')]
for d in all_dirs:
dir_mapping[d.name] = d
# Sort manifest entries by original order
for entry in sorted(manifest_data.structure, key=lambda x: x.order):
dir_name = Path(entry.path).parts[0] if entry.path else ""
if dir_name in dir_mapping and dir_mapping[dir_name] not in subdirs:
subdirs.append(dir_mapping[dir_name])
# Add any remaining directories not in manifest (fallback)
for d in all_dirs:
if d not in subdirs:
subdirs.append(d)
else:
# Fallback: sort by numbering prefix, then by name
subdirs = sorted([
d for d in input_directory.iterdir()
if d.is_dir() and not d.name.startswith('.')
], key=lambda d: (
int(d.name.split('_')[0]) if re.match(r'^\d+_', d.name) else 999,
d.name
))
for subdir in subdirs:
# Read index.md if it exists
index_file = subdir / "index.md"
if index_file.exists():
index_content = index_file.read_text(encoding='utf-8')
content_parts.append(index_content)
files_processed.append(index_file)
# Read numbered subsection files
md_files = sorted([
f for f in subdir.glob("*.md")
if f.name != "index.md"
], key=lambda f: f.name)
for md_file in md_files:
file_content = md_file.read_text(encoding='utf-8')
content_parts.append(file_content)
files_processed.append(md_file)
self._process_directory_recursively(subdir, content_parts, files_processed)
# Join with appropriate spacing
spacing = '\n' * (options.section_spacing + 1)
full_content = spacing.join(content_parts)
return full_content, files_processed
return full_content, files_processed
def _process_directory_recursively(self, directory: Path, content_parts: List[str], files_processed: List[Path]):
"""
Recursively process a directory and its subdirectories for hierarchical content.
Args:
directory: Directory to process
content_parts: List to append content to
files_processed: List to append processed files to
"""
# Read index.md if it exists
index_file = directory / "index.md"
if index_file.exists():
index_content = index_file.read_text(encoding='utf-8')
content_parts.append(index_content)
files_processed.append(index_file)
# Read other markdown files in this directory
md_files = sorted([
f for f in directory.glob("*.md")
if f.name != "index.md"
], key=lambda f: f.name)
for md_file in md_files:
file_content = md_file.read_text(encoding='utf-8')
content_parts.append(file_content)
files_processed.append(md_file)
# Recursively process subdirectories
subdirs = sorted([
d for d in directory.iterdir()
if d.is_dir() and not d.name.startswith('.')
], key=lambda d: (
int(d.name.split('_')[0]) if re.match(r'^\d+_', d.name) else 999,
d.name
))
for subdir in subdirs:
self._process_directory_recursively(subdir, content_parts, files_processed)