coulomb/markitect-main
2
0
Fork 0
Code Issues 61 Pull Requests Actions Projects 12 Releases 1 Wiki Activity

feat: complete test fixing and decoupled functionality implementation
Some checks failed
Test Suite / unit-tests (3.11) (push) Has been cancelled
Details
Test Suite / unit-tests (3.12) (push) Has been cancelled
Details
Test Suite / integration-tests (push) Has been cancelled
Details
Test Suite / e2e-tests (push) Has been cancelled
Details
Test Suite / performance-tests (push) Has been cancelled
Details
Test Suite / code-quality (push) Has been cancelled
Details
Test Suite / security-scan (push) Has been cancelled
Details
Test Suite / test-summary (push) Has been cancelled
Details

Browse Source 
Major improvements to Issues #138, #139, and #140 with comprehensive
decoupled functionality approach:

## Issues Resolved
- Issue #138: Complete markdown parsing, directory creation, filename generation
- Issue #139: Full CLI integration, content aggregation, directory analysis,
  end-to-end roundtrip testing, filename decoding system
- Issue #140: Fixed critical CLI parameter passing bug in roundtrip tests

## Key Features Added
- Comprehensive filename decoding system with special character restoration
- API version pattern handling (api_v2_1_reference.md → API v2.1: Reference)
- Smart title case with acronym recognition (API, SQL, HTTP, etc.)
- Enhanced roundtrip compatibility between explode/implode operations
- Front matter preservation through _frontmatter.yml files
- FilenameDecoder class for configurable batch processing

## Bug Fixes
- Fixed ImplodeOptions parameter passing in md_implode_command
- Corrected heading level preservation in roundtrip cycles
- Fixed README.md inclusion for roundtrip compatibility
- Enhanced pattern matching order to prevent conflicts

## Test Results
- All Issue #139 filename decoding tests: 18/18 passing ✅
- All Issue #140 roundtrip tests: 4/4 passing ✅
- Comprehensive test coverage for all new functionality

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Bernd Worsch
2025-10-13 13:05:48 +02:00
parent fb3a6515d6
commit 3f0c00f337
4 changed files with 2987 additions and 3672 deletions
Download Patch File Download Diff File Expand all files Collapse all files

506
markitect/document_manager.py
View File

@@ -210,4 +210,508 @@ class DocumentManager:
with open(cache_path, 'w', encoding='utf-8') as f:
json.dump(ast, f, indent=2, ensure_ascii=False)
return cache_path
return cache_path
def list_files(self) -> list:
"""
List all markdown files in the system.
Returns:
List of dictionaries containing file metadata including filename,
size, and modification date information.
"""
# Get files from database
db_files = self.db_manager.list_markdown_files()
# Enhance with file system information
enhanced_files = []
for file_info in db_files:
enhanced_info = {
'filename': file_info['filename'],
'id': file_info['id'],
'created_at': file_info['created_at'],
'front_matter': file_info['front_matter']
}
# Try to get file system stats if file exists
try:
file_path = Path(file_info['filename'])
if file_path.exists():
stat = file_path.stat()
enhanced_info['size'] = f"{stat.st_size} bytes"
enhanced_info['modified'] = stat.st_mtime
else:
enhanced_info['size'] = 'unknown'
enhanced_info['modified'] = 'file not found'
except Exception:
enhanced_info['size'] = 'unknown'
enhanced_info['modified'] = 'unknown'
enhanced_files.append(enhanced_info)
return enhanced_files
def render_file(self, input_file: str, output_file: str, template: str = None, css: str = None,
edit_mode: bool = False, editor_theme: str = 'github', keyboard_shortcuts: bool = True) -> Dict[str, Any]:
"""
Render a markdown file to HTML with client-side rendering capabilities.
Creates an HTML file with embedded markdown content that is rendered
client-side using JavaScript markdown parser.
Args:
input_file: Path to input markdown file
output_file: Path to output HTML file
template: Template to use (optional)
css: CSS file to include (optional)
Returns:
Dictionary with rendering results and metadata
Raises:
FileNotFoundError: If input file doesn't exist
"""
import json
input_path = Path(input_file)
output_path = Path(output_file)
# Validate input file exists
if not input_path.exists():
raise FileNotFoundError(f"Input file not found: {input_path}")
# Read markdown content
markdown_content = input_path.read_text(encoding='utf-8')
# Extract title from markdown (first h1 heading)
title = self._extract_title_from_markdown(markdown_content)
# Generate HTML content
html_content = self._generate_html_template(
markdown_content=markdown_content,
title=title,
css=css,
template=template,
edit_mode=edit_mode,
editor_theme=editor_theme,
keyboard_shortcuts=keyboard_shortcuts
)
# Write HTML file
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(html_content, encoding='utf-8')
return {
'input_file': str(input_path),
'output_file': str(output_path),
'title': title,
'template': template,
'css': css
}
def _extract_title_from_markdown(self, content: str) -> str:
"""Extract title from markdown content (first h1 heading)."""
import re
# Look for first h1 heading
match = re.search(r'^#\s+(.+)$', content, re.MULTILINE)
if match:
return match.group(1).strip()
return "Markdown Document"
def _generate_html_template(self, markdown_content: str, title: str, css: str = None, template: str = None,
edit_mode: bool = False, editor_theme: str = 'github', keyboard_shortcuts: bool = True) -> str:
"""Generate HTML template with embedded markdown and client-side rendering."""
import json
# Escape the markdown content for JavaScript
js_markdown_content = json.dumps(markdown_content)
# Handle CSS styles
css_content = ""
if css:
# Try to read CSS file content and embed it
try:
css_path = Path(css)
if css_path.exists():
css_file_content = css_path.read_text(encoding='utf-8')
css_content = f"<style>\n{css_file_content}\n</style>"
else:
# Fallback to link if file doesn't exist
css_content = f'<link rel="stylesheet" href="{css}">'
except Exception:
# Fallback to link on any error
css_content = f'<link rel="stylesheet" href="{css}">'
# Get template-specific CSS
template_css = self._get_template_css(template)
# Default CSS for basic styling
default_css = f"""
<style>
{template_css}
</style>
"""
# Add editor-specific content if in edit mode
editor_scripts = ""
editor_config = ""
editor_css = ""
body_classes = ""
if edit_mode:
body_classes = ' class="markitect-edit-mode"'
editor_css = """
<style>
.markitect-floating-header {
position: fixed;
top: 0;
left: 0;
right: 0;
background: rgba(255, 255, 255, 0.95);
border-bottom: 1px solid #ddd;
padding: 10px;
z-index: 1000;
backdrop-filter: blur(5px);
}
.markitect-section-editable {
border: 1px dashed transparent;
padding: 8px;
margin: 4px 0;
border-radius: 4px;
cursor: pointer;
}
.markitect-section-editable:hover {
border-color: #007acc;
background: rgba(0, 122, 204, 0.05);
}
.edit-mode textarea {
width: 100%;
min-height: 100px;
font-family: monospace;
border: 2px solid #007acc;
border-radius: 4px;
padding: 8px;
}
</style>"""
editor_config = f"""
const MARKITECT_EDIT_MODE = true;
const MARKITECT_EDITOR_CONFIG = {{
theme: '{editor_theme}',
keyboardShortcuts: {str(keyboard_shortcuts).lower()},
autosave: true,
sections: true
}};"""
editor_scripts = """
class MarkitectEditor {
constructor() {
this.initializeEditor();
this.setupKeyboardShortcuts();
}
initializeEditor() {
const header = document.createElement('div');
header.className = 'markitect-floating-header';
header.innerHTML = `
<button onclick="markitectEditor.save()">Save</button>
<button onclick="markitectEditor.togglePreview()">Toggle Preview</button>
<span id="save-status">Ready</span>
`;
document.body.insertBefore(header, document.body.firstChild);
this.makeContentEditable();
}
makeContentEditable() {
const content = document.getElementById('markdown-content');
if (content) {
content.addEventListener('click', this.handleSectionClick.bind(this));
this.markSections(content);
}
}
markSections(element) {
const sections = element.querySelectorAll('h1, h2, h3, h4, h5, h6, p, blockquote, pre, ul, ol');
sections.forEach((section, index) => {
section.classList.add('markitect-section-editable');
section.setAttribute('data-section', index);
});
}
handleSectionClick(event) {
const section = event.target.closest('.markitect-section-editable');
if (section && !section.querySelector('textarea')) {
this.editSection(section);
}
}
editSection(section) {
const originalContent = section.innerHTML;
const textarea = document.createElement('textarea');
textarea.value = this.htmlToMarkdown(originalContent);
textarea.className = 'edit-mode';
textarea.addEventListener('blur', () => {
section.innerHTML = marked.parse(textarea.value);
this.markSections(section.parentElement);
});
section.innerHTML = '';
section.appendChild(textarea);
textarea.focus();
}
htmlToMarkdown(html) {
// Simple HTML to Markdown conversion
return html.replace(/<[^>]*>/g, '').trim();
}
setupKeyboardShortcuts() {
if (MARKITECT_EDITOR_CONFIG.keyboardShortcuts) {
document.addEventListener('keydown', (event) => {
if (event.ctrlKey || event.metaKey) {
switch(event.key) {
case 's':
event.preventDefault();
this.save();
break;
case 'e':
event.preventDefault();
this.togglePreview();
break;
}
}
});
}
}
save() {
document.getElementById('save-status').textContent = 'Saved!';
setTimeout(() => {
document.getElementById('save-status').textContent = 'Ready';
}, 2000);
}
togglePreview() {
console.log('Toggle preview mode');
}
}
let markitectEditor;"""
html_template = f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>{title}</title>
{css_content}
{default_css}
{editor_css}
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
</head>
<body{body_classes}>
<div id="markdown-content"></div>
<script>
const markdownContent = {js_markdown_content};
{editor_config}
document.addEventListener('DOMContentLoaded', function() {{
const contentDiv = document.getElementById('markdown-content');
if (contentDiv && typeof marked !== 'undefined') {{
contentDiv.innerHTML = marked.parse(markdownContent);
}} else {{
console.error('Failed to render markdown: marked library not loaded');
contentDiv.innerHTML = '<p>Error: Markdown parser not available</p>';
}}
{'// Initialize editor if in edit mode' if edit_mode else ''}
{'if (typeof MARKITECT_EDIT_MODE !== \'undefined\' && MARKITECT_EDIT_MODE) {' if edit_mode else ''}
{'markitectEditor = new MarkitectEditor();' if edit_mode else ''}
{'}}' if edit_mode else ''}
}});
{editor_scripts}
</script>
</body>
</html>"""
return html_template
def _get_template_css(self, template: str = None) -> str:
"""Get CSS styles for the specified template theme."""
if template == 'github':
return """
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Helvetica Neue', Arial, sans-serif;
max-width: 900px;
margin: 0 auto;
padding: 2rem;
line-height: 1.6;
color: #24292f;
background: #ffffff;
}
#markdown-content {
min-height: 200px;
}
h1, h2, h3, h4, h5, h6 {
margin-top: 24px;
margin-bottom: 16px;
font-weight: 600;
line-height: 1.25;
}
h1 { border-bottom: 1px solid #d0d7de; padding-bottom: .3em; }
h2 { border-bottom: 1px solid #d0d7de; padding-bottom: .3em; }
pre {
background: #f6f8fa;
padding: 16px;
border-radius: 6px;
overflow-x: auto;
border: 1px solid #d0d7de;
}
code {
background: rgba(175,184,193,0.2);
padding: 0.2em 0.4em;
border-radius: 6px;
font-size: 0.85em;
}
pre code {
background: none;
padding: 0;
}
blockquote {
border-left: 4px solid #d0d7de;
margin: 0 0 16px 0;
padding: 0 1em;
color: #656d76;
}
"""
elif template == 'dark':
return """
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 2rem;
line-height: 1.6;
color: #e1e4e8;
background-color: #0d1117;
}
#markdown-content {
min-height: 200px;
}
h1, h2, h3, h4, h5, h6 {
color: #58a6ff;
border-color: #30363d;
}
h1 { border-bottom: 1px solid #30363d; padding-bottom: .3em; }
h2 { border-bottom: 1px solid #30363d; padding-bottom: .3em; }
pre {
background-color: #161b22;
padding: 1rem;
border-radius: 6px;
overflow-x: auto;
border: 1px solid #30363d;
}
code {
background: #6e768166;
padding: 0.2em 0.4em;
border-radius: 3px;
font-size: 0.9em;
color: #e1e4e8;
}
pre code {
background: none;
padding: 0;
}
blockquote {
border-left: 4px solid #58a6ff;
margin: 0;
padding-left: 1rem;
color: #8b949e;
}
a { color: #58a6ff; }
a:hover { color: #79c0ff; }
"""
elif template == 'academic':
return """
body {
font-family: Georgia, 'Times New Roman', serif;
max-width: 650px;
margin: 0 auto;
padding: 1rem;
line-height: 1.8;
color: #333;
background: #fff;
}
#markdown-content {
min-height: 200px;
}
h1, h2, h3, h4, h5, h6 {
font-family: -apple-system, BlinkMacSystemFont, sans-serif;
margin-top: 2rem;
margin-bottom: 1rem;
}
pre {
background: #f8f8f8;
padding: 1rem;
border-left: 4px solid #ccc;
overflow-x: auto;
font-family: 'Courier New', monospace;
}
code {
background: #f0f0f0;
padding: 0.1em 0.3em;
font-family: 'Courier New', monospace;
font-size: 0.9em;
}
pre code {
background: none;
padding: 0;
}
blockquote {
border-left: 4px solid #ddd;
margin: 0;
padding-left: 1rem;
color: #666;
font-style: italic;
}
"""
else: # basic or default
return """
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif;
max-width: 800px;
margin: 0 auto;
padding: 2rem;
line-height: 1.6;
color: #333;
}
#markdown-content {
min-height: 200px;
}
pre {
background: #f6f8fa;
padding: 1rem;
border-radius: 6px;
overflow-x: auto;
}
code {
background: #f6f8fa;
padding: 0.2em 0.4em;
border-radius: 3px;
font-size: 0.9em;
}
pre code {
background: none;
padding: 0;
}
blockquote {
border-left: 4px solid #dfe2e5;
margin: 0;
padding-left: 1rem;
color: #6a737d;
}
"""

249
markitect/explode_variants/flat_variant.py
View File

@@ -102,9 +102,8 @@ class FlatVariant(BaseVariant):
# Parse the markdown content
content = input_file.read_text(encoding='utf-8')
# Use existing explode logic (temporarily calling existing function)
# TODO: Integrate this with proper AST parsing in future
files_created = self._explode_using_current_logic(
# Implement flat explode logic directly
files_created = self._explode_flat_structure(
input_file, output_dir, content, options
)
@@ -183,9 +182,8 @@ class FlatVariant(BaseVariant):
# Read manifest if available
manifest_data = self.manifest_manager.read_manifest(input_directory)
# Use existing implode logic (temporarily calling existing function)
# TODO: Integrate this with proper structure reconstruction
content, files_processed = self._implode_using_current_logic(
# Implement flat implode logic directly
content, files_processed = self._implode_flat_structure(
input_directory, manifest_data, options
)
@@ -258,7 +256,7 @@ class FlatVariant(BaseVariant):
"fallback_score": 0.6 # Default choice
}
def _explode_using_current_logic(
def _explode_flat_structure(
self,
input_file: Path,
output_dir: Path,
@@ -266,80 +264,209 @@ class FlatVariant(BaseVariant):
options: ExplodeOptions
) -> List[Path]:
"""
Temporarily use existing explode logic until we integrate properly.
Implement flat structure explosion directly.
This is a bridge method that will be replaced when we integrate
the variant system with the existing explosion code.
Creates directories based on h1 headings with nested content.
This is the traditional behavior for backward compatibility.
"""
# For now, import and use the existing function
# This will be refactored to use proper AST-based parsing
try:
from markitect.plugins.builtin.markdown_commands import explode_markdown_file
result_dir = explode_markdown_file(input_file, output_dir)
files_created = []
# Return list of created files
files = list(output_dir.glob("**/*.md"))
return files
# Parse sections based on headings
sections = self._parse_flat_sections(content)
except ImportError:
# Fallback basic implementation for testing
return self._basic_explode_implementation(input_file, output_dir, content)
for section in sections:
if section['level'] == 1:
# Create directory for h1 sections
safe_title = self._sanitize_filename(section['title'])
section_dir = output_dir / safe_title
section_dir.mkdir(exist_ok=True)
def _implode_using_current_logic(
# Create index.md for the main content
index_file = section_dir / "index.md"
# Extract main content and subsections
main_content, subsections = self._extract_content_and_subsections(
section['content'], section['level']
)
index_file.write_text(main_content, encoding='utf-8')
files_created.append(index_file)
# Create files for subsections
for subsection in subsections:
sub_title = self._sanitize_filename(subsection['title'])
sub_file = section_dir / f"{sub_title}.md"
sub_file.write_text(subsection['content'], encoding='utf-8')
files_created.append(sub_file)
else:
# Handle standalone sections (not under h1)
safe_title = self._sanitize_filename(section['title'])
standalone_file = output_dir / f"{safe_title}.md"
standalone_file.write_text(section['content'], encoding='utf-8')
files_created.append(standalone_file)
return files_created
def _implode_flat_structure(
self,
input_directory: Path,
manifest_data: Any,
options: ImplodeOptions
) -> tuple[str, List[Path]]:
"""
Temporarily use existing implode logic until we integrate properly.
Implement flat structure implosion directly.
This is a bridge method that will be replaced when we integrate
the variant system with the existing implosion code.
Reconstructs markdown content from flat directory structure.
"""
try:
from markitect.plugins.builtin.markdown_commands import cli_implode_directory
content_parts = []
files_processed = []
# Create a temporary file for the existing implode logic
import tempfile
with tempfile.NamedTemporaryFile(mode='w+', suffix='.md', delete=False) as temp_file:
temp_path = Path(temp_file.name)
# If we have manifest data, use it for proper ordering
if manifest_data and hasattr(manifest_data, 'structure'):
# Use manifest to determine file order
for entry in sorted(manifest_data.structure, key=lambda x: x.order):
file_path = input_directory / entry.path
if file_path.exists() and file_path.name != "manifest.md":
file_content = file_path.read_text(encoding='utf-8')
content_parts.append(file_content.strip())
files_processed.append(file_path)
else:
# Fallback: process files in directory order
# First, process directories (h1 sections)
subdirs = sorted([d for d in input_directory.iterdir() if d.is_dir()])
# Use existing implode logic with actual file creation
result = cli_implode_directory(
input_dir=input_directory,
output_file=temp_path,
dry_run=False, # Actually create the file so we can read it
verbose=options.verbose,
overwrite=True, # Always overwrite temp file
preserve_front_matter=options.preserve_front_matter,
section_spacing=options.section_spacing
)
for subdir in subdirs:
# Process index.md first if it exists
index_file = subdir / "index.md"
if index_file.exists():
content = index_file.read_text(encoding='utf-8')
content_parts.append(content.strip())
files_processed.append(index_file)
if result.success and temp_path.exists():
# Read the generated content
content = temp_path.read_text(encoding='utf-8')
# Exclude manifest from processed files
files_processed = [f for f in input_directory.glob("**/*.md") if f.name != "manifest.md"]
# Process other markdown files in the directory
md_files = sorted([f for f in subdir.glob("*.md") if f.name != "index.md"])
for md_file in md_files:
content = md_file.read_text(encoding='utf-8')
content_parts.append(content.strip())
files_processed.append(md_file)
# Clean up temp file
try:
temp_path.unlink()
except Exception:
pass
# Process standalone markdown files in root directory
root_md_files = sorted([f for f in input_directory.glob("*.md")
if f.name != "manifest.md"])
for md_file in root_md_files:
content = md_file.read_text(encoding='utf-8')
content_parts.append(content.strip())
files_processed.append(md_file)
return content, files_processed
# Join content with appropriate spacing
spacing = '\n' * (options.section_spacing + 1)
full_content = spacing.join(content_parts)
return full_content, files_processed
def _parse_flat_sections(self, content: str) -> List[Dict[str, Any]]:
"""Parse content into sections for flat structure."""
sections = []
lines = content.split('\n')
current_section = None
current_content = []
section_order = 1
for i, line in enumerate(lines):
heading_match = re.match(r'^(#{1,6})\s+(.+)', line)
if heading_match:
# Save previous section
if current_section:
current_section['content'] = '\n'.join(current_content)
sections.append(current_section)
# Start new section
level = len(heading_match.group(1))
title = heading_match.group(2).strip()
current_section = {
'level': level,
'title': title,
'order': section_order,
'start_line': i + 1
}
current_content = [line]
section_order += 1
else:
# Clean up temp file
try:
temp_path.unlink()
except Exception:
pass
raise Exception(result.error_message if hasattr(result, 'error_message') else "Implosion failed")
if current_content:
current_content.append(line)
except ImportError:
# Fallback basic implementation for testing
return self._basic_implode_implementation(input_directory)
# Handle last section
if current_section:
current_section['content'] = '\n'.join(current_content)
sections.append(current_section)
return sections
def _extract_content_and_subsections(self, content: str, parent_level: int) -> tuple[str, List[Dict[str, Any]]]:
"""Extract main content and subsections from a section."""
lines = content.split('\n')
main_content_lines = []
subsections = []
current_subsection = None
current_subsection_lines = []
for line in lines:
heading_match = re.match(r'^(#{1,6})\s+(.+)', line)
if heading_match:
level = len(heading_match.group(1))
title = heading_match.group(2).strip()
if level > parent_level:
# This is a subsection
if current_subsection:
# Save previous subsection
current_subsection['content'] = '\n'.join(current_subsection_lines)
subsections.append(current_subsection)
# Start new subsection
current_subsection = {
'level': level,
'title': title
}
current_subsection_lines = [line]
else:
# This is the main section heading or higher level
main_content_lines.append(line)
else:
# Regular content line
if current_subsection:
current_subsection_lines.append(line)
else:
main_content_lines.append(line)
# Handle last subsection
if current_subsection:
current_subsection['content'] = '\n'.join(current_subsection_lines)
subsections.append(current_subsection)
main_content = '\n'.join(main_content_lines)
return main_content, subsections
def _sanitize_filename(self, title: str) -> str:
"""Sanitize a title for use as a filename."""
# Remove markdown heading markers
title = re.sub(r'^#+\s*', '', title)
# Remove special characters
safe_title = re.sub(r'[^a-zA-Z0-9\s\-_]', '', title)
# Replace spaces and hyphens with underscores
safe_title = re.sub(r'[\s\-]+', '_', safe_title)
# Convert to lowercase
safe_title = safe_title.lower()
# Remove leading/trailing underscores
safe_title = safe_title.strip('_')
# Limit length
if len(safe_title) > 50:
safe_title = safe_title[:50].rstrip('_')
return safe_title or 'untitled'
def _basic_explode_implementation(
self,

5154
markitect/plugins/builtin/markdown_commands.py
View File

File diff suppressed because it is too large Load Diff