diff --git a/README.html b/README.html new file mode 100644 index 00000000..6feee6a4 --- /dev/null +++ b/README.html @@ -0,0 +1,68 @@ + + + + + + README + + + +
+ + + + + \ No newline at end of file diff --git a/cost_notes/issue_135_cost_2025-10-07.md b/cost_notes/issue_135_cost_2025-10-07.md new file mode 100644 index 00000000..da1136c8 --- /dev/null +++ b/cost_notes/issue_135_cost_2025-10-07.md @@ -0,0 +1,139 @@ +--- +note_type: "issue_cost_tracking" +issue_id: 135 +issue_title: "Instant Markdown base and publication directory" +session_date: "2025-10-07" +claude_model: "claude-sonnet-4" +total_cost_eur: 0.4416 +total_cost_usd: 0.48 +total_tokens: 63000 +generated_at: "2025-10-07T11:30:00.000000" +--- + +# Issue #135 Implementation Cost +**Issue**: Instant Markdown base and publication directory +**Date**: 2025-10-07 +**Claude Model**: claude-sonnet-4 + +## Cost Summary +- **Total Cost**: €0.4416 ($0.48 USD) +- **Token Usage**: 63,000 tokens +- **Input Tokens**: 38,000 tokens @ $3.00/M +- **Output Tokens**: 25,000 tokens @ $15.00/M + +## Cost Breakdown + +| Component | Tokens | Rate ($/M) | Cost (USD) | Cost (EUR) | +|-----------|--------|------------|------------|------------| +| Input | 38,000 | $3.00 | $0.1140 | €0.1049 | +| Output | 25,000 | $15.00 | $0.3750 | €0.3450 | +| **Total** | 63,000 | - | $0.4890 | €0.4499 | + +## Implementation Summary +Complete TDD8 workflow implementation of instant markdown base and publication directory functionality. Extended md-render command to support both single files and directory processing with publication directory management, environment variable override, and CLI flags for behavior control. Generated comprehensive test suite with 18 tests covering all scenarios from publication directory management to edge cases. + +## Key Features Delivered +- Publication directory support with ~/Notes/ default and MARKITECT_PUBLICATION_DIR override +- Single file processing with --use-publication-dir flag +- Directory processing with recursive traversal and structure preservation +- --dont-use-publication-dir flag for placing HTML next to MD files +- Comprehensive CLI integration with detailed help documentation +- 9 new helper functions for directory/file processing +- Full backward compatibility maintained +- Extensive test coverage (18 tests, 100% pass rate) + +## Technical Implementation Details +- Modified md-render command with new CLI options and directory support +- Added publication directory management functions (get_publication_directory, normalize_publication_path, ensure_publication_directory) +- Implemented file processing functions (process_single_file, process_directory, find_markdown_files) +- Created utility functions (get_output_filename, get_relative_output_path, _render_single_markdown_file) +- Updated command help documentation with examples and usage patterns +- Comprehensive error handling and edge case management + +## Test Coverage Breakdown +1. **Publication Directory Management** (4 tests): Default directory, environment variable override, directory creation, path normalization +2. **Single File Processing** (3 tests): Default behavior, publication directory usage, naming conventions +3. **Directory Processing** (4 tests): Publication directory with structure, HTML next to MD, recursive traversal, structure preservation +4. **CLI Integration** (4 tests): Flag presence, directory input support, environment variable integration +5. **Edge Cases** (3 tests): Empty directories, mixed content, error handling + +## Cost Allocation +This cost has been allocated to the 'AI & ML Services' category as a one-time expense for issue #135 implementation using full TDD8 methodology including requirements analysis, test design, implementation, refactoring, documentation, and integration. + +## Implementation Methodology +- **TDD8 Workflow**: Complete ISSUE→TEST→RED→GREEN→REFACTOR→DOCUMENT→REFINE→PUBLISH cycle +- **Test-Driven Approach**: 18 tests written first (RED state), then implementation (GREEN state) +- **Code Quality**: Refactoring phase ensured clean, maintainable code +- **Documentation**: Comprehensive implementation and test plan documentation +- **Integration**: Full CLI integration with help text and examples + +## Performance Metrics +- **Development Time**: Full TDD8 cycle implementation +- **Test Success Rate**: 18/18 tests passing (100%) +- **Code Quality**: Clean, well-documented, modular implementation +- **CLI Integration**: Complete with comprehensive help documentation +- **Backward Compatibility**: Maintained for all existing functionality + +## Notes +- Currency conversion rate: 1 USD = 0.920 EUR +- Pricing based on claude-sonnet-4 rates as of 2025-10-07 +- Token counts estimated based on comprehensive TDD8 implementation session +- Includes requirements engineering, test generation, RED-GREEN-REFACTOR cycle, documentation, and final integration +- Higher token count than typical due to extensive directory processing logic and comprehensive test coverage + + \ No newline at end of file diff --git a/markitect/plugins/builtin/markdown_commands.py b/markitect/plugins/builtin/markdown_commands.py index ea68ae0b..fbdfd8f1 100644 --- a/markitect/plugins/builtin/markdown_commands.py +++ b/markitect/plugins/builtin/markdown_commands.py @@ -8,6 +8,7 @@ replacing the legacy unprefixed commands for better namespace consistency. import click import json import os +import re import tempfile from pathlib import Path from typing import Dict, Any @@ -43,7 +44,8 @@ class MarkdownCommandsPlugin(CommandPlugin): 'md-ingest': md_ingest_command, 'md-get': md_get_command, 'md-list': md_list_command, - 'md-render': md_render_command + 'md-render': md_render_command, + 'md-index': md_index_command } @@ -400,6 +402,81 @@ def md_render_command(ctx, input_file, output, template, css, edit, editor_theme raise click.Abort() +@click.command() +@click.argument('directory', type=click.Path(exists=True)) +@click.option('--output', '-o', type=click.Path(), help='Output index file path (defaults to directory/index.html)') +@click.option('--template', type=click.Choice(['basic', 'github', 'academic', 'dark']), + default='basic', help='HTML template: basic (default), github, academic, or dark theme') +@click.option('--recursive', '-r', is_flag=True, help='Include HTML files from subdirectories') +@click.pass_context +def md_index_command(ctx, directory, output, template, recursive): + """ + Generate an index page for HTML files in a directory. + + Creates an HTML index page that lists all HTML files found in the specified + directory, providing navigation links to each file. The index page uses the + same template system as md-render for consistent styling. + + DIRECTORY: Path to the directory containing HTML files + + Examples: + # Generate index for current directory + markitect md-index . + + # Generate index with custom output file + markitect md-index docs/ --output docs/contents.html + + # Generate index with GitHub template + markitect md-index notes/ --template github + + # Include subdirectories recursively + markitect md-index docs/ --recursive + """ + config = ctx.obj or {} + try: + directory_path = Path(directory) + + if config.get('verbose', False): + click.echo(f"Generating index for directory: {directory_path}") + + # Determine output file + if output: + output_path = Path(output) + else: + output_path = directory_path / "index.html" + + # Find and filter HTML files + html_files = find_html_files(directory_path, recursive=recursive) + html_files = [f for f in html_files if f != output_path] + + if config.get('verbose', False): + click.echo(f"Found {len(html_files)} HTML file(s)") + + # Prepare file info for template + file_infos = _prepare_file_infos(html_files, output_path) + + # Generate and write index HTML + directory_name = directory_path.name or "Directory" + index_title = f"{directory_name} - Index" + index_html = generate_index_html(file_infos, index_title, template) + + # Ensure output directory exists and write file + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(index_html, encoding='utf-8') + + click.echo(f"✓ Index generated: {output_path}") + + if config.get('verbose', False): + click.echo(f" Template: {template}") + click.echo(f" Files indexed: {len(file_infos)}") + if recursive: + click.echo(f" Recursive: enabled") + + except Exception as e: + click.echo(f"Error generating index: {e}", err=True) + raise click.Abort() + + def _render_single_markdown_file(input_path, output_path, template, css, edit, editor_theme, keyboard_shortcuts, config): """Render a single markdown file to HTML.""" # Read markdown file @@ -1020,4 +1097,205 @@ def process_directory(input_dir, use_publication_dir, publication_dir): output_files.append(output_file) - return output_files \ No newline at end of file + return output_files + + +# Index generation functions for Issue #136 +def find_html_files(directory, recursive=False): + """Find all HTML files in a directory.""" + directory = Path(directory) + html_files = [] + + if recursive: + for pattern in ['*.html', '*.htm']: + html_files.extend(directory.rglob(pattern)) + else: + for pattern in ['*.html', '*.htm']: + html_files.extend(directory.glob(pattern)) + + return sorted(html_files) + + +# HTML parsing patterns for index generation +HTML_TITLE_PATTERN = re.compile(r']*>(.*?)', re.IGNORECASE | re.DOTALL) +HTML_H1_PATTERN = re.compile(r']*>(.*?)', re.IGNORECASE | re.DOTALL) +HTML_TAG_PATTERN = re.compile(r'<[^>]+>') + + +def extract_html_title(html_file): + """Extract title from HTML file, falling back to H1 tag or filename.""" + try: + content = html_file.read_text(encoding='utf-8') + + # Try to extract from title tag + title_match = HTML_TITLE_PATTERN.search(content) + if title_match: + return title_match.group(1).strip() + + # Try to extract from H1 tag + h1_match = HTML_H1_PATTERN.search(content) + if h1_match: + # Remove HTML tags from H1 content + h1_text = HTML_TAG_PATTERN.sub('', h1_match.group(1)) + return h1_text.strip() + + # Fallback to filename + return html_file.stem + + except Exception: + # If any error occurs, fallback to filename + return html_file.stem + + +def generate_index_html(html_files, title, template="basic"): + """Generate HTML index page with links to HTML files.""" + # Get template styles from existing TEMPLATE_STYLES + styles = TEMPLATE_STYLES.get(template, TEMPLATE_STYLES['basic']) + + # Generate links list + links_html = "" + if html_files: + links_html = "" + else: + links_html = "

No HTML files found in this directory.

" + + # Generate HTML template + html_template = ''' + + + + + {title} + + + +

{title}

+ +
+

📁 Directory Index - Navigate through the available HTML pages

+
+ +

Available Pages

+ {links_html} + +
+

+ Generated with MarkiTect • {file_count} file(s) +

+ +''' + + return html_template.format( + title=title, + links_html=links_html, + file_count=len(html_files), + **styles + ) + + +def _prepare_file_infos(html_files, output_path): + """Prepare file information for template generation.""" + file_infos = [] + for html_file in html_files: + title = extract_html_title(html_file) + + # Calculate relative path from output directory to HTML file + try: + relative_path = html_file.relative_to(output_path.parent) + except ValueError: + # If files are in different directory trees, use filename + relative_path = html_file.name + + file_infos.append({ + 'path': html_file, + 'title': title, + 'relative_path': str(relative_path) + }) + return file_infos + + +def process_directory_for_index(directory, index_filename="index.html", template="basic", recursive=False): + """Process directory and generate index file.""" + directory = Path(directory) + output_path = directory / index_filename + + if not directory.exists() or not directory.is_dir(): + raise FileNotFoundError(f"Directory not found: {directory}") + + # Find and filter HTML files + html_files = find_html_files(directory, recursive=recursive) + html_files = [f for f in html_files if f != output_path] + + # Prepare file info for template + file_infos = _prepare_file_infos(html_files, output_path) + + # Generate and write index HTML + directory_name = directory.name or "Directory" + index_title = f"{directory_name} - Index" + index_html = generate_index_html(file_infos, index_title, template) + + # Ensure output directory exists and write file + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(index_html, encoding='utf-8') + + return output_path \ No newline at end of file diff --git a/tests/test_issue_136_index_generation.py b/tests/test_issue_136_index_generation.py new file mode 100644 index 00000000..0739b1ed --- /dev/null +++ b/tests/test_issue_136_index_generation.py @@ -0,0 +1,539 @@ +#!/usr/bin/env python3 +""" +Test suite for Issue #136: Index page for notes in a directory + +This test suite validates the index page generation functionality for HTML files, +including directory scanning, HTML generation, and CLI integration. + +TDD8 Workflow: ISSUE→TEST→RED→GREEN→REFACTOR→DOCUMENT→REFINE→PUBLISH +State: RED (Tests should fail initially) +""" + +import pytest +import tempfile +import os +import shutil +from pathlib import Path +from unittest.mock import patch, MagicMock +import subprocess +import re +from html.parser import HTMLParser + + +class SimpleHTMLParser(HTMLParser): + """Simple HTML parser to extract title and links for testing.""" + + def __init__(self): + super().__init__() + self.title = None + self.links = [] + self.in_title = False + + def handle_starttag(self, tag, attrs): + if tag == 'title': + self.in_title = True + elif tag == 'a': + href = dict(attrs).get('href', '') + self.links.append({'href': href, 'text': ''}) + + def handle_endtag(self, tag): + if tag == 'title': + self.in_title = False + + def handle_data(self, data): + if self.in_title: + self.title = data.strip() + elif self.links and not self.links[-1]['text']: + self.links[-1]['text'] = data.strip() + + +class TestHTMLFileDiscovery: + """Test HTML file discovery and processing.""" + + def setup_method(self): + """Set up test environment with temporary directories and files.""" + self.temp_dir = tempfile.mkdtemp() + self.test_dir = Path(self.temp_dir) / "test_notes" + self.test_dir.mkdir() + + # Create test HTML files + (self.test_dir / "index.html").write_text(""" + +Index Page +

Index Page

Main index

+""") + + (self.test_dir / "document1.html").write_text(""" + +Document One +

Document One

Content here

+""") + + (self.test_dir / "notes.html").write_text(""" + +My Notes +

My Notes

Note content

+""") + + # Create subdirectory with HTML files + sub_dir = self.test_dir / "subdir" + sub_dir.mkdir() + (sub_dir / "subdoc.html").write_text(""" + +Sub Document +

Sub Document

Sub content

+""") + + # Create non-HTML files (should be ignored) + (self.test_dir / "readme.txt").write_text("Not HTML") + (self.test_dir / "image.png").write_bytes(b"fake image data") + + def teardown_method(self): + """Clean up test environment.""" + shutil.rmtree(self.temp_dir) + + def test_find_html_files_in_directory(self): + """Test finding all HTML files in a directory.""" + from markitect.plugins.builtin.markdown_commands import find_html_files + + html_files = find_html_files(self.test_dir) + + expected_files = [ + self.test_dir / "index.html", + self.test_dir / "document1.html", + self.test_dir / "notes.html" + ] + + assert len(html_files) == 3 + for expected_file in expected_files: + assert expected_file in html_files + + def test_find_html_files_recursively(self): + """Test finding HTML files recursively in subdirectories.""" + from markitect.plugins.builtin.markdown_commands import find_html_files + + html_files = find_html_files(self.test_dir, recursive=True) + + expected_files = [ + self.test_dir / "index.html", + self.test_dir / "document1.html", + self.test_dir / "notes.html", + self.test_dir / "subdir" / "subdoc.html" + ] + + assert len(html_files) == 4 + for expected_file in expected_files: + assert expected_file in html_files + + def test_extract_title_from_html_file(self): + """Test extracting title from HTML file.""" + from markitect.plugins.builtin.markdown_commands import extract_html_title + + title = extract_html_title(self.test_dir / "document1.html") + assert title == "Document One" + + title = extract_html_title(self.test_dir / "notes.html") + assert title == "My Notes" + + def test_extract_title_from_h1_if_no_title_tag(self): + """Test extracting title from H1 tag if no title tag exists.""" + from markitect.plugins.builtin.markdown_commands import extract_html_title + + # Create HTML file without title tag + no_title_file = self.test_dir / "no_title.html" + no_title_file.write_text(""" + + +

Header Title

Content

+""") + + title = extract_html_title(no_title_file) + assert title == "Header Title" + + def test_extract_title_fallback_to_filename(self): + """Test falling back to filename if no title or H1 found.""" + from markitect.plugins.builtin.markdown_commands import extract_html_title + + # Create HTML file without title or H1 + plain_file = self.test_dir / "plain_file.html" + plain_file.write_text(""" + + +

Just content

+""") + + title = extract_html_title(plain_file) + assert title == "plain_file" + + +class TestIndexPageGeneration: + """Test index page HTML generation.""" + + def setup_method(self): + """Set up test environment.""" + self.temp_dir = tempfile.mkdtemp() + self.test_dir = Path(self.temp_dir) / "test_notes" + self.test_dir.mkdir() + + def teardown_method(self): + """Clean up test environment.""" + shutil.rmtree(self.temp_dir) + + def test_generate_index_html_structure(self): + """Test generating basic index HTML structure.""" + from markitect.plugins.builtin.markdown_commands import generate_index_html + + html_files = [ + {"path": self.test_dir / "doc1.html", "title": "Document One", "relative_path": "doc1.html"}, + {"path": self.test_dir / "doc2.html", "title": "Document Two", "relative_path": "doc2.html"} + ] + + html_content = generate_index_html(html_files, "Test Directory Index") + + # Parse HTML to verify structure + parser = SimpleHTMLParser() + parser.feed(html_content) + + assert parser.title == "Test Directory Index" + + # Check for navigation list + assert "