coulomb/markitect-filter
2
0
Fork 0
generated from coulomb/repo-seed
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add source attachment metadata compatibility

Browse Source
This commit is contained in:
Bernd Worsch
2026-05-15 14:36:24 +02:00
parent afa51f8764
commit ad137b214f
13 changed files with 724 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/markitect_filter/__init__.py
View File

@@ -1,5 +1,10 @@
"""Concrete source-format adapters for Markitect."""
from markitect_filter.adapters import epub3_adapter_descriptor, pdf_adapter_descriptor
from markitect_filter.assets import SOURCE_ATTACHMENT_METADATA_VERSION
__all__ = ["epub3_adapter_descriptor", "pdf_adapter_descriptor"]
__all__ = [
"SOURCE_ATTACHMENT_METADATA_VERSION",
"epub3_adapter_descriptor",
"pdf_adapter_descriptor",
]

11
src/markitect_filter/adapters.py
View File

@@ -41,12 +41,15 @@ def epub3_adapter_descriptor() -> SourceAdapterDescriptor:
},
quality_profile={
"text_extraction": "stdlib-xhtml",
"images": "metadata-only",
"styles": "ignored",
"images": "metadata-with-digest",
"styles": "metadata-with-digest",
"fonts": "metadata-with-digest",
"attachments": "read-side-source-assets",
},
metadata={
"format": "EPUB3",
"dependency_profile": "stdlib",
"render_asset_manifest_compatible": True,
},
)
@@ -96,12 +99,14 @@ def pdf_adapter_descriptor() -> SourceAdapterDescriptor:
},
quality_profile={
"text_extraction": "stdlib-pdf-text",
"images": "diagnostic-only",
"attachments": "metadata-with-digest",
"images": "signal-only",
"styles": "ignored",
"tables": "plain-text-only",
},
metadata={
"format": "PDF",
"dependency_profile": "stdlib",
"render_asset_manifest_compatible": True,
},
)

88
src/markitect_filter/assets.py Normal file
View File

@@ -0,0 +1,88 @@
"""Read-side source asset metadata helpers."""
from __future__ import annotations
import hashlib
import mimetypes
import posixpath
from pathlib import PurePosixPath
from typing import Any
from markitect_tool.source import SourceAsset
SOURCE_ATTACHMENT_METADATA_VERSION = "markitect-filter.source-attachment.v1"
def bytes_digest(data: bytes) -> str:
"""Return a Markitect-compatible sha256 digest for source bytes."""
return "sha256:" + hashlib.sha256(data).hexdigest()
def source_member_uri(container_uri: str, member_path: str) -> str:
"""Return a stable URI for a source package member."""
return f"{container_uri}!/{member_path}"
def source_asset_from_member(
*,
container_uri: str,
member_path: str,
data: bytes,
media_type: str | None,
source_adapter: str,
source_role: str,
metadata: dict[str, Any] | None = None,
) -> SourceAsset:
"""Build read-side metadata for a package member without extracting it."""
name = PurePosixPath(member_path).name or member_path
extension = PurePosixPath(name).suffix.lower() or None
resolved_media_type = media_type or mimetypes.guess_type(name)[0] or "application/octet-stream"
return SourceAsset(
uri=source_member_uri(container_uri, member_path),
path=member_path,
name=name,
media_type=resolved_media_type,
extension=extension,
size=len(data),
digest=bytes_digest(data),
metadata={
"schema_version": SOURCE_ATTACHMENT_METADATA_VERSION,
"source_adapter": source_adapter,
"source_role": source_role,
"package_path": posixpath.normpath(member_path),
**(metadata or {}),
},
)
def source_asset_signal(
*,
container_uri: str,
signal_id: str,
media_type: str,
source_adapter: str,
source_role: str,
digest_parts: list[bytes],
metadata: dict[str, Any] | None = None,
) -> SourceAsset:
"""Build metadata for a detected source resource signal without bytes."""
digest = bytes_digest(b"\n".join(digest_parts))
return SourceAsset(
uri=f"{container_uri}#{signal_id}",
path=signal_id,
name=signal_id.rsplit("/", 1)[-1],
media_type=media_type,
digest=digest,
metadata={
"schema_version": SOURCE_ATTACHMENT_METADATA_VERSION,
"source_adapter": source_adapter,
"source_role": source_role,
"signal_only": True,
**(metadata or {}),
},
)

123
src/markitect_filter/epub3.py
View File

@@ -28,12 +28,29 @@ from markitect_tool.source import (
)
from markitect_filter.adapters import epub3_adapter_descriptor
from markitect_filter.assets import source_asset_from_member
XHTML_MEDIA_TYPES = {
"application/xhtml+xml",
"text/html",
}
EPUB_ATTACHMENT_MEDIA_PREFIXES = (
"audio/",
"font/",
"image/",
"video/",
)
EPUB_ATTACHMENT_MEDIA_TYPES = {
"application/font-sfnt",
"application/vnd.ms-opentype",
"application/x-font-ttf",
"font/otf",
"font/ttf",
"font/woff",
"font/woff2",
"text/css",
}
BOILERPLATE_HINTS = {
"cover",
"nav",
@@ -101,11 +118,15 @@ class Epub3ReadAdapter:
asset=request.asset,
adapter=_adapter_info(request.options),
metadata=metadata,
capabilities=["read"],
capabilities=["read", "attachments"],
quality=NormalizationQuality(
lossiness="unknown" if has_error(diagnostics) else "low",
confidence=0.9 if not has_error(diagnostics) else 0.0,
warnings=_warning_count(diagnostics),
metadata={
"manifest_items": len(package.manifest) if package else 0,
"attachment_candidates": _attachment_candidate_count(package.manifest) if package else 0,
},
),
diagnostics=diagnostics,
valid=not has_error(diagnostics),
@@ -122,6 +143,7 @@ class Epub3ReadAdapter:
skip_boilerplate = bool(request.options.get("skip_boilerplate", True))
try:
with zipfile.ZipFile(Path(request.asset.path or request.asset.uri)) as archive:
attachments = _extract_attachments(archive, request.asset, package, diagnostics)
for order, item_id in enumerate(package.spine):
item = package.manifest.get(item_id)
if item is None:
@@ -187,7 +209,10 @@ class Epub3ReadAdapter:
confidence=0.9 if not has_error(diagnostics) else 0.0,
skipped_items=sum(1 for diagnostic in diagnostics if diagnostic.code == "source.epub3.skipped_boilerplate"),
warnings=_warning_count(diagnostics),
metadata={"extraction": "epub3-stdlib-xhtml"},
metadata={
"extraction": "epub3-stdlib-xhtml",
"attachment_count": len(attachments),
},
)
adapter = _adapter_info(request.options)
document = NormalizedMarkdownDocument(
@@ -203,9 +228,10 @@ class Epub3ReadAdapter:
source_uri=request.asset.uri,
source_path=request.asset.path,
digest=request.asset.digest,
metadata={"rootfile": package.rootfile_path},
metadata={"rootfile": package.rootfile_path, "attachment_count": len(attachments)},
)
],
attachments=attachments,
adapter=adapter,
cache_key=normalization_cache_key(
asset=request.asset,
@@ -393,6 +419,97 @@ def _extract_nav_labels(
return labels
def _extract_attachments(
archive: zipfile.ZipFile,
asset: SourceAsset,
package: EpubPackage,
diagnostics: list[Diagnostic],
) -> list[SourceAsset]:
attachments: list[SourceAsset] = []
for item in sorted(package.manifest.values(), key=lambda value: value.get("href", "")):
media_type = item.get("media_type", "")
href = item.get("href", "")
if not href or media_type in XHTML_MEDIA_TYPES:
continue
package_path = _resolve_package_path(package.rootfile_path, href)
if not _is_attachment_media(media_type):
diagnostics.append(
_warning(
asset,
"source.epub3.skipped_resource",
f"Skipped unsupported EPUB resource media type `{media_type}`.",
details={
"href": href,
"package_path": package_path,
"media_type": media_type,
"manifest_id": item.get("id"),
},
)
)
continue
try:
data = archive.read(package_path)
except KeyError:
diagnostics.append(
_warning(
asset,
"source.epub3.missing_resource",
f"EPUB resource `{package_path}` is declared but missing.",
details={
"href": href,
"package_path": package_path,
"media_type": media_type,
"manifest_id": item.get("id"),
},
)
)
continue
attachments.append(
source_asset_from_member(
container_uri=asset.uri,
member_path=package_path,
data=data,
media_type=media_type,
source_adapter="source.epub3",
source_role=_attachment_role(media_type),
metadata={
"href": href,
"manifest_id": item.get("id"),
"properties": item.get("properties", ""),
"container_path": asset.path,
"render_manifest_compatible": True,
},
)
)
return attachments
def _attachment_candidate_count(manifest: dict[str, dict[str, str]]) -> int:
return sum(1 for item in manifest.values() if _is_attachment_media(item.get("media_type", "")))
def _is_attachment_media(media_type: str) -> bool:
normalized = media_type.lower()
return normalized in EPUB_ATTACHMENT_MEDIA_TYPES or any(
normalized.startswith(prefix) for prefix in EPUB_ATTACHMENT_MEDIA_PREFIXES
)
def _attachment_role(media_type: str) -> str:
normalized = media_type.lower()
if normalized.startswith("image/"):
return "image"
if normalized == "text/css":
return "stylesheet"
if normalized.startswith("font/") or "font" in normalized or "opentype" in normalized:
return "font"
if normalized.startswith("audio/"):
return "audio"
if normalized.startswith("video/"):
return "video"
return "package-resource"
def _extract_segment(
archive: zipfile.ZipFile,
asset: SourceAsset,

163
src/markitect_filter/pdf.py
View File

@@ -26,6 +26,7 @@ from markitect_tool.source import (
)
from markitect_filter.adapters import pdf_adapter_descriptor
from markitect_filter.assets import bytes_digest, source_asset_signal, source_asset_from_member
PDF_HEADER_RE = re.compile(rb"%PDF-\d\.\d")
@@ -36,6 +37,9 @@ PAGES_TYPE_RE = re.compile(rb"/Type\s*/Pages\b")
REF_RE = re.compile(rb"(\d+)\s+\d+\s+R")
INFO_REF_RE = re.compile(rb"/Info\s+(\d+)\s+\d+\s+R")
COUNT_RE = re.compile(rb"/Count\s+(\d+)")
EMBEDDED_FILE_RE = re.compile(rb"/Type\s*/EmbeddedFile\b")
FILESPEC_RE = re.compile(rb"/Type\s*/Filespec\b")
EMBEDDED_FILE_REF_RE = re.compile(rb"/EF\s*<<.*?/F\s+(\d+)\s+\d+\s+R", re.DOTALL)
@dataclass(frozen=True)
@@ -53,6 +57,7 @@ class PdfPackage:
encrypted: bool
pages: list[PdfPage]
diagnostics: list[Diagnostic]
attachments: list[SourceAsset]
class PdfReadAdapter:
@@ -92,7 +97,7 @@ class PdfReadAdapter:
asset=request.asset,
adapter=_adapter_info(request.options),
metadata=package.metadata,
capabilities=["read"],
capabilities=["read", "attachments"],
quality=NormalizationQuality(
lossiness="unknown" if has_error(diagnostics) else "medium",
confidence=_confidence(package, diagnostics),
@@ -102,6 +107,9 @@ class PdfReadAdapter:
"page_count": package.page_count,
"pages_with_text": extracted_pages,
"encrypted": package.encrypted,
"attachment_count": len(package.attachments),
"image_signal_count": _attachment_role_count(package.attachments, "image-signal"),
"embedded_file_count": _attachment_role_count(package.attachments, "embedded-file"),
},
),
diagnostics=diagnostics,
@@ -188,6 +196,9 @@ class PdfReadAdapter:
"selected_pages": [page.number for page in selected_pages],
"pages_extracted": len(segments),
"page_coverage": page_coverage,
"attachment_count": len(package.attachments),
"image_signal_count": _attachment_role_count(package.attachments, "image-signal"),
"embedded_file_count": _attachment_role_count(package.attachments, "embedded-file"),
},
)
document = NormalizedMarkdownDocument(
@@ -203,9 +214,10 @@ class PdfReadAdapter:
source_uri=request.asset.uri,
source_path=request.asset.path,
digest=request.asset.digest,
metadata={"page_count": package.page_count},
metadata={"page_count": package.page_count, "attachment_count": len(package.attachments)},
)
],
attachments=package.attachments,
adapter=_adapter_info(request.options),
cache_key=normalization_cache_key(
asset=request.asset,
@@ -227,6 +239,7 @@ def _load_pdf(asset: SourceAsset) -> PdfPackage:
page_count=0,
encrypted=False,
pages=[],
attachments=[],
diagnostics=[
_pdf_error(
asset,
@@ -243,6 +256,7 @@ def _load_pdf(asset: SourceAsset) -> PdfPackage:
page_count=0,
encrypted=False,
pages=[],
attachments=[],
diagnostics=[_malformed(asset, "PDF does not start with a PDF header.")],
)
@@ -255,6 +269,7 @@ def _load_pdf(asset: SourceAsset) -> PdfPackage:
page_count=_page_count(objects),
encrypted=True,
pages=[],
attachments=[],
diagnostics=[
_pdf_error(
asset,
@@ -267,10 +282,30 @@ def _load_pdf(asset: SourceAsset) -> PdfPackage:
page_ids = _page_object_ids(objects)
page_count = _page_count(objects) or len(page_ids)
pages: list[PdfPage] = []
attachments = _embedded_file_assets(objects, asset, diagnostics)
for page_number, object_id in enumerate(page_ids, start=1):
page_body = objects[object_id]
page_diagnostics: list[Diagnostic] = []
content_ids = _content_refs(page_body)
image_object_ids = _image_object_ids(page_body, objects, content_ids)
if image_object_ids:
attachments.append(
_image_signal_asset(
asset,
page_number=page_number,
page_object_id=object_id,
image_object_ids=image_object_ids,
digest_parts=[page_body, *[objects.get(image_id, b"") for image_id in image_object_ids]],
)
)
page_diagnostics.append(
_warning(
asset,
"source.pdf.image_resource_signal",
f"PDF page {page_number} references image resources; binary extraction is not performed.",
details={"page": page_number, "image_objects": image_object_ids},
)
)
text_parts: list[str] = []
if not content_ids and STREAM_RE.search(page_body):
stream = _stream_data(page_body, asset, page_diagnostics)
@@ -312,6 +347,126 @@ def _load_pdf(asset: SourceAsset) -> PdfPackage:
encrypted=False,
pages=pages,
diagnostics=diagnostics,
attachments=attachments,
)
def _embedded_file_assets(
objects: dict[int, bytes],
asset: SourceAsset,
diagnostics: list[Diagnostic],
) -> list[SourceAsset]:
file_names = _embedded_file_names(objects)
attachments: list[SourceAsset] = []
for object_id, body in sorted(objects.items()):
if not EMBEDDED_FILE_RE.search(body):
continue
attachment_diagnostics: list[Diagnostic] = []
stream = _stream_data(body, asset, attachment_diagnostics)
diagnostics.extend(attachment_diagnostics)
if not stream:
diagnostics.append(
_warning(
asset,
"source.pdf.embedded_file_unreadable",
f"PDF embedded file object {object_id} does not expose readable bytes.",
details={"object_id": object_id},
)
)
continue
name = file_names.get(object_id) or f"embedded-{object_id}.bin"
attachments.append(
source_asset_from_member(
container_uri=asset.uri,
member_path=f"embedded/{name}",
data=stream,
media_type=None,
source_adapter="source.pdf",
source_role="embedded-file",
metadata={
"container_path": asset.path,
"pdf_object": object_id,
"embedded_file_name": name,
"render_manifest_compatible": True,
},
)
)
return attachments
def _embedded_file_names(objects: dict[int, bytes]) -> dict[int, str]:
names: dict[int, str] = {}
for body in objects.values():
if not FILESPEC_RE.search(body):
continue
ref_match = EMBEDDED_FILE_REF_RE.search(body)
if ref_match is None:
continue
object_id = int(ref_match.group(1))
names[object_id] = _file_spec_name(body) or f"embedded-{object_id}.bin"
return names
def _file_spec_name(body: bytes) -> str | None:
for key in ("UF", "F"):
literal_match = re.search(rb"/" + key.encode("ascii") + rb"\s*(\()", body)
if literal_match:
value, _ = _read_literal_string(body, literal_match.start(1))
else:
value = _metadata_value(body, key)
if value:
return re.sub(r"[\\/:]+", "-", value).strip() or None
return None
def _image_object_ids(
page_body: bytes,
objects: dict[int, bytes],
content_ids: list[int],
) -> list[int]:
page_refs = {
int(match.group(1))
for match in REF_RE.finditer(page_body)
}
image_refs = sorted(
object_id
for object_id in page_refs
if re.search(rb"/Subtype\s*/Image\b", objects.get(object_id, b""))
)
if image_refs:
return image_refs
haystack = page_body + b"\n" + b"\n".join(objects.get(ref, b"") for ref in content_ids)
if not re.search(rb"/Subtype\s*/Image\b|\bDo\b", haystack):
return []
return sorted(
object_id
for object_id, body in objects.items()
if re.search(rb"/Subtype\s*/Image\b", body)
)
def _image_signal_asset(
asset: SourceAsset,
*,
page_number: int,
page_object_id: int,
image_object_ids: list[int],
digest_parts: list[bytes],
) -> SourceAsset:
return source_asset_signal(
container_uri=asset.uri,
signal_id=f"page-{page_number:04d}/image-signal",
media_type="application/x.markitect-pdf-image-signal",
source_adapter="source.pdf",
source_role="image-signal",
digest_parts=digest_parts or [bytes_digest(",".join(map(str, image_object_ids)).encode("ascii")).encode("ascii")],
metadata={
"container_path": asset.path,
"page": page_number,
"pdf_object": page_object_id,
"image_objects": image_object_ids,
"render_manifest_compatible": True,
},
)
@@ -733,6 +888,10 @@ def _confidence(package: PdfPackage, diagnostics: list[Diagnostic]) -> float:
return max(0.1, 0.75 * coverage)
def _attachment_role_count(attachments: list[SourceAsset], role: str) -> int:
return sum(1 for attachment in attachments if attachment.metadata.get("source_role") == role)
def _warning(
asset: SourceAsset,
code: str,