kindle-zotero_importer/src/kindle_zotero_importer/epub_position.py
Utku Bilen Demir 830b27e510 0.6.4: incremental id-based import, Mappings/Integrated, black iA Duo, resizable tables
- id-based incremental (new_ids - prev_integrated_ids) with --full flag, kindle-id tag, deletions handling
- Mappings tab with delete to return to Conflicts, sorted by updated_at newest first
- Integrated tab with Added On / Integrated dates and Citekey
- black over white iA Writer Duo theme, candidate-list per-row Use buttons, resizable/sortable tables
- Settings editable + save, Artifacts Open/Reveal, Re-import bar with last file reuse
- Fix override single-field validation and ignore cleanup
2026-09-16 14:44:31 +02:00

266 lines
8.5 KiB
Python

from __future__ import annotations
from dataclasses import dataclass
from html import unescape
from html.parser import HTMLParser
import posixpath
import zipfile
import xml.etree.ElementTree as ET
from typing import Any
CONTAINER_PATH = "META-INF/container.xml"
@dataclass(frozen=True)
class TextNode:
spine_index: int
cfi_parent_path: str
text_step: str
text: str
start: int
end: int
def add_epub_positions(plan: dict[str, Any]) -> dict[str, Any]:
cache: dict[str, list[TextNode]] = {}
positioned = []
for item in plan["items"]:
positioned.append(_position_item(item, cache))
status_counts: dict[str, int] = {}
for item in positioned:
status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1
updated = dict(plan)
updated["items"] = positioned
updated["status_counts"] = status_counts
return updated
def _position_item(
item: dict[str, Any], cache: dict[str, list[TextNode]]
) -> dict[str, Any]:
if item.get("status") != "ready-for-positioning":
return item
attachment = item.get("zotero", {}).get("attachment")
if not attachment or attachment.get("content_type") != "application/epub+zip":
return item
if item["clipping"]["kind"] != "highlight" or not item["clipping"].get("text"):
return _with_problem(item, "epub-position-skipped-non-highlight")
path = attachment.get("resolved_path") or attachment.get("path")
if not path:
return _with_problem(item, "epub-position-missing-path")
try:
text_nodes = cache.setdefault(path, extract_epub_text_nodes(path))
position = find_epub_cfi(text_nodes, item["clipping"]["text"])
except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch
return _with_problem(item, f"epub-position-error:{error}")
if not position:
return _with_problem(item, "epub-text-not-found")
updated = dict(item)
annotation = dict(updated["annotation"])
annotation["position"] = position["selector"]
annotation["sortIndex"] = position["sortIndex"]
updated["annotation"] = annotation
updated["status"] = "positioned"
return updated
def extract_epub_text_nodes(path: str) -> list[TextNode]:
with zipfile.ZipFile(path) as epub:
opf_path = _opf_path(epub)
spine_items = _spine_items(epub, opf_path)
nodes: list[TextNode] = []
cursor = 0
for spine_index, item_path in enumerate(spine_items):
try:
xml = epub.read(item_path).decode("utf-8", errors="replace")
except KeyError:
continue
for cfi_parent_path, text_step, text in _xhtml_text_nodes(xml):
if not text.strip():
continue
start = cursor
cursor += len(text)
nodes.append(
TextNode(
spine_index, cfi_parent_path, text_step, text, start, cursor
)
)
cursor += 1
return nodes
def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, Any] | None:
haystack = " ".join(node.text for node in text_nodes)
normalized_haystack, haystack_map = _normalize_with_map(haystack)
normalized_quote, _ = _normalize_with_map(quote)
if not normalized_quote:
return None
match_start = normalized_haystack.find(normalized_quote)
if match_start < 0:
return None
match_end = match_start + len(normalized_quote)
raw_start = haystack_map[match_start]
raw_end = haystack_map[match_end - 1] + 1
# Account for the spaces inserted between text nodes in haystack construction.
adjusted_nodes = _nodes_for_joined_text(text_nodes)
start_node, start_offset = _node_at(adjusted_nodes, raw_start)
end_node, end_offset = _node_at(adjusted_nodes, raw_end)
if not start_node or not end_node:
return None
if start_node.spine_index != end_node.spine_index:
return None
spine_step = 2 * (start_node.spine_index + 1)
if start_node.cfi_parent_path == end_node.cfi_parent_path:
value = (
f"epubcfi(/6/{spine_step}!{start_node.cfi_parent_path},"
f"/{start_node.text_step}:{start_offset},/{end_node.text_step}:{end_offset})"
)
else:
value = (
f"epubcfi(/6/{spine_step}!"
f"{start_node.cfi_parent_path}/{start_node.text_step}:{start_offset},"
f"{end_node.cfi_parent_path}/{end_node.text_step}:{end_offset})"
)
return {
"selector": {
"type": "FragmentSelector",
"conformsTo": "http://www.idpf.org/epub/linking/cfi/epub-cfi.html",
"value": value,
},
"sortIndex": f"{spine_step:05d}|{raw_start:08d}",
}
def _opf_path(epub: zipfile.ZipFile) -> str:
root = ET.fromstring(epub.read(CONTAINER_PATH))
rootfile = root.find(".//{*}rootfile")
if rootfile is None:
raise ValueError("EPUB container has no rootfile")
return rootfile.attrib["full-path"]
def _spine_items(epub: zipfile.ZipFile, opf_path: str) -> list[str]:
root = ET.fromstring(epub.read(opf_path))
manifest = {
item.attrib["id"]: item.attrib["href"]
for item in root.findall(".//{*}manifest/{*}item")
}
base = posixpath.dirname(opf_path)
paths = []
for itemref in root.findall(".//{*}spine/{*}itemref"):
href = manifest.get(itemref.attrib.get("idref", ""))
if href:
paths.append(posixpath.normpath(posixpath.join(base, href)))
return paths
def _xhtml_text_nodes(xml: str) -> list[tuple[str, str, str]]:
parser = _CFIHTMLParser()
parser.feed(xml)
return parser.nodes
class _CFIHTMLParser(HTMLParser):
def __init__(self) -> None:
super().__init__(convert_charrefs=True)
self.stack: list[dict[str, Any]] = []
self.nodes: list[tuple[str, str, str]] = []
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
del attrs
tag = tag.lower()
if not self.stack:
path: list[str] = [] if tag == "html" else ["2"]
else:
parent = self.stack[-1]
parent["element_count"] += 1
path = [*parent["path"], str(2 * parent["element_count"])]
self.stack.append(
{"tag": tag, "path": path, "element_count": 0, "text_count": 0}
)
def handle_endtag(self, tag: str) -> None:
tag = tag.lower()
for index in range(len(self.stack) - 1, -1, -1):
if self.stack[index]["tag"] == tag:
del self.stack[index:]
return
def handle_data(self, data: str) -> None:
if not self.stack or not data.strip():
return
current = self.stack[-1]
current["text_count"] += 1
text_step = str(2 * current["text_count"] - 1)
self.nodes.append(
("/" + "/".join(current["path"]), text_step, _clean_text(data))
)
def _clean_text(text: str) -> str:
return unescape(text).replace("\xa0", " ")
def _normalize_with_map(text: str) -> tuple[str, list[int]]:
normalized = []
mapping = []
last_was_space = False
for index, char in enumerate(text):
if char.isspace():
if not last_was_space and normalized:
normalized.append(" ")
mapping.append(index)
last_was_space = True
continue
normalized.append(char.casefold())
mapping.append(index)
last_was_space = False
if normalized and normalized[-1] == " ":
normalized.pop()
mapping.pop()
return "".join(normalized), mapping
def _nodes_for_joined_text(text_nodes: list[TextNode]) -> list[TextNode]:
nodes = []
cursor = 0
for node in text_nodes:
start = cursor
end = start + len(node.text)
nodes.append(
TextNode(
node.spine_index,
node.cfi_parent_path,
node.text_step,
node.text,
start,
end,
)
)
cursor = end + 1
return nodes
def _node_at(nodes: list[TextNode], offset: int) -> tuple[TextNode | None, int]:
for node in nodes:
if node.start <= offset <= node.end:
return node, max(0, min(offset - node.start, len(node.text)))
return None, 0
def _with_problem(item: dict[str, Any], problem: str) -> dict[str, Any]:
updated = dict(item)
updated["status"] = problem
updated["problems"] = [*item.get("problems", []), problem]
return updated