from __future__ import annotations from dataclasses import dataclass import hashlib import os import re import shutil import subprocess import tempfile import unicodedata import xml.etree.ElementTree as ET from pathlib import Path from typing import Any _TOOL_PATHS = { "pdftotext": (str(Path.home() / "bin" / "pdftotext"), "/opt/homebrew/bin/pdftotext"), "pdftohtml": (str(Path.home() / "bin" / "pdftohtml"), "/opt/homebrew/bin/pdftohtml"), "pdfinfo": (str(Path.home() / "bin" / "pdfinfo"), "/opt/homebrew/bin/pdfinfo"), "qpdf": (str(Path.home() / "bin" / "qpdf"), "/opt/homebrew/bin/qpdf"), } def _tool(name: str) -> str: resolved = shutil.which(name) if resolved: return resolved for candidate in _TOOL_PATHS.get(name, ()): if os.path.isfile(candidate) and os.access(candidate, os.X_OK): return candidate return name def _missing_required_tools() -> list[str]: missing = [] for name in ("pdftotext", "pdftohtml", "pdfinfo"): path = _tool(name) if path == name and shutil.which(name) is None: missing.append(name) return missing @dataclass(frozen=True) class PDFWord: text: str left: float top: float width: float height: float start: int end: int @dataclass(frozen=True) class PDFPageXML: index: int width: float height: float words: list[PDFWord] def add_pdf_positions(plan: dict[str, Any]) -> dict[str, Any]: text_cache: dict[str, list[str]] = {} xml_cache: dict[tuple[str, int], PDFPageXML] = {} size_cache: dict[tuple[str, int], tuple[float, float]] = {} decrypted_cache: dict[str, str] = {} positioned = [] missing_tools = _missing_required_tools() with tempfile.TemporaryDirectory(prefix="kindle-zotero-pdf-") as temp_dir: for item in plan["items"]: attachment = (item.get("zotero") or {}).get("attachment") or {} if ( missing_tools and item.get("status") == "ready-for-positioning" and attachment.get("content_type") == "application/pdf" ): positioned.append( _with_problem(item, f"pdf-tools-missing:{','.join(missing_tools)}") ) else: positioned.append( _position_item( item, text_cache, xml_cache, size_cache, decrypted_cache, temp_dir, ) ) status_counts: dict[str, int] = {} for item in positioned: status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1 updated = dict(plan) updated["items"] = positioned updated["status_counts"] = status_counts return updated def _position_item( item: dict[str, Any], text_cache: dict[str, list[str]], xml_cache: dict[tuple[str, int], PDFPageXML], size_cache: dict[tuple[str, int], tuple[float, float]], decrypted_cache: dict[str, str], temp_dir: str, ) -> dict[str, Any]: if item.get("status") != "ready-for-positioning": return item attachment = item.get("zotero", {}).get("attachment") if not attachment or attachment.get("content_type") != "application/pdf": return item if item["clipping"]["kind"] != "highlight" or not item["clipping"].get("text"): return _with_problem(item, "pdf-position-skipped-non-highlight") path = attachment.get("resolved_path") or attachment.get("path") if not path or path.startswith("attachments:"): return _with_problem(item, "pdf-position-missing-path") try: text_pages, extraction_path = _text_pages_for_path( path, text_cache, decrypted_cache, temp_dir ) page_index = find_pdf_text_page( text_pages, item["clipping"]["text"], item["clipping"].get("page") ) if page_index is None: failed = _with_problem(item, "pdf-text-not-found") hint = _page_hint_text(text_pages, item["clipping"].get("page")) if hint: failed = dict(failed) failed["page_hint"] = hint return failed page_xml = xml_cache.setdefault( (extraction_path, page_index), extract_pdf_page_xml(extraction_path, page_index), ) position = find_pdf_rects( extraction_path, page_xml, item["clipping"]["text"], size_cache ) except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch return _with_problem(item, f"pdf-position-error:{error}") if not position: return _with_problem(item, "pdf-rects-not-found") updated = dict(item) annotation = dict(updated["annotation"]) annotation["position"] = { "pageIndex": position["pageIndex"], "rects": position["rects"], } annotation["pageLabel"] = item["clipping"].get("page") or str( position["pageIndex"] + 1 ) annotation["sortIndex"] = position.get("sortIndex") updated["annotation"] = annotation updated["status"] = "positioned" if position.get("partial"): updated["problems"] = [*item.get("problems", []), "partial-text-match"] return updated def extract_pdf_text_pages(path: str) -> list[str]: result = subprocess.run( [_tool("pdftotext"), "-enc", "UTF-8", path, "-"], check=True, capture_output=True, text=True, ) return result.stdout.split("\f") def _text_pages_for_path( path: str, text_cache: dict[str, list[str]], decrypted_cache: dict[str, str], temp_dir: str, ) -> tuple[list[str], str]: if path in text_cache: return text_cache[path], decrypted_cache.get(path, path) try: text_cache[path] = extract_pdf_text_pages(path) return text_cache[path], path except subprocess.CalledProcessError as error: if not _is_pdf_permission_error(error): raise decrypted_path = _decrypted_pdf_path(path, decrypted_cache, temp_dir) text_cache[path] = extract_pdf_text_pages(decrypted_path) return text_cache[path], decrypted_path def _decrypted_pdf_path( path: str, decrypted_cache: dict[str, str], temp_dir: str ) -> str: cached = decrypted_cache.get(path) if cached: return cached digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:16] base_name = os.path.splitext(os.path.basename(path))[0] or "document" output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf") subprocess.run( [_tool("qpdf"), "--decrypt", path, output_path], check=True, capture_output=True, text=True, ) decrypted_cache[path] = output_path return output_path def _is_pdf_permission_error(error: subprocess.CalledProcessError) -> bool: output = "\n".join(part for part in (error.stdout, error.stderr) if part) return "Permission Error" in output and "Copying of text" in output def find_pdf_text_page( pages: list[str], quote: str, kindle_page: str | None = None ) -> int | None: normalized_quote, _ = _normalize_with_map(quote) if not normalized_quote: return None page_order = list(range(len(pages))) if kindle_page and kindle_page.isdigit(): index = int(kindle_page) - 1 if 0 <= index < len(pages): page_order.remove(index) page_order.insert(0, index) for index in page_order: normalized_page, _ = _normalize_with_map(pages[index]) if normalized_quote in normalized_page: return index # Fallback: alphanumeric-only search tolerates PDF extraction artifacts # such as "o f" for "of", hyphenation, and punctuation differences. stripped_quote, _ = _strip_with_map(quote) if stripped_quote and len(stripped_quote) >= 20: for index in page_order: stripped_page, _ = _strip_with_map(pages[index]) if stripped_quote in stripped_page: return index # Partial fallback: long highlights may span pages or contain a # divergent middle (edition differences). Anchor to either half so # Zotero-style word search behaviour is approximated. for index in page_order: stripped_page, _ = _strip_with_map(pages[index]) if _partial_stripped_hit(stripped_page, stripped_quote): return index # Shingle fallback: scattered OCR substitutions (e.g. Kindle # "yadsınaraz" vs PDF "yadsınamaz") break every long substring but # leave most 6-word runs intact. Rank pages by shingle hits. quote_words = _quote_words(normalized_quote) if len(quote_words) >= 6: required = 2 if len(quote_words) >= 12 else 1 best_index: int | None = None best_hits = 0 for index in page_order: normalized_page, _ = _normalize_with_map(pages[index]) hits = _shingle_hits(normalized_page, quote_words) if hits > best_hits: best_hits = hits best_index = index if best_index is not None and best_hits >= required: return best_index return None def extract_pdf_page_xml(path: str, page_index: int) -> PDFPageXML: page_number = page_index + 1 result = subprocess.run( [ _tool("pdftohtml"), "-f", str(page_number), "-l", str(page_number), "-xml", "-stdout", path, ], check=True, capture_output=True, text=True, ) root = ET.fromstring(result.stdout) page_el = root.find("page") if page_el is None: raise ValueError(f"no XML page for PDF page {page_number}") cursor = 0 words: list[PDFWord] = [] for text_el in page_el.findall("text"): text = "".join(text_el.itertext()).strip() if not text: continue for word in text.split(): start = cursor end = start + len(word) words.append( PDFWord( text=word, left=float(text_el.attrib["left"]), top=float(text_el.attrib["top"]), width=float(text_el.attrib["width"]), height=float(text_el.attrib["height"]), start=start, end=end, ) ) cursor = end + 1 return PDFPageXML( index=page_index, width=float(page_el.attrib["width"]), height=float(page_el.attrib["height"]), words=words, ) def find_pdf_rects( path: str, page: PDFPageXML, quote: str, size_cache: dict[tuple[str, int], tuple[float, float]], ) -> dict[str, Any] | None: normalized_quote, _ = _normalize_with_map(quote) text = " ".join(word.text for word in page.words) normalized_text, text_map = _normalize_with_map(text) start = normalized_text.find(normalized_quote) raw_start: int | None = None raw_end: int | None = None partial = False if start >= 0: end = start + len(normalized_quote) raw_start = text_map[start] raw_end = text_map[end - 1] + 1 else: stripped_quote, _ = _strip_with_map(quote) if stripped_quote and len(stripped_quote) >= 20: stripped_text, stripped_map = _strip_with_map(text) strip_start = stripped_text.find(stripped_quote) if strip_start >= 0: strip_end = strip_start + len(stripped_quote) raw_start = stripped_map[strip_start] raw_end = stripped_map[strip_end - 1] + 1 else: hit = _partial_stripped_range(stripped_text, stripped_quote) if hit is not None: strip_start, strip_end = hit raw_start = stripped_map[strip_start] raw_end = stripped_map[strip_end - 1] + 1 partial = True else: span = _shingle_span(normalized_text, _quote_words(normalized_quote)) if span is not None: raw_start = text_map[span[0]] raw_end = text_map[span[1] - 1] + 1 partial = True if raw_start is None or raw_end is None: return None matched_words = [ word for word in page.words if word.end >= raw_start and word.start <= raw_end ] if not matched_words: return None size_key = (path, page.index) if size_key not in size_cache: size_cache[size_key] = _pdf_page_size(path, page.index + 1, page.width, page.height) pdf_width, pdf_height = size_cache[size_key] rects = _word_rects(matched_words, page, pdf_width, pdf_height) return { "pageIndex": page.index, "rects": rects, "sortIndex": f"{page.index:05d}|{int(rects[0][1]):06d}|{int(rects[0][0]):05d}", "partial": partial, } def _word_rects( words: list[PDFWord], page: PDFPageXML, pdf_width: float, pdf_height: float ) -> list[list[float]]: lines: dict[int, list[PDFWord]] = {} for word in words: lines.setdefault(round(word.top), []).append(word) scale_x = pdf_width / page.width scale_y = pdf_height / page.height rects = [] for top in sorted(lines): line_words = lines[top] left = min(word.left for word in line_words) right = max(word.left + word.width for word in line_words) line_top = min(word.top for word in line_words) line_bottom = max(word.top + word.height for word in line_words) rects.append( [ round(left * scale_x, 3), round(pdf_height - line_bottom * scale_y, 3), round(right * scale_x, 3), round(pdf_height - line_top * scale_y, 3), ] ) return rects def _pdf_page_size( path: str, page_number: int, fallback_width: float, fallback_height: float ) -> tuple[float, float]: result = subprocess.run( [_tool("pdfinfo"), "-f", str(page_number), "-l", str(page_number), path], check=True, capture_output=True, text=True, ) match = re.search( rf"Page\s+{page_number}\s+size:\s+([0-9.]+)\s+x\s+([0-9.]+)\s+pts", result.stdout, ) if not match: match = re.search( r"Page size:\s+([0-9.]+)\s+x\s+([0-9.]+)\s+pts", result.stdout ) if not match: return fallback_width, fallback_height return float(match.group(1)), float(match.group(2)) def _normalize_with_map(text: str) -> tuple[str, list[int]]: text = unicodedata.normalize("NFKC", text) text = ( text.replace("’", "'").replace("‘", "'").replace("“", '"').replace("”", '"') .replace("–", "-").replace("—", "-").replace("\xa0", " ").replace("\xad", "") ) normalized = [] mapping = [] last_was_space = False for index, char in enumerate(text): if char.isspace(): if not last_was_space and normalized: normalized.append(" ") mapping.append(index) last_was_space = True continue normalized.append(char.casefold()) mapping.append(index) last_was_space = False if normalized and normalized[-1] == " ": normalized.pop() mapping.pop() return "".join(normalized), mapping def _strip_with_map(text: str) -> tuple[str, list[int]]: """Alphanumeric-only folding with index mapping. Removes spaces and punctuation so PDF artifacts like "o f" for "of", hyphenated line breaks, and curly-quote/dash variants still match. Mapping points back into the input string for rect recovery. """ text = unicodedata.normalize("NFKC", text).casefold() stripped: list[str] = [] mapping: list[int] = [] for index, char in enumerate(text): if char.isalnum(): stripped.append(char) mapping.append(index) return "".join(stripped), mapping def _partial_stripped_range( haystack: str, quote: str, min_len: int = 30 ) -> tuple[int, int] | None: """Best half-quote hit for long highlights spanning pages/editions.""" if len(quote) < min_len * 2: return None mid = len(quote) // 2 for part in (quote[:mid], quote[mid:]): if len(part) < min_len: continue start = haystack.find(part) if start >= 0: return start, start + len(part) return None def _partial_stripped_hit(haystack: str, quote: str, min_len: int = 30) -> bool: return _partial_stripped_range(haystack, quote, min_len) is not None def _quote_words(normalized: str, width: int = 6) -> list[str]: del width words: list[str] = [] for token in normalized.split(): cleaned = "".join(char for char in token if char.isalnum()) if cleaned: words.append(cleaned) return words def _shingle_hits(normalized_page: str, quote_words: list[str], width: int = 6) -> int: hits = 0 seen: set[tuple[str, ...]] = set() for start in range(len(quote_words) - width + 1): shingle = tuple(quote_words[start : start + width]) if shingle in seen: continue seen.add(shingle) if " ".join(shingle) in normalized_page: hits += 1 return hits def _shingle_span( normalized_text: str, quote_words: list[str], width: int = 6 ) -> tuple[int, int] | None: first: int | None = None last: int | None = None for start in range(len(quote_words) - width + 1): needle = " ".join(quote_words[start : start + width]) at = normalized_text.find(needle) if at >= 0: end = at + len(needle) first = at if first is None else min(first, at) last = end if last is None else max(last, end) if first is None or last is None: return None return first, last def _page_hint_text( pages: list[str], kindle_page: str | None, radius: int = 1, limit: int = 1200 ) -> dict[str, Any] | None: """Snippet of the PDF pages around the Kindle page hint. Lets the user eyeball which part a `pdf-text-not-found` highlight belongs to and place it manually. Truncated to keep artifacts small. """ if not kindle_page or not kindle_page.isdigit(): return None center = int(kindle_page) - 1 if center < 0 or center >= len(pages): return None parts = [] for index in range(max(0, center - radius), min(len(pages), center + radius + 1)): snippet = " ".join(pages[index].split()) if snippet: parts.append(f"[PDF p.{index + 1}] {snippet[:limit]}") if not parts: return None return { "kindle_page": kindle_page, "pdf_pages": list( range(max(1, center - radius + 1), min(len(pages), center + radius + 1) + 1) ), "text": "\n\n".join(parts)[: limit * 2 + 64], } def _with_problem(item: dict[str, Any], problem: str) -> dict[str, Any]: updated = dict(item) updated["status"] = problem updated["problems"] = [*item.get("problems", []), problem] return updated