From d8b273ad6e198d817f4fa7342b6c4c7939757538 Mon Sep 17 00:00:00 2001 From: Utku Bilen Demir <84389167+UtkuBilenDemir@users.noreply.github.com> Date: Wed, 27 May 2026 10:49:04 +0200 Subject: [PATCH] Initial Kindle Zotero importer baseline --- .gitignore | 11 + README.md | 87 ++++++ docs/import-plan.md | 56 ++++ main.py | 4 + pyproject.toml | 17 ++ src/kindle_zotero_importer/__init__.py | 5 + src/kindle_zotero_importer/cli.py | 283 ++++++++++++++++++ src/kindle_zotero_importer/clippings.py | 118 ++++++++ src/kindle_zotero_importer/epub_position.py | 271 +++++++++++++++++ src/kindle_zotero_importer/final_plan.py | 51 ++++ src/kindle_zotero_importer/import_plan.py | 271 +++++++++++++++++ src/kindle_zotero_importer/matcher.py | 209 +++++++++++++ src/kindle_zotero_importer/mismatch_review.py | 238 +++++++++++++++ src/kindle_zotero_importer/overrides.py | 123 ++++++++ src/kindle_zotero_importer/pdf_position.py | 280 +++++++++++++++++ src/kindle_zotero_importer/zotero_index.py | 183 +++++++++++ zotero-writer.js | 156 ++++++++++ 17 files changed, 2363 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/import-plan.md create mode 100644 main.py create mode 100644 pyproject.toml create mode 100644 src/kindle_zotero_importer/__init__.py create mode 100644 src/kindle_zotero_importer/cli.py create mode 100644 src/kindle_zotero_importer/clippings.py create mode 100644 src/kindle_zotero_importer/epub_position.py create mode 100644 src/kindle_zotero_importer/final_plan.py create mode 100644 src/kindle_zotero_importer/import_plan.py create mode 100644 src/kindle_zotero_importer/matcher.py create mode 100644 src/kindle_zotero_importer/mismatch_review.py create mode 100644 src/kindle_zotero_importer/overrides.py create mode 100644 src/kindle_zotero_importer/pdf_position.py create mode 100644 src/kindle_zotero_importer/zotero_index.py create mode 100644 zotero-writer.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1f733a9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +clippings.json +zotero-index.json +matches.json +match-overrides.generated.json +!match-overrides.json +import-plan*.json +*.import-plan.json +__pycache__/ +*.py[cod] +.venv/ +.pytest_cache/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..8133499 --- /dev/null +++ b/README.md @@ -0,0 +1,87 @@ +# Kindle Zotero Importer + +Prototype importer for turning Kindle `My Clippings.txt` entries into a JSON import plan that Zotero can later write as native annotations. + +The project intentionally keeps Zotero writes out of Python. Python parses, matches, and plans. Zotero JavaScript, and eventually a Zotero plugin, will perform annotation creation through Zotero's own APIs. + +## Current Milestone + +Parse Kindle clippings and emit a stable JSON preview: + +```sh +python -m kindle_zotero_importer parse "/path/to/My Clippings.txt" --output clippings.json +``` + +Index Zotero items and PDF/EPUB attachments read-only: + +```sh +python -m kindle_zotero_importer index-zotero --output zotero-index.json +``` + +Match parsed Kindle titles to Zotero items: + +```sh +python -m kindle_zotero_importer match clippings.json zotero-index.json --overrides match-overrides.json --output matches.json +``` + +Generate reusable manual overrides for ambiguous/unmatched titles: + +```sh +python -m kindle_zotero_importer generate-overrides matches.json --output match-overrides.generated.json --pretty +``` + +Apply reviewed overrides: + +```sh +python -m kindle_zotero_importer match clippings.json zotero-index.json --overrides match-overrides.json --output matches.json +``` + +Keep confirmed mappings in `match-overrides.json`. The `match-overrides.generated.json` file is only a disposable review skeleton and can be regenerated from the current `matches.json` at any time. + +If a matched Zotero item has multiple usable PDF/EPUB attachments, add the selected attachment to the same override entry: + +```json +{ + "clipping_title": "Example Kindle Title", + "resolution": { + "citation_key": "example2024", + "attachment_key": "ABC12345" + } +} +``` + +Use either `attachment_key` or `attachment_item_id`. Attachment choices are listed in `docs/mismatch-review.md` when an item is ambiguous. + +For cumulative Kindle exports, rerun the whole pipeline against the latest `My Clippings.txt`. Previously confirmed overrides in `match-overrides.json` will be applied automatically to old and new clippings. + +Review unresolved cases in `docs/mismatch-review.md` after each run. Prioritize: + +1. `matched-title-no-attachment`: add/link the missing PDF or EPUB in Zotero. +2. `unmatched-title`: add a confirmed title mapping to `match-overrides.json`. +3. position failures: improve text/position matching or handle manually. + +Regenerate the copy-friendly review report: + +```sh +python -m kindle_zotero_importer review-mismatches import-plan.positioned.json matches.json --output docs/mismatch-review.md +``` + +Each reviewed title includes a YAML-style block with standalone keys such as `clipping_title`, `citation_key`, `zotero_item_id`, `zotero_key`, and `override_entry` for easy copying. + +Generate a preliminary import plan for matched titles: + +```sh +python -m kindle_zotero_importer plan clippings.json zotero-index.json matches.json --output import-plan.json --pretty +``` + +Position annotations and export the Zotero writer plan: + +```sh +python -m kindle_zotero_importer position-epub import-plan.json --output import-plan.epub.json --pretty +python -m kindle_zotero_importer position-pdf import-plan.epub.json --output import-plan.positioned.json --pretty +python -m kindle_zotero_importer finalize import-plan.positioned.json --output import-plan.final.json --pretty +``` + +## Safety Rule + +Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes. diff --git a/docs/import-plan.md b/docs/import-plan.md new file mode 100644 index 0000000..7102963 --- /dev/null +++ b/docs/import-plan.md @@ -0,0 +1,56 @@ +# Import Plan Contract + +The long-term plugin boundary is a JSON contract. Python can generate this contract during prototyping; a future Zotero plugin can either consume it or reimplement the same planning logic in TypeScript. + +## Current Parser Output + +```json +{ + "format": "kindle-zotero-importer.clippings.v1", + "count": 1, + "clippings": [ + { + "id": "stable-hash-prefix", + "title": "Book Title (Author)", + "kind": "highlight", + "text": "Highlighted text", + "raw_detail": "- Your Highlight on page 10 | Location 100-101 | Added on ...", + "page": "10", + "location": "100-101", + "added_on": "Kindle date string", + "added_on_iso": "Parsed ISO datetime when recognized" + } + ] +} +``` + +## Future Import Plan + +The final writer-facing plan should add matched Zotero targets and annotation positions: + +```json +{ + "format": "kindle-zotero-importer.import-plan.v1", + "items": [ + { + "clipping_id": "stable-hash-prefix", + "status": "matched", + "confidence": "exact-text-match", + "zotero": { + "parent_item_id": 51395, + "attachment_item_id": 51402, + "attachment_type": "application/epub+zip" + }, + "annotation": { + "type": "highlight", + "text": "Highlighted text", + "comment": null, + "color": "#ffd400", + "pageLabel": "", + "sortIndex": "00014|00047607", + "position": {} + } + } + ] +} +``` diff --git a/main.py b/main.py new file mode 100644 index 0000000..ff0cc64 --- /dev/null +++ b/main.py @@ -0,0 +1,4 @@ +from kindle_zotero_importer.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8f51b33 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "kindle-zotero-importer" +version = "0.1.0" +description = "Generate Zotero annotation import plans from Kindle My Clippings.txt files." +readme = "README.md" +requires-python = ">=3.11" +dependencies = [] + +[project.scripts] +kindle-zotero-importer = "kindle_zotero_importer.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/kindle_zotero_importer"] diff --git a/src/kindle_zotero_importer/__init__.py b/src/kindle_zotero_importer/__init__.py new file mode 100644 index 0000000..f2dd245 --- /dev/null +++ b/src/kindle_zotero_importer/__init__.py @@ -0,0 +1,5 @@ +"""Kindle to Zotero annotation import planning.""" + +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/src/kindle_zotero_importer/cli.py b/src/kindle_zotero_importer/cli.py new file mode 100644 index 0000000..86d4896 --- /dev/null +++ b/src/kindle_zotero_importer/cli.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +import argparse +import json +import sys + +from .clippings import clippings_to_jsonable, load_clippings +from .epub_position import add_epub_positions +from .final_plan import build_final_writer_plan +from .import_plan import build_import_plan +from .matcher import build_match_report, load_json +from .mismatch_review import build_mismatch_review +from .overrides import generate_override_skeleton, load_overrides +from .pdf_position import add_pdf_positions +from .zotero_index import build_zotero_index + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="kindle-zotero-importer", + description="Generate Zotero import-plan data from Kindle clippings.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + parse_parser = subparsers.add_parser( + "parse", help="Parse a Kindle My Clippings.txt file" + ) + parse_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt") + parse_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + parse_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + index_parser = subparsers.add_parser( + "index-zotero", help="Index Zotero items and attachments read-only" + ) + index_parser.add_argument( + "--db", default="/home/user/Zotero/zotero.sqlite", help="Path to zotero.sqlite" + ) + index_parser.add_argument( + "--storage-root", + default="/home/user/Zotero/storage", + help="Path to Zotero storage directory for storage: attachments", + ) + index_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + index_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + match_parser = subparsers.add_parser( + "match", help="Match parsed Kindle clipping titles to Zotero items" + ) + match_parser.add_argument("clippings_json", help="Path to parsed clippings JSON") + match_parser.add_argument("zotero_index_json", help="Path to Zotero index JSON") + match_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + match_parser.add_argument( + "--overrides", help="Path to reusable match overrides JSON" + ) + match_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + overrides_parser = subparsers.add_parser( + "generate-overrides", + help="Generate a reusable override skeleton from ambiguous/unmatched matches", + ) + overrides_parser.add_argument("matches_json", help="Path to match report JSON") + overrides_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + overrides_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + plan_parser = subparsers.add_parser( + "plan", help="Generate a preliminary Zotero annotation import plan" + ) + plan_parser.add_argument("clippings_json", help="Path to parsed clippings JSON") + plan_parser.add_argument("zotero_index_json", help="Path to Zotero index JSON") + plan_parser.add_argument("matches_json", help="Path to match report JSON") + plan_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + plan_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + epub_parser = subparsers.add_parser( + "position-epub", help="Add EPUB CFI positions to an import plan" + ) + epub_parser.add_argument( + "import_plan_json", help="Path to preliminary import plan JSON" + ) + epub_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + epub_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + pdf_parser = subparsers.add_parser( + "position-pdf", help="Add PDF page/rect positions to an import plan" + ) + pdf_parser.add_argument("import_plan_json", help="Path to import plan JSON") + pdf_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + pdf_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + final_parser = subparsers.add_parser( + "finalize", + help="Export positioned annotations for the Zotero JavaScript writer", + ) + final_parser.add_argument( + "positioned_plan_json", help="Path to positioned import plan JSON" + ) + final_parser.add_argument( + "--output", "-o", help="Write JSON to this file instead of stdout" + ) + final_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON output" + ) + + review_parser = subparsers.add_parser( + "review-mismatches", + help="Generate a copy-friendly mismatch review Markdown file", + ) + review_parser.add_argument( + "positioned_plan_json", help="Path to positioned import plan JSON" + ) + review_parser.add_argument("matches_json", help="Path to match report JSON") + review_parser.add_argument( + "--output", "-o", help="Write Markdown to this file instead of stdout" + ) + + args = parser.parse_args(argv) + + if args.command == "parse": + payload = clippings_to_jsonable(load_clippings(args.clippings_file)) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "index-zotero": + payload = build_zotero_index(args.db, args.storage_root) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "match": + overrides = ( + load_overrides(load_json(args.overrides)) if args.overrides else None + ) + payload = build_match_report( + load_json(args.clippings_json), load_json(args.zotero_index_json), overrides + ) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "generate-overrides": + payload = generate_override_skeleton(load_json(args.matches_json)) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "plan": + payload = build_import_plan( + load_json(args.clippings_json), + load_json(args.zotero_index_json), + load_json(args.matches_json), + ) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "position-epub": + payload = add_epub_positions(load_json(args.import_plan_json)) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "position-pdf": + payload = add_pdf_positions(load_json(args.import_plan_json)) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "finalize": + payload = build_final_writer_plan(load_json(args.positioned_plan_json)) + json_text = json.dumps( + payload, ensure_ascii=False, indent=2 if args.pretty else None + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + return 0 + + if args.command == "review-mismatches": + text = build_mismatch_review( + load_json(args.positioned_plan_json), load_json(args.matches_json) + ) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(text) + else: + sys.stdout.write(text) + return 0 + + parser.error(f"unknown command: {args.command}") + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/kindle_zotero_importer/clippings.py b/src/kindle_zotero_importer/clippings.py new file mode 100644 index 0000000..03f64ad --- /dev/null +++ b/src/kindle_zotero_importer/clippings.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import datetime +import hashlib +import re +from typing import Any + + +ENTRY_SEPARATOR = "==========" +DETAIL_RE = re.compile( + r"^- Your (?PHighlight|Note|Bookmark)" + r"(?: on page (?P[^|]+?))?" + r"(?: (?:\| location|at location) (?P[^|]+?))?" + r"(?: \| Added on (?P.+))?$", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class Clipping: + id: str + title: str + kind: str + text: str + raw_detail: str + page: str | None = None + location: str | None = None + added_on: str | None = None + added_on_iso: str | None = None + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + +def parse_clippings_text(text: str) -> list[Clipping]: + entries = [entry.strip() for entry in text.split(ENTRY_SEPARATOR) if entry.strip()] + clippings: list[Clipping] = [] + + for entry in entries: + lines = entry.splitlines() + if len(lines) < 2: + continue + + title = lines[0].strip() + raw_detail = lines[1].strip() + body = "\n".join(line.rstrip() for line in lines[2:]).strip() + match = DETAIL_RE.match(raw_detail) + + if match: + kind = match.group("kind").lower() + page = _clean(match.group("page")) + location = _clean(match.group("location")) + added_on = _clean(match.group("date")) + else: + kind = "unknown" + page = None + location = None + added_on = None + + clippings.append( + Clipping( + id=_stable_id(title, raw_detail, body), + title=title, + kind=kind, + text=body, + raw_detail=raw_detail, + page=page, + location=location, + added_on=added_on, + added_on_iso=_parse_kindle_date(added_on), + ) + ) + + return clippings + + +def load_clippings(path: str) -> list[Clipping]: + with open(path, "r", encoding="utf-8-sig") as file: + return parse_clippings_text(file.read()) + + +def clippings_to_jsonable(clippings: list[Clipping]) -> dict[str, Any]: + return { + "format": "kindle-zotero-importer.clippings.v1", + "count": len(clippings), + "clippings": [clipping.to_dict() for clipping in clippings], + } + + +def _clean(value: str | None) -> str | None: + if value is None: + return None + cleaned = value.strip() + return cleaned or None + + +def _stable_id(title: str, raw_detail: str, text: str) -> str: + digest = hashlib.sha256( + f"{title}\n{raw_detail}\n{text}".encode("utf-8") + ).hexdigest() + return digest[:16] + + +def _parse_kindle_date(value: str | None) -> str | None: + if not value: + return None + + for fmt in ( + "%A, %B %d, %Y %I:%M:%S %p", + "%A, %d %B %Y %H:%M:%S", + "%A, %B %d, %Y %H:%M:%S", + ): + try: + return datetime.strptime(value, fmt).isoformat() + except ValueError: + pass + return None diff --git a/src/kindle_zotero_importer/epub_position.py b/src/kindle_zotero_importer/epub_position.py new file mode 100644 index 0000000..8639ee3 --- /dev/null +++ b/src/kindle_zotero_importer/epub_position.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +from dataclasses import dataclass +from html import unescape +from html.parser import HTMLParser +import posixpath +import re +import zipfile +import xml.etree.ElementTree as ET +from typing import Any + + +CONTAINER_PATH = "META-INF/container.xml" + + +@dataclass(frozen=True) +class TextNode: + spine_index: int + cfi_parent_path: str + text_step: str + text: str + start: int + end: int + + +def add_epub_positions(plan: dict[str, Any]) -> dict[str, Any]: + cache: dict[str, list[TextNode]] = {} + positioned = [] + for item in plan["items"]: + positioned.append(_position_item(item, cache)) + + status_counts: dict[str, int] = {} + for item in positioned: + status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1 + + updated = dict(plan) + updated["items"] = positioned + updated["status_counts"] = status_counts + return updated + + +def _position_item( + item: dict[str, Any], cache: dict[str, list[TextNode]] +) -> dict[str, Any]: + if item.get("status") != "ready-for-positioning": + return item + attachment = item.get("zotero", {}).get("attachment") + if not attachment or attachment.get("content_type") != "application/epub+zip": + return item + if item["clipping"]["kind"] != "highlight" or not item["clipping"].get("text"): + return _with_problem(item, "epub-position-skipped-non-highlight") + + path = attachment.get("resolved_path") or attachment.get("path") + if not path: + return _with_problem(item, "epub-position-missing-path") + + try: + text_nodes = cache.setdefault(path, extract_epub_text_nodes(path)) + position = find_epub_cfi(text_nodes, item["clipping"]["text"]) + except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch + return _with_problem(item, f"epub-position-error:{error}") + + if not position: + return _with_problem(item, "epub-text-not-found") + + updated = dict(item) + annotation = dict(updated["annotation"]) + annotation["position"] = position + annotation["sortIndex"] = _epub_sort_index(position["value"]) + updated["annotation"] = annotation + updated["status"] = "positioned" + return updated + + +def extract_epub_text_nodes(path: str) -> list[TextNode]: + with zipfile.ZipFile(path) as epub: + opf_path = _opf_path(epub) + spine_items = _spine_items(epub, opf_path) + nodes: list[TextNode] = [] + cursor = 0 + for spine_index, item_path in enumerate(spine_items): + try: + xml = epub.read(item_path).decode("utf-8", errors="replace") + except KeyError: + continue + for cfi_parent_path, text_step, text in _xhtml_text_nodes(xml): + if not text.strip(): + continue + start = cursor + cursor += len(text) + nodes.append( + TextNode( + spine_index, cfi_parent_path, text_step, text, start, cursor + ) + ) + cursor += 1 + return nodes + + +def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, str] | None: + haystack = " ".join(node.text for node in text_nodes) + normalized_haystack, haystack_map = _normalize_with_map(haystack) + normalized_quote, _ = _normalize_with_map(quote) + if not normalized_quote: + return None + + match_start = normalized_haystack.find(normalized_quote) + if match_start < 0: + return None + match_end = match_start + len(normalized_quote) + raw_start = haystack_map[match_start] + raw_end = haystack_map[match_end - 1] + 1 + + # Account for the spaces inserted between text nodes in haystack construction. + adjusted_nodes = _nodes_for_joined_text(text_nodes) + start_node, start_offset = _node_at(adjusted_nodes, raw_start) + end_node, end_offset = _node_at(adjusted_nodes, raw_end) + if not start_node or not end_node: + return None + + if start_node.spine_index != end_node.spine_index: + return None + + spine_step = 2 * (start_node.spine_index + 1) + if start_node.cfi_parent_path == end_node.cfi_parent_path: + value = ( + f"epubcfi(/6/{spine_step}!{start_node.cfi_parent_path}," + f"/{start_node.text_step}:{start_offset},/{end_node.text_step}:{end_offset})" + ) + else: + value = ( + f"epubcfi(/6/{spine_step}!" + f"{start_node.cfi_parent_path}/{start_node.text_step}:{start_offset}," + f"{end_node.cfi_parent_path}/{end_node.text_step}:{end_offset})" + ) + return { + "type": "FragmentSelector", + "conformsTo": "http://www.idpf.org/epub/linking/cfi/epub-cfi.html", + "value": value, + } + + +def _opf_path(epub: zipfile.ZipFile) -> str: + root = ET.fromstring(epub.read(CONTAINER_PATH)) + rootfile = root.find(".//{*}rootfile") + if rootfile is None: + raise ValueError("EPUB container has no rootfile") + return rootfile.attrib["full-path"] + + +def _spine_items(epub: zipfile.ZipFile, opf_path: str) -> list[str]: + root = ET.fromstring(epub.read(opf_path)) + manifest = { + item.attrib["id"]: item.attrib["href"] + for item in root.findall(".//{*}manifest/{*}item") + } + base = posixpath.dirname(opf_path) + paths = [] + for itemref in root.findall(".//{*}spine/{*}itemref"): + href = manifest.get(itemref.attrib.get("idref", "")) + if href: + paths.append(posixpath.normpath(posixpath.join(base, href))) + return paths + + +def _xhtml_text_nodes(xml: str) -> list[tuple[str, str, str]]: + parser = _CFIHTMLParser() + parser.feed(xml) + return parser.nodes + + +class _CFIHTMLParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.stack: list[dict[str, Any]] = [] + self.nodes: list[tuple[str, str, str]] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + del attrs + tag = tag.lower() + if not self.stack: + path: list[str] = [] if tag == "html" else ["2"] + else: + parent = self.stack[-1] + parent["element_count"] += 1 + path = [*parent["path"], str(2 * parent["element_count"])] + self.stack.append( + {"tag": tag, "path": path, "element_count": 0, "text_count": 0} + ) + + def handle_endtag(self, tag: str) -> None: + tag = tag.lower() + for index in range(len(self.stack) - 1, -1, -1): + if self.stack[index]["tag"] == tag: + del self.stack[index:] + return + + def handle_data(self, data: str) -> None: + if not self.stack or not data.strip(): + return + current = self.stack[-1] + current["text_count"] += 1 + text_step = str(2 * current["text_count"] - 1) + self.nodes.append( + ("/" + "/".join(current["path"]), text_step, _clean_text(data)) + ) + + +def _clean_text(text: str) -> str: + return unescape(text).replace("\xa0", " ") + + +def _normalize_with_map(text: str) -> tuple[str, list[int]]: + normalized = [] + mapping = [] + last_was_space = False + for index, char in enumerate(text): + if char.isspace(): + if not last_was_space and normalized: + normalized.append(" ") + mapping.append(index) + last_was_space = True + continue + normalized.append(char.casefold()) + mapping.append(index) + last_was_space = False + if normalized and normalized[-1] == " ": + normalized.pop() + mapping.pop() + return "".join(normalized), mapping + + +def _nodes_for_joined_text(text_nodes: list[TextNode]) -> list[TextNode]: + nodes = [] + cursor = 0 + for node in text_nodes: + start = cursor + end = start + len(node.text) + nodes.append( + TextNode( + node.spine_index, + node.cfi_parent_path, + node.text_step, + node.text, + start, + end, + ) + ) + cursor = end + 1 + return nodes + + +def _node_at(nodes: list[TextNode], offset: int) -> tuple[TextNode | None, int]: + for node in nodes: + if node.start <= offset <= node.end: + return node, max(0, min(offset - node.start, len(node.text))) + return None, 0 + + +def _epub_sort_index(cfi: str) -> str: + numbers = [int(number) for number in re.findall(r"/([0-9]+)", cfi)] + spine = numbers[1] if len(numbers) > 1 else 0 + content = numbers[-1] if numbers else 0 + return f"{spine:05d}|{content:08d}" + + +def _with_problem(item: dict[str, Any], problem: str) -> dict[str, Any]: + updated = dict(item) + updated["status"] = problem + updated["problems"] = [*item.get("problems", []), problem] + return updated diff --git a/src/kindle_zotero_importer/final_plan.py b/src/kindle_zotero_importer/final_plan.py new file mode 100644 index 0000000..682882e --- /dev/null +++ b/src/kindle_zotero_importer/final_plan.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +from typing import Any + + +FINAL_FORMAT = "kindle-zotero-importer.zotero-writer-plan.v1" + + +def build_final_writer_plan(positioned_plan: dict[str, Any]) -> dict[str, Any]: + annotations = [] + skipped: dict[str, int] = {} + for item in positioned_plan["items"]: + if item.get("status") != "positioned": + skipped[item["status"]] = skipped.get(item["status"], 0) + 1 + continue + attachment = item["zotero"]["attachment"] + annotation = item["annotation"] + if not annotation.get("position"): + skipped["positioned-missing-writer-fields"] = ( + skipped.get("positioned-missing-writer-fields", 0) + 1 + ) + continue + annotations.append( + { + "clipping_id": item["clipping"]["id"], + "clipping_title": item["clipping"]["title"], + "attachment_item_id": attachment["item_id"], + "attachment_key": attachment["key"], + "parent_item_id": item["zotero"]["parent_item_id"], + "parent_key": item["zotero"]["parent_key"], + "citation_key": item["zotero"].get("citation_key"), + "annotation": { + "type": annotation["type"], + "text": annotation.get("text") or "", + "comment": annotation.get("comment") or "", + "color": annotation.get("color") or "#ffd400", + "pageLabel": annotation.get("pageLabel") or "", + "sortIndex": annotation.get("sortIndex"), + "position": annotation["position"], + "tags": annotation.get("tags") or [{"name": "kindle-import"}], + }, + } + ) + + return { + "format": FINAL_FORMAT, + "source_format": positioned_plan.get("format"), + "annotation_count": len(annotations), + "skipped_counts": skipped, + "annotations": annotations, + } diff --git a/src/kindle_zotero_importer/import_plan.py b/src/kindle_zotero_importer/import_plan.py new file mode 100644 index 0000000..bff5685 --- /dev/null +++ b/src/kindle_zotero_importer/import_plan.py @@ -0,0 +1,271 @@ +from __future__ import annotations + +import re +from typing import Any + + +PLAN_FORMAT = "kindle-zotero-importer.import-plan.v1" + + +def build_import_plan( + clippings_payload: dict[str, Any], + zotero_index: dict[str, Any], + match_report: dict[str, Any], +) -> dict[str, Any]: + items_by_id = {item["item_id"]: item for item in zotero_index["items"]} + matches_by_title = { + match["clipping_title"]: match for match in match_report["matches"] + } + overrides_by_title = { + match["clipping_title"]: (match.get("override") or {}) + for match in match_report["matches"] + } + clippings = _attach_notes_to_highlights(clippings_payload["clippings"]) + plan_items = [ + _plan_clipping(clipping, matches_by_title, items_by_id, overrides_by_title) + for clipping in clippings + if clipping["kind"] in {"highlight", "note"} + ] + status_counts: dict[str, int] = {} + for item in plan_items: + status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1 + + return { + "format": PLAN_FORMAT, + "source": { + "clippings_format": clippings_payload.get("format"), + "zotero_index_format": zotero_index.get("format"), + "match_report_format": match_report.get("format"), + }, + "count": len(plan_items), + "status_counts": status_counts, + "items": plan_items, + } + + +def _plan_clipping( + clipping: dict[str, Any], + matches_by_title: dict[str, dict[str, Any]], + items_by_id: dict[int, dict[str, Any]], + overrides_by_title: dict[str, dict[str, Any]], +) -> dict[str, Any]: + base = { + "clipping": clipping, + "status": "unmatched-title", + "match": None, + "zotero": None, + "annotation": None, + "problems": [], + } + match = matches_by_title.get(clipping["title"]) + if not match or match.get("status") != "matched" or not match.get("candidates"): + return base + + candidate = match["candidates"][0] + if not candidate.get("item_id"): + base["match"] = candidate + base["problems"] = [candidate.get("reason", "unresolved-match")] + return base + + zotero_item = items_by_id.get(candidate["item_id"]) + if not zotero_item: + base["status"] = "missing-zotero-item" + base["match"] = candidate + base["problems"] = ["matched item is absent from Zotero index"] + return base + + attachment, attachment_status, problems = _choose_attachment( + zotero_item, overrides_by_title.get(clipping["title"]) or {} + ) + status = "ready-for-positioning" if attachment else attachment_status + return { + "clipping": clipping, + "status": status, + "match": candidate, + "zotero": { + "parent_item_id": zotero_item["item_id"], + "parent_key": zotero_item["key"], + "parent_title": zotero_item.get("title"), + "citation_key": zotero_item["fields"].get("citationKey"), + "attachment": attachment, + "attachment_choices": zotero_item.get("attachments", []), + "expected_attachment_type": _expected_attachment_type( + clipping, zotero_item.get("attachments", []) + ), + }, + "annotation": _annotation_stub(clipping) if attachment else None, + "problems": problems, + } + + +def _choose_attachment( + zotero_item: dict[str, Any], + override: dict[str, Any] | None = None, +) -> tuple[dict[str, Any] | None, str, list[str]]: + attachments = zotero_item.get("attachments", []) + if not attachments: + return ( + None, + "matched-title-no-attachment", + ["matched Zotero item has no PDF/EPUB attachment"], + ) + if override: + attachment, problem = _attachment_from_override(attachments, override) + if attachment: + return ( + attachment, + "ready-for-positioning", + ["selected attachment by override"], + ) + if problem: + return None, "matched-title-attachment-override-unresolved", [problem] + if len(attachments) == 1: + return attachments[0], "ready-for-positioning", [] + + epubs = [ + attachment + for attachment in attachments + if attachment.get("content_type") == "application/epub+zip" + ] + if len(epubs) == 1: + return ( + epubs[0], + "ready-for-positioning", + ["multiple attachments; selected sole EPUB"], + ) + + return ( + None, + "matched-title-ambiguous-attachment", + [f"matched Zotero item has {len(attachments)} PDF/EPUB attachments"], + ) + + +def _expected_attachment_type( + clipping: dict[str, Any], attachments: list[dict[str, Any]] +) -> str: + content_types = {attachment.get("content_type") for attachment in attachments} + if len(content_types) == 1: + content_type = next(iter(content_types)) + if content_type == "application/epub+zip": + return "epub" + if content_type == "application/pdf": + return "pdf" + if clipping.get("page") and not clipping.get("location"): + return "pdf-preferred" + if clipping.get("location") and not clipping.get("page"): + return "epub-preferred" + return "pdf-or-epub" + + +def _attachment_from_override( + attachments: list[dict[str, Any]], override: dict[str, Any] +) -> tuple[dict[str, Any] | None, str | None]: + if "attachment_item_id" in override: + value = int(override["attachment_item_id"]) + matches = [ + attachment for attachment in attachments if attachment["item_id"] == value + ] + if len(matches) == 1: + return matches[0], None + return ( + None, + f"attachment_item_id override did not match one attachment: {value}", + ) + if "attachment_key" in override: + value = str(override["attachment_key"]).casefold() + matches = [ + attachment + for attachment in attachments + if attachment["key"].casefold() == value + ] + if len(matches) == 1: + return matches[0], None + return ( + None, + f"attachment_key override did not match one attachment: {override['attachment_key']}", + ) + return None, None + + +def _annotation_stub(clipping: dict[str, Any]) -> dict[str, Any]: + annotation_type = "highlight" if clipping["kind"] == "highlight" else "note" + return { + "type": annotation_type, + "text": clipping["text"] if annotation_type == "highlight" else "", + "comment": clipping.get("comment") + or (clipping["text"] if annotation_type == "note" else None), + "color": "#ffd400", + "pageLabel": clipping.get("page") or "", + "sortIndex": None, + "position": None, + "tags": [{"name": "kindle-import"}], + } + + +def _attach_notes_to_highlights( + clippings: list[dict[str, Any]], +) -> list[dict[str, Any]]: + updated = [dict(clipping) for clipping in clippings] + attached_note_ids = set() + for note_index, note in enumerate(updated): + if note["kind"] != "note" or not note.get("text"): + continue + highlight_index = _matching_highlight_index(updated, note_index, note) + if highlight_index is None: + continue + highlight = dict(updated[highlight_index]) + comments = [ + comment for comment in (highlight.get("comment"), note["text"]) if comment + ] + highlight["comment"] = "\n\n".join(comments) + highlight["note_ids"] = [*highlight.get("note_ids", []), note["id"]] + updated[highlight_index] = highlight + attached_note_ids.add(note["id"]) + return [ + clipping + for clipping in updated + if clipping["kind"] != "note" or clipping["id"] not in attached_note_ids + ] + + +def _matching_highlight_index( + clippings: list[dict[str, Any]], note_index: int, note: dict[str, Any] +) -> int | None: + note_location = _range_start(note.get("location")) + if note_location is not None: + for index, clipping in enumerate(clippings): + if clipping["kind"] != "highlight" or clipping["title"] != note["title"]: + continue + highlight_range = _number_range(clipping.get("location")) + if ( + highlight_range + and highlight_range[0] <= note_location <= highlight_range[1] + ): + return index + + note_page = _range_start(note.get("page")) + if note_page is None: + return None + for index in range(note_index - 1, -1, -1): + clipping = clippings[index] + if clipping["kind"] != "highlight" or clipping["title"] != note["title"]: + continue + highlight_page = _number_range(clipping.get("page")) + if highlight_page and highlight_page[0] <= note_page <= highlight_page[1]: + return index + return None + + +def _range_start(value: str | None) -> int | None: + number_range = _number_range(value) + return number_range[0] if number_range else None + + +def _number_range(value: str | None) -> tuple[int, int] | None: + if not value: + return None + numbers = [int(number) for number in re.findall(r"\d+", value)] + if not numbers: + return None + return numbers[0], numbers[-1] diff --git a/src/kindle_zotero_importer/matcher.py b/src/kindle_zotero_importer/matcher.py new file mode 100644 index 0000000..20b9069 --- /dev/null +++ b/src/kindle_zotero_importer/matcher.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from difflib import SequenceMatcher +import json +import re +import unicodedata +from typing import Any + +from .overrides import resolve_override + + +@dataclass(frozen=True) +class MatchCandidate: + item_id: int + key: str + title: str | None + citation_key: str | None + creators: list[str] + attachment_count: int + score: float + reason: str + + +@dataclass(frozen=True) +class TitleMatch: + clipping_title: str + clipping_count: int + candidates: list[MatchCandidate] + + +def load_json(path: str) -> dict[str, Any]: + with open(path, "r", encoding="utf-8") as file: + return json.load(file) + + +def build_match_report( + clippings_payload: dict[str, Any], + zotero_index: dict[str, Any], + overrides: dict[str, dict[str, Any]] | None = None, + max_candidates: int = 5, +) -> dict[str, Any]: + title_counts = _clipping_title_counts(clippings_payload) + matches = [ + _match_title( + title, + count, + zotero_index["items"], + max_candidates, + overrides.get(title) if overrides else None, + ) + for title, count in sorted( + title_counts.items(), key=lambda item: item[0].lower() + ) + ] + status_counts = _status_counts(matches) + return { + "format": "kindle-zotero-importer.match-report.v1", + "clipping_title_count": len(matches), + "matched_title_count": status_counts["matched"], + "ambiguous_title_count": status_counts["ambiguous"], + "unmatched_title_count": status_counts["unmatched"], + "matches": [ + { + "clipping_title": match.clipping_title, + "clipping_count": match.clipping_count, + "status": _match_status(match), + "override": overrides.get(match.clipping_title) if overrides else None, + "candidates": [asdict(candidate) for candidate in match.candidates], + } + for match in matches + ], + } + + +def _clipping_title_counts(clippings_payload: dict[str, Any]) -> dict[str, int]: + counts: dict[str, int] = {} + for clipping in clippings_payload["clippings"]: + title = clipping["title"] + counts[title] = counts.get(title, 0) + 1 + return counts + + +def _match_title( + clipping_title: str, + count: int, + items: list[dict[str, Any]], + max_candidates: int, + override: dict[str, Any] | None = None, +) -> TitleMatch: + if override: + item, reason = resolve_override(clipping_title, override, items) + if item: + return TitleMatch(clipping_title, count, [_candidate(item, 1.0, reason)]) + return TitleMatch( + clipping_title, + count, + [ + MatchCandidate( + item_id=0, + key="", + title=None, + citation_key=None, + creators=[], + attachment_count=0, + score=0.0, + reason=reason, + ) + ], + ) + + citekey = _extract_citekey(clipping_title) + normalized_clipping_title = _normalize_title(clipping_title) + candidates: list[MatchCandidate] = [] + + for item in items: + item_citekey = item["fields"].get("citationKey") + if citekey and item_citekey and citekey.casefold() == item_citekey.casefold(): + candidates.append(_candidate(item, 1.0, "citationKey")) + continue + + item_title = item.get("title") + if not item_title: + continue + + normalized_item_title = _normalize_title(item_title) + if not normalized_item_title: + continue + + if normalized_item_title == normalized_clipping_title: + candidates.append(_candidate(item, 0.98, "title-exact")) + elif normalized_item_title in normalized_clipping_title: + candidates.append( + _candidate(item, 0.92, "title-contained-in-clipping-title") + ) + elif normalized_clipping_title in normalized_item_title: + candidates.append( + _candidate(item, 0.88, "clipping-title-contained-in-title") + ) + else: + score = SequenceMatcher( + None, normalized_clipping_title, normalized_item_title + ).ratio() + if score >= 0.82: + candidates.append(_candidate(item, round(score, 4), "title-fuzzy")) + + candidates.sort( + key=lambda candidate: (candidate.score, candidate.attachment_count), + reverse=True, + ) + return TitleMatch(clipping_title, count, candidates[:max_candidates]) + + +def _candidate(item: dict[str, Any], score: float, reason: str) -> MatchCandidate: + return MatchCandidate( + item_id=item["item_id"], + key=item["key"], + title=item.get("title"), + citation_key=item["fields"].get("citationKey"), + creators=item.get("creators", []), + attachment_count=len(item.get("attachments", [])), + score=score, + reason=reason, + ) + + +def _extract_citekey(title: str) -> str | None: + stripped = title.strip() + if re.fullmatch(r"@?[A-Za-z][A-Za-z0-9_:\-.]+", stripped): + return stripped.removeprefix("@") + + match = re.search(r"(?:^|[\s\[(])@([A-Za-z][A-Za-z0-9_:\-.]+)(?:$|[\s\])])", title) + if match: + return match.group(1) + + match = re.match(r"^([A-Za-z][A-Za-z'\-]+)_([0-9]{4})(?:_|\b)", stripped) + if match: + return f"{match.group(1)}{match.group(2)}" + return None + + +def _normalize_title(value: str) -> str: + value = unicodedata.normalize("NFKD", value) + value = "".join( + character for character in value if not unicodedata.combining(character) + ) + value = value.casefold() + value = re.sub(r"\([^)]*\)", " ", value) + value = re.sub(r"\bby\b.*$", " ", value) + value = re.sub(r"\bz-lib\.org\b", " ", value) + value = re.sub(r"[^a-z0-9]+", " ", value) + return " ".join(value.split()) + + +def _status_counts(matches: list[TitleMatch]) -> dict[str, int]: + counts = {"matched": 0, "ambiguous": 0, "unmatched": 0} + for match in matches: + counts[_match_status(match)] += 1 + return counts + + +def _match_status(match: TitleMatch) -> str: + if not match.candidates: + return "unmatched" + if len(match.candidates) == 1: + return "matched" + if match.candidates[0].score > match.candidates[1].score: + return "matched" + return "ambiguous" diff --git a/src/kindle_zotero_importer/mismatch_review.py b/src/kindle_zotero_importer/mismatch_review.py new file mode 100644 index 0000000..bfa2104 --- /dev/null +++ b/src/kindle_zotero_importer/mismatch_review.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +from collections import defaultdict +import json +from typing import Any + + +SECTIONS = [ + ( + "Matched Zotero Item But No Attachment", + "matched-title-no-attachment", + "Attach/link a PDF or EPUB to the listed Zotero item, then re-index Zotero.", + ), + ( + "Unmatched Kindle Titles", + "unmatched-title", + "Add one confirmed mapping to match-overrides.json using citation_key, zotero_key, or zotero_item_id.", + ), + ( + "Matched Attachment But Missing File Path", + "pdf-position-missing-path", + "Fix Zotero linked-file/storage path, then re-index Zotero.", + ), + ( + "Attachment Found But Text Position Failed", + "epub-text-not-found", + "Needs looser EPUB text matching or manual review.", + ), + ( + "PDF Text Not Found", + "pdf-text-not-found", + "Needs looser PDF text matching/OCR/page-offset handling.", + ), + ( + "PDF Rectangles Not Found", + "pdf-rects-not-found", + "Needs improved PDF rectangle recovery after text/page match.", + ), +] + + +def build_mismatch_review( + positioned_plan: dict[str, Any], match_report: dict[str, Any] +) -> str: + match_by_title = { + match["clipping_title"]: match for match in match_report["matches"] + } + by_status: dict[str, list[dict[str, Any]]] = defaultdict(list) + for item in positioned_plan["items"]: + if item.get("status") != "positioned": + by_status[item.get("status", "unknown")].append(item) + + lines = [ + "# Mismatch Review", + "", + "Persistent rule: confirmed title matches go in `match-overrides.json`, not `match-overrides.generated.json`. The generated file can be replaced at any time.", + "", + ] + for heading, status, instruction in SECTIONS: + summaries = _title_summaries(by_status.get(status, []), match_by_title) + lines += [ + f"## {heading}", + "", + f"Status: `{status}`", + f"Clippings: {len(by_status.get(status, []))}", + f"Unique titles: {len(summaries)}", + f"Action: {instruction}", + "", + ] + for summary in summaries[:30]: + lines += _summary_lines(status, summary) + if len(summaries) > 30: + lines += [f"_Showing first 30 of {len(summaries)} unique titles._", ""] + + return "\n".join(lines).rstrip() + "\n" + + +def _title_summaries( + items: list[dict[str, Any]], match_by_title: dict[str, dict[str, Any]] +) -> list[dict[str, Any]]: + summaries: dict[str, dict[str, Any]] = {} + for item in items: + clipping = item["clipping"] + title = clipping["title"] + summary = summaries.setdefault( + title, + { + "title": title, + "count": 0, + "kinds": defaultdict(int), + "examples": [], + "zotero": item.get("zotero"), + "match": match_by_title.get(title), + }, + ) + summary["count"] += 1 + summary["kinds"][clipping.get("kind")] += 1 + if len(summary["examples"]) < 3: + summary["examples"].append( + { + "kind": clipping.get("kind"), + "page": clipping.get("page"), + "location": clipping.get("location"), + "text": (clipping.get("text") or "")[:180], + } + ) + return sorted( + summaries.values(), + key=lambda summary: (-summary["count"], summary["title"].casefold()), + ) + + +def _summary_lines(status: str, summary: dict[str, Any]) -> list[str]: + match = summary.get("match") or {} + candidates = match.get("candidates") or [] + best = candidates[0] if candidates else {} + zotero = summary.get("zotero") or {} + citation_key = zotero.get("citation_key") or best.get("citation_key") or "" + zotero_item_id = zotero.get("parent_item_id") or best.get("item_id") or "" + zotero_key = best.get("key") or "" + attachment_count = best.get("attachment_count") + expected_attachment_type = zotero.get( + "expected_attachment_type" + ) or _expected_attachment_type(status, summary) + override_resolution = _override_resolution(citation_key, zotero_key, zotero_item_id) + + lines = [f"### {summary['title']}", "", "```yaml"] + lines += [ + f"clipping_title: {_yaml_string(summary['title'])}", + f"status: {status}", + f"clipping_count: {summary['count']}", + "kinds: " + _yaml_inline_map(summary["kinds"]), + f"citation_key: {_yaml_string(citation_key)}", + f"zotero_item_id: {_yaml_value(zotero_item_id)}", + f"zotero_key: {_yaml_string(zotero_key)}", + f"zotero_title: {_yaml_string(zotero.get('parent_title') or best.get('title') or '')}", + f"attachment_count: {_yaml_value(attachment_count)}", + f"expected_attachment_type: {_yaml_string(expected_attachment_type)}", + f"match_score: {_yaml_value(best.get('score'))}", + f"match_reason: {_yaml_string(best.get('reason') or '')}", + ] + attachments = zotero.get("attachment_choices") or [] + if attachments: + lines.append("attachment_choices:") + for attachment in attachments: + lines += [ + f" - attachment_item_id: {attachment.get('item_id')}", + f" attachment_key: {_yaml_string(attachment.get('key') or '')}", + f" attachment_title: {_yaml_string(attachment.get('title') or '')}", + f" content_type: {_yaml_string(attachment.get('content_type') or '')}", + f" path: {_yaml_string(attachment.get('resolved_path') or attachment.get('path') or '')}", + ] + if override_resolution: + lines += [ + "override_entry:", + f" clipping_title: {_yaml_string(summary['title'])}", + " resolution:", + f" {override_resolution[0]}: {_yaml_value(override_resolution[1])}", + ] + if attachments: + lines += [ + ' attachment_key: "PASTE_SELECTED_ATTACHMENT_KEY"', + " attachment_item_id: null", + ] + lines += ["```", ""] + + if candidates: + lines += ["Candidates:"] + for candidate in candidates[:5]: + lines.append( + "- " + f"citation_key: `{candidate.get('citation_key')}`, " + f"zotero_item_id: `{candidate.get('item_id')}`, " + f"zotero_key: `{candidate.get('key')}`, " + f"attachments: `{candidate.get('attachment_count')}`, " + f"score: `{candidate.get('score')}`, " + f"title: {candidate.get('title')}" + ) + lines.append("") + + for example in summary["examples"]: + text = example["text"].replace("\n", " ") + lines.append( + f"- Example: {example['kind']} page `{example['page']}` loc `{example['location']}`: {text}" + ) + lines.append("") + return lines + + +def _override_resolution( + citation_key: str, zotero_key: str, zotero_item_id: str | int +) -> tuple[str, str | int] | None: + if citation_key: + return "citation_key", citation_key + if zotero_key: + return "zotero_key", zotero_key + if zotero_item_id: + return "zotero_item_id", zotero_item_id + return None + + +def _expected_attachment_type(status: str, summary: dict[str, Any]) -> str: + if status.startswith("epub-"): + return "epub" + if status.startswith("pdf-"): + return "pdf" + pages = 0 + locations = 0 + for example in summary["examples"]: + if example.get("page"): + pages += 1 + if example.get("location"): + locations += 1 + if pages and not locations: + return "pdf-preferred" + if locations and not pages: + return "epub-preferred" + return "pdf-or-epub" + + +def _yaml_inline_map(values: dict[str, int]) -> str: + return ( + "{" + + ", ".join(f"{key}: {value}" for key, value in sorted(values.items())) + + "}" + ) + + +def _yaml_value(value: Any) -> str: + if value is None or value == "": + return "null" + if isinstance(value, (int, float)): + return str(value) + return _yaml_string(str(value)) + + +def _yaml_string(value: str) -> str: + return json.dumps(value, ensure_ascii=False) diff --git a/src/kindle_zotero_importer/overrides.py b/src/kindle_zotero_importer/overrides.py new file mode 100644 index 0000000..437d53a --- /dev/null +++ b/src/kindle_zotero_importer/overrides.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + + +OVERRIDES_FORMAT = "kindle-zotero-importer.match-overrides.v1" + + +class OverrideError(ValueError): + pass + + +def load_overrides(payload: dict[str, Any]) -> dict[str, dict[str, Any]]: + if payload.get("format") != OVERRIDES_FORMAT: + raise OverrideError(f"unsupported overrides format: {payload.get('format')}") + + overrides: dict[str, dict[str, Any]] = {} + for entry in payload.get("overrides", []): + clipping_title = entry.get("clipping_title") + if not clipping_title: + raise OverrideError("override missing clipping_title") + + resolution = _clean_resolution(entry.get("resolution") or {}) + if not resolution: + continue + item_fields = [ + key + for key in ("citation_key", "zotero_key", "zotero_item_id") + if key in resolution + ] + if len(item_fields) != 1: + raise OverrideError( + f"override for {clipping_title!r} must contain exactly one item resolution field" + ) + overrides[clipping_title] = resolution + + return overrides + + +def resolve_override( + clipping_title: str, resolution: dict[str, Any], items: list[dict[str, Any]] +) -> tuple[dict[str, Any] | None, str]: + if "citation_key" in resolution: + value = str(resolution["citation_key"]).casefold() + return _find_item( + items, + lambda item: (item["fields"].get("citationKey") or "").casefold() == value, + f"override-citation-key:{resolution['citation_key']}", + clipping_title, + ) + if "zotero_key" in resolution: + value = str(resolution["zotero_key"]).casefold() + return _find_item( + items, + lambda item: item["key"].casefold() == value, + f"override-zotero-key:{resolution['zotero_key']}", + clipping_title, + ) + if "zotero_item_id" in resolution: + value = int(resolution["zotero_item_id"]) + return _find_item( + items, + lambda item: item["item_id"] == value, + f"override-zotero-item-id:{value}", + clipping_title, + ) + raise OverrideError(f"unsupported override resolution for {clipping_title!r}") + + +def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]: + overrides = [] + for match in match_report.get("matches", []): + if match.get("status") == "matched": + continue + overrides.append( + { + "clipping_title": match["clipping_title"], + "resolution": { + "citation_key": "", + "zotero_key": "", + "zotero_item_id": None, + "attachment_key": "", + "attachment_item_id": None, + }, + "notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments.", + "candidates": match.get("candidates", []), + } + ) + + return {"format": OVERRIDES_FORMAT, "overrides": overrides} + + +def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]: + cleaned: dict[str, Any] = {} + for key in ( + "citation_key", + "zotero_key", + "zotero_item_id", + "attachment_key", + "attachment_item_id", + ): + value = resolution.get(key) + if value is None: + continue + if isinstance(value, str) and not value.strip(): + continue + cleaned[key] = value.strip() if isinstance(value, str) else value + return cleaned + + +def _find_item( + items: list[dict[str, Any]], + predicate: Callable[[dict[str, Any]], bool], + reason: str, + clipping_title: str, +) -> tuple[dict[str, Any] | None, str]: + matches = [item for item in items if predicate(item)] + if len(matches) == 1: + return matches[0], reason + if not matches: + return None, f"override-unresolved:{clipping_title}" + return None, f"override-ambiguous:{clipping_title}" diff --git a/src/kindle_zotero_importer/pdf_position.py b/src/kindle_zotero_importer/pdf_position.py new file mode 100644 index 0000000..8dc5bda --- /dev/null +++ b/src/kindle_zotero_importer/pdf_position.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +from dataclasses import dataclass +import re +import subprocess +import xml.etree.ElementTree as ET +from typing import Any + + +@dataclass(frozen=True) +class PDFWord: + text: str + left: float + top: float + width: float + height: float + start: int + end: int + + +@dataclass(frozen=True) +class PDFPageXML: + index: int + width: float + height: float + words: list[PDFWord] + + +def add_pdf_positions(plan: dict[str, Any]) -> dict[str, Any]: + text_cache: dict[str, list[str]] = {} + xml_cache: dict[tuple[str, int], PDFPageXML] = {} + size_cache: dict[tuple[str, int], tuple[float, float]] = {} + positioned = [] + for item in plan["items"]: + positioned.append(_position_item(item, text_cache, xml_cache, size_cache)) + + status_counts: dict[str, int] = {} + for item in positioned: + status_counts[item["status"]] = status_counts.get(item["status"], 0) + 1 + + updated = dict(plan) + updated["items"] = positioned + updated["status_counts"] = status_counts + return updated + + +def _position_item( + item: dict[str, Any], + text_cache: dict[str, list[str]], + xml_cache: dict[tuple[str, int], PDFPageXML], + size_cache: dict[tuple[str, int], tuple[float, float]], +) -> dict[str, Any]: + if item.get("status") != "ready-for-positioning": + return item + attachment = item.get("zotero", {}).get("attachment") + if not attachment or attachment.get("content_type") != "application/pdf": + return item + if item["clipping"]["kind"] != "highlight" or not item["clipping"].get("text"): + return _with_problem(item, "pdf-position-skipped-non-highlight") + + path = attachment.get("resolved_path") or attachment.get("path") + if not path or path.startswith("attachments:"): + return _with_problem(item, "pdf-position-missing-path") + + try: + text_pages = text_cache.setdefault(path, extract_pdf_text_pages(path)) + page_index = find_pdf_text_page( + text_pages, item["clipping"]["text"], item["clipping"].get("page") + ) + if page_index is None: + return _with_problem(item, "pdf-text-not-found") + page_xml = xml_cache.setdefault( + (path, page_index), extract_pdf_page_xml(path, page_index) + ) + position = find_pdf_rects(path, page_xml, item["clipping"]["text"], size_cache) + except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch + return _with_problem(item, f"pdf-position-error:{error}") + + if not position: + return _with_problem(item, "pdf-rects-not-found") + + updated = dict(item) + annotation = dict(updated["annotation"]) + annotation["position"] = { + "pageIndex": position["pageIndex"], + "rects": position["rects"], + } + annotation["pageLabel"] = item["clipping"].get("page") or str( + position["pageIndex"] + 1 + ) + annotation["sortIndex"] = position.get("sortIndex") + updated["annotation"] = annotation + updated["status"] = "positioned" + return updated + + +def extract_pdf_text_pages(path: str) -> list[str]: + result = subprocess.run( + ["pdftotext", "-enc", "UTF-8", path, "-"], + check=True, + capture_output=True, + text=True, + ) + return result.stdout.split("\f") + + +def find_pdf_text_page( + pages: list[str], quote: str, kindle_page: str | None = None +) -> int | None: + normalized_quote, _ = _normalize_with_map(quote) + if not normalized_quote: + return None + page_order = list(range(len(pages))) + if kindle_page and kindle_page.isdigit(): + index = int(kindle_page) - 1 + if 0 <= index < len(pages): + page_order.remove(index) + page_order.insert(0, index) + for index in page_order: + normalized_page, _ = _normalize_with_map(pages[index]) + if normalized_quote in normalized_page: + return index + return None + + +def extract_pdf_page_xml(path: str, page_index: int) -> PDFPageXML: + page_number = page_index + 1 + result = subprocess.run( + [ + "pdftohtml", + "-f", + str(page_number), + "-l", + str(page_number), + "-xml", + "-stdout", + path, + ], + check=True, + capture_output=True, + text=True, + ) + root = ET.fromstring(result.stdout) + page_el = root.find("page") + if page_el is None: + raise ValueError(f"no XML page for PDF page {page_number}") + cursor = 0 + words: list[PDFWord] = [] + for text_el in page_el.findall("text"): + text = "".join(text_el.itertext()).strip() + if not text: + continue + for word in text.split(): + start = cursor + end = start + len(word) + words.append( + PDFWord( + text=word, + left=float(text_el.attrib["left"]), + top=float(text_el.attrib["top"]), + width=float(text_el.attrib["width"]), + height=float(text_el.attrib["height"]), + start=start, + end=end, + ) + ) + cursor = end + 1 + return PDFPageXML( + index=page_index, + width=float(page_el.attrib["width"]), + height=float(page_el.attrib["height"]), + words=words, + ) + + +def find_pdf_rects( + path: str, + page: PDFPageXML, + quote: str, + size_cache: dict[tuple[str, int], tuple[float, float]], +) -> dict[str, Any] | None: + normalized_quote, _ = _normalize_with_map(quote) + text = " ".join(word.text for word in page.words) + normalized_text, text_map = _normalize_with_map(text) + start = normalized_text.find(normalized_quote) + if start < 0: + return None + end = start + len(normalized_quote) + raw_start = text_map[start] + raw_end = text_map[end - 1] + 1 + matched_words = [ + word for word in page.words if word.end >= raw_start and word.start <= raw_end + ] + if not matched_words: + return None + pdf_width, pdf_height = size_cache.setdefault( + (path, page.index), + _pdf_page_size(path, page.index + 1, page.width, page.height), + ) + rects = _word_rects(matched_words, page, pdf_width, pdf_height) + return { + "pageIndex": page.index, + "rects": rects, + "sortIndex": f"{page.index:05d}|{int(rects[0][1]):06d}|{int(rects[0][0]):05d}", + } + + +def _word_rects( + words: list[PDFWord], page: PDFPageXML, pdf_width: float, pdf_height: float +) -> list[list[float]]: + lines: dict[int, list[PDFWord]] = {} + for word in words: + lines.setdefault(round(word.top), []).append(word) + scale_x = pdf_width / page.width + scale_y = pdf_height / page.height + rects = [] + for top in sorted(lines): + line_words = lines[top] + left = min(word.left for word in line_words) + right = max(word.left + word.width for word in line_words) + line_top = min(word.top for word in line_words) + line_bottom = max(word.top + word.height for word in line_words) + rects.append( + [ + round(left * scale_x, 3), + round(pdf_height - line_bottom * scale_y, 3), + round(right * scale_x, 3), + round(pdf_height - line_top * scale_y, 3), + ] + ) + return rects + + +def _pdf_page_size( + path: str, page_number: int, fallback_width: float, fallback_height: float +) -> tuple[float, float]: + result = subprocess.run( + ["pdfinfo", "-f", str(page_number), "-l", str(page_number), path], + check=True, + capture_output=True, + text=True, + ) + match = re.search( + rf"Page\s+{page_number}\s+size:\s+([0-9.]+)\s+x\s+([0-9.]+)\s+pts", + result.stdout, + ) + if not match: + match = re.search( + r"Page size:\s+([0-9.]+)\s+x\s+([0-9.]+)\s+pts", result.stdout + ) + if not match: + return fallback_width, fallback_height + return float(match.group(1)), float(match.group(2)) + + +def _normalize_with_map(text: str) -> tuple[str, list[int]]: + normalized = [] + mapping = [] + last_was_space = False + for index, char in enumerate(text): + if char.isspace(): + if not last_was_space and normalized: + normalized.append(" ") + mapping.append(index) + last_was_space = True + continue + normalized.append(char.casefold()) + mapping.append(index) + last_was_space = False + if normalized and normalized[-1] == " ": + normalized.pop() + mapping.pop() + return "".join(normalized), mapping + + +def _with_problem(item: dict[str, Any], problem: str) -> dict[str, Any]: + updated = dict(item) + updated["status"] = problem + updated["problems"] = [*item.get("problems", []), problem] + return updated diff --git a/src/kindle_zotero_importer/zotero_index.py b/src/kindle_zotero_importer/zotero_index.py new file mode 100644 index 0000000..d1a60be --- /dev/null +++ b/src/kindle_zotero_importer/zotero_index.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from pathlib import Path +import sqlite3 +from typing import Any + + +METADATA_FIELDS = ( + "title", + "publicationTitle", + "shortTitle", + "DOI", + "ISBN", + "date", + "url", + "citationKey", +) + + +@dataclass(frozen=True) +class ZoteroAttachment: + item_id: int + key: str + title: str | None + content_type: str | None + path: str | None + resolved_path: str | None + + +@dataclass(frozen=True) +class ZoteroItem: + item_id: int + key: str + item_type: str + title: str | None + creators: list[str] = field(default_factory=list) + fields: dict[str, str] = field(default_factory=dict) + attachments: list[ZoteroAttachment] = field(default_factory=list) + + +def build_zotero_index(db_path: str, storage_root: str | None = None) -> dict[str, Any]: + db_uri = f"file:{Path(db_path)}?mode=ro&immutable=1" + with sqlite3.connect(db_uri, uri=True) as connection: + connection.row_factory = sqlite3.Row + items = _load_parent_items(connection) + creators = _load_creators(connection) + attachments = _load_attachments(connection, storage_root) + + indexed_items: list[ZoteroItem] = [] + for row in items: + fields = _metadata_for_item(row) + indexed_items.append( + ZoteroItem( + item_id=row["itemID"], + key=row["key"], + item_type=row["typeName"], + title=fields.get("title"), + creators=creators.get(row["itemID"], []), + fields=fields, + attachments=attachments.get(row["itemID"], []), + ) + ) + + return { + "format": "kindle-zotero-importer.zotero-index.v1", + "database": str(db_path), + "count": len(indexed_items), + "attachment_count": sum(len(item.attachments) for item in indexed_items), + "items": [_item_to_dict(item) for item in indexed_items], + } + + +def _load_parent_items(connection: sqlite3.Connection) -> list[sqlite3.Row]: + field_columns = ",\n".join( + f"MAX(CASE WHEN fc.fieldName = '{field_name}' THEN idv.value END) AS {field_name}" + for field_name in METADATA_FIELDS + ) + query = f""" + SELECT + i.itemID, + i.key, + it.typeName, + {field_columns} + FROM items i + JOIN itemTypesCombined it ON it.itemTypeID = i.itemTypeID + LEFT JOIN itemData id ON id.itemID = i.itemID + LEFT JOIN fieldsCombined fc ON fc.fieldID = id.fieldID + LEFT JOIN itemDataValues idv ON idv.valueID = id.valueID + WHERE it.typeName NOT IN ('attachment', 'note', 'annotation') + AND i.itemID NOT IN (SELECT itemID FROM deletedItems) + GROUP BY i.itemID + ORDER BY title COLLATE NOCASE + """ + return list(connection.execute(query)) + + +def _load_creators(connection: sqlite3.Connection) -> dict[int, list[str]]: + rows = connection.execute( + """ + SELECT ic.itemID, c.firstName, c.lastName, c.fieldMode + FROM itemCreators ic + JOIN creators c ON c.creatorID = ic.creatorID + ORDER BY ic.itemID, ic.orderIndex + """ + ) + creators: dict[int, list[str]] = {} + for row in rows: + creators.setdefault(row["itemID"], []).append(_format_creator(row)) + return creators + + +def _load_attachments( + connection: sqlite3.Connection, storage_root: str | None +) -> dict[int, list[ZoteroAttachment]]: + rows = connection.execute( + """ + SELECT + a.parentItemID, + a.itemID, + i.key, + a.contentType, + a.path, + idv.value AS title + FROM itemAttachments a + JOIN items i ON i.itemID = a.itemID + LEFT JOIN itemData id ON id.itemID = a.itemID + LEFT JOIN fieldsCombined fc ON fc.fieldID = id.fieldID AND fc.fieldName = 'title' + LEFT JOIN itemDataValues idv ON idv.valueID = id.valueID + WHERE a.parentItemID IS NOT NULL + AND a.contentType IN ('application/pdf', 'application/epub+zip') + AND a.itemID NOT IN (SELECT itemID FROM deletedItems) + ORDER BY a.parentItemID, title COLLATE NOCASE + """ + ) + attachments: dict[int, list[ZoteroAttachment]] = {} + for row in rows: + attachment = ZoteroAttachment( + item_id=row["itemID"], + key=row["key"], + title=row["title"], + content_type=row["contentType"], + path=row["path"], + resolved_path=_resolve_attachment_path( + row["path"], row["key"], storage_root + ), + ) + attachments.setdefault(row["parentItemID"], []).append(attachment) + return attachments + + +def _metadata_for_item(row: sqlite3.Row) -> dict[str, str]: + return { + field_name: row[field_name] for field_name in METADATA_FIELDS if row[field_name] + } + + +def _format_creator(row: sqlite3.Row) -> str: + first_name = (row["firstName"] or "").strip() + last_name = (row["lastName"] or "").strip() + if row["fieldMode"] == 1: + return last_name or first_name + return " ".join(part for part in (first_name, last_name) if part) + + +def _resolve_attachment_path( + path: str | None, key: str, storage_root: str | None +) -> str | None: + if not path: + return None + if path.startswith("storage:"): + if not storage_root: + return path + return str(Path(storage_root) / key / path.removeprefix("storage:")) + if path.startswith("attachments:"): + return path + return path + + +def _item_to_dict(item: ZoteroItem) -> dict[str, Any]: + data = asdict(item) + data["attachments"] = [asdict(attachment) for attachment in item.attachments] + return data diff --git a/zotero-writer.js b/zotero-writer.js new file mode 100644 index 0000000..3b968f7 --- /dev/null +++ b/zotero-writer.js @@ -0,0 +1,156 @@ +// Run in Zotero via Tools > Developer > Run JavaScript. +// Set PLAN_PATH to the generated import-plan.final.json path. +// Keep DRY_RUN=true until the preview output looks correct. + +const PLAN_PATH = "/home/user/Library/Mobile Documents/iCloud~md~obsidian/Documents/rhizome/06_projects/UTI/kindle-zotero-importer/import-plan.final.json"; +const DRY_RUN = false; + +const text = await Zotero.File.getContentsAsync(PLAN_PATH); +const plan = JSON.parse(text); + +if (plan.format !== "kindle-zotero-importer.zotero-writer-plan.v1") { + throw new Error(`Unsupported plan format: ${plan.format}`); +} + +const results = { + dryRun: DRY_RUN, + total: plan.annotations.length, + created: 0, + skippedExisting: 0, + updatedComments: 0, + removedDuplicates: 0, + failed: [], + preview: [], +}; + +const existingByAttachment = new Map(); + +function normalizePosition(position) { + if (!position) { + return ""; + } + if (typeof position === "string") { + try { + return JSON.stringify(JSON.parse(position)); + } catch (_error) { + return position; + } + } + return JSON.stringify(position); +} + +function annotationFingerprint(annotation) { + return `${annotation.text || ""}\u0000${normalizePosition(annotation.position)}`; +} + +function existingAnnotationFingerprint(annotation) { + return `${annotation.annotationText || ""}\u0000${normalizePosition(annotation.annotationPosition)}`; +} + +async function getExistingAnnotations(attachment) { + if (!existingByAttachment.has(attachment.id)) { + const annotations = attachment.getAnnotations ? attachment.getAnnotations() : []; + const byFingerprint = new Map(); + + for (const annotation of annotations) { + const fingerprint = existingAnnotationFingerprint(annotation); + const kept = byFingerprint.get(fingerprint); + if (!kept) { + byFingerprint.set(fingerprint, annotation); + continue; + } + + const keptHasComment = Boolean(kept.annotationComment); + const currentHasComment = Boolean(annotation.annotationComment); + const duplicate = keptHasComment || !currentHasComment ? annotation : kept; + const replacement = duplicate === annotation ? kept : annotation; + + byFingerprint.set(fingerprint, replacement); + if (!DRY_RUN) { + await duplicate.eraseTx(); + } + results.removedDuplicates += 1; + } + + existingByAttachment.set(attachment.id, byFingerprint); + } + return existingByAttachment.get(attachment.id); +} + +function mergeComments(existingComment, newComment) { + const seen = new Set(); + const merged = []; + for (const comment of [existingComment, newComment]) { + for (const part of String(comment || "").split(/\n\s*\n/)) { + const cleaned = part.trim(); + if (!cleaned || seen.has(cleaned)) { + continue; + } + seen.add(cleaned); + merged.push(cleaned); + } + } + return merged.join("\n\n"); +} + +for (const entry of plan.annotations) { + try { + const attachment = Zotero.Items.get(entry.attachment_item_id); + if (!attachment) { + throw new Error(`Attachment not found: ${entry.attachment_item_id}`); + } + + const annotation = { + key: Zotero.DataObjectUtilities.generateKey(), + ...entry.annotation, + }; + if (!annotation.sortIndex) { + delete annotation.sortIndex; + } + + const existingAnnotations = await getExistingAnnotations(attachment); + const fingerprint = annotationFingerprint(annotation); + const existingAnnotation = existingAnnotations.get(fingerprint); + if (existingAnnotation) { + const mergedComment = mergeComments(existingAnnotation.annotationComment, annotation.comment); + if (mergedComment && existingAnnotation.saveTx && existingAnnotation.annotationComment !== mergedComment) { + existingAnnotation.annotationComment = mergedComment; + if (!DRY_RUN) { + await existingAnnotation.saveTx(); + } + results.updatedComments += 1; + } + results.skippedExisting += 1; + continue; + } + + if (DRY_RUN) { + results.preview.push({ + clipping_id: entry.clipping_id, + attachment_item_id: entry.attachment_item_id, + type: annotation.type, + text: annotation.text.slice(0, 120), + pageLabel: annotation.pageLabel, + sortIndex: annotation.sortIndex, + position: annotation.position, + }); + continue; + } + + await Zotero.Annotations.saveFromJSON(attachment, annotation); + existingAnnotations.set(fingerprint, { + annotationComment: annotation.comment || "", + annotationPosition: annotation.position, + annotationText: annotation.text || "", + }); + results.created += 1; + } catch (error) { + results.failed.push({ + clipping_id: entry.clipping_id, + attachment_item_id: entry.attachment_item_id, + message: String(error), + }); + } +} + +return JSON.stringify(results, null, 2);