diff --git a/README.md b/README.md index 8133499..af8e82a 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Generate reusable manual overrides for ambiguous/unmatched titles: python -m kindle_zotero_importer generate-overrides matches.json --output match-overrides.generated.json --pretty ``` +Generated overrides are designed for batch editing: unresolved titles are sorted by clipping count, unmatched titles default to `ignore: true`, and already-entered overrides are kept at the end as an audit trail. Replace `ignore` with a confirmed `citation_key`, `zotero_key`, or `zotero_item_id` to import a title instead of discarding it. + Apply reviewed overrides: ```sh @@ -38,6 +40,17 @@ python -m kindle_zotero_importer match clippings.json zotero-index.json --overri Keep confirmed mappings in `match-overrides.json`. The `match-overrides.generated.json` file is only a disposable review skeleton and can be regenerated from the current `matches.json` at any time. +To permanently skip a Kindle title that should not be imported, add an ignore override: + +```json +{ + "clipping_title": "Example Kindle Title", + "resolution": { + "ignore": true + } +} +``` + If a matched Zotero item has multiple usable PDF/EPUB attachments, add the selected attachment to the same override entry: ```json @@ -82,6 +95,8 @@ python -m kindle_zotero_importer position-pdf import-plan.epub.json --output imp python -m kindle_zotero_importer finalize import-plan.positioned.json --output import-plan.final.json --pretty ``` +PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a PDF permits viewing but blocks text copying, the importer retries extraction through a temporary `qpdf --decrypt` copy. The Zotero attachment itself is not modified. + ## Safety Rule Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes. diff --git a/src/kindle_zotero_importer/__main__.py b/src/kindle_zotero_importer/__main__.py new file mode 100644 index 0000000..42fbee7 --- /dev/null +++ b/src/kindle_zotero_importer/__main__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +from .cli import main + + +raise SystemExit(main()) diff --git a/src/kindle_zotero_importer/import_plan.py b/src/kindle_zotero_importer/import_plan.py index bff5685..258ce20 100644 --- a/src/kindle_zotero_importer/import_plan.py +++ b/src/kindle_zotero_importer/import_plan.py @@ -58,6 +58,11 @@ def _plan_clipping( "problems": [], } match = matches_by_title.get(clipping["title"]) + if match and match.get("status") == "ignored": + base["status"] = "ignored-title" + base["match"] = None + base["problems"] = ["ignored by match override"] + return base if not match or match.get("status") != "matched" or not match.get("candidates"): return base diff --git a/src/kindle_zotero_importer/matcher.py b/src/kindle_zotero_importer/matcher.py index 20b9069..2689f4a 100644 --- a/src/kindle_zotero_importer/matcher.py +++ b/src/kindle_zotero_importer/matcher.py @@ -27,6 +27,7 @@ class TitleMatch: clipping_title: str clipping_count: int candidates: list[MatchCandidate] + status_override: str | None = None def load_json(path: str) -> dict[str, Any]: @@ -60,6 +61,7 @@ def build_match_report( "matched_title_count": status_counts["matched"], "ambiguous_title_count": status_counts["ambiguous"], "unmatched_title_count": status_counts["unmatched"], + "ignored_title_count": status_counts["ignored"], "matches": [ { "clipping_title": match.clipping_title, @@ -89,6 +91,8 @@ def _match_title( override: dict[str, Any] | None = None, ) -> TitleMatch: if override: + if override.get("ignore") is True: + return TitleMatch(clipping_title, count, [], "ignored") item, reason = resolve_override(clipping_title, override, items) if item: return TitleMatch(clipping_title, count, [_candidate(item, 1.0, reason)]) @@ -193,13 +197,15 @@ def _normalize_title(value: str) -> str: def _status_counts(matches: list[TitleMatch]) -> dict[str, int]: - counts = {"matched": 0, "ambiguous": 0, "unmatched": 0} + counts = {"matched": 0, "ambiguous": 0, "unmatched": 0, "ignored": 0} for match in matches: counts[_match_status(match)] += 1 return counts def _match_status(match: TitleMatch) -> str: + if match.status_override: + return match.status_override if not match.candidates: return "unmatched" if len(match.candidates) == 1: diff --git a/src/kindle_zotero_importer/mismatch_review.py b/src/kindle_zotero_importer/mismatch_review.py index bfa2104..2a7056d 100644 --- a/src/kindle_zotero_importer/mismatch_review.py +++ b/src/kindle_zotero_importer/mismatch_review.py @@ -36,6 +36,11 @@ SECTIONS = [ "pdf-rects-not-found", "Needs improved PDF rectangle recovery after text/page match.", ), + ( + "PDF Position Errors", + "pdf-position-error:", + "Inspect the PDF extraction command error, file validity, encryption, or permissions.", + ), ] @@ -57,12 +62,13 @@ def build_mismatch_review( "", ] for heading, status, instruction in SECTIONS: - summaries = _title_summaries(by_status.get(status, []), match_by_title) + section_items = _items_for_status(by_status, status) + summaries = _title_summaries(section_items, match_by_title) lines += [ f"## {heading}", "", f"Status: `{status}`", - f"Clippings: {len(by_status.get(status, []))}", + f"Clippings: {len(section_items)}", f"Unique titles: {len(summaries)}", f"Action: {instruction}", "", @@ -75,6 +81,18 @@ def build_mismatch_review( return "\n".join(lines).rstrip() + "\n" +def _items_for_status( + by_status: dict[str, list[dict[str, Any]]], status: str +) -> list[dict[str, Any]]: + if status.endswith(":"): + items: list[dict[str, Any]] = [] + for item_status, status_items in by_status.items(): + if item_status.startswith(status): + items.extend(status_items) + return items + return by_status.get(status, []) + + def _title_summaries( items: list[dict[str, Any]], match_by_title: dict[str, dict[str, Any]] ) -> list[dict[str, Any]]: diff --git a/src/kindle_zotero_importer/overrides.py b/src/kindle_zotero_importer/overrides.py index 437d53a..f6dec7b 100644 --- a/src/kindle_zotero_importer/overrides.py +++ b/src/kindle_zotero_importer/overrides.py @@ -24,6 +24,25 @@ def load_overrides(payload: dict[str, Any]) -> dict[str, dict[str, Any]]: resolution = _clean_resolution(entry.get("resolution") or {}) if not resolution: continue + if resolution.get("ignore") is True: + conflicting_fields = [ + key + for key in ( + "citation_key", + "zotero_key", + "zotero_item_id", + "attachment_key", + "attachment_item_id", + ) + if key in resolution + ] + if conflicting_fields: + raise OverrideError( + f"ignore override for {clipping_title!r} cannot contain other resolution fields" + ) + overrides[clipping_title] = resolution + continue + item_fields = [ key for key in ("citation_key", "zotero_key", "zotero_item_id") @@ -70,20 +89,71 @@ def resolve_override( def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]: overrides = [] - for match in match_report.get("matches", []): - if match.get("status") == "matched": + matches = match_report.get("matches", []) + review_matches = [ + match for match in matches if match.get("status") not in {"matched", "ignored"} + ] + resolved_overrides = [ + match + for match in matches + if match.get("status") in {"matched", "ignored"} and match.get("override") + ] + sorted_matches = [ + *sorted(review_matches, key=_review_sort_key), + *sorted(resolved_overrides, key=_review_sort_key), + ] + + for match in sorted_matches: + if match.get("status") == "unmatched": + overrides.append( + { + "clipping_title": match["clipping_title"], + "review": { + "status": match.get("status"), + "clipping_count": match.get("clipping_count"), + }, + "resolution": { + "ignore": True, + "citation_key": "", + }, + "notes": "Default is to discard unmatched titles. To import, change ignore to false and fill citation_key, or replace it with zotero_key/zotero_item_id.", + "candidates": match.get("candidates", []), + } + ) + continue + if match.get("status") in {"matched", "ignored"}: + resolution = dict(match.get("override") or {}) + if resolution.get("ignore") is True: + resolution.setdefault("citation_key", "") + overrides.append( + { + "clipping_title": match["clipping_title"], + "review": { + "status": match.get("status"), + "clipping_count": match.get("clipping_count"), + }, + "resolution": resolution, + "notes": "Already resolved by match-overrides.json. Kept at the end for review/audit.", + "candidates": match.get("candidates", []), + } + ) continue overrides.append( { "clipping_title": match["clipping_title"], + "review": { + "status": match.get("status"), + "clipping_count": match.get("clipping_count"), + }, "resolution": { + "ignore": False, "citation_key": "", "zotero_key": "", "zotero_item_id": None, "attachment_key": "", "attachment_item_id": None, }, - "notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments.", + "notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments. Use ignore: true for works that should not be imported.", "candidates": match.get("candidates", []), } ) @@ -91,9 +161,14 @@ def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]: return {"format": OVERRIDES_FORMAT, "overrides": overrides} +def _review_sort_key(match: dict[str, Any]) -> tuple[int, str]: + return -int(match.get("clipping_count") or 0), match["clipping_title"].casefold() + + def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]: cleaned: dict[str, Any] = {} for key in ( + "ignore", "citation_key", "zotero_key", "zotero_item_id", @@ -105,6 +180,8 @@ def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]: continue if isinstance(value, str) and not value.strip(): continue + if key == "ignore" and value is not True: + continue cleaned[key] = value.strip() if isinstance(value, str) else value return cleaned diff --git a/src/kindle_zotero_importer/pdf_position.py b/src/kindle_zotero_importer/pdf_position.py index 8dc5bda..5bc52b5 100644 --- a/src/kindle_zotero_importer/pdf_position.py +++ b/src/kindle_zotero_importer/pdf_position.py @@ -1,8 +1,11 @@ from __future__ import annotations from dataclasses import dataclass +import hashlib +import os import re import subprocess +import tempfile import xml.etree.ElementTree as ET from typing import Any @@ -30,9 +33,20 @@ def add_pdf_positions(plan: dict[str, Any]) -> dict[str, Any]: text_cache: dict[str, list[str]] = {} xml_cache: dict[tuple[str, int], PDFPageXML] = {} size_cache: dict[tuple[str, int], tuple[float, float]] = {} + decrypted_cache: dict[str, str] = {} positioned = [] - for item in plan["items"]: - positioned.append(_position_item(item, text_cache, xml_cache, size_cache)) + with tempfile.TemporaryDirectory(prefix="kindle-zotero-pdf-") as temp_dir: + for item in plan["items"]: + positioned.append( + _position_item( + item, + text_cache, + xml_cache, + size_cache, + decrypted_cache, + temp_dir, + ) + ) status_counts: dict[str, int] = {} for item in positioned: @@ -49,6 +63,8 @@ def _position_item( text_cache: dict[str, list[str]], xml_cache: dict[tuple[str, int], PDFPageXML], size_cache: dict[tuple[str, int], tuple[float, float]], + decrypted_cache: dict[str, str], + temp_dir: str, ) -> dict[str, Any]: if item.get("status") != "ready-for-positioning": return item @@ -63,16 +79,21 @@ def _position_item( return _with_problem(item, "pdf-position-missing-path") try: - text_pages = text_cache.setdefault(path, extract_pdf_text_pages(path)) + text_pages, extraction_path = _text_pages_for_path( + path, text_cache, decrypted_cache, temp_dir + ) page_index = find_pdf_text_page( text_pages, item["clipping"]["text"], item["clipping"].get("page") ) if page_index is None: return _with_problem(item, "pdf-text-not-found") page_xml = xml_cache.setdefault( - (path, page_index), extract_pdf_page_xml(path, page_index) + (extraction_path, page_index), + extract_pdf_page_xml(extraction_path, page_index), + ) + position = find_pdf_rects( + extraction_path, page_xml, item["clipping"]["text"], size_cache ) - position = find_pdf_rects(path, page_xml, item["clipping"]["text"], size_cache) except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch return _with_problem(item, f"pdf-position-error:{error}") @@ -104,6 +125,52 @@ def extract_pdf_text_pages(path: str) -> list[str]: return result.stdout.split("\f") +def _text_pages_for_path( + path: str, + text_cache: dict[str, list[str]], + decrypted_cache: dict[str, str], + temp_dir: str, +) -> tuple[list[str], str]: + if path in text_cache: + return text_cache[path], decrypted_cache.get(path, path) + + try: + text_cache[path] = extract_pdf_text_pages(path) + return text_cache[path], path + except subprocess.CalledProcessError as error: + if not _is_pdf_permission_error(error): + raise + + decrypted_path = _decrypted_pdf_path(path, decrypted_cache, temp_dir) + text_cache[path] = extract_pdf_text_pages(decrypted_path) + return text_cache[path], decrypted_path + + +def _decrypted_pdf_path( + path: str, decrypted_cache: dict[str, str], temp_dir: str +) -> str: + cached = decrypted_cache.get(path) + if cached: + return cached + + digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:16] + base_name = os.path.splitext(os.path.basename(path))[0] or "document" + output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf") + subprocess.run( + ["qpdf", "--decrypt", path, output_path], + check=True, + capture_output=True, + text=True, + ) + decrypted_cache[path] = output_path + return output_path + + +def _is_pdf_permission_error(error: subprocess.CalledProcessError) -> bool: + output = "\n".join(part for part in (error.stdout, error.stderr) if part) + return "Permission Error" in output and "Copying of text" in output + + def find_pdf_text_page( pages: list[str], quote: str, kindle_page: str | None = None ) -> int | None: