Improve override review and PDF positioning

This commit is contained in:
Utku Bilen Demir 2026-06-03 10:51:15 +02:00
parent d8b273ad6e
commit 0838e6049a
7 changed files with 205 additions and 11 deletions

View file

@ -30,6 +30,8 @@ Generate reusable manual overrides for ambiguous/unmatched titles:
python -m kindle_zotero_importer generate-overrides matches.json --output match-overrides.generated.json --pretty python -m kindle_zotero_importer generate-overrides matches.json --output match-overrides.generated.json --pretty
``` ```
Generated overrides are designed for batch editing: unresolved titles are sorted by clipping count, unmatched titles default to `ignore: true`, and already-entered overrides are kept at the end as an audit trail. Replace `ignore` with a confirmed `citation_key`, `zotero_key`, or `zotero_item_id` to import a title instead of discarding it.
Apply reviewed overrides: Apply reviewed overrides:
```sh ```sh
@ -38,6 +40,17 @@ python -m kindle_zotero_importer match clippings.json zotero-index.json --overri
Keep confirmed mappings in `match-overrides.json`. The `match-overrides.generated.json` file is only a disposable review skeleton and can be regenerated from the current `matches.json` at any time. Keep confirmed mappings in `match-overrides.json`. The `match-overrides.generated.json` file is only a disposable review skeleton and can be regenerated from the current `matches.json` at any time.
To permanently skip a Kindle title that should not be imported, add an ignore override:
```json
{
"clipping_title": "Example Kindle Title",
"resolution": {
"ignore": true
}
}
```
If a matched Zotero item has multiple usable PDF/EPUB attachments, add the selected attachment to the same override entry: If a matched Zotero item has multiple usable PDF/EPUB attachments, add the selected attachment to the same override entry:
```json ```json
@ -82,6 +95,8 @@ python -m kindle_zotero_importer position-pdf import-plan.epub.json --output imp
python -m kindle_zotero_importer finalize import-plan.positioned.json --output import-plan.final.json --pretty python -m kindle_zotero_importer finalize import-plan.positioned.json --output import-plan.final.json --pretty
``` ```
PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a PDF permits viewing but blocks text copying, the importer retries extraction through a temporary `qpdf --decrypt` copy. The Zotero attachment itself is not modified.
## Safety Rule ## Safety Rule
Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes. Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes.

View file

@ -0,0 +1,6 @@
from __future__ import annotations
from .cli import main
raise SystemExit(main())

View file

@ -58,6 +58,11 @@ def _plan_clipping(
"problems": [], "problems": [],
} }
match = matches_by_title.get(clipping["title"]) match = matches_by_title.get(clipping["title"])
if match and match.get("status") == "ignored":
base["status"] = "ignored-title"
base["match"] = None
base["problems"] = ["ignored by match override"]
return base
if not match or match.get("status") != "matched" or not match.get("candidates"): if not match or match.get("status") != "matched" or not match.get("candidates"):
return base return base

View file

@ -27,6 +27,7 @@ class TitleMatch:
clipping_title: str clipping_title: str
clipping_count: int clipping_count: int
candidates: list[MatchCandidate] candidates: list[MatchCandidate]
status_override: str | None = None
def load_json(path: str) -> dict[str, Any]: def load_json(path: str) -> dict[str, Any]:
@ -60,6 +61,7 @@ def build_match_report(
"matched_title_count": status_counts["matched"], "matched_title_count": status_counts["matched"],
"ambiguous_title_count": status_counts["ambiguous"], "ambiguous_title_count": status_counts["ambiguous"],
"unmatched_title_count": status_counts["unmatched"], "unmatched_title_count": status_counts["unmatched"],
"ignored_title_count": status_counts["ignored"],
"matches": [ "matches": [
{ {
"clipping_title": match.clipping_title, "clipping_title": match.clipping_title,
@ -89,6 +91,8 @@ def _match_title(
override: dict[str, Any] | None = None, override: dict[str, Any] | None = None,
) -> TitleMatch: ) -> TitleMatch:
if override: if override:
if override.get("ignore") is True:
return TitleMatch(clipping_title, count, [], "ignored")
item, reason = resolve_override(clipping_title, override, items) item, reason = resolve_override(clipping_title, override, items)
if item: if item:
return TitleMatch(clipping_title, count, [_candidate(item, 1.0, reason)]) return TitleMatch(clipping_title, count, [_candidate(item, 1.0, reason)])
@ -193,13 +197,15 @@ def _normalize_title(value: str) -> str:
def _status_counts(matches: list[TitleMatch]) -> dict[str, int]: def _status_counts(matches: list[TitleMatch]) -> dict[str, int]:
counts = {"matched": 0, "ambiguous": 0, "unmatched": 0} counts = {"matched": 0, "ambiguous": 0, "unmatched": 0, "ignored": 0}
for match in matches: for match in matches:
counts[_match_status(match)] += 1 counts[_match_status(match)] += 1
return counts return counts
def _match_status(match: TitleMatch) -> str: def _match_status(match: TitleMatch) -> str:
if match.status_override:
return match.status_override
if not match.candidates: if not match.candidates:
return "unmatched" return "unmatched"
if len(match.candidates) == 1: if len(match.candidates) == 1:

View file

@ -36,6 +36,11 @@ SECTIONS = [
"pdf-rects-not-found", "pdf-rects-not-found",
"Needs improved PDF rectangle recovery after text/page match.", "Needs improved PDF rectangle recovery after text/page match.",
), ),
(
"PDF Position Errors",
"pdf-position-error:",
"Inspect the PDF extraction command error, file validity, encryption, or permissions.",
),
] ]
@ -57,12 +62,13 @@ def build_mismatch_review(
"", "",
] ]
for heading, status, instruction in SECTIONS: for heading, status, instruction in SECTIONS:
summaries = _title_summaries(by_status.get(status, []), match_by_title) section_items = _items_for_status(by_status, status)
summaries = _title_summaries(section_items, match_by_title)
lines += [ lines += [
f"## {heading}", f"## {heading}",
"", "",
f"Status: `{status}`", f"Status: `{status}`",
f"Clippings: {len(by_status.get(status, []))}", f"Clippings: {len(section_items)}",
f"Unique titles: {len(summaries)}", f"Unique titles: {len(summaries)}",
f"Action: {instruction}", f"Action: {instruction}",
"", "",
@ -75,6 +81,18 @@ def build_mismatch_review(
return "\n".join(lines).rstrip() + "\n" return "\n".join(lines).rstrip() + "\n"
def _items_for_status(
by_status: dict[str, list[dict[str, Any]]], status: str
) -> list[dict[str, Any]]:
if status.endswith(":"):
items: list[dict[str, Any]] = []
for item_status, status_items in by_status.items():
if item_status.startswith(status):
items.extend(status_items)
return items
return by_status.get(status, [])
def _title_summaries( def _title_summaries(
items: list[dict[str, Any]], match_by_title: dict[str, dict[str, Any]] items: list[dict[str, Any]], match_by_title: dict[str, dict[str, Any]]
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:

View file

@ -24,6 +24,25 @@ def load_overrides(payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
resolution = _clean_resolution(entry.get("resolution") or {}) resolution = _clean_resolution(entry.get("resolution") or {})
if not resolution: if not resolution:
continue continue
if resolution.get("ignore") is True:
conflicting_fields = [
key
for key in (
"citation_key",
"zotero_key",
"zotero_item_id",
"attachment_key",
"attachment_item_id",
)
if key in resolution
]
if conflicting_fields:
raise OverrideError(
f"ignore override for {clipping_title!r} cannot contain other resolution fields"
)
overrides[clipping_title] = resolution
continue
item_fields = [ item_fields = [
key key
for key in ("citation_key", "zotero_key", "zotero_item_id") for key in ("citation_key", "zotero_key", "zotero_item_id")
@ -70,20 +89,71 @@ def resolve_override(
def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]: def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]:
overrides = [] overrides = []
for match in match_report.get("matches", []): matches = match_report.get("matches", [])
if match.get("status") == "matched": review_matches = [
match for match in matches if match.get("status") not in {"matched", "ignored"}
]
resolved_overrides = [
match
for match in matches
if match.get("status") in {"matched", "ignored"} and match.get("override")
]
sorted_matches = [
*sorted(review_matches, key=_review_sort_key),
*sorted(resolved_overrides, key=_review_sort_key),
]
for match in sorted_matches:
if match.get("status") == "unmatched":
overrides.append(
{
"clipping_title": match["clipping_title"],
"review": {
"status": match.get("status"),
"clipping_count": match.get("clipping_count"),
},
"resolution": {
"ignore": True,
"citation_key": "",
},
"notes": "Default is to discard unmatched titles. To import, change ignore to false and fill citation_key, or replace it with zotero_key/zotero_item_id.",
"candidates": match.get("candidates", []),
}
)
continue
if match.get("status") in {"matched", "ignored"}:
resolution = dict(match.get("override") or {})
if resolution.get("ignore") is True:
resolution.setdefault("citation_key", "")
overrides.append(
{
"clipping_title": match["clipping_title"],
"review": {
"status": match.get("status"),
"clipping_count": match.get("clipping_count"),
},
"resolution": resolution,
"notes": "Already resolved by match-overrides.json. Kept at the end for review/audit.",
"candidates": match.get("candidates", []),
}
)
continue continue
overrides.append( overrides.append(
{ {
"clipping_title": match["clipping_title"], "clipping_title": match["clipping_title"],
"review": {
"status": match.get("status"),
"clipping_count": match.get("clipping_count"),
},
"resolution": { "resolution": {
"ignore": False,
"citation_key": "", "citation_key": "",
"zotero_key": "", "zotero_key": "",
"zotero_item_id": None, "zotero_item_id": None,
"attachment_key": "", "attachment_key": "",
"attachment_item_id": None, "attachment_item_id": None,
}, },
"notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments.", "notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments. Use ignore: true for works that should not be imported.",
"candidates": match.get("candidates", []), "candidates": match.get("candidates", []),
} }
) )
@ -91,9 +161,14 @@ def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]:
return {"format": OVERRIDES_FORMAT, "overrides": overrides} return {"format": OVERRIDES_FORMAT, "overrides": overrides}
def _review_sort_key(match: dict[str, Any]) -> tuple[int, str]:
return -int(match.get("clipping_count") or 0), match["clipping_title"].casefold()
def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]: def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]:
cleaned: dict[str, Any] = {} cleaned: dict[str, Any] = {}
for key in ( for key in (
"ignore",
"citation_key", "citation_key",
"zotero_key", "zotero_key",
"zotero_item_id", "zotero_item_id",
@ -105,6 +180,8 @@ def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]:
continue continue
if isinstance(value, str) and not value.strip(): if isinstance(value, str) and not value.strip():
continue continue
if key == "ignore" and value is not True:
continue
cleaned[key] = value.strip() if isinstance(value, str) else value cleaned[key] = value.strip() if isinstance(value, str) else value
return cleaned return cleaned

View file

@ -1,8 +1,11 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
import hashlib
import os
import re import re
import subprocess import subprocess
import tempfile
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from typing import Any from typing import Any
@ -30,9 +33,20 @@ def add_pdf_positions(plan: dict[str, Any]) -> dict[str, Any]:
text_cache: dict[str, list[str]] = {} text_cache: dict[str, list[str]] = {}
xml_cache: dict[tuple[str, int], PDFPageXML] = {} xml_cache: dict[tuple[str, int], PDFPageXML] = {}
size_cache: dict[tuple[str, int], tuple[float, float]] = {} size_cache: dict[tuple[str, int], tuple[float, float]] = {}
decrypted_cache: dict[str, str] = {}
positioned = [] positioned = []
with tempfile.TemporaryDirectory(prefix="kindle-zotero-pdf-") as temp_dir:
for item in plan["items"]: for item in plan["items"]:
positioned.append(_position_item(item, text_cache, xml_cache, size_cache)) positioned.append(
_position_item(
item,
text_cache,
xml_cache,
size_cache,
decrypted_cache,
temp_dir,
)
)
status_counts: dict[str, int] = {} status_counts: dict[str, int] = {}
for item in positioned: for item in positioned:
@ -49,6 +63,8 @@ def _position_item(
text_cache: dict[str, list[str]], text_cache: dict[str, list[str]],
xml_cache: dict[tuple[str, int], PDFPageXML], xml_cache: dict[tuple[str, int], PDFPageXML],
size_cache: dict[tuple[str, int], tuple[float, float]], size_cache: dict[tuple[str, int], tuple[float, float]],
decrypted_cache: dict[str, str],
temp_dir: str,
) -> dict[str, Any]: ) -> dict[str, Any]:
if item.get("status") != "ready-for-positioning": if item.get("status") != "ready-for-positioning":
return item return item
@ -63,16 +79,21 @@ def _position_item(
return _with_problem(item, "pdf-position-missing-path") return _with_problem(item, "pdf-position-missing-path")
try: try:
text_pages = text_cache.setdefault(path, extract_pdf_text_pages(path)) text_pages, extraction_path = _text_pages_for_path(
path, text_cache, decrypted_cache, temp_dir
)
page_index = find_pdf_text_page( page_index = find_pdf_text_page(
text_pages, item["clipping"]["text"], item["clipping"].get("page") text_pages, item["clipping"]["text"], item["clipping"].get("page")
) )
if page_index is None: if page_index is None:
return _with_problem(item, "pdf-text-not-found") return _with_problem(item, "pdf-text-not-found")
page_xml = xml_cache.setdefault( page_xml = xml_cache.setdefault(
(path, page_index), extract_pdf_page_xml(path, page_index) (extraction_path, page_index),
extract_pdf_page_xml(extraction_path, page_index),
)
position = find_pdf_rects(
extraction_path, page_xml, item["clipping"]["text"], size_cache
) )
position = find_pdf_rects(path, page_xml, item["clipping"]["text"], size_cache)
except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch
return _with_problem(item, f"pdf-position-error:{error}") return _with_problem(item, f"pdf-position-error:{error}")
@ -104,6 +125,52 @@ def extract_pdf_text_pages(path: str) -> list[str]:
return result.stdout.split("\f") return result.stdout.split("\f")
def _text_pages_for_path(
path: str,
text_cache: dict[str, list[str]],
decrypted_cache: dict[str, str],
temp_dir: str,
) -> tuple[list[str], str]:
if path in text_cache:
return text_cache[path], decrypted_cache.get(path, path)
try:
text_cache[path] = extract_pdf_text_pages(path)
return text_cache[path], path
except subprocess.CalledProcessError as error:
if not _is_pdf_permission_error(error):
raise
decrypted_path = _decrypted_pdf_path(path, decrypted_cache, temp_dir)
text_cache[path] = extract_pdf_text_pages(decrypted_path)
return text_cache[path], decrypted_path
def _decrypted_pdf_path(
path: str, decrypted_cache: dict[str, str], temp_dir: str
) -> str:
cached = decrypted_cache.get(path)
if cached:
return cached
digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:16]
base_name = os.path.splitext(os.path.basename(path))[0] or "document"
output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf")
subprocess.run(
["qpdf", "--decrypt", path, output_path],
check=True,
capture_output=True,
text=True,
)
decrypted_cache[path] = output_path
return output_path
def _is_pdf_permission_error(error: subprocess.CalledProcessError) -> bool:
output = "\n".join(part for part in (error.stdout, error.stderr) if part)
return "Permission Error" in output and "Copying of text" in output
def find_pdf_text_page( def find_pdf_text_page(
pages: list[str], quote: str, kindle_page: str | None = None pages: list[str], quote: str, kindle_page: str | None = None
) -> int | None: ) -> int | None: