Improve override review and PDF positioning

This commit is contained in:
Utku Bilen Demir 2026-06-03 10:51:15 +02:00
parent d8b273ad6e
commit 0838e6049a
7 changed files with 205 additions and 11 deletions

View file

@ -30,6 +30,8 @@ Generate reusable manual overrides for ambiguous/unmatched titles:
python -m kindle_zotero_importer generate-overrides matches.json --output match-overrides.generated.json --pretty
```
Generated overrides are designed for batch editing: unresolved titles are sorted by clipping count, unmatched titles default to `ignore: true`, and already-entered overrides are kept at the end as an audit trail. Replace `ignore` with a confirmed `citation_key`, `zotero_key`, or `zotero_item_id` to import a title instead of discarding it.
Apply reviewed overrides:
```sh
@ -38,6 +40,17 @@ python -m kindle_zotero_importer match clippings.json zotero-index.json --overri
Keep confirmed mappings in `match-overrides.json`. The `match-overrides.generated.json` file is only a disposable review skeleton and can be regenerated from the current `matches.json` at any time.
To permanently skip a Kindle title that should not be imported, add an ignore override:
```json
{
"clipping_title": "Example Kindle Title",
"resolution": {
"ignore": true
}
}
```
If a matched Zotero item has multiple usable PDF/EPUB attachments, add the selected attachment to the same override entry:
```json
@ -82,6 +95,8 @@ python -m kindle_zotero_importer position-pdf import-plan.epub.json --output imp
python -m kindle_zotero_importer finalize import-plan.positioned.json --output import-plan.final.json --pretty
```
PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a PDF permits viewing but blocks text copying, the importer retries extraction through a temporary `qpdf --decrypt` copy. The Zotero attachment itself is not modified.
## Safety Rule
Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes.

View file

@ -0,0 +1,6 @@
from __future__ import annotations
from .cli import main
raise SystemExit(main())

View file

@ -58,6 +58,11 @@ def _plan_clipping(
"problems": [],
}
match = matches_by_title.get(clipping["title"])
if match and match.get("status") == "ignored":
base["status"] = "ignored-title"
base["match"] = None
base["problems"] = ["ignored by match override"]
return base
if not match or match.get("status") != "matched" or not match.get("candidates"):
return base

View file

@ -27,6 +27,7 @@ class TitleMatch:
clipping_title: str
clipping_count: int
candidates: list[MatchCandidate]
status_override: str | None = None
def load_json(path: str) -> dict[str, Any]:
@ -60,6 +61,7 @@ def build_match_report(
"matched_title_count": status_counts["matched"],
"ambiguous_title_count": status_counts["ambiguous"],
"unmatched_title_count": status_counts["unmatched"],
"ignored_title_count": status_counts["ignored"],
"matches": [
{
"clipping_title": match.clipping_title,
@ -89,6 +91,8 @@ def _match_title(
override: dict[str, Any] | None = None,
) -> TitleMatch:
if override:
if override.get("ignore") is True:
return TitleMatch(clipping_title, count, [], "ignored")
item, reason = resolve_override(clipping_title, override, items)
if item:
return TitleMatch(clipping_title, count, [_candidate(item, 1.0, reason)])
@ -193,13 +197,15 @@ def _normalize_title(value: str) -> str:
def _status_counts(matches: list[TitleMatch]) -> dict[str, int]:
counts = {"matched": 0, "ambiguous": 0, "unmatched": 0}
counts = {"matched": 0, "ambiguous": 0, "unmatched": 0, "ignored": 0}
for match in matches:
counts[_match_status(match)] += 1
return counts
def _match_status(match: TitleMatch) -> str:
if match.status_override:
return match.status_override
if not match.candidates:
return "unmatched"
if len(match.candidates) == 1:

View file

@ -36,6 +36,11 @@ SECTIONS = [
"pdf-rects-not-found",
"Needs improved PDF rectangle recovery after text/page match.",
),
(
"PDF Position Errors",
"pdf-position-error:",
"Inspect the PDF extraction command error, file validity, encryption, or permissions.",
),
]
@ -57,12 +62,13 @@ def build_mismatch_review(
"",
]
for heading, status, instruction in SECTIONS:
summaries = _title_summaries(by_status.get(status, []), match_by_title)
section_items = _items_for_status(by_status, status)
summaries = _title_summaries(section_items, match_by_title)
lines += [
f"## {heading}",
"",
f"Status: `{status}`",
f"Clippings: {len(by_status.get(status, []))}",
f"Clippings: {len(section_items)}",
f"Unique titles: {len(summaries)}",
f"Action: {instruction}",
"",
@ -75,6 +81,18 @@ def build_mismatch_review(
return "\n".join(lines).rstrip() + "\n"
def _items_for_status(
by_status: dict[str, list[dict[str, Any]]], status: str
) -> list[dict[str, Any]]:
if status.endswith(":"):
items: list[dict[str, Any]] = []
for item_status, status_items in by_status.items():
if item_status.startswith(status):
items.extend(status_items)
return items
return by_status.get(status, [])
def _title_summaries(
items: list[dict[str, Any]], match_by_title: dict[str, dict[str, Any]]
) -> list[dict[str, Any]]:

View file

@ -24,6 +24,25 @@ def load_overrides(payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
resolution = _clean_resolution(entry.get("resolution") or {})
if not resolution:
continue
if resolution.get("ignore") is True:
conflicting_fields = [
key
for key in (
"citation_key",
"zotero_key",
"zotero_item_id",
"attachment_key",
"attachment_item_id",
)
if key in resolution
]
if conflicting_fields:
raise OverrideError(
f"ignore override for {clipping_title!r} cannot contain other resolution fields"
)
overrides[clipping_title] = resolution
continue
item_fields = [
key
for key in ("citation_key", "zotero_key", "zotero_item_id")
@ -70,20 +89,71 @@ def resolve_override(
def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]:
overrides = []
for match in match_report.get("matches", []):
if match.get("status") == "matched":
matches = match_report.get("matches", [])
review_matches = [
match for match in matches if match.get("status") not in {"matched", "ignored"}
]
resolved_overrides = [
match
for match in matches
if match.get("status") in {"matched", "ignored"} and match.get("override")
]
sorted_matches = [
*sorted(review_matches, key=_review_sort_key),
*sorted(resolved_overrides, key=_review_sort_key),
]
for match in sorted_matches:
if match.get("status") == "unmatched":
overrides.append(
{
"clipping_title": match["clipping_title"],
"review": {
"status": match.get("status"),
"clipping_count": match.get("clipping_count"),
},
"resolution": {
"ignore": True,
"citation_key": "",
},
"notes": "Default is to discard unmatched titles. To import, change ignore to false and fill citation_key, or replace it with zotero_key/zotero_item_id.",
"candidates": match.get("candidates", []),
}
)
continue
if match.get("status") in {"matched", "ignored"}:
resolution = dict(match.get("override") or {})
if resolution.get("ignore") is True:
resolution.setdefault("citation_key", "")
overrides.append(
{
"clipping_title": match["clipping_title"],
"review": {
"status": match.get("status"),
"clipping_count": match.get("clipping_count"),
},
"resolution": resolution,
"notes": "Already resolved by match-overrides.json. Kept at the end for review/audit.",
"candidates": match.get("candidates", []),
}
)
continue
overrides.append(
{
"clipping_title": match["clipping_title"],
"review": {
"status": match.get("status"),
"clipping_count": match.get("clipping_count"),
},
"resolution": {
"ignore": False,
"citation_key": "",
"zotero_key": "",
"zotero_item_id": None,
"attachment_key": "",
"attachment_item_id": None,
},
"notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments.",
"notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments. Use ignore: true for works that should not be imported.",
"candidates": match.get("candidates", []),
}
)
@ -91,9 +161,14 @@ def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]:
return {"format": OVERRIDES_FORMAT, "overrides": overrides}
def _review_sort_key(match: dict[str, Any]) -> tuple[int, str]:
return -int(match.get("clipping_count") or 0), match["clipping_title"].casefold()
def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]:
cleaned: dict[str, Any] = {}
for key in (
"ignore",
"citation_key",
"zotero_key",
"zotero_item_id",
@ -105,6 +180,8 @@ def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]:
continue
if isinstance(value, str) and not value.strip():
continue
if key == "ignore" and value is not True:
continue
cleaned[key] = value.strip() if isinstance(value, str) else value
return cleaned

View file

@ -1,8 +1,11 @@
from __future__ import annotations
from dataclasses import dataclass
import hashlib
import os
import re
import subprocess
import tempfile
import xml.etree.ElementTree as ET
from typing import Any
@ -30,9 +33,20 @@ def add_pdf_positions(plan: dict[str, Any]) -> dict[str, Any]:
text_cache: dict[str, list[str]] = {}
xml_cache: dict[tuple[str, int], PDFPageXML] = {}
size_cache: dict[tuple[str, int], tuple[float, float]] = {}
decrypted_cache: dict[str, str] = {}
positioned = []
with tempfile.TemporaryDirectory(prefix="kindle-zotero-pdf-") as temp_dir:
for item in plan["items"]:
positioned.append(_position_item(item, text_cache, xml_cache, size_cache))
positioned.append(
_position_item(
item,
text_cache,
xml_cache,
size_cache,
decrypted_cache,
temp_dir,
)
)
status_counts: dict[str, int] = {}
for item in positioned:
@ -49,6 +63,8 @@ def _position_item(
text_cache: dict[str, list[str]],
xml_cache: dict[tuple[str, int], PDFPageXML],
size_cache: dict[tuple[str, int], tuple[float, float]],
decrypted_cache: dict[str, str],
temp_dir: str,
) -> dict[str, Any]:
if item.get("status") != "ready-for-positioning":
return item
@ -63,16 +79,21 @@ def _position_item(
return _with_problem(item, "pdf-position-missing-path")
try:
text_pages = text_cache.setdefault(path, extract_pdf_text_pages(path))
text_pages, extraction_path = _text_pages_for_path(
path, text_cache, decrypted_cache, temp_dir
)
page_index = find_pdf_text_page(
text_pages, item["clipping"]["text"], item["clipping"].get("page")
)
if page_index is None:
return _with_problem(item, "pdf-text-not-found")
page_xml = xml_cache.setdefault(
(path, page_index), extract_pdf_page_xml(path, page_index)
(extraction_path, page_index),
extract_pdf_page_xml(extraction_path, page_index),
)
position = find_pdf_rects(
extraction_path, page_xml, item["clipping"]["text"], size_cache
)
position = find_pdf_rects(path, page_xml, item["clipping"]["text"], size_cache)
except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch
return _with_problem(item, f"pdf-position-error:{error}")
@ -104,6 +125,52 @@ def extract_pdf_text_pages(path: str) -> list[str]:
return result.stdout.split("\f")
def _text_pages_for_path(
path: str,
text_cache: dict[str, list[str]],
decrypted_cache: dict[str, str],
temp_dir: str,
) -> tuple[list[str], str]:
if path in text_cache:
return text_cache[path], decrypted_cache.get(path, path)
try:
text_cache[path] = extract_pdf_text_pages(path)
return text_cache[path], path
except subprocess.CalledProcessError as error:
if not _is_pdf_permission_error(error):
raise
decrypted_path = _decrypted_pdf_path(path, decrypted_cache, temp_dir)
text_cache[path] = extract_pdf_text_pages(decrypted_path)
return text_cache[path], decrypted_path
def _decrypted_pdf_path(
path: str, decrypted_cache: dict[str, str], temp_dir: str
) -> str:
cached = decrypted_cache.get(path)
if cached:
return cached
digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:16]
base_name = os.path.splitext(os.path.basename(path))[0] or "document"
output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf")
subprocess.run(
["qpdf", "--decrypt", path, output_path],
check=True,
capture_output=True,
text=True,
)
decrypted_cache[path] = output_path
return output_path
def _is_pdf_permission_error(error: subprocess.CalledProcessError) -> bool:
output = "\n".join(part for part in (error.stdout, error.stderr) if part)
return "Permission Error" in output and "Copying of text" in output
def find_pdf_text_page(
pages: list[str], quote: str, kindle_page: str | None = None
) -> int | None: