Improve override review and PDF positioning
This commit is contained in:
parent
d8b273ad6e
commit
0838e6049a
7 changed files with 205 additions and 11 deletions
15
README.md
15
README.md
|
|
@ -30,6 +30,8 @@ Generate reusable manual overrides for ambiguous/unmatched titles:
|
|||
python -m kindle_zotero_importer generate-overrides matches.json --output match-overrides.generated.json --pretty
|
||||
```
|
||||
|
||||
Generated overrides are designed for batch editing: unresolved titles are sorted by clipping count, unmatched titles default to `ignore: true`, and already-entered overrides are kept at the end as an audit trail. Replace `ignore` with a confirmed `citation_key`, `zotero_key`, or `zotero_item_id` to import a title instead of discarding it.
|
||||
|
||||
Apply reviewed overrides:
|
||||
|
||||
```sh
|
||||
|
|
@ -38,6 +40,17 @@ python -m kindle_zotero_importer match clippings.json zotero-index.json --overri
|
|||
|
||||
Keep confirmed mappings in `match-overrides.json`. The `match-overrides.generated.json` file is only a disposable review skeleton and can be regenerated from the current `matches.json` at any time.
|
||||
|
||||
To permanently skip a Kindle title that should not be imported, add an ignore override:
|
||||
|
||||
```json
|
||||
{
|
||||
"clipping_title": "Example Kindle Title",
|
||||
"resolution": {
|
||||
"ignore": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If a matched Zotero item has multiple usable PDF/EPUB attachments, add the selected attachment to the same override entry:
|
||||
|
||||
```json
|
||||
|
|
@ -82,6 +95,8 @@ python -m kindle_zotero_importer position-pdf import-plan.epub.json --output imp
|
|||
python -m kindle_zotero_importer finalize import-plan.positioned.json --output import-plan.final.json --pretty
|
||||
```
|
||||
|
||||
PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a PDF permits viewing but blocks text copying, the importer retries extraction through a temporary `qpdf --decrypt` copy. The Zotero attachment itself is not modified.
|
||||
|
||||
## Safety Rule
|
||||
|
||||
Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes.
|
||||
|
|
|
|||
6
src/kindle_zotero_importer/__main__.py
Normal file
6
src/kindle_zotero_importer/__main__.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from .cli import main
|
||||
|
||||
|
||||
raise SystemExit(main())
|
||||
|
|
@ -58,6 +58,11 @@ def _plan_clipping(
|
|||
"problems": [],
|
||||
}
|
||||
match = matches_by_title.get(clipping["title"])
|
||||
if match and match.get("status") == "ignored":
|
||||
base["status"] = "ignored-title"
|
||||
base["match"] = None
|
||||
base["problems"] = ["ignored by match override"]
|
||||
return base
|
||||
if not match or match.get("status") != "matched" or not match.get("candidates"):
|
||||
return base
|
||||
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ class TitleMatch:
|
|||
clipping_title: str
|
||||
clipping_count: int
|
||||
candidates: list[MatchCandidate]
|
||||
status_override: str | None = None
|
||||
|
||||
|
||||
def load_json(path: str) -> dict[str, Any]:
|
||||
|
|
@ -60,6 +61,7 @@ def build_match_report(
|
|||
"matched_title_count": status_counts["matched"],
|
||||
"ambiguous_title_count": status_counts["ambiguous"],
|
||||
"unmatched_title_count": status_counts["unmatched"],
|
||||
"ignored_title_count": status_counts["ignored"],
|
||||
"matches": [
|
||||
{
|
||||
"clipping_title": match.clipping_title,
|
||||
|
|
@ -89,6 +91,8 @@ def _match_title(
|
|||
override: dict[str, Any] | None = None,
|
||||
) -> TitleMatch:
|
||||
if override:
|
||||
if override.get("ignore") is True:
|
||||
return TitleMatch(clipping_title, count, [], "ignored")
|
||||
item, reason = resolve_override(clipping_title, override, items)
|
||||
if item:
|
||||
return TitleMatch(clipping_title, count, [_candidate(item, 1.0, reason)])
|
||||
|
|
@ -193,13 +197,15 @@ def _normalize_title(value: str) -> str:
|
|||
|
||||
|
||||
def _status_counts(matches: list[TitleMatch]) -> dict[str, int]:
|
||||
counts = {"matched": 0, "ambiguous": 0, "unmatched": 0}
|
||||
counts = {"matched": 0, "ambiguous": 0, "unmatched": 0, "ignored": 0}
|
||||
for match in matches:
|
||||
counts[_match_status(match)] += 1
|
||||
return counts
|
||||
|
||||
|
||||
def _match_status(match: TitleMatch) -> str:
|
||||
if match.status_override:
|
||||
return match.status_override
|
||||
if not match.candidates:
|
||||
return "unmatched"
|
||||
if len(match.candidates) == 1:
|
||||
|
|
|
|||
|
|
@ -36,6 +36,11 @@ SECTIONS = [
|
|||
"pdf-rects-not-found",
|
||||
"Needs improved PDF rectangle recovery after text/page match.",
|
||||
),
|
||||
(
|
||||
"PDF Position Errors",
|
||||
"pdf-position-error:",
|
||||
"Inspect the PDF extraction command error, file validity, encryption, or permissions.",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -57,12 +62,13 @@ def build_mismatch_review(
|
|||
"",
|
||||
]
|
||||
for heading, status, instruction in SECTIONS:
|
||||
summaries = _title_summaries(by_status.get(status, []), match_by_title)
|
||||
section_items = _items_for_status(by_status, status)
|
||||
summaries = _title_summaries(section_items, match_by_title)
|
||||
lines += [
|
||||
f"## {heading}",
|
||||
"",
|
||||
f"Status: `{status}`",
|
||||
f"Clippings: {len(by_status.get(status, []))}",
|
||||
f"Clippings: {len(section_items)}",
|
||||
f"Unique titles: {len(summaries)}",
|
||||
f"Action: {instruction}",
|
||||
"",
|
||||
|
|
@ -75,6 +81,18 @@ def build_mismatch_review(
|
|||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def _items_for_status(
|
||||
by_status: dict[str, list[dict[str, Any]]], status: str
|
||||
) -> list[dict[str, Any]]:
|
||||
if status.endswith(":"):
|
||||
items: list[dict[str, Any]] = []
|
||||
for item_status, status_items in by_status.items():
|
||||
if item_status.startswith(status):
|
||||
items.extend(status_items)
|
||||
return items
|
||||
return by_status.get(status, [])
|
||||
|
||||
|
||||
def _title_summaries(
|
||||
items: list[dict[str, Any]], match_by_title: dict[str, dict[str, Any]]
|
||||
) -> list[dict[str, Any]]:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,25 @@ def load_overrides(payload: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
|||
resolution = _clean_resolution(entry.get("resolution") or {})
|
||||
if not resolution:
|
||||
continue
|
||||
if resolution.get("ignore") is True:
|
||||
conflicting_fields = [
|
||||
key
|
||||
for key in (
|
||||
"citation_key",
|
||||
"zotero_key",
|
||||
"zotero_item_id",
|
||||
"attachment_key",
|
||||
"attachment_item_id",
|
||||
)
|
||||
if key in resolution
|
||||
]
|
||||
if conflicting_fields:
|
||||
raise OverrideError(
|
||||
f"ignore override for {clipping_title!r} cannot contain other resolution fields"
|
||||
)
|
||||
overrides[clipping_title] = resolution
|
||||
continue
|
||||
|
||||
item_fields = [
|
||||
key
|
||||
for key in ("citation_key", "zotero_key", "zotero_item_id")
|
||||
|
|
@ -70,20 +89,71 @@ def resolve_override(
|
|||
|
||||
def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]:
|
||||
overrides = []
|
||||
for match in match_report.get("matches", []):
|
||||
if match.get("status") == "matched":
|
||||
matches = match_report.get("matches", [])
|
||||
review_matches = [
|
||||
match for match in matches if match.get("status") not in {"matched", "ignored"}
|
||||
]
|
||||
resolved_overrides = [
|
||||
match
|
||||
for match in matches
|
||||
if match.get("status") in {"matched", "ignored"} and match.get("override")
|
||||
]
|
||||
sorted_matches = [
|
||||
*sorted(review_matches, key=_review_sort_key),
|
||||
*sorted(resolved_overrides, key=_review_sort_key),
|
||||
]
|
||||
|
||||
for match in sorted_matches:
|
||||
if match.get("status") == "unmatched":
|
||||
overrides.append(
|
||||
{
|
||||
"clipping_title": match["clipping_title"],
|
||||
"review": {
|
||||
"status": match.get("status"),
|
||||
"clipping_count": match.get("clipping_count"),
|
||||
},
|
||||
"resolution": {
|
||||
"ignore": True,
|
||||
"citation_key": "",
|
||||
},
|
||||
"notes": "Default is to discard unmatched titles. To import, change ignore to false and fill citation_key, or replace it with zotero_key/zotero_item_id.",
|
||||
"candidates": match.get("candidates", []),
|
||||
}
|
||||
)
|
||||
continue
|
||||
if match.get("status") in {"matched", "ignored"}:
|
||||
resolution = dict(match.get("override") or {})
|
||||
if resolution.get("ignore") is True:
|
||||
resolution.setdefault("citation_key", "")
|
||||
overrides.append(
|
||||
{
|
||||
"clipping_title": match["clipping_title"],
|
||||
"review": {
|
||||
"status": match.get("status"),
|
||||
"clipping_count": match.get("clipping_count"),
|
||||
},
|
||||
"resolution": resolution,
|
||||
"notes": "Already resolved by match-overrides.json. Kept at the end for review/audit.",
|
||||
"candidates": match.get("candidates", []),
|
||||
}
|
||||
)
|
||||
continue
|
||||
overrides.append(
|
||||
{
|
||||
"clipping_title": match["clipping_title"],
|
||||
"review": {
|
||||
"status": match.get("status"),
|
||||
"clipping_count": match.get("clipping_count"),
|
||||
},
|
||||
"resolution": {
|
||||
"ignore": False,
|
||||
"citation_key": "",
|
||||
"zotero_key": "",
|
||||
"zotero_item_id": None,
|
||||
"attachment_key": "",
|
||||
"attachment_item_id": None,
|
||||
},
|
||||
"notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments.",
|
||||
"notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments. Use ignore: true for works that should not be imported.",
|
||||
"candidates": match.get("candidates", []),
|
||||
}
|
||||
)
|
||||
|
|
@ -91,9 +161,14 @@ def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]:
|
|||
return {"format": OVERRIDES_FORMAT, "overrides": overrides}
|
||||
|
||||
|
||||
def _review_sort_key(match: dict[str, Any]) -> tuple[int, str]:
|
||||
return -int(match.get("clipping_count") or 0), match["clipping_title"].casefold()
|
||||
|
||||
|
||||
def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]:
|
||||
cleaned: dict[str, Any] = {}
|
||||
for key in (
|
||||
"ignore",
|
||||
"citation_key",
|
||||
"zotero_key",
|
||||
"zotero_item_id",
|
||||
|
|
@ -105,6 +180,8 @@ def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]:
|
|||
continue
|
||||
if isinstance(value, str) and not value.strip():
|
||||
continue
|
||||
if key == "ignore" and value is not True:
|
||||
continue
|
||||
cleaned[key] = value.strip() if isinstance(value, str) else value
|
||||
return cleaned
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
|
|
@ -30,9 +33,20 @@ def add_pdf_positions(plan: dict[str, Any]) -> dict[str, Any]:
|
|||
text_cache: dict[str, list[str]] = {}
|
||||
xml_cache: dict[tuple[str, int], PDFPageXML] = {}
|
||||
size_cache: dict[tuple[str, int], tuple[float, float]] = {}
|
||||
decrypted_cache: dict[str, str] = {}
|
||||
positioned = []
|
||||
with tempfile.TemporaryDirectory(prefix="kindle-zotero-pdf-") as temp_dir:
|
||||
for item in plan["items"]:
|
||||
positioned.append(_position_item(item, text_cache, xml_cache, size_cache))
|
||||
positioned.append(
|
||||
_position_item(
|
||||
item,
|
||||
text_cache,
|
||||
xml_cache,
|
||||
size_cache,
|
||||
decrypted_cache,
|
||||
temp_dir,
|
||||
)
|
||||
)
|
||||
|
||||
status_counts: dict[str, int] = {}
|
||||
for item in positioned:
|
||||
|
|
@ -49,6 +63,8 @@ def _position_item(
|
|||
text_cache: dict[str, list[str]],
|
||||
xml_cache: dict[tuple[str, int], PDFPageXML],
|
||||
size_cache: dict[tuple[str, int], tuple[float, float]],
|
||||
decrypted_cache: dict[str, str],
|
||||
temp_dir: str,
|
||||
) -> dict[str, Any]:
|
||||
if item.get("status") != "ready-for-positioning":
|
||||
return item
|
||||
|
|
@ -63,16 +79,21 @@ def _position_item(
|
|||
return _with_problem(item, "pdf-position-missing-path")
|
||||
|
||||
try:
|
||||
text_pages = text_cache.setdefault(path, extract_pdf_text_pages(path))
|
||||
text_pages, extraction_path = _text_pages_for_path(
|
||||
path, text_cache, decrypted_cache, temp_dir
|
||||
)
|
||||
page_index = find_pdf_text_page(
|
||||
text_pages, item["clipping"]["text"], item["clipping"].get("page")
|
||||
)
|
||||
if page_index is None:
|
||||
return _with_problem(item, "pdf-text-not-found")
|
||||
page_xml = xml_cache.setdefault(
|
||||
(path, page_index), extract_pdf_page_xml(path, page_index)
|
||||
(extraction_path, page_index),
|
||||
extract_pdf_page_xml(extraction_path, page_index),
|
||||
)
|
||||
position = find_pdf_rects(
|
||||
extraction_path, page_xml, item["clipping"]["text"], size_cache
|
||||
)
|
||||
position = find_pdf_rects(path, page_xml, item["clipping"]["text"], size_cache)
|
||||
except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch
|
||||
return _with_problem(item, f"pdf-position-error:{error}")
|
||||
|
||||
|
|
@ -104,6 +125,52 @@ def extract_pdf_text_pages(path: str) -> list[str]:
|
|||
return result.stdout.split("\f")
|
||||
|
||||
|
||||
def _text_pages_for_path(
|
||||
path: str,
|
||||
text_cache: dict[str, list[str]],
|
||||
decrypted_cache: dict[str, str],
|
||||
temp_dir: str,
|
||||
) -> tuple[list[str], str]:
|
||||
if path in text_cache:
|
||||
return text_cache[path], decrypted_cache.get(path, path)
|
||||
|
||||
try:
|
||||
text_cache[path] = extract_pdf_text_pages(path)
|
||||
return text_cache[path], path
|
||||
except subprocess.CalledProcessError as error:
|
||||
if not _is_pdf_permission_error(error):
|
||||
raise
|
||||
|
||||
decrypted_path = _decrypted_pdf_path(path, decrypted_cache, temp_dir)
|
||||
text_cache[path] = extract_pdf_text_pages(decrypted_path)
|
||||
return text_cache[path], decrypted_path
|
||||
|
||||
|
||||
def _decrypted_pdf_path(
|
||||
path: str, decrypted_cache: dict[str, str], temp_dir: str
|
||||
) -> str:
|
||||
cached = decrypted_cache.get(path)
|
||||
if cached:
|
||||
return cached
|
||||
|
||||
digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:16]
|
||||
base_name = os.path.splitext(os.path.basename(path))[0] or "document"
|
||||
output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf")
|
||||
subprocess.run(
|
||||
["qpdf", "--decrypt", path, output_path],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
decrypted_cache[path] = output_path
|
||||
return output_path
|
||||
|
||||
|
||||
def _is_pdf_permission_error(error: subprocess.CalledProcessError) -> bool:
|
||||
output = "\n".join(part for part in (error.stdout, error.stderr) if part)
|
||||
return "Permission Error" in output and "Copying of text" in output
|
||||
|
||||
|
||||
def find_pdf_text_page(
|
||||
pages: list[str], quote: str, kindle_page: str | None = None
|
||||
) -> int | None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue