- src/kindle_zotero_importer/cli.py: incremental re-queue now also checks new notes (not yet integrated) with bracket colour for same title, so a past highlight getting a new [r] note will recolour without Full re-import - Bump to 0.6.13
623 lines
28 KiB
Python
623 lines
28 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from .clippings import clippings_to_jsonable, load_clippings
|
|
from .epub_position import add_epub_positions
|
|
from .final_plan import build_final_writer_plan
|
|
from .import_plan import DEFAULT_COLOUR_MAP, build_import_plan
|
|
from .matcher import build_match_report, load_json
|
|
from .mismatch_review import build_mismatch_review
|
|
from .overrides import generate_override_skeleton, load_overrides
|
|
from .pdf_position import add_pdf_positions
|
|
from .zotero_index import build_zotero_index
|
|
|
|
|
|
def _load_colour_map(workdir: Path) -> dict[str, str] | None:
|
|
# 1) workdir/colour-map.json (editable via Settings)
|
|
p = workdir / "colour-map.json"
|
|
if p.exists():
|
|
try:
|
|
data = json.loads(p.read_text(encoding="utf-8"))
|
|
if isinstance(data, dict):
|
|
return {str(k).lower(): str(v) for k, v in data.items()}
|
|
except Exception:
|
|
pass
|
|
# 2) plugin-config.json colourMap field
|
|
cfg = workdir / "plugin-config.json"
|
|
if cfg.exists():
|
|
try:
|
|
data = json.loads(cfg.read_text(encoding="utf-8"))
|
|
m = data.get("colourMap") or data.get("colorMap")
|
|
if isinstance(m, dict):
|
|
return {str(k).lower(): str(v) for k, v in m.items()}
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(
|
|
prog="kindle-zotero-importer",
|
|
description="Minimalist Kindle → Zotero importer. Primary command is `run`; others are developer utilities.",
|
|
)
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
parse_parser = subparsers.add_parser(
|
|
"parse", help="Parse a Kindle My Clippings.txt file"
|
|
)
|
|
parse_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt")
|
|
parse_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
parse_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
index_parser = subparsers.add_parser(
|
|
"index-zotero", help="Index Zotero items and attachments read-only"
|
|
)
|
|
index_parser.add_argument(
|
|
"--db", default="/home/user/Zotero/zotero.sqlite", help="Path to zotero.sqlite"
|
|
)
|
|
index_parser.add_argument(
|
|
"--storage-root",
|
|
default="/home/user/Zotero/storage",
|
|
help="Path to Zotero storage directory for storage: attachments",
|
|
)
|
|
index_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
index_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
match_parser = subparsers.add_parser(
|
|
"match", help="Match parsed Kindle clipping titles to Zotero items"
|
|
)
|
|
match_parser.add_argument("clippings_json", help="Path to parsed clippings JSON")
|
|
match_parser.add_argument("zotero_index_json", help="Path to Zotero index JSON")
|
|
match_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
match_parser.add_argument(
|
|
"--overrides", help="Path to reusable match overrides JSON"
|
|
)
|
|
match_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
overrides_parser = subparsers.add_parser(
|
|
"generate-overrides",
|
|
help="Generate a reusable override skeleton from ambiguous/unmatched matches",
|
|
)
|
|
overrides_parser.add_argument("matches_json", help="Path to match report JSON")
|
|
overrides_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
overrides_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
plan_parser = subparsers.add_parser(
|
|
"plan", help="Generate a preliminary Zotero annotation import plan"
|
|
)
|
|
plan_parser.add_argument("clippings_json", help="Path to parsed clippings JSON")
|
|
plan_parser.add_argument("zotero_index_json", help="Path to Zotero index JSON")
|
|
plan_parser.add_argument("matches_json", help="Path to match report JSON")
|
|
plan_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
plan_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
epub_parser = subparsers.add_parser(
|
|
"position-epub", help="Add EPUB CFI positions to an import plan"
|
|
)
|
|
epub_parser.add_argument(
|
|
"import_plan_json", help="Path to preliminary import plan JSON"
|
|
)
|
|
epub_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
epub_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
pdf_parser = subparsers.add_parser(
|
|
"position-pdf", help="Add PDF page/rect positions to an import plan"
|
|
)
|
|
pdf_parser.add_argument("import_plan_json", help="Path to import plan JSON")
|
|
pdf_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
pdf_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
final_parser = subparsers.add_parser(
|
|
"finalize",
|
|
help="Export positioned annotations for the Zotero JavaScript writer",
|
|
)
|
|
final_parser.add_argument(
|
|
"positioned_plan_json", help="Path to positioned import plan JSON"
|
|
)
|
|
final_parser.add_argument(
|
|
"--output", "-o", help="Write JSON to this file instead of stdout"
|
|
)
|
|
final_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON output"
|
|
)
|
|
|
|
review_parser = subparsers.add_parser(
|
|
"review-mismatches",
|
|
help="Generate a copy-friendly mismatch review Markdown file",
|
|
)
|
|
review_parser.add_argument(
|
|
"positioned_plan_json", help="Path to positioned import plan JSON"
|
|
)
|
|
review_parser.add_argument("matches_json", help="Path to match report JSON")
|
|
review_parser.add_argument(
|
|
"--output", "-o", help="Write Markdown to this file instead of stdout"
|
|
)
|
|
|
|
run_parser = subparsers.add_parser(
|
|
"run", help="Run the full importer pipeline for a Kindle My Clippings.txt file"
|
|
)
|
|
run_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt")
|
|
run_parser.add_argument(
|
|
"--workdir",
|
|
default=".",
|
|
help="Directory for generated importer artifacts",
|
|
)
|
|
run_parser.add_argument(
|
|
"--db", default="/home/user/Zotero/zotero.sqlite", help="Path to zotero.sqlite"
|
|
)
|
|
run_parser.add_argument(
|
|
"--storage-root",
|
|
default="/home/user/Zotero/storage",
|
|
help="Path to Zotero storage directory for storage: attachments",
|
|
)
|
|
run_parser.add_argument(
|
|
"--overrides",
|
|
default="match-overrides.json",
|
|
help="Path to reusable match overrides JSON, relative to workdir unless absolute",
|
|
)
|
|
run_parser.add_argument(
|
|
"--summary-output",
|
|
help="Write pipeline summary JSON to this path instead of stdout",
|
|
)
|
|
run_parser.add_argument(
|
|
"--progress-output",
|
|
help="Write machine-readable pipeline progress JSON to this path",
|
|
)
|
|
run_parser.add_argument(
|
|
"--pretty", action="store_true", help="Pretty-print JSON artifacts"
|
|
)
|
|
run_parser.add_argument(
|
|
"--full",
|
|
action="store_true",
|
|
help="Force full re-import from scratch, ignoring incremental state (processes all clippings)",
|
|
)
|
|
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.command == "parse":
|
|
payload = clippings_to_jsonable(load_clippings(args.clippings_file))
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "index-zotero":
|
|
payload = build_zotero_index(args.db, args.storage_root)
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "match":
|
|
overrides = (
|
|
load_overrides(load_json(args.overrides)) if args.overrides else None
|
|
)
|
|
payload = build_match_report(
|
|
load_json(args.clippings_json), load_json(args.zotero_index_json), overrides
|
|
)
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "generate-overrides":
|
|
payload = generate_override_skeleton(load_json(args.matches_json))
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "plan":
|
|
payload = build_import_plan(
|
|
load_json(args.clippings_json),
|
|
load_json(args.zotero_index_json),
|
|
load_json(args.matches_json),
|
|
)
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "position-epub":
|
|
payload = add_epub_positions(load_json(args.import_plan_json))
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "position-pdf":
|
|
payload = add_pdf_positions(load_json(args.import_plan_json))
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "finalize":
|
|
payload = build_final_writer_plan(load_json(args.positioned_plan_json))
|
|
json_text = json.dumps(
|
|
payload, ensure_ascii=False, indent=2 if args.pretty else None
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
return 0
|
|
|
|
if args.command == "review-mismatches":
|
|
text = build_mismatch_review(
|
|
load_json(args.positioned_plan_json), load_json(args.matches_json)
|
|
)
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as file:
|
|
file.write(text)
|
|
else:
|
|
sys.stdout.write(text)
|
|
return 0
|
|
|
|
if args.command == "run":
|
|
workdir = Path(args.workdir).expanduser().resolve()
|
|
workdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
progress_path = (
|
|
Path(args.progress_output).expanduser().resolve()
|
|
if args.progress_output
|
|
else None
|
|
)
|
|
|
|
def report_progress(percent: int, stage: str, detail: str) -> None:
|
|
if not progress_path:
|
|
return
|
|
temporary_path = progress_path.with_suffix(progress_path.suffix + ".tmp")
|
|
temporary_path.write_text(
|
|
json.dumps(
|
|
{"percent": percent, "stage": stage, "detail": detail},
|
|
ensure_ascii=False,
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
temporary_path.replace(progress_path)
|
|
|
|
def artifact(name: str) -> Path:
|
|
return workdir / name
|
|
|
|
overrides_path = Path(args.overrides).expanduser()
|
|
if not overrides_path.is_absolute():
|
|
overrides_path = workdir / overrides_path
|
|
colour_map = _load_colour_map(workdir)
|
|
|
|
report_progress(2, "Reading clippings", "Parsing My Clippings.txt")
|
|
new_clippings_list = load_clippings(args.clippings_file)
|
|
new_clippings = clippings_to_jsonable(new_clippings_list)
|
|
new_ids = {c["id"] for c in new_clippings.get("clippings", [])}
|
|
|
|
# Incremental state: previous clippings and previous integrated ids
|
|
is_incremental = not args.full
|
|
prev_clippings_ids: set[str] = set()
|
|
prev_integrated_ids: set[str] = set()
|
|
prev_final_plan = None
|
|
if is_incremental:
|
|
prev_clippings_path = artifact("clippings.json")
|
|
prev_final_path = artifact("import-plan.final.json")
|
|
if prev_clippings_path.exists():
|
|
try:
|
|
prev_clippings_ids = {c["id"] for c in load_json(str(prev_clippings_path)).get("clippings", [])}
|
|
except Exception:
|
|
prev_clippings_ids = set()
|
|
if prev_final_path.exists():
|
|
try:
|
|
prev_final_plan = load_json(str(prev_final_path))
|
|
prev_integrated_ids = {a.get("clipping_id") for a in prev_final_plan.get("annotations", []) if a.get("clipping_id")}
|
|
except Exception:
|
|
prev_final_plan = None
|
|
prev_integrated_ids = set()
|
|
# If no previous state, fall back to full
|
|
if not prev_clippings_ids and not prev_integrated_ids:
|
|
is_incremental = False
|
|
|
|
if is_incremental:
|
|
# New or previously not integrated (skipped) -> need to (re)process
|
|
to_process_ids = set(new_ids - prev_integrated_ids)
|
|
# Deletions: previously integrated but now absent (removed or changed text)
|
|
deletions_ids = sorted(prev_integrated_ids - new_ids)
|
|
# Also re-queue any previously integrated clippings whose title now has an override
|
|
# (mapping may have changed, need to move annotation). We need overrides dict for this.
|
|
# Load overrides early for this check (already loaded later, but do here for incremental decision)
|
|
# We will reload overrides here if not yet loaded
|
|
try:
|
|
_ov_for_inc = load_overrides(load_json(str(overrides_path))) if overrides_path.exists() else {}
|
|
except Exception:
|
|
_ov_for_inc = {}
|
|
for c in new_clippings_list:
|
|
if c.id in prev_integrated_ids and c.title in _ov_for_inc:
|
|
to_process_ids.add(c.id)
|
|
# Also re-queue highlights whose bracket colour would change the annotation
|
|
# (so you can recolour a past highlight by adding [r]/[o] etc. to its note or highlight text)
|
|
if colour_map:
|
|
import re as _re
|
|
# Build map of new notes by title for quick lookup
|
|
new_notes_by_title: dict[str, list] = {}
|
|
for n in new_clippings_list:
|
|
if n.kind == "note" and n.id not in prev_integrated_ids:
|
|
# new note (not yet integrated) — check if it has a bracket colour
|
|
m = _re.match(r"^\s*\[([^\]]+)\]\s*", n.text or "")
|
|
if m and m.group(1).strip().lower() in {k.lower() for k in colour_map.keys()}:
|
|
new_notes_by_title.setdefault(n.title, []).append(n)
|
|
for c in new_clippings_list:
|
|
if c.id not in prev_integrated_ids:
|
|
continue
|
|
if c.kind != "highlight":
|
|
continue
|
|
has_bracket = False
|
|
# check highlight text itself
|
|
for t in [c.text or "", getattr(c, "comment", "") or ""]:
|
|
m = _re.match(r"^\s*\[([^\]]+)\]\s*", t)
|
|
if m and m.group(1).strip().lower() in {k.lower() for k in colour_map.keys()}:
|
|
has_bracket = True
|
|
break
|
|
# also check if a *new* note for same title would attach to this highlight (by location/page proximity)
|
|
# Approximate: any new note with same title is considered for re-queue (conservative, ensures recolours)
|
|
if not has_bracket and c.title in new_notes_by_title:
|
|
has_bracket = True
|
|
if has_bracket:
|
|
to_process_ids.add(c.id)
|
|
# If colourMap itself changed since last run, re-queue all prev integrated (so Settings change recolours)
|
|
try:
|
|
prev_map = (prev_final_plan or {}).get("colourMap") or (prev_final_plan or {}).get("colorMap")
|
|
if colour_map is not None and prev_map is not None and dict(colour_map) != dict(prev_map):
|
|
to_process_ids.update(prev_integrated_ids)
|
|
elif colour_map is not None and prev_map is None and len(colour_map) != len(DEFAULT_COLOUR_MAP):
|
|
# custom map added where previously default — re-queue to apply
|
|
to_process_ids.update(prev_integrated_ids)
|
|
except Exception:
|
|
pass
|
|
# Also handle deleted overrides: if a previously integrated title's override was deleted,
|
|
# its annotations should be deleted. Detect via previous final plan's titles vs current overrides.
|
|
# For incremental, we don't have previous overrides, so we handle deletions via the main deletions list
|
|
# (IDs no longer present). For deleted overrides where ID still present but now unmatched,
|
|
# the match report will make them not positioned, and they will be in to_process but then skipped;
|
|
# we need to ensure they are deleted. We do this by checking if a previously integrated clipping's
|
|
# title is now not in overrides and would be unmatched -> treat as to_delete.
|
|
# Simplify: any prev_integrated clipping whose title is not in current new match's "matched" will be
|
|
# handled as not in to_process_ids? Actually it is still in new_ids, but we re-queued it above only if title in overrides.
|
|
# For now, rely on full re-import for deleted-override cleanup.
|
|
# Build filtered clippings for this delta run
|
|
filtered_list = [c for c in new_clippings_list if c.id in to_process_ids]
|
|
# If nothing to do and no deletions, still need to report but skip expensive steps
|
|
if not filtered_list and not deletions_ids:
|
|
report_progress(15, "Matching titles", "No new highlights — incremental skip")
|
|
# Still need to generate artifacts for consistency (empty delta)
|
|
zotero_index = build_zotero_index(args.db, args.storage_root)
|
|
overrides = None
|
|
if overrides_path.exists():
|
|
overrides = load_overrides(load_json(str(overrides_path)))
|
|
clippings = new_clippings
|
|
matches = build_match_report(clippings, zotero_index, overrides)
|
|
override_skeleton = generate_override_skeleton(matches)
|
|
plan = {"format": "kindle-zotero-importer.import-plan.v1", "items": []}
|
|
positioned_plan = {"format": plan["format"], "items": []}
|
|
final_plan = {"format": FINAL_FORMAT, "source_format": positioned_plan.get("format"), "annotation_count": 0, "skipped_counts": {}, "annotations": [], "deletions": [], "is_incremental": True, "incremental_stats": {"new_clippings": len(new_ids), "prev_integrated": len(prev_integrated_ids), "to_process": 0, "deletions": 0}}
|
|
final_plan["colourMap"] = colour_map if colour_map is not None else DEFAULT_COLOUR_MAP
|
|
final_plan["colorMap"] = final_plan["colourMap"]
|
|
mismatch_review = build_mismatch_review(positioned_plan, matches)
|
|
# Will write artifacts below with empty delta
|
|
else:
|
|
report_progress(8, "Indexing Zotero", "Reading library items and attachments (incremental)")
|
|
zotero_index = build_zotero_index(args.db, args.storage_root)
|
|
overrides = None
|
|
if overrides_path.exists():
|
|
overrides = load_overrides(load_json(str(overrides_path)))
|
|
# Only the delta clippings go through matching/positioning
|
|
delta_clippings = clippings_to_jsonable(filtered_list)
|
|
report_progress(15, "Matching titles", f"Matching {len(filtered_list)} new/pending highlights (incremental)")
|
|
matches = build_match_report(delta_clippings, zotero_index, overrides)
|
|
override_skeleton = generate_override_skeleton(matches)
|
|
# For artifact completeness, also build full matches for summary? Use delta matches for now
|
|
# But we need full matches for review? Keep delta
|
|
report_progress(22, "Building plan", "Selecting attachments (incremental)")
|
|
plan = build_import_plan(delta_clippings, zotero_index, matches, colour_map)
|
|
report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files (incremental)")
|
|
epub_plan = add_epub_positions(plan)
|
|
report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files (incremental)")
|
|
positioned_plan = add_pdf_positions(epub_plan)
|
|
report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
|
|
final_plan = build_final_writer_plan(positioned_plan)
|
|
# Attach deletions for writer
|
|
final_plan["deletions"] = deletions_ids
|
|
final_plan["is_incremental"] = True
|
|
final_plan["incremental_stats"] = {
|
|
"new_clippings": len(new_ids),
|
|
"prev_integrated": len(prev_integrated_ids),
|
|
"to_process": len(to_process_ids),
|
|
"deletions": len(deletions_ids),
|
|
}
|
|
final_plan["colourMap"] = colour_map if colour_map is not None else DEFAULT_COLOUR_MAP
|
|
final_plan["colorMap"] = final_plan["colourMap"]
|
|
mismatch_review = build_mismatch_review(positioned_plan, matches)
|
|
# For outputs, clippings should be the full new set (for next diff), but plan artifacts are delta
|
|
clippings = new_clippings
|
|
else:
|
|
# Full re-import
|
|
report_progress(8, "Indexing Zotero", "Reading library items and attachments")
|
|
zotero_index = build_zotero_index(args.db, args.storage_root)
|
|
overrides = None
|
|
if overrides_path.exists():
|
|
overrides = load_overrides(load_json(str(overrides_path)))
|
|
clippings = new_clippings
|
|
report_progress(15, "Matching titles", "Matching Kindle titles to Zotero items")
|
|
matches = build_match_report(clippings, zotero_index, overrides)
|
|
override_skeleton = generate_override_skeleton(matches)
|
|
report_progress(22, "Building plan", "Selecting attachments")
|
|
plan = build_import_plan(clippings, zotero_index, matches, colour_map)
|
|
report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files")
|
|
epub_plan = add_epub_positions(plan)
|
|
report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files; this is usually the longest stage")
|
|
positioned_plan = add_pdf_positions(epub_plan)
|
|
report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
|
|
final_plan = build_final_writer_plan(positioned_plan)
|
|
final_plan["deletions"] = []
|
|
final_plan["is_incremental"] = False
|
|
final_plan["colourMap"] = colour_map if colour_map is not None else DEFAULT_COLOUR_MAP
|
|
final_plan["colorMap"] = final_plan["colourMap"]
|
|
mismatch_review = build_mismatch_review(positioned_plan, matches)
|
|
|
|
indent = 2 if args.pretty else None
|
|
outputs = {
|
|
"clippings": artifact("clippings.json"),
|
|
"zotero_index": artifact("zotero-index.json"),
|
|
"matches": artifact("matches.json"),
|
|
"generated_overrides": artifact("match-overrides.generated.json"),
|
|
"import_plan": artifact("import-plan.json"),
|
|
"epub_plan": artifact("import-plan.epub.json"),
|
|
"positioned_plan": artifact("import-plan.positioned.json"),
|
|
"final_plan": artifact("import-plan.final.json"),
|
|
"mismatch_review": workdir / "docs" / "mismatch-review.md",
|
|
}
|
|
outputs["mismatch_review"].parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
report_progress(95, "Saving results", "Writing import artifacts")
|
|
for key, payload in [
|
|
("clippings", clippings),
|
|
("zotero_index", zotero_index),
|
|
("matches", matches),
|
|
("generated_overrides", override_skeleton),
|
|
("import_plan", plan),
|
|
("epub_plan", epub_plan),
|
|
("positioned_plan", positioned_plan),
|
|
("final_plan", final_plan),
|
|
]:
|
|
with open(outputs[key], "w", encoding="utf-8") as file:
|
|
json.dump(payload, file, ensure_ascii=False, indent=indent)
|
|
file.write("\n")
|
|
|
|
with open(outputs["mismatch_review"], "w", encoding="utf-8") as file:
|
|
file.write(mismatch_review)
|
|
|
|
status_counts: dict[str, int] = {}
|
|
for item in positioned_plan.get("items", []):
|
|
status = item.get("status", "unknown")
|
|
status_counts[status] = status_counts.get(status, 0) + 1
|
|
|
|
match_counts: dict[str, int] = {}
|
|
for match in matches.get("matches", []):
|
|
status = match.get("status", "unknown")
|
|
match_counts[status] = match_counts.get(status, 0) + 1
|
|
|
|
summary = {
|
|
"format": "kindle-zotero-importer.pipeline-summary.v1",
|
|
"workdir": str(workdir),
|
|
"overrides_path": str(overrides_path),
|
|
"outputs": {key: str(path) for key, path in outputs.items()},
|
|
"counts": {
|
|
"clippings": len(clippings.get("clippings", [])),
|
|
"unique_titles": len(matches.get("matches", [])),
|
|
"final_annotations": final_plan.get("annotation_count", 0),
|
|
"match_statuses": match_counts,
|
|
"plan_statuses": status_counts,
|
|
"skipped_final": final_plan.get("skipped_counts", {}),
|
|
"is_incremental": final_plan.get("is_incremental", False),
|
|
"incremental_stats": final_plan.get("incremental_stats", {}),
|
|
"deletions": len(final_plan.get("deletions", [])),
|
|
},
|
|
}
|
|
json_text = json.dumps(summary, ensure_ascii=False, indent=2)
|
|
if args.summary_output:
|
|
with open(args.summary_output, "w", encoding="utf-8") as file:
|
|
file.write(json_text)
|
|
file.write("\n")
|
|
else:
|
|
sys.stdout.write(json_text)
|
|
sys.stdout.write("\n")
|
|
report_progress(100, "Pipeline complete", "Ready to import annotations into Zotero")
|
|
return 0
|
|
|
|
parser.error(f"unknown command: {args.command}")
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|