kindle-zotero_importer/src/kindle_zotero_importer/cli.py
Utku Bilen Demir f7d0e2f73a Kanzi 0.6.13: incremental also catches new notes with [r]/[o] etc.
- src/kindle_zotero_importer/cli.py: incremental re-queue now also checks new notes (not yet integrated) with bracket colour for same title, so a past highlight getting a new [r] note will recolour without Full re-import
- Bump to 0.6.13
2026-09-23 14:00:45 +02:00

623 lines
28 KiB
Python

from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from .clippings import clippings_to_jsonable, load_clippings
from .epub_position import add_epub_positions
from .final_plan import build_final_writer_plan
from .import_plan import DEFAULT_COLOUR_MAP, build_import_plan
from .matcher import build_match_report, load_json
from .mismatch_review import build_mismatch_review
from .overrides import generate_override_skeleton, load_overrides
from .pdf_position import add_pdf_positions
from .zotero_index import build_zotero_index
def _load_colour_map(workdir: Path) -> dict[str, str] | None:
# 1) workdir/colour-map.json (editable via Settings)
p = workdir / "colour-map.json"
if p.exists():
try:
data = json.loads(p.read_text(encoding="utf-8"))
if isinstance(data, dict):
return {str(k).lower(): str(v) for k, v in data.items()}
except Exception:
pass
# 2) plugin-config.json colourMap field
cfg = workdir / "plugin-config.json"
if cfg.exists():
try:
data = json.loads(cfg.read_text(encoding="utf-8"))
m = data.get("colourMap") or data.get("colorMap")
if isinstance(m, dict):
return {str(k).lower(): str(v) for k, v in m.items()}
except Exception:
pass
return None
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
prog="kindle-zotero-importer",
description="Minimalist Kindle → Zotero importer. Primary command is `run`; others are developer utilities.",
)
subparsers = parser.add_subparsers(dest="command", required=True)
parse_parser = subparsers.add_parser(
"parse", help="Parse a Kindle My Clippings.txt file"
)
parse_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt")
parse_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
parse_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
index_parser = subparsers.add_parser(
"index-zotero", help="Index Zotero items and attachments read-only"
)
index_parser.add_argument(
"--db", default="/home/user/Zotero/zotero.sqlite", help="Path to zotero.sqlite"
)
index_parser.add_argument(
"--storage-root",
default="/home/user/Zotero/storage",
help="Path to Zotero storage directory for storage: attachments",
)
index_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
index_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
match_parser = subparsers.add_parser(
"match", help="Match parsed Kindle clipping titles to Zotero items"
)
match_parser.add_argument("clippings_json", help="Path to parsed clippings JSON")
match_parser.add_argument("zotero_index_json", help="Path to Zotero index JSON")
match_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
match_parser.add_argument(
"--overrides", help="Path to reusable match overrides JSON"
)
match_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
overrides_parser = subparsers.add_parser(
"generate-overrides",
help="Generate a reusable override skeleton from ambiguous/unmatched matches",
)
overrides_parser.add_argument("matches_json", help="Path to match report JSON")
overrides_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
overrides_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
plan_parser = subparsers.add_parser(
"plan", help="Generate a preliminary Zotero annotation import plan"
)
plan_parser.add_argument("clippings_json", help="Path to parsed clippings JSON")
plan_parser.add_argument("zotero_index_json", help="Path to Zotero index JSON")
plan_parser.add_argument("matches_json", help="Path to match report JSON")
plan_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
plan_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
epub_parser = subparsers.add_parser(
"position-epub", help="Add EPUB CFI positions to an import plan"
)
epub_parser.add_argument(
"import_plan_json", help="Path to preliminary import plan JSON"
)
epub_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
epub_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
pdf_parser = subparsers.add_parser(
"position-pdf", help="Add PDF page/rect positions to an import plan"
)
pdf_parser.add_argument("import_plan_json", help="Path to import plan JSON")
pdf_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
pdf_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
final_parser = subparsers.add_parser(
"finalize",
help="Export positioned annotations for the Zotero JavaScript writer",
)
final_parser.add_argument(
"positioned_plan_json", help="Path to positioned import plan JSON"
)
final_parser.add_argument(
"--output", "-o", help="Write JSON to this file instead of stdout"
)
final_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON output"
)
review_parser = subparsers.add_parser(
"review-mismatches",
help="Generate a copy-friendly mismatch review Markdown file",
)
review_parser.add_argument(
"positioned_plan_json", help="Path to positioned import plan JSON"
)
review_parser.add_argument("matches_json", help="Path to match report JSON")
review_parser.add_argument(
"--output", "-o", help="Write Markdown to this file instead of stdout"
)
run_parser = subparsers.add_parser(
"run", help="Run the full importer pipeline for a Kindle My Clippings.txt file"
)
run_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt")
run_parser.add_argument(
"--workdir",
default=".",
help="Directory for generated importer artifacts",
)
run_parser.add_argument(
"--db", default="/home/user/Zotero/zotero.sqlite", help="Path to zotero.sqlite"
)
run_parser.add_argument(
"--storage-root",
default="/home/user/Zotero/storage",
help="Path to Zotero storage directory for storage: attachments",
)
run_parser.add_argument(
"--overrides",
default="match-overrides.json",
help="Path to reusable match overrides JSON, relative to workdir unless absolute",
)
run_parser.add_argument(
"--summary-output",
help="Write pipeline summary JSON to this path instead of stdout",
)
run_parser.add_argument(
"--progress-output",
help="Write machine-readable pipeline progress JSON to this path",
)
run_parser.add_argument(
"--pretty", action="store_true", help="Pretty-print JSON artifacts"
)
run_parser.add_argument(
"--full",
action="store_true",
help="Force full re-import from scratch, ignoring incremental state (processes all clippings)",
)
args = parser.parse_args(argv)
if args.command == "parse":
payload = clippings_to_jsonable(load_clippings(args.clippings_file))
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "index-zotero":
payload = build_zotero_index(args.db, args.storage_root)
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "match":
overrides = (
load_overrides(load_json(args.overrides)) if args.overrides else None
)
payload = build_match_report(
load_json(args.clippings_json), load_json(args.zotero_index_json), overrides
)
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "generate-overrides":
payload = generate_override_skeleton(load_json(args.matches_json))
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "plan":
payload = build_import_plan(
load_json(args.clippings_json),
load_json(args.zotero_index_json),
load_json(args.matches_json),
)
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "position-epub":
payload = add_epub_positions(load_json(args.import_plan_json))
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "position-pdf":
payload = add_pdf_positions(load_json(args.import_plan_json))
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "finalize":
payload = build_final_writer_plan(load_json(args.positioned_plan_json))
json_text = json.dumps(
payload, ensure_ascii=False, indent=2 if args.pretty else None
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
return 0
if args.command == "review-mismatches":
text = build_mismatch_review(
load_json(args.positioned_plan_json), load_json(args.matches_json)
)
if args.output:
with open(args.output, "w", encoding="utf-8") as file:
file.write(text)
else:
sys.stdout.write(text)
return 0
if args.command == "run":
workdir = Path(args.workdir).expanduser().resolve()
workdir.mkdir(parents=True, exist_ok=True)
progress_path = (
Path(args.progress_output).expanduser().resolve()
if args.progress_output
else None
)
def report_progress(percent: int, stage: str, detail: str) -> None:
if not progress_path:
return
temporary_path = progress_path.with_suffix(progress_path.suffix + ".tmp")
temporary_path.write_text(
json.dumps(
{"percent": percent, "stage": stage, "detail": detail},
ensure_ascii=False,
)
+ "\n",
encoding="utf-8",
)
temporary_path.replace(progress_path)
def artifact(name: str) -> Path:
return workdir / name
overrides_path = Path(args.overrides).expanduser()
if not overrides_path.is_absolute():
overrides_path = workdir / overrides_path
colour_map = _load_colour_map(workdir)
report_progress(2, "Reading clippings", "Parsing My Clippings.txt")
new_clippings_list = load_clippings(args.clippings_file)
new_clippings = clippings_to_jsonable(new_clippings_list)
new_ids = {c["id"] for c in new_clippings.get("clippings", [])}
# Incremental state: previous clippings and previous integrated ids
is_incremental = not args.full
prev_clippings_ids: set[str] = set()
prev_integrated_ids: set[str] = set()
prev_final_plan = None
if is_incremental:
prev_clippings_path = artifact("clippings.json")
prev_final_path = artifact("import-plan.final.json")
if prev_clippings_path.exists():
try:
prev_clippings_ids = {c["id"] for c in load_json(str(prev_clippings_path)).get("clippings", [])}
except Exception:
prev_clippings_ids = set()
if prev_final_path.exists():
try:
prev_final_plan = load_json(str(prev_final_path))
prev_integrated_ids = {a.get("clipping_id") for a in prev_final_plan.get("annotations", []) if a.get("clipping_id")}
except Exception:
prev_final_plan = None
prev_integrated_ids = set()
# If no previous state, fall back to full
if not prev_clippings_ids and not prev_integrated_ids:
is_incremental = False
if is_incremental:
# New or previously not integrated (skipped) -> need to (re)process
to_process_ids = set(new_ids - prev_integrated_ids)
# Deletions: previously integrated but now absent (removed or changed text)
deletions_ids = sorted(prev_integrated_ids - new_ids)
# Also re-queue any previously integrated clippings whose title now has an override
# (mapping may have changed, need to move annotation). We need overrides dict for this.
# Load overrides early for this check (already loaded later, but do here for incremental decision)
# We will reload overrides here if not yet loaded
try:
_ov_for_inc = load_overrides(load_json(str(overrides_path))) if overrides_path.exists() else {}
except Exception:
_ov_for_inc = {}
for c in new_clippings_list:
if c.id in prev_integrated_ids and c.title in _ov_for_inc:
to_process_ids.add(c.id)
# Also re-queue highlights whose bracket colour would change the annotation
# (so you can recolour a past highlight by adding [r]/[o] etc. to its note or highlight text)
if colour_map:
import re as _re
# Build map of new notes by title for quick lookup
new_notes_by_title: dict[str, list] = {}
for n in new_clippings_list:
if n.kind == "note" and n.id not in prev_integrated_ids:
# new note (not yet integrated) — check if it has a bracket colour
m = _re.match(r"^\s*\[([^\]]+)\]\s*", n.text or "")
if m and m.group(1).strip().lower() in {k.lower() for k in colour_map.keys()}:
new_notes_by_title.setdefault(n.title, []).append(n)
for c in new_clippings_list:
if c.id not in prev_integrated_ids:
continue
if c.kind != "highlight":
continue
has_bracket = False
# check highlight text itself
for t in [c.text or "", getattr(c, "comment", "") or ""]:
m = _re.match(r"^\s*\[([^\]]+)\]\s*", t)
if m and m.group(1).strip().lower() in {k.lower() for k in colour_map.keys()}:
has_bracket = True
break
# also check if a *new* note for same title would attach to this highlight (by location/page proximity)
# Approximate: any new note with same title is considered for re-queue (conservative, ensures recolours)
if not has_bracket and c.title in new_notes_by_title:
has_bracket = True
if has_bracket:
to_process_ids.add(c.id)
# If colourMap itself changed since last run, re-queue all prev integrated (so Settings change recolours)
try:
prev_map = (prev_final_plan or {}).get("colourMap") or (prev_final_plan or {}).get("colorMap")
if colour_map is not None and prev_map is not None and dict(colour_map) != dict(prev_map):
to_process_ids.update(prev_integrated_ids)
elif colour_map is not None and prev_map is None and len(colour_map) != len(DEFAULT_COLOUR_MAP):
# custom map added where previously default — re-queue to apply
to_process_ids.update(prev_integrated_ids)
except Exception:
pass
# Also handle deleted overrides: if a previously integrated title's override was deleted,
# its annotations should be deleted. Detect via previous final plan's titles vs current overrides.
# For incremental, we don't have previous overrides, so we handle deletions via the main deletions list
# (IDs no longer present). For deleted overrides where ID still present but now unmatched,
# the match report will make them not positioned, and they will be in to_process but then skipped;
# we need to ensure they are deleted. We do this by checking if a previously integrated clipping's
# title is now not in overrides and would be unmatched -> treat as to_delete.
# Simplify: any prev_integrated clipping whose title is not in current new match's "matched" will be
# handled as not in to_process_ids? Actually it is still in new_ids, but we re-queued it above only if title in overrides.
# For now, rely on full re-import for deleted-override cleanup.
# Build filtered clippings for this delta run
filtered_list = [c for c in new_clippings_list if c.id in to_process_ids]
# If nothing to do and no deletions, still need to report but skip expensive steps
if not filtered_list and not deletions_ids:
report_progress(15, "Matching titles", "No new highlights — incremental skip")
# Still need to generate artifacts for consistency (empty delta)
zotero_index = build_zotero_index(args.db, args.storage_root)
overrides = None
if overrides_path.exists():
overrides = load_overrides(load_json(str(overrides_path)))
clippings = new_clippings
matches = build_match_report(clippings, zotero_index, overrides)
override_skeleton = generate_override_skeleton(matches)
plan = {"format": "kindle-zotero-importer.import-plan.v1", "items": []}
positioned_plan = {"format": plan["format"], "items": []}
final_plan = {"format": FINAL_FORMAT, "source_format": positioned_plan.get("format"), "annotation_count": 0, "skipped_counts": {}, "annotations": [], "deletions": [], "is_incremental": True, "incremental_stats": {"new_clippings": len(new_ids), "prev_integrated": len(prev_integrated_ids), "to_process": 0, "deletions": 0}}
final_plan["colourMap"] = colour_map if colour_map is not None else DEFAULT_COLOUR_MAP
final_plan["colorMap"] = final_plan["colourMap"]
mismatch_review = build_mismatch_review(positioned_plan, matches)
# Will write artifacts below with empty delta
else:
report_progress(8, "Indexing Zotero", "Reading library items and attachments (incremental)")
zotero_index = build_zotero_index(args.db, args.storage_root)
overrides = None
if overrides_path.exists():
overrides = load_overrides(load_json(str(overrides_path)))
# Only the delta clippings go through matching/positioning
delta_clippings = clippings_to_jsonable(filtered_list)
report_progress(15, "Matching titles", f"Matching {len(filtered_list)} new/pending highlights (incremental)")
matches = build_match_report(delta_clippings, zotero_index, overrides)
override_skeleton = generate_override_skeleton(matches)
# For artifact completeness, also build full matches for summary? Use delta matches for now
# But we need full matches for review? Keep delta
report_progress(22, "Building plan", "Selecting attachments (incremental)")
plan = build_import_plan(delta_clippings, zotero_index, matches, colour_map)
report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files (incremental)")
epub_plan = add_epub_positions(plan)
report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files (incremental)")
positioned_plan = add_pdf_positions(epub_plan)
report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
final_plan = build_final_writer_plan(positioned_plan)
# Attach deletions for writer
final_plan["deletions"] = deletions_ids
final_plan["is_incremental"] = True
final_plan["incremental_stats"] = {
"new_clippings": len(new_ids),
"prev_integrated": len(prev_integrated_ids),
"to_process": len(to_process_ids),
"deletions": len(deletions_ids),
}
final_plan["colourMap"] = colour_map if colour_map is not None else DEFAULT_COLOUR_MAP
final_plan["colorMap"] = final_plan["colourMap"]
mismatch_review = build_mismatch_review(positioned_plan, matches)
# For outputs, clippings should be the full new set (for next diff), but plan artifacts are delta
clippings = new_clippings
else:
# Full re-import
report_progress(8, "Indexing Zotero", "Reading library items and attachments")
zotero_index = build_zotero_index(args.db, args.storage_root)
overrides = None
if overrides_path.exists():
overrides = load_overrides(load_json(str(overrides_path)))
clippings = new_clippings
report_progress(15, "Matching titles", "Matching Kindle titles to Zotero items")
matches = build_match_report(clippings, zotero_index, overrides)
override_skeleton = generate_override_skeleton(matches)
report_progress(22, "Building plan", "Selecting attachments")
plan = build_import_plan(clippings, zotero_index, matches, colour_map)
report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files")
epub_plan = add_epub_positions(plan)
report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files; this is usually the longest stage")
positioned_plan = add_pdf_positions(epub_plan)
report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
final_plan = build_final_writer_plan(positioned_plan)
final_plan["deletions"] = []
final_plan["is_incremental"] = False
final_plan["colourMap"] = colour_map if colour_map is not None else DEFAULT_COLOUR_MAP
final_plan["colorMap"] = final_plan["colourMap"]
mismatch_review = build_mismatch_review(positioned_plan, matches)
indent = 2 if args.pretty else None
outputs = {
"clippings": artifact("clippings.json"),
"zotero_index": artifact("zotero-index.json"),
"matches": artifact("matches.json"),
"generated_overrides": artifact("match-overrides.generated.json"),
"import_plan": artifact("import-plan.json"),
"epub_plan": artifact("import-plan.epub.json"),
"positioned_plan": artifact("import-plan.positioned.json"),
"final_plan": artifact("import-plan.final.json"),
"mismatch_review": workdir / "docs" / "mismatch-review.md",
}
outputs["mismatch_review"].parent.mkdir(parents=True, exist_ok=True)
report_progress(95, "Saving results", "Writing import artifacts")
for key, payload in [
("clippings", clippings),
("zotero_index", zotero_index),
("matches", matches),
("generated_overrides", override_skeleton),
("import_plan", plan),
("epub_plan", epub_plan),
("positioned_plan", positioned_plan),
("final_plan", final_plan),
]:
with open(outputs[key], "w", encoding="utf-8") as file:
json.dump(payload, file, ensure_ascii=False, indent=indent)
file.write("\n")
with open(outputs["mismatch_review"], "w", encoding="utf-8") as file:
file.write(mismatch_review)
status_counts: dict[str, int] = {}
for item in positioned_plan.get("items", []):
status = item.get("status", "unknown")
status_counts[status] = status_counts.get(status, 0) + 1
match_counts: dict[str, int] = {}
for match in matches.get("matches", []):
status = match.get("status", "unknown")
match_counts[status] = match_counts.get(status, 0) + 1
summary = {
"format": "kindle-zotero-importer.pipeline-summary.v1",
"workdir": str(workdir),
"overrides_path": str(overrides_path),
"outputs": {key: str(path) for key, path in outputs.items()},
"counts": {
"clippings": len(clippings.get("clippings", [])),
"unique_titles": len(matches.get("matches", [])),
"final_annotations": final_plan.get("annotation_count", 0),
"match_statuses": match_counts,
"plan_statuses": status_counts,
"skipped_final": final_plan.get("skipped_counts", {}),
"is_incremental": final_plan.get("is_incremental", False),
"incremental_stats": final_plan.get("incremental_stats", {}),
"deletions": len(final_plan.get("deletions", [])),
},
}
json_text = json.dumps(summary, ensure_ascii=False, indent=2)
if args.summary_output:
with open(args.summary_output, "w", encoding="utf-8") as file:
file.write(json_text)
file.write("\n")
else:
sys.stdout.write(json_text)
sys.stdout.write("\n")
report_progress(100, "Pipeline complete", "Ready to import annotations into Zotero")
return 0
parser.error(f"unknown command: {args.command}")
return 2
if __name__ == "__main__":
raise SystemExit(main())