kindle-zotero_importer/src/kindle_zotero_importer/clippings.py
Utku Bilen Demir deaa105be2 Kanzi 0.6.20: kanzi.xpi only, dedup identical highlights, pagination 50
- scripts/build_plugin.py: OUTPUT kanzi.xpi (was kindle-zotero-importer.xpi), clean legacy
- plugin/manifest.json: homepage/update_url -> kanzi repo, name Kanzi 0.6.20
- README: Install kanzi.xpi from kanzi/releases, build -> dist/kanzi.xpi
- updates.json: 0.6.20 link -> kanzi/kanzi.xpi
- src/kindle_zotero_importer/clippings.py: dedup_clippings() collapse identical highlights (same title + norm text) to fix badiou yellow×3
- plugin/bootstrap.js: norm dedup for existing yellow over yellow, mergeComments bracket stripped
2026-09-24 01:01:49 +02:00

151 lines
4.1 KiB
Python

from __future__ import annotations
from dataclasses import asdict, dataclass
from datetime import datetime
import hashlib
import re
from typing import Any
ENTRY_SEPARATOR = "=========="
DETAIL_RE = re.compile(
r"^- Your (?P<kind>Highlight|Note|Bookmark)"
r"(?: on page (?P<page>[^|]+?))?"
r"(?: (?:\| location|at location) (?P<location>[^|]+?))?"
r"(?: \| Added on (?P<date>.+))?$",
re.IGNORECASE,
)
@dataclass(frozen=True)
class Clipping:
id: str
title: str
kind: str
text: str
raw_detail: str
page: str | None = None
location: str | None = None
added_on: str | None = None
added_on_iso: str | None = None
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def parse_clippings_text(text: str) -> list[Clipping]:
entries = [entry.strip() for entry in text.split(ENTRY_SEPARATOR) if entry.strip()]
clippings: list[Clipping] = []
for entry in entries:
lines = entry.splitlines()
if len(lines) < 2:
continue
title = lines[0].strip()
raw_detail = lines[1].strip()
body = "\n".join(line.rstrip() for line in lines[2:]).strip()
match = DETAIL_RE.match(raw_detail)
if match:
kind = match.group("kind").lower()
page = _clean(match.group("page"))
location = _clean(match.group("location"))
added_on = _clean(match.group("date"))
else:
kind = "unknown"
page = None
location = None
added_on = None
clippings.append(
Clipping(
id=_stable_id(title, raw_detail, body),
title=title,
kind=kind,
text=body,
raw_detail=raw_detail,
page=page,
location=location,
added_on=added_on,
added_on_iso=_parse_kindle_date(added_on),
)
)
return clippings
def load_clippings(path: str) -> list[Clipping]:
with open(path, "r", encoding="utf-8-sig") as file:
return parse_clippings_text(file.read())
def _norm_text(text: str) -> str:
import unicodedata
t = unicodedata.normalize("NFKC", text or "").lower()
t = re.sub(r"-\s*\n\s*", "", t)
t = re.sub(r"[^\w\s]+", " ", t)
t = re.sub(r"\s+", " ", t).strip()
return t
def dedup_clippings(clippings: list[Clipping]) -> list[Clipping]:
seen: dict[tuple[str, str], Clipping] = {}
for c in clippings:
if c.kind not in {"highlight", "note"}:
# keep bookmarks etc. as is
key = (c.title, c.id)
if key not in seen:
seen[key] = c
continue
key = (c.title, _norm_text(c.text))
# keep earliest added_on, or first seen
if key not in seen:
seen[key] = c
else:
# if same norm text, keep the one with earlier added_on or shorter location range
prev = seen[key]
# prefer the one with more complete text (longer) or earlier date
if len(c.text) > len(prev.text):
seen[key] = c
return list(seen.values())
def clippings_to_jsonable(clippings: list[Clipping]) -> dict[str, Any]:
clippings = dedup_clippings(clippings)
return {
"format": "kindle-zotero-importer.clippings.v1",
"count": len(clippings),
"clippings": [clipping.to_dict() for clipping in clippings],
}
def _clean(value: str | None) -> str | None:
if value is None:
return None
cleaned = value.strip()
return cleaned or None
def _stable_id(title: str, raw_detail: str, text: str) -> str:
digest = hashlib.sha256(
f"{title}\n{raw_detail}\n{text}".encode("utf-8")
).hexdigest()
return digest[:16]
def _parse_kindle_date(value: str | None) -> str | None:
if not value:
return None
for fmt in (
"%A, %B %d, %Y %I:%M:%S %p",
"%A, %d %B %Y %H:%M:%S",
"%A, %B %d, %Y %H:%M:%S",
):
try:
return datetime.strptime(value, fmt).isoformat()
except ValueError:
pass
return None