diff --git a/README.md b/README.md index cef28bd..c4384e5 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Import Kindle `My Clippings.txt` highlights into Zotero as native annotations, d # Install -1. Download `kindle-zotero-importer.xpi` from [Releases](../../releases) (latest `0.6.19`, or `0.6.18-beta` for preview). +1. Download `kanzi.xpi` from [Releases](https://github.com/UtkuBilenDemir/kanzi/releases) (latest `0.6.20`, or `0.6.19-beta` for preview). 2. In Zotero: `Tools` → `Plugins` → `gear` → `Install Plugin From File…` → pick the `.xpi` → restart Zotero. 3. `Tools` → `Kanzi…` to open the manager. @@ -79,7 +79,7 @@ Incremental is default. Only new, changed or colour-changed highlights are repos # For developers / CLI ```sh -python scripts/build_plugin.py # → dist/kindle-zotero-importer.xpi +python scripts/build_plugin.py # → dist/kanzi.xpi python -m kindle_zotero_importer run "/path/to/My Clippings.txt" --workdir . --pretty python -m kindle_zotero_importer run ... --full # ignore incremental and reprocess all ``` @@ -88,7 +88,7 @@ PDF positioning uses Poppler (`pdftotext`, `pdftohtml`, `pdfinfo`) plus `qpdf` f # Releases -Stable releases are on `Releases` with `kindle-zotero-importer.xpi` and `updates.json` attached. Pre-releases (`beta`) are marked `Pre-release` on GitHub. Zotero auto-updates from `releases/latest/download/updates.json`. +Stable releases are on `Releases` with `kanzi.xpi` and `updates.json` attached. Pre-releases (`beta`) are marked `Pre-release` on GitHub. Zotero auto-updates from `releases/latest/download/updates.json`. # Support diff --git a/plugin/bootstrap.js b/plugin/bootstrap.js index edefe9c..911c989 100644 --- a/plugin/bootstrap.js +++ b/plugin/bootstrap.js @@ -701,12 +701,34 @@ var KindleZoteroImporter = { } const existingByAttachment = new Map(); + const normForDedup = (s) => String(s || "").toLowerCase().normalize("NFKC").replace(/-\s*\n\s*/g, "").replace(/[^\w\s]+/g, " ").replace(/\s+/g, " ").trim(); const getExistingAnnotations = async (attachment) => { if (!existingByAttachment.has(attachment.id)) { const annotations = attachment.getAnnotations ? attachment.getAnnotations() : []; const byFingerprint = new Map(); + const byNorm = new Map(); for (const annotation of annotations) { + const norm = normForDedup(annotation.annotationText || ""); + const dupNorm = byNorm.get(norm); + if (dupNorm && norm) { + // collapse identical highlights (e.g. My Clippings duplicates "forged its might…" x5) + const kept = dupNorm; + const keptHasComment = Boolean(kept.annotationComment); + const curHasComment = Boolean(annotation.annotationComment); + const duplicate = keptHasComment || !curHasComment ? annotation : kept; + const replacement = duplicate === annotation ? kept : annotation; + byNorm.set(norm, replacement); + // also update fingerprint map + const fpDup = this.existingAnnotationFingerprint(duplicate); + const fpKeep = this.existingAnnotationFingerprint(replacement); + byFingerprint.delete(fpDup); + byFingerprint.set(fpKeep, replacement); + if (!dryRun) await duplicate.eraseTx(); + results.removedDuplicates += 1; + continue; + } + if (norm) byNorm.set(norm, annotation); const fingerprint = this.existingAnnotationFingerprint(annotation); const kept = byFingerprint.get(fingerprint); if (!kept) { diff --git a/plugin/manifest.json b/plugin/manifest.json index ea8959a..aa1a732 100644 --- a/plugin/manifest.json +++ b/plugin/manifest.json @@ -1,14 +1,14 @@ { "manifest_version": 2, "name": "Kanzi", - "version": "0.6.19", + "version": "0.6.20", "description": "Kindle annotations to Zotero — My Clippings.txt → native annotations.", - "homepage_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer", + "homepage_url": "https://github.com/UtkuBilenDemir/kanzi", "author": "Utku Bilen Demir", "applications": { "zotero": { "id": "kindlezoteroimporter@utkubilen.de", - "update_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/latest/download/updates.json", + "update_url": "https://github.com/UtkuBilenDemir/kanzi/releases/latest/download/updates.json", "strict_min_version": "6.999", "strict_max_version": "10.0.*" } diff --git a/scripts/build_plugin.py b/scripts/build_plugin.py index 3815701..ceafd68 100644 --- a/scripts/build_plugin.py +++ b/scripts/build_plugin.py @@ -8,13 +8,15 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] PLUGIN_DIR = ROOT / "plugin" DIST_DIR = ROOT / "dist" -OUTPUT = DIST_DIR / "kindle-zotero-importer.xpi" +OUTPUT = DIST_DIR / "kanzi.xpi" +LEGACY_OUTPUT = DIST_DIR / "kindle-zotero-importer.xpi" def main() -> int: DIST_DIR.mkdir(exist_ok=True) - if OUTPUT.exists(): - OUTPUT.unlink() + for p in [OUTPUT, LEGACY_OUTPUT]: + if p.exists(): + p.unlink() manifest_path = PLUGIN_DIR / "manifest.json" manifest = json.loads(manifest_path.read_text(encoding="utf-8")) diff --git a/src/kindle_zotero_importer/clippings.py b/src/kindle_zotero_importer/clippings.py index 03f64ad..37ecfa0 100644 --- a/src/kindle_zotero_importer/clippings.py +++ b/src/kindle_zotero_importer/clippings.py @@ -80,7 +80,40 @@ def load_clippings(path: str) -> list[Clipping]: return parse_clippings_text(file.read()) +def _norm_text(text: str) -> str: + import unicodedata + + t = unicodedata.normalize("NFKC", text or "").lower() + t = re.sub(r"-\s*\n\s*", "", t) + t = re.sub(r"[^\w\s]+", " ", t) + t = re.sub(r"\s+", " ", t).strip() + return t + + +def dedup_clippings(clippings: list[Clipping]) -> list[Clipping]: + seen: dict[tuple[str, str], Clipping] = {} + for c in clippings: + if c.kind not in {"highlight", "note"}: + # keep bookmarks etc. as is + key = (c.title, c.id) + if key not in seen: + seen[key] = c + continue + key = (c.title, _norm_text(c.text)) + # keep earliest added_on, or first seen + if key not in seen: + seen[key] = c + else: + # if same norm text, keep the one with earlier added_on or shorter location range + prev = seen[key] + # prefer the one with more complete text (longer) or earlier date + if len(c.text) > len(prev.text): + seen[key] = c + return list(seen.values()) + + def clippings_to_jsonable(clippings: list[Clipping]) -> dict[str, Any]: + clippings = dedup_clippings(clippings) return { "format": "kindle-zotero-importer.clippings.v1", "count": len(clippings), diff --git a/updates.json b/updates.json index 7e7e4a2..4a4af8c 100644 --- a/updates.json +++ b/updates.json @@ -2,6 +2,15 @@ "addons": { "kindlezoteroimporter@utkubilen.de": { "updates": [ + { + "version": "0.6.20", + "update_link": "https://github.com/UtkuBilenDemir/kanzi/releases/download/v0.6.20/kanzi.xpi", + "applications": { + "zotero": { + "strict_min_version": "6.999" + } + } + }, { "version": "0.6.19", "update_link": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/download/v0.6.19/kindle-zotero-importer.xpi",