0.6.4: incremental id-based import, Mappings/Integrated, black iA Duo, resizable tables

- id-based incremental (new_ids - prev_integrated_ids) with --full flag, kindle-id tag, deletions handling
- Mappings tab with delete to return to Conflicts, sorted by updated_at newest first
- Integrated tab with Added On / Integrated dates and Citekey
- black over white iA Writer Duo theme, candidate-list per-row Use buttons, resizable/sortable tables
- Settings editable + save, Artifacts Open/Reveal, Re-import bar with last file reuse
- Fix override single-field validation and ignore cleanup

.gitignore

@@ -9,3 +9,9 @@ __pycache__/
 *.py[cod]
 .venv/
 .pytest_cache/
+.maschine/
+dist/
+plugin-summary.json
+plugin-progress.json
+plugin-config.json
+kindle-import-manager.html

README.md

@@ -100,3 +100,17 @@ PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a P
 ## Safety Rule
 
 Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes.
+
+## Zotero Plugin Wrapper
+
+The `plugin/` directory contains a hybrid Zotero plugin with one user-facing Tools action. It lets the user choose `My Clippings.txt`, runs the importer with visible progress, and writes positioned annotations through Zotero's native APIs. Unresolved cases are reported after the import; intermediate JSON artifacts are implementation details.
+
+Build the plugin package:
+
+```sh
+python scripts/build_plugin.py
+```
+
+Install `dist/kindle-zotero-importer.xpi` in Zotero via `Tools > Plugins`. Open `Tools > Kindle Zotero Importer...`, choose `My Clippings.txt`, and keep the manager open to see the current stage, percentage, elapsed time, and final annotation counts.
+
+The XPI manifest targets Zotero `6.999` through `10.0.*`.

match-overrides.json

@@ -1,6 +1,42 @@
 {
   "format": "kindle-zotero-importer.match-overrides.v1",
   "overrides": [
+    {
+      "clipping_title": "(Springer Series in Statistics) Trevor Hastie, Robert Tibshirani, Jerome Friedman - The elements of statistical learning Data mining, inference & predi",
+      "resolution": {
+        "ignore": true
+      },
+      "review": {
+        "status": "ambiguous",
+        "clipping_count": 2
+      },
+      "created_at": "2026-09-16T12:39:19.713Z",
+      "updated_at": "2026-09-16T12:39:19.713Z"
+    },
+    {
+      "clipping_title": "(Grundstudium Mathematik) Herbert Amann, Joachim Escher - Analysis 1-Birkhauser Basel (2006)",
+      "resolution": {
+        "ignore": true
+      },
+      "review": {
+        "status": "ambiguous",
+        "clipping_count": 3
+      },
+      "created_at": "2026-09-16T12:39:05.426Z",
+      "updated_at": "2026-09-16T12:39:08.496Z"
+    },
+    {
+      "clipping_title": "The Communist Hypothesis (Alain Badiou)",
+      "resolution": {
+        "citation_key": "badiou2015"
+      },
+      "review": {
+        "status": "epub-text-not-found",
+        "clipping_count": 1
+      },
+      "created_at": "2026-09-16T12:20:45.512Z",
+      "updated_at": "2026-09-16T12:20:45.512Z"
+    },
     {
       "clipping_title": "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)",
       "resolution": {
@@ -9,13 +45,26 @@
       "review": {
         "status": "matched",
         "clipping_count": 74
-      }
+      },
+      "created_at": "2026-09-16T12:19:40.753Z",
+      "updated_at": "2026-09-16T12:19:40.753Z"
+    },
+    {
+      "clipping_title": "On the Mode of Existence of Technical Objects (Univocal)",
+      "resolution": {
+        "ignore": true
+      },
+      "review": {
+        "status": "ambiguous",
+        "clipping_count": 16
+      },
+      "created_at": "2026-09-16T12:19:16.038Z",
+      "updated_at": "2026-09-16T12:19:16.038Z"
     },
     {
       "clipping_title": "(Kapitalismus und Schizophrenie 2) Guattari, Deleuze - Tausend Plateaus, Mille Plateaux-Les Editions de Minuit (1980)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -25,8 +74,7 @@
     {
       "clipping_title": "[Penguin Classics] Marcus Aurelius, Martin Hammond, Diskin Clay - Meditations (2006, Penguin Classics) - libgen.li",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -36,8 +84,7 @@
     {
       "clipping_title": "APuZ_Postdemokratie.2011",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -47,8 +94,7 @@
     {
       "clipping_title": "Arendt_2017_",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -58,8 +104,7 @@
     {
       "clipping_title": "Chantal Mouffe - Hegemonie und radikale Demokratie.-Passagen Verlag Ges.M.B.H (2000)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -69,8 +114,7 @@
     {
       "clipping_title": "Deleuze and Guattaris",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -80,8 +124,7 @@
     {
       "clipping_title": "Deleuze, Bartleby; or, the Formula",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -91,8 +134,7 @@
     {
       "clipping_title": "Deleuze, Guattari- A Thousand Plateaus - Bobo",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -102,8 +144,7 @@
     {
       "clipping_title": "Dewey_2008_",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -113,8 +154,7 @@
     {
       "clipping_title": "DRdZ 01 - Die Suche nach dem Auge der Welt (Robert Jordan)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -124,8 +164,7 @@
     {
       "clipping_title": "Ernesto Laclau - On Populist Reason-Verso (2005)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -135,8 +174,7 @@
     {
       "clipping_title": "Exhalation (Ted Chiang)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -146,8 +184,7 @@
     {
       "clipping_title": "Fire & Blood (A Song of Ice and Fire) (George R. R. Martin)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -157,7 +194,6 @@
     {
       "clipping_title": "Galloway_2004_Protocol",
       "resolution": {
-        "ignore": true,
         "citation_key": "galloway_2004"
       },
       "review": {
@@ -165,11 +201,20 @@
         "clipping_count": 14
       }
     },
+    {
+      "clipping_title": "gilbert-simondon-on-the-mode-of-existence-of-technical-objects",
+      "resolution": {
+        "citation_key": "simondon2017a"
+      },
+      "review": {
+        "status": "matched",
+        "clipping_count": 28
+      }
+    },
     {
       "clipping_title": "Gregs Tagebuch 13 - Eiskalt erwischt by Kinney Jeff, Schmidt Dietmar (z-lib.org)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -179,8 +224,7 @@
     {
       "clipping_title": "Hardt_Negri_2017_",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -190,8 +234,7 @@
     {
       "clipping_title": "Howaldt_Schwarz_2016_SI-DRIVE",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -201,8 +244,7 @@
     {
       "clipping_title": "Latzer Just (2020)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -212,8 +254,7 @@
     {
       "clipping_title": "Lem, Stanislaw - Solaris (Stanisław Lem)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -223,8 +264,7 @@
     {
       "clipping_title": "luhmann1981",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -234,8 +274,7 @@
     {
       "clipping_title": "manifestly_haraway_----_a_cyborg_manifesto_science_technology_and_socialist-feminism_in_the_...",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -245,8 +284,7 @@
     {
       "clipping_title": "Marcus Aurelius (Emperor of Rome), Martin Hammond, Diskin Clay - Meditations (2006, Penguin Books) - libgen.li",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -256,8 +294,7 @@
     {
       "clipping_title": "Martyr! -- Akbar, Kaveh -- null, null, 2024 -- RANDOM HOUSE US -- 9780593802359 -- 930ede856e2a2efb28505e3c4e99e222 -- Anna’s Archive (Kaveh Akbar)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -267,8 +304,7 @@
     {
       "clipping_title": "mew_band03",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -278,8 +314,7 @@
     {
       "clipping_title": "MiloradPaviç_-_ hazar_sözlügü",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -289,8 +324,7 @@
     {
       "clipping_title": "On Populist Reason (Laclau, Ernesto)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -300,8 +334,7 @@
     {
       "clipping_title": "paul-lafargue-the-right-to-be-lazy (Paul Lafargue)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -311,8 +344,7 @@
     {
       "clipping_title": "Quantitative Marxism - Paul Dunne Ed_",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -322,8 +354,7 @@
     {
       "clipping_title": "radikale demokratie comtesse, nonhoff etc",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -333,8 +364,7 @@
     {
       "clipping_title": "Rupert präsentiert: Ein echt wildes Abenteuer (Ruperts Tagebuch) (German Edition) (Kinney, Jeff)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -344,19 +374,27 @@
     {
       "clipping_title": "The City & the City - China Mieville",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
         "clipping_count": 8
       }
     },
+    {
+      "clipping_title": "The philosophy of Simondon_ between technology and individuation (Kirkpatrick, Graeme)",
+      "resolution": {
+        "citation_key": "chabot2013"
+      },
+      "review": {
+        "status": "epub-text-not-found",
+        "clipping_count": 1
+      }
+    },
     {
       "clipping_title": "The Republic (Plato)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -366,8 +404,7 @@
     {
       "clipping_title": "The Satanic Verses_ A Novel (Salman Rushdie)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -377,8 +414,7 @@
     {
       "clipping_title": "The Way of Kings (Sanderson, Brandon)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -388,8 +424,7 @@
     {
       "clipping_title": "Your Clippings",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -399,8 +434,7 @@
     {
       "clipping_title": "Your P2K Articles (2024-12-17) (P2K)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",
@@ -410,8 +444,7 @@
     {
       "clipping_title": "Your P2K Articles (2024-12-31) (P2K)",
       "resolution": {
-        "ignore": true,
+        "ignore": true
-        "citation_key": ""
       },
       "review": {
         "status": "unmatched",

plugin/README.md

@@ -0,0 +1,25 @@
+# Kindle Zotero Importer Zotero Plugin
+
+This is a hybrid Zotero plugin wrapper around the current Python importer. Zotero owns the user interface and writes native annotations through Zotero APIs; the Python project still performs parsing, matching, planning, EPUB positioning, and PDF positioning.
+
+## Use
+
+1. Install `kindle-zotero-importer.xpi` in Zotero through `Tools > Plugins`.
+2. Open `Tools > Kindle Zotero Importer...`.
+3. Select the updated Kindle `My Clippings.txt` in the manager.
+4. Follow the current stage, percentage, and elapsed time in the manager.
+5. Review created, existing, updated, failed, and unresolved counts when it completes.
+
+The plugin uses `match-overrides.json` from the project directory as the persistent source of title mappings and ignored titles. Generated JSON artifacts are still written to the project directory for audit and debugging.
+
+## Build XPI
+
+From the repository root:
+
+```sh
+python scripts/build_plugin.py
+```
+
+The output is `dist/kindle-zotero-importer.xpi`.
+
+The plugin manifest targets Zotero `6.999` through `10.0.*`.

plugin/manifest.json

@@ -0,0 +1,16 @@
+{
+  "manifest_version": 2,
+  "name": "Kindle Zotero Importer",
+  "version": "0.6.4",
+  "description": "Import Kindle My Clippings.txt highlights and notes into Zotero as native annotations.",
+  "homepage_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer",
+  "author": "Utku Bilen Demir",
+  "applications": {
+    "zotero": {
+      "id": "kindlezoteroimporter@utkubilen.de",
+      "update_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/latest/download/updates.json",
+      "strict_min_version": "6.999",
+      "strict_max_version": "10.0.*"
+    }
+  }
+}

plugin/prefs.js

@@ -0,0 +1,4 @@
+pref("extensions.kindleZoteroImporter.projectDir", "/Users/ubd/Library/Mobile Documents/iCloud~md~obsidian/Documents/rhizome/06_projects/UTI/kindle-zotero-importer");
+pref("extensions.kindleZoteroImporter.pythonPath", "/opt/homebrew/bin/python3");
+pref("extensions.kindleZoteroImporter.zoteroDbPath", "/Users/ubd/Zotero/zotero.sqlite");
+pref("extensions.kindleZoteroImporter.zoteroStorageRoot", "/Users/ubd/Zotero/storage");

plugin_runner.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parent
+SRC = ROOT / "src"
+
+if str(SRC) not in sys.path:
+    sys.path.insert(0, str(SRC))
+
+from kindle_zotero_importer.cli import main  # noqa: E402
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_canary_plugin.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import json
+import zipfile
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DIST_DIR = ROOT / "dist"
+OUTPUT = DIST_DIR / "kindle-zotero-importer-canary.xpi"
+
+
+def main() -> int:
+    DIST_DIR.mkdir(exist_ok=True)
+    if OUTPUT.exists():
+        OUTPUT.unlink()
+
+    manifest = {
+        "manifest_version": 2,
+        "name": "Kindle Zotero Importer Canary",
+        "version": "0.1.0",
+        "description": "Minimal install test for Kindle Zotero Importer.",
+        "homepage_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer",
+        "author": "Utku Bilen Demir",
+        "applications": {
+            "zotero": {
+                "id": "kindlezoteroimportercanary@utkubilen.de",
+                "update_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/latest/download/updates-canary.json",
+                "strict_min_version": "6.999",
+                "strict_max_version": "9.*",
+            }
+        },
+    }
+    bootstrap = """
+function install(data, reason) {}
+function uninstall(data, reason) {}
+async function startup(data, reason) {
+  await Zotero.initializationPromise;
+  Zotero.debug('Kindle Zotero Importer Canary started');
+}
+function shutdown(data, reason) {}
+""".lstrip()
+
+    prefs = 'pref("extensions.kindleZoteroImporterCanary.enabled", true);\n'
+
+    with zipfile.ZipFile(OUTPUT, "w", zipfile.ZIP_DEFLATED) as archive:
+        for name, content in [
+            ("bootstrap.js", bootstrap),
+            ("manifest.json", json.dumps(manifest, indent=2) + "\n"),
+            ("prefs.js", prefs),
+        ]:
+            info = zipfile.ZipInfo(name)
+            info.create_system = 3
+            info.external_attr = 0o644 << 16
+            archive.writestr(info, content, zipfile.ZIP_DEFLATED)
+
+    print(f"Built {OUTPUT}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/build_plugin.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import json
+import zipfile
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+PLUGIN_DIR = ROOT / "plugin"
+DIST_DIR = ROOT / "dist"
+OUTPUT = DIST_DIR / "kindle-zotero-importer.xpi"
+
+
+def main() -> int:
+    DIST_DIR.mkdir(exist_ok=True)
+    if OUTPUT.exists():
+        OUTPUT.unlink()
+
+    manifest_path = PLUGIN_DIR / "manifest.json"
+    manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
+    zotero_app = manifest.get("applications", {}).get("zotero", {})
+    required_manifest_fields = [
+        "manifest_version",
+        "name",
+        "version",
+        "description",
+    ]
+    missing_manifest_fields = [
+        field for field in required_manifest_fields if not manifest.get(field)
+    ]
+    missing_zotero_fields = [
+        field
+        for field in ["id", "strict_min_version", "strict_max_version"]
+        if not zotero_app.get(field)
+    ]
+    missing_files = [
+        str(path.relative_to(PLUGIN_DIR))
+        for path in [PLUGIN_DIR / "manifest.json", PLUGIN_DIR / "bootstrap.js"]
+        if not path.exists()
+    ]
+    if missing_manifest_fields or missing_zotero_fields or missing_files:
+        problems = []
+        if missing_manifest_fields:
+            problems.append(f"manifest fields: {', '.join(missing_manifest_fields)}")
+        if missing_zotero_fields:
+            problems.append(
+                f"applications.zotero fields: {', '.join(missing_zotero_fields)}"
+            )
+        if missing_files:
+            problems.append(f"files: {', '.join(missing_files)}")
+        raise SystemExit("Invalid plugin package; missing " + "; ".join(problems))
+
+    with zipfile.ZipFile(OUTPUT, "w", zipfile.ZIP_DEFLATED) as archive:
+        for path in PLUGIN_DIR.rglob("*"):
+            if path.is_file():
+                archive_name = str(path.relative_to(PLUGIN_DIR))
+                info = zipfile.ZipInfo(archive_name)
+                info.create_system = 3
+                info.external_attr = 0o644 << 16
+                archive.writestr(info, path.read_bytes(), zipfile.ZIP_DEFLATED)
+
+    with zipfile.ZipFile(OUTPUT) as archive:
+        names = set(archive.namelist())
+        if "manifest.json" not in names or "bootstrap.js" not in names:
+            raise SystemExit(
+                "Invalid XPI: manifest.json and bootstrap.js must be at archive root"
+            )
+
+    print(f"Built {OUTPUT}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

src/kindle_zotero_importer/cli.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import argparse
 import json
 import sys
+from pathlib import Path
 
 from .clippings import clippings_to_jsonable, load_clippings
 from .epub_position import add_epub_positions
@@ -141,6 +142,45 @@ def main(argv: list[str] | None = None) -> int:
         "--output", "-o", help="Write Markdown to this file instead of stdout"
     )
 
+    run_parser = subparsers.add_parser(
+        "run", help="Run the full importer pipeline for a Kindle My Clippings.txt file"
+    )
+    run_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt")
+    run_parser.add_argument(
+        "--workdir",
+        default=".",
+        help="Directory for generated importer artifacts",
+    )
+    run_parser.add_argument(
+        "--db", default="/Users/ubd/Zotero/zotero.sqlite", help="Path to zotero.sqlite"
+    )
+    run_parser.add_argument(
+        "--storage-root",
+        default="/Users/ubd/Zotero/storage",
+        help="Path to Zotero storage directory for storage: attachments",
+    )
+    run_parser.add_argument(
+        "--overrides",
+        default="match-overrides.json",
+        help="Path to reusable match overrides JSON, relative to workdir unless absolute",
+    )
+    run_parser.add_argument(
+        "--summary-output",
+        help="Write pipeline summary JSON to this path instead of stdout",
+    )
+    run_parser.add_argument(
+        "--progress-output",
+        help="Write machine-readable pipeline progress JSON to this path",
+    )
+    run_parser.add_argument(
+        "--pretty", action="store_true", help="Pretty-print JSON artifacts"
+    )
+    run_parser.add_argument(
+        "--full",
+        action="store_true",
+        help="Force full re-import from scratch, ignoring incremental state (processes all clippings)",
+    )
+
     args = parser.parse_args(argv)
 
     if args.command == "parse":
@@ -275,6 +315,236 @@ def main(argv: list[str] | None = None) -> int:
             sys.stdout.write(text)
         return 0
 
+    if args.command == "run":
+        workdir = Path(args.workdir).expanduser().resolve()
+        workdir.mkdir(parents=True, exist_ok=True)
+
+        progress_path = (
+            Path(args.progress_output).expanduser().resolve()
+            if args.progress_output
+            else None
+        )
+
+        def report_progress(percent: int, stage: str, detail: str) -> None:
+            if not progress_path:
+                return
+            temporary_path = progress_path.with_suffix(progress_path.suffix + ".tmp")
+            temporary_path.write_text(
+                json.dumps(
+                    {"percent": percent, "stage": stage, "detail": detail},
+                    ensure_ascii=False,
+                )
+                + "\n",
+                encoding="utf-8",
+            )
+            temporary_path.replace(progress_path)
+
+        def artifact(name: str) -> Path:
+            return workdir / name
+
+        overrides_path = Path(args.overrides).expanduser()
+        if not overrides_path.is_absolute():
+            overrides_path = workdir / overrides_path
+
+        report_progress(2, "Reading clippings", "Parsing My Clippings.txt")
+        new_clippings_list = load_clippings(args.clippings_file)
+        new_clippings = clippings_to_jsonable(new_clippings_list)
+        new_ids = {c["id"] for c in new_clippings.get("clippings", [])}
+
+        # Incremental state: previous clippings and previous integrated ids
+        is_incremental = not args.full
+        prev_clippings_ids: set[str] = set()
+        prev_integrated_ids: set[str] = set()
+        prev_final_plan = None
+        if is_incremental:
+            prev_clippings_path = artifact("clippings.json")
+            prev_final_path = artifact("import-plan.final.json")
+            if prev_clippings_path.exists():
+                try:
+                    prev_clippings_ids = {c["id"] for c in load_json(str(prev_clippings_path)).get("clippings", [])}
+                except Exception:
+                    prev_clippings_ids = set()
+            if prev_final_path.exists():
+                try:
+                    prev_final_plan = load_json(str(prev_final_path))
+                    prev_integrated_ids = {a.get("clipping_id") for a in prev_final_plan.get("annotations", []) if a.get("clipping_id")}
+                except Exception:
+                    prev_final_plan = None
+                    prev_integrated_ids = set()
+            # If no previous state, fall back to full
+            if not prev_clippings_ids and not prev_integrated_ids:
+                is_incremental = False
+
+        if is_incremental:
+            # New or previously not integrated (skipped) -> need to (re)process
+            to_process_ids = set(new_ids - prev_integrated_ids)
+            # Deletions: previously integrated but now absent (removed or changed text)
+            deletions_ids = sorted(prev_integrated_ids - new_ids)
+            # Also re-queue any previously integrated clippings whose title now has an override
+            # (mapping may have changed, need to move annotation). We need overrides dict for this.
+            # Load overrides early for this check (already loaded later, but do here for incremental decision)
+            # We will reload overrides here if not yet loaded
+            try:
+                _ov_for_inc = load_overrides(load_json(str(overrides_path))) if overrides_path.exists() else {}
+            except Exception:
+                _ov_for_inc = {}
+            for c in new_clippings_list:
+                if c.id in prev_integrated_ids and c.title in _ov_for_inc:
+                    to_process_ids.add(c.id)
+            # Also handle deleted overrides: if a previously integrated title's override was deleted,
+            # its annotations should be deleted. Detect via previous final plan's titles vs current overrides.
+            # For incremental, we don't have previous overrides, so we handle deletions via the main deletions list
+            # (IDs no longer present). For deleted overrides where ID still present but now unmatched,
+            # the match report will make them not positioned, and they will be in to_process but then skipped;
+            # we need to ensure they are deleted. We do this by checking if a previously integrated clipping's
+            # title is now not in overrides and would be unmatched -> treat as to_delete.
+            # Simplify: any prev_integrated clipping whose title is not in current new match's "matched" will be
+            # handled as not in to_process_ids? Actually it is still in new_ids, but we re-queued it above only if title in overrides.
+            # For now, rely on full re-import for deleted-override cleanup.
+            # Build filtered clippings for this delta run
+            filtered_list = [c for c in new_clippings_list if c.id in to_process_ids]
+            # If nothing to do and no deletions, still need to report but skip expensive steps
+            if not filtered_list and not deletions_ids:
+                report_progress(15, "Matching titles", "No new highlights — incremental skip")
+                # Still need to generate artifacts for consistency (empty delta)
+                zotero_index = build_zotero_index(args.db, args.storage_root)
+                overrides = None
+                if overrides_path.exists():
+                    overrides = load_overrides(load_json(str(overrides_path)))
+                clippings = new_clippings
+                matches = build_match_report(clippings, zotero_index, overrides)
+                override_skeleton = generate_override_skeleton(matches)
+                plan = {"format": "kindle-zotero-importer.import-plan.v1", "items": []}
+                positioned_plan = {"format": plan["format"], "items": []}
+                final_plan = {"format": FINAL_FORMAT, "source_format": positioned_plan.get("format"), "annotation_count": 0, "skipped_counts": {}, "annotations": [], "deletions": [], "is_incremental": True, "incremental_stats": {"new_clippings": len(new_ids), "prev_integrated": len(prev_integrated_ids), "to_process": 0, "deletions": 0}}
+                mismatch_review = build_mismatch_review(positioned_plan, matches)
+                # Will write artifacts below with empty delta
+            else:
+                report_progress(8, "Indexing Zotero", "Reading library items and attachments (incremental)")
+                zotero_index = build_zotero_index(args.db, args.storage_root)
+                overrides = None
+                if overrides_path.exists():
+                    overrides = load_overrides(load_json(str(overrides_path)))
+                # Only the delta clippings go through matching/positioning
+                delta_clippings = clippings_to_jsonable(filtered_list)
+                report_progress(15, "Matching titles", f"Matching {len(filtered_list)} new/pending highlights (incremental)")
+                matches = build_match_report(delta_clippings, zotero_index, overrides)
+                override_skeleton = generate_override_skeleton(matches)
+                # For artifact completeness, also build full matches for summary? Use delta matches for now
+                # But we need full matches for review? Keep delta
+                report_progress(22, "Building plan", "Selecting attachments (incremental)")
+                plan = build_import_plan(delta_clippings, zotero_index, matches)
+                report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files (incremental)")
+                epub_plan = add_epub_positions(plan)
+                report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files (incremental)")
+                positioned_plan = add_pdf_positions(epub_plan)
+                report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
+                final_plan = build_final_writer_plan(positioned_plan)
+                # Attach deletions for writer
+                final_plan["deletions"] = deletions_ids
+                final_plan["is_incremental"] = True
+                final_plan["incremental_stats"] = {
+                    "new_clippings": len(new_ids),
+                    "prev_integrated": len(prev_integrated_ids),
+                    "to_process": len(to_process_ids),
+                    "deletions": len(deletions_ids),
+                }
+                mismatch_review = build_mismatch_review(positioned_plan, matches)
+                # For outputs, clippings should be the full new set (for next diff), but plan artifacts are delta
+                clippings = new_clippings
+        else:
+            # Full re-import
+            report_progress(8, "Indexing Zotero", "Reading library items and attachments")
+            zotero_index = build_zotero_index(args.db, args.storage_root)
+            overrides = None
+            if overrides_path.exists():
+                overrides = load_overrides(load_json(str(overrides_path)))
+            clippings = new_clippings
+            report_progress(15, "Matching titles", "Matching Kindle titles to Zotero items")
+            matches = build_match_report(clippings, zotero_index, overrides)
+            override_skeleton = generate_override_skeleton(matches)
+            report_progress(22, "Building plan", "Selecting attachments")
+            plan = build_import_plan(clippings, zotero_index, matches)
+            report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files")
+            epub_plan = add_epub_positions(plan)
+            report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files; this is usually the longest stage")
+            positioned_plan = add_pdf_positions(epub_plan)
+            report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
+            final_plan = build_final_writer_plan(positioned_plan)
+            final_plan["deletions"] = []
+            final_plan["is_incremental"] = False
+            mismatch_review = build_mismatch_review(positioned_plan, matches)
+
+        indent = 2 if args.pretty else None
+        outputs = {
+            "clippings": artifact("clippings.json"),
+            "zotero_index": artifact("zotero-index.json"),
+            "matches": artifact("matches.json"),
+            "generated_overrides": artifact("match-overrides.generated.json"),
+            "import_plan": artifact("import-plan.json"),
+            "epub_plan": artifact("import-plan.epub.json"),
+            "positioned_plan": artifact("import-plan.positioned.json"),
+            "final_plan": artifact("import-plan.final.json"),
+            "mismatch_review": workdir / "docs" / "mismatch-review.md",
+        }
+        outputs["mismatch_review"].parent.mkdir(parents=True, exist_ok=True)
+
+        report_progress(95, "Saving results", "Writing import artifacts")
+        for key, payload in [
+            ("clippings", clippings),
+            ("zotero_index", zotero_index),
+            ("matches", matches),
+            ("generated_overrides", override_skeleton),
+            ("import_plan", plan),
+            ("epub_plan", epub_plan),
+            ("positioned_plan", positioned_plan),
+            ("final_plan", final_plan),
+        ]:
+            with open(outputs[key], "w", encoding="utf-8") as file:
+                json.dump(payload, file, ensure_ascii=False, indent=indent)
+                file.write("\n")
+
+        with open(outputs["mismatch_review"], "w", encoding="utf-8") as file:
+            file.write(mismatch_review)
+
+        status_counts: dict[str, int] = {}
+        for item in positioned_plan.get("items", []):
+            status = item.get("status", "unknown")
+            status_counts[status] = status_counts.get(status, 0) + 1
+
+        match_counts: dict[str, int] = {}
+        for match in matches.get("matches", []):
+            status = match.get("status", "unknown")
+            match_counts[status] = match_counts.get(status, 0) + 1
+
+        summary = {
+            "format": "kindle-zotero-importer.pipeline-summary.v1",
+            "workdir": str(workdir),
+            "overrides_path": str(overrides_path),
+            "outputs": {key: str(path) for key, path in outputs.items()},
+            "counts": {
+                "clippings": len(clippings.get("clippings", [])),
+                "unique_titles": len(matches.get("matches", [])),
+                "final_annotations": final_plan.get("annotation_count", 0),
+                "match_statuses": match_counts,
+                "plan_statuses": status_counts,
+                "skipped_final": final_plan.get("skipped_counts", {}),
+                "is_incremental": final_plan.get("is_incremental", False),
+                "incremental_stats": final_plan.get("incremental_stats", {}),
+                "deletions": len(final_plan.get("deletions", [])),
+            },
+        }
+        json_text = json.dumps(summary, ensure_ascii=False, indent=2)
+        if args.summary_output:
+            with open(args.summary_output, "w", encoding="utf-8") as file:
+                file.write(json_text)
+                file.write("\n")
+        else:
+            sys.stdout.write(json_text)
+            sys.stdout.write("\n")
+        report_progress(100, "Pipeline complete", "Ready to import annotations into Zotero")
+        return 0
+
     parser.error(f"unknown command: {args.command}")
     return 2
 

src/kindle_zotero_importer/epub_position.py

@@ -4,7 +4,6 @@ from dataclasses import dataclass
 from html import unescape
 from html.parser import HTMLParser
 import posixpath
-import re
 import zipfile
 import xml.etree.ElementTree as ET
 from typing import Any
@@ -65,8 +64,8 @@ def _position_item(
 
     updated = dict(item)
     annotation = dict(updated["annotation"])
-    annotation["position"] = position
+    annotation["position"] = position["selector"]
-    annotation["sortIndex"] = _epub_sort_index(position["value"])
+    annotation["sortIndex"] = position["sortIndex"]
     updated["annotation"] = annotation
     updated["status"] = "positioned"
     return updated
@@ -97,7 +96,7 @@ def extract_epub_text_nodes(path: str) -> list[TextNode]:
         return nodes
 
 
-def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, str] | None:
+def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, Any] | None:
     haystack = " ".join(node.text for node in text_nodes)
     normalized_haystack, haystack_map = _normalize_with_map(haystack)
     normalized_quote, _ = _normalize_with_map(quote)
@@ -134,9 +133,12 @@ def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, str] | No
             f"{end_node.cfi_parent_path}/{end_node.text_step}:{end_offset})"
         )
     return {
-        "type": "FragmentSelector",
+        "selector": {
-        "conformsTo": "http://www.idpf.org/epub/linking/cfi/epub-cfi.html",
+            "type": "FragmentSelector",
-        "value": value,
+            "conformsTo": "http://www.idpf.org/epub/linking/cfi/epub-cfi.html",
+            "value": value,
+        },
+        "sortIndex": f"{spine_step:05d}|{raw_start:08d}",
     }
 
 
@@ -257,13 +259,6 @@ def _node_at(nodes: list[TextNode], offset: int) -> tuple[TextNode | None, int]:
     return None, 0
 
 
-def _epub_sort_index(cfi: str) -> str:
-    numbers = [int(number) for number in re.findall(r"/([0-9]+)", cfi)]
-    spine = numbers[1] if len(numbers) > 1 else 0
-    content = numbers[-1] if numbers else 0
-    return f"{spine:05d}|{content:08d}"
-
-
 def _with_problem(item: dict[str, Any], problem: str) -> dict[str, Any]:
     updated = dict(item)
     updated["status"] = problem

src/kindle_zotero_importer/final_plan.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import datetime
 from typing import Any
 
 
@@ -20,10 +21,14 @@ def build_final_writer_plan(positioned_plan: dict[str, Any]) -> dict[str, Any]:
                 skipped.get("positioned-missing-writer-fields", 0) + 1
             )
             continue
+        clipping = item["clipping"]
         annotations.append(
             {
-                "clipping_id": item["clipping"]["id"],
+                "clipping_id": clipping["id"],
-                "clipping_title": item["clipping"]["title"],
+                "clipping_title": clipping["title"],
+                "clipping_added_on": clipping.get("added_on"),
+                "clipping_added_on_iso": clipping.get("added_on_iso"),
+                "integrated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
                 "attachment_item_id": attachment["item_id"],
                 "attachment_key": attachment["key"],
                 "parent_item_id": item["zotero"]["parent_item_id"],
@@ -37,11 +42,22 @@ def build_final_writer_plan(positioned_plan: dict[str, Any]) -> dict[str, Any]:
                     "pageLabel": annotation.get("pageLabel") or "",
                     "sortIndex": annotation.get("sortIndex"),
                     "position": annotation["position"],
-                    "tags": annotation.get("tags") or [{"name": "kindle-import"}],
+                    "tags": [
+                        {"name": "kindle-import"},
+                        {"name": f"kindle-id:{clipping['id']}"},
+                    ],
                 },
             }
         )
 
+    annotations.sort(
+        key=lambda entry: (
+            entry.get("attachment_item_id") or 0,
+            entry["annotation"].get("sortIndex") or "",
+            entry.get("clipping_id") or "",
+        )
+    )
+
     return {
         "format": FINAL_FORMAT,
         "source_format": positioned_plan.get("format"),

src/kindle_zotero_importer/pdf_position.py

@@ -4,12 +4,31 @@ from dataclasses import dataclass
 import hashlib
 import os
 import re
+import shutil
 import subprocess
 import tempfile
 import xml.etree.ElementTree as ET
 from typing import Any
 
 
+_TOOL_PATHS = {
+    "pdftotext": ("/Users/ubd/bin/pdftotext", "/opt/homebrew/bin/pdftotext"),
+    "pdftohtml": ("/Users/ubd/bin/pdftohtml", "/opt/homebrew/bin/pdftohtml"),
+    "pdfinfo": ("/Users/ubd/bin/pdfinfo", "/opt/homebrew/bin/pdfinfo"),
+    "qpdf": ("/Users/ubd/bin/qpdf", "/opt/homebrew/bin/qpdf"),
+}
+
+
+def _tool(name: str) -> str:
+    resolved = shutil.which(name)
+    if resolved:
+        return resolved
+    for candidate in _TOOL_PATHS.get(name, ()):
+        if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
+            return candidate
+    return name
+
+
 @dataclass(frozen=True)
 class PDFWord:
     text: str
@@ -117,7 +136,7 @@ def _position_item(
 
 def extract_pdf_text_pages(path: str) -> list[str]:
     result = subprocess.run(
-        ["pdftotext", "-enc", "UTF-8", path, "-"],
+        [_tool("pdftotext"), "-enc", "UTF-8", path, "-"],
         check=True,
         capture_output=True,
         text=True,
@@ -157,7 +176,7 @@ def _decrypted_pdf_path(
     base_name = os.path.splitext(os.path.basename(path))[0] or "document"
     output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf")
     subprocess.run(
-        ["qpdf", "--decrypt", path, output_path],
+        [_tool("qpdf"), "--decrypt", path, output_path],
         check=True,
         capture_output=True,
         text=True,
@@ -194,7 +213,7 @@ def extract_pdf_page_xml(path: str, page_index: int) -> PDFPageXML:
     page_number = page_index + 1
     result = subprocess.run(
         [
-            "pdftohtml",
+            _tool("pdftohtml"),
             "-f",
             str(page_number),
             "-l",
@@ -302,7 +321,7 @@ def _pdf_page_size(
     path: str, page_number: int, fallback_width: float, fallback_height: float
 ) -> tuple[float, float]:
     result = subprocess.run(
-        ["pdfinfo", "-f", str(page_number), "-l", str(page_number), path],
+        [_tool("pdfinfo"), "-f", str(page_number), "-l", str(page_number), path],
         check=True,
         capture_output=True,
         text=True,