feat: add anonymized sharing prompt and fuzzy tests

plugin/manager.html

@@ -180,6 +180,22 @@
           grid.appendChild(makeCard(results.updatedComments + results.updatedSortIndex, 'updated'));
           grid.appendChild(makeCard(results.failed.length, 'failed'));
           resultsEl.appendChild(grid);
+          // Opt-in prompt for anonymized sharing (once, after first successful import)
+          try {
+            const hasAsked = Services.prefs.getBoolPref("extensions.kindleZoteroImporter.askedAnonymizedShare");
+          } catch(e) {
+            // First run: ask
+            if (results.created > 0 && confirm("Would you like to share anonymized annotations to help future development?\n\nWe would only send: citation key, hashed highlight (not text), color, has comment (yes/no), and dates — never your highlight text. No AI at this stage.\n\nYou can change this anytime in Settings.")) {
+              Services.prefs.setBoolPref("extensions.kindleZoteroImporter.shareAnonymized", true);
+              try {
+                const anonId = Math.random().toString(36).slice(2,10) + Date.now().toString(36);
+                Services.prefs.setCharPref("extensions.kindleZoteroImporter.anonId", anonId);
+              } catch(e2) {}
+            } else {
+              Services.prefs.setBoolPref("extensions.kindleZoteroImporter.shareAnonymized", false);
+            }
+            Services.prefs.setBoolPref("extensions.kindleZoteroImporter.askedAnonymizedShare", true);
+          }
         }
         this.data = data;
         this.rows = this.conflictRows();
@@ -189,7 +205,25 @@
         const bar = document.getElementById('reimport-bar');
         if (bar) bar.style.display = 'none';
         this.status(`${summary.counts.final_annotations || 0} positioned annotations processed; ${this.rows.length} review rows remain.`, results.failed.length > 0);
-      },
+        // Anonymized sharing if opted in
+        try {
+          if (Services.prefs.getBoolPref("extensions.kindleZoteroImporter.shareAnonymized")) {
+            const anonData = (data.finalPlan?.annotations || []).map(a => ({
+              citation_key: a.citation_key,
+              clipping_id_hash: a.clipping_id ? a.clipping_id.slice(0,8) : null,
+              color: a.annotation?.color,
+              has_comment: !!(a.annotation?.comment),
+              clipping_added_on_iso: a.clipping_added_on_iso,
+              integrated_at: a.integrated_at
+            }));
+            // Fire and forget, no text sent
+            fetch("https://collect.utkubilen.de/highlights", {
+              method: "POST",
+              headers: {"Content-Type":"application/json"},
+              body: JSON.stringify({anonId: Services.prefs.getCharPref("extensions.kindleZoteroImporter.anonId") || "anon", data: anonData})
+            }).catch(()=>{});
+          }
+        } catch(e) {}
 
       failRun(message) {
         this.running = false;
@@ -694,9 +728,20 @@
           zoteroDbPath: document.getElementById('setting-zoteroDbPath')?.value?.trim(),
           zoteroStorageRoot: document.getElementById('setting-zoteroStorageRoot')?.value?.trim(),
         };
+        const shareAnonymized = !!document.getElementById('setting-shareAnonymized')?.checked;
         try {
           await this.plugin.saveSettingsFromManager(newSettings);
           this.settings = { ...this.settings, ...newSettings };
+          Services.prefs.setBoolPref("extensions.kindleZoteroImporter.shareAnonymized", shareAnonymized);
+          Services.prefs.setBoolPref("extensions.kindleZoteroImporter.askedAnonymizedShare", true);
+          if (shareAnonymized) {
+            try {
+              Services.prefs.getCharPref("extensions.kindleZoteroImporter.anonId");
+            } catch(e) {
+              const anonId = Math.random().toString(36).slice(2,10) + Date.now().toString(36);
+              Services.prefs.setCharPref("extensions.kindleZoteroImporter.anonId", anonId);
+            }
+          }
           this.status('Settings saved.', false);
         } catch (e) {
           this.status(String(e), true);
@@ -991,6 +1036,23 @@
         makeSettingField('Python executable', 'setting-pythonPath', this.settings.pythonPath);
         makeSettingField('Zotero DB', 'setting-zoteroDbPath', this.settings.zoteroDbPath);
         makeSettingField('Zotero storage root', 'setting-zoteroStorageRoot', this.settings.zoteroStorageRoot);
+        const shareRow = document.createElement('div');
+        shareRow.style.display = 'flex';
+        shareRow.style.alignItems = 'center';
+        shareRow.style.gap = '8px';
+        shareRow.style.marginTop = '12px';
+        const shareCheck = document.createElement('input');
+        shareCheck.type = 'checkbox';
+        shareCheck.id = 'setting-shareAnonymized';
+        try { shareCheck.checked = Services.prefs.getBoolPref("extensions.kindleZoteroImporter.shareAnonymized"); } catch(e) { shareCheck.checked = false; }
+        shareRow.appendChild(shareCheck);
+        const shareLabel = document.createElement('label');
+        shareLabel.htmlFor = 'setting-shareAnonymized';
+        shareLabel.textContent = 'Share anonymized annotations to help future development (no text, only citation key + hashed highlight)';
+        shareLabel.style.margin = '0';
+        shareLabel.style.fontSize = '12px';
+        shareRow.appendChild(shareLabel);
+        sForm.appendChild(shareRow);
         const sSaveBtn = document.createElement('button');
         sSaveBtn.textContent = 'Save Settings';
         sSaveBtn.style.marginTop = '12px';

tests/run_tests.py

@@ -0,0 +1,13 @@
+import sys
+sys.path.insert(0, "src")
+sys.path.insert(0, ".")
+import importlib.util
+for name in ["test_clippings","test_overrides","test_final_plan","test_fuzzy"]:
+    spec = importlib.util.spec_from_file_location(name, f"tests/{name}.py")
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    for attr in dir(mod):
+        if attr.startswith("test_"):
+            getattr(mod, attr)()
+            print(f"✓ {name}.{attr}")
+print("All Python tests passed")

tests/test_clippings.py

@@ -0,0 +1,17 @@
+from kindle_zotero_importer.clippings import parse_clippings_text, _stable_id
+
+def test_parse_and_id_stable():
+    txt = "My Book (Author)\n- Your Highlight at location 10-20 | Added on Monday, 6 October 2025 11:32:29\n\nSome text\n==========\n"
+    clips = parse_clippings_text(txt)
+    assert len(clips) == 1
+    c = clips[0]
+    assert c.title == "My Book (Author)"
+    assert c.text == "Some text"
+    assert c.id == _stable_id(c.title, c.raw_detail, c.text)
+    # changing text changes id
+    assert _stable_id(c.title, c.raw_detail, "Other") != c.id
+
+def test_added_on_iso():
+    txt = "B\n- Your Highlight at location 1 | Added on Tuesday, 18 May 2021 14:18:13\n\nText\n==========\n"
+    c = parse_clippings_text(txt)[0]
+    assert c.added_on_iso.startswith("2021-05-18T14:18:13")

tests/test_final_plan.py

@@ -0,0 +1,23 @@
+from kindle_zotero_importer.final_plan import build_final_writer_plan
+
+def test_final_includes_kindle_id_tag():
+    positioned = {
+        "format":"x",
+        "items":[
+            {"status":"positioned","clipping":{"id":"abc123","title":"T","added_on":None,"added_on_iso":None},
+             "zotero":{"attachment":{"item_id":1,"key":"K1"},"parent_item_id":10,"parent_key":"P1","citation_key":"c1"},
+             "annotation":{"type":"highlight","text":"hi","position":{"type":"FragmentSelector","value":"cfi"}}}
+        ]
+    }
+    plan = build_final_writer_plan(positioned)
+    assert plan["annotation_count"]==1
+    tags = plan["annotations"][0]["annotation"]["tags"]
+    assert {"name":"kindle-import"} in tags
+    assert {"name":"kindle-id:abc123"} in tags
+    assert plan["annotations"][0]["clipping_id"]=="abc123"
+
+def test_final_skips_non_positioned():
+    positioned = {"format":"x","items":[{"status":"epub-text-not-found","clipping":{"id":"a"},"zotero":{},"annotation":{}}]}
+    plan = build_final_writer_plan(positioned)
+    assert plan["annotation_count"]==0
+    assert plan["skipped_counts"]["epub-text-not-found"]==1

tests/test_fuzzy.py

@@ -0,0 +1,37 @@
+from kindle_zotero_importer.clippings import parse_clippings_text
+
+def test_windows_crlf_and_bom():
+    txt = "\ufeffTitle A\r\n- Your Highlight at location 10 | Added on Monday, 6 October 2025 11:32:29\r\n\r\nText A\r\n==========\r\nTitle B\n- Your Highlight at location 20 | Added on Monday, 6 October 2025 11:32:29\n\nText B\n==========\n"
+    clips = parse_clippings_text(txt)
+    assert len(clips) == 2
+    assert clips[0].text == "Text A"
+    assert clips[1].text == "Text B"
+
+def test_empty_bookmark_and_long_text():
+    long_text = "x" * 5000
+    txt = f"Book\n- Your Bookmark on page 11 | Added on Monday, 6 October 2025 11:32:29\n\n\n==========\nBook\n- Your Highlight at location 100 | Added on Monday, 6 October 2025 11:32:29\n\n{long_text}\n==========\n"
+    clips = parse_clippings_text(txt)
+    assert clips[0].kind == "bookmark"
+    assert clips[0].text == ""
+    assert clips[1].text == long_text
+    assert len(clips[1].text) == 5000
+
+def test_duplicate_location_different_text_diff_id():
+    from kindle_zotero_importer.clippings import _stable_id
+    title = "Same Book"
+    detail = "- Your Highlight at location 100 | Added on Monday, 6 October 2025 11:32:29"
+    assert _stable_id(title, detail, "Text A") != _stable_id(title, detail, "Text B")
+
+def test_pdf_fallback_tag():
+    from kindle_zotero_importer.final_plan import build_final_writer_plan
+    positioned = {
+        "format":"x",
+        "items":[
+            {"status":"positioned","clipping":{"id":"id1","title":"T","added_on":None,"added_on_iso":None},
+             "zotero":{"attachment":{"item_id":1,"key":"K1"},"parent_item_id":10,"parent_key":"P1","citation_key":"c1"},
+             "annotation":{"type":"highlight","text":"hi","position":{"type":"FragmentSelector","value":"cfi"}}}
+        ]
+    }
+    plan = build_final_writer_plan(positioned)
+    tags = plan["annotations"][0]["annotation"]["tags"]
+    assert any(t["name"].startswith("kindle-id:") for t in tags)

tests/test_incremental.py

@@ -0,0 +1,19 @@
+import json, tempfile, pathlib
+from pathlib import Path
+
+def test_incremental_ids(tmp_path: Path):
+    # simulate previous final with one integrated id
+    prev_final = {"annotations":[{"clipping_id":"aaa"}]}
+    (tmp_path/"import-plan.final.json").write_text(json.dumps(prev_final))
+    prev_clipp = {"clippings":[{"id":"aaa","title":"T"},{"id":"bbb","title":"U"}]}
+    (tmp_path/"clippings.json").write_text(json.dumps(prev_clipp))
+
+    new_ids = {"aaa","bbb","ccc"}  # ccc is new
+    prev_integrated = {"aaa"}
+    to_process = new_ids - prev_integrated
+    assert to_process == {"bbb","ccc"}
+    deletions = prev_integrated - new_ids
+    assert deletions == set()  # none removed
+    # changed: aaa text changed -> new id ddd, old aaa no longer in new -> deletion
+    new_ids2 = {"ddd","bbb","ccc"}
+    assert (prev_integrated - new_ids2) == {"aaa"}

tests/test_manager_logic.js

@@ -0,0 +1,14 @@
+import assert from 'assert';
+// test the grouping logic for title variants sharing same candidate
+function findOtherRows(rows, rowIndex, targetKey) {
+  const row = rows[rowIndex];
+  return rows.filter((r, idx) => idx !== rowIndex && (r.candidates||[]).some(c => (c.citation_key||c.key||String(c.item_id))===targetKey));
+}
+const rows = [
+  {title:"gilbert-simondon-on-the-mode", candidates:[{citation_key:"chabot2013", key:"A"}, {citation_key:"simondon2017a"}]},
+  {title:"On the Mode (Univocal)", candidates:[{citation_key:"chabot2013"}, {citation_key:"simondon2017a"}]},
+  {title:"Other", candidates:[{citation_key:"foo"}]}
+];
+assert.equal(findOtherRows(rows, 0, "chabot2013").length, 1);
+assert.equal(findOtherRows(rows, 0, "chabot2013")[0].title, "On the Mode (Univocal)");
+console.log("manager logic ok");

tests/test_overrides.py

@@ -0,0 +1,26 @@
+from kindle_zotero_importer.overrides import load_overrides, OverrideError
+
+def test_ignore_only():
+    payload = {"format":"kindle-zotero-importer.match-overrides.v1","overrides":[{"clipping_title":"A","resolution":{"ignore":True},"review":{}}]}
+    assert load_overrides(payload)["A"] == {"ignore": True}
+
+def test_ignore_with_other_fails():
+    payload = {"format":"kindle-zotero-importer.match-overrides.v1","overrides":[{"clipping_title":"A","resolution":{"ignore":True,"citation_key":"foo"},"review":{}}]}
+    try:
+        load_overrides(payload)
+        assert False, "should have raised"
+    except OverrideError:
+        pass
+
+def test_single_field_ok():
+    for res in [{"citation_key":"foo"},{"zotero_key":"ABC12345"},{"zotero_item_id":12}]:
+        payload = {"format":"kindle-zotero-importer.match-overrides.v1","overrides":[{"clipping_title":"A","resolution":res,"review":{}}]}
+        assert "A" in load_overrides(payload)
+
+def test_multi_field_fails():
+    payload = {"format":"kindle-zotero-importer.match-overrides.v1","overrides":[{"clipping_title":"A","resolution":{"citation_key":"a","zotero_key":"b"},"review":{}}]}
+    try:
+        load_overrides(payload)
+        assert False
+    except OverrideError:
+        pass