From aa3fc5ebc51bc563a35e10c9df5cb371bd029f12 Mon Sep 17 00:00:00 2001 From: Utku Bilen Demir <84389167+UtkuBilenDemir@users.noreply.github.com> Date: Thu, 15 Oct 2026 14:01:05 +0200 Subject: [PATCH] Kanzi 0.7.1: safer imports and resilient positioning --- README.md | 39 +- plugin/bootstrap.js | 514 +++++++++- plugin/manager.html | 967 ++++++++++++++++-- plugin/manifest.json | 2 +- pyproject.toml | 2 +- src/kindle_zotero_importer/cli.py | 189 +++- src/kindle_zotero_importer/epub_position.py | 100 +- src/kindle_zotero_importer/final_plan.py | 1 + src/kindle_zotero_importer/import_plan.py | 87 +- src/kindle_zotero_importer/pdf_position.py | 195 +++- src/kindle_zotero_importer/zotero_index.py | 15 + tests/fixtures/windows-smoke/My Clippings.txt | 15 + tests/fixtures/windows-smoke/checklist.txt | 36 + tests/fixtures/windows-smoke/kanzi-dummy.epub | Bin 0 -> 1143 bytes tests/fixtures/windows-smoke/kanzi-dummy.pdf | 33 + tests/fixtures/windows-smoke/make_fixtures.py | 93 ++ tests/test_fuzzy.py | 198 +++- tests/test_incremental.py | 114 ++- tests/test_manager_logic.js | 382 ++++++- 19 files changed, 2823 insertions(+), 159 deletions(-) create mode 100644 tests/fixtures/windows-smoke/My Clippings.txt create mode 100644 tests/fixtures/windows-smoke/checklist.txt create mode 100644 tests/fixtures/windows-smoke/kanzi-dummy.epub create mode 100644 tests/fixtures/windows-smoke/kanzi-dummy.pdf create mode 100644 tests/fixtures/windows-smoke/make_fixtures.py diff --git a/README.md b/README.md index b70d891..e506907 100644 --- a/README.md +++ b/README.md @@ -17,16 +17,39 @@ Import Kindle `My Clippings.txt` highlights into Zotero as native annotations, d 2. In Zotero: `Tools` → `Plugins` → `gear` → `Install Plugin From File…` → pick the `.xpi` → restart Zotero. 3. `Tools` → `Kanzi…` to open the manager. -Works with Zotero 7 to 10. Kanzi packages its importer source, but requires Python 3.11 or newer on the host. EPUB imports need no external positioning tools. PDF imports additionally require Poppler (`pdftotext`, `pdftohtml`, `pdfinfo`); `qpdf` is recommended for encrypted PDFs. macOS and Linux are tested. Windows requires setting the full Python executable path in `Settings` and installing the PDF tools separately. +Works with Zotero 7 to 10. Kanzi packages its importer source, but requires Python 3.11 or newer on the host. EPUB imports need no external positioning tools. PDF imports additionally require Poppler (`pdftotext`, `pdftohtml`, `pdfinfo`); `qpdf` is recommended for encrypted PDFs. Kanzi detects standard Python installations automatically on macOS, Linux and Windows. + +## Windows + +Open PowerShell and install the host tools: + +```powershell +winget install --id Python.Python.3.13 --exact +winget install --id oschwartz10612.Poppler --exact +winget install --id QPDF.QPDF --exact # Optional: handles copy-protected PDFs +``` + +Fully quit and reopen Zotero after installation so it receives the updated `PATH`, then install `kanzi.xpi` and open `Tools` → `Kanzi…`. Kanzi checks standard per-user and system Python locations as well as `PATH`; you should not need to enter an executable path manually. EPUB-only imports require Python but not Poppler. + +To verify the installation in a new PowerShell window: + +```powershell +python --version +pdftotext -v +pdftohtml -v +pdfinfo -v +qpdf --version # Optional +``` # Use 1. **Choose your file.** Click `Choose My Clippings.txt` and pick your *cumulative* `My Clippings.txt` from Kindle (`Documents/My Clippings.txt`). Keep the window open. You will see live progress. -2. **Check Integrated.** After the run, `Integrated` shows what was imported. Columns are `Highlight Text`, `Citekey`, `Added On`, `Integrated` and `Page`. Use the filter to find anything. -3. **Fix Conflicts.** If a Kindle title did not match a Zotero item, `Conflicts` shows it with up to three suggestions `citekey · title (score%)`. Click `Use` on the right one. Or type any citation key, Zotero key or item ID under `Use Custom`. Or `Ignore Title` to skip it forever. If you map `chabot2013` for one `Simondon` variant, it will offer to apply the same mapping to the other variants with that candidate. -4. **Re-import.** After you have fixed one or more titles, click `Re-import with saved overrides`. It reuses the last file, or asks for it. Only new or changed highlights are repositioned. Already integrated ones are skipped, so the second run is fast. Use `Retry unresolved attachments and positions` for prior failures without rebuilding successful annotations. Tick `Full re-import from scratch` only if you want to rebuild everything. -5. **Mappings.** See all titles you have approved or ignored, newest first, with date. Use `Delete` to send any entry back to `Conflicts` for rematching. -6. **Settings and Artifacts.** Change `Python` or `Zotero DB` paths and `Save Settings`. Or use `Open` or `Reveal` for any generated file, such as `mismatch-review.md` or `match-overrides.json`. +2. **Review before writing.** New highlights are grouped by title and selected by default. Untick individual highlights or whole titles, then press `Integrate selected`. Skipped highlights return on the next run. `Cancel` and closing the manager write nothing to Zotero and do not mark pending highlights as integrated. +3. **Check Integrated.** After integration, `Integrated` shows what was imported. Columns are `Highlight Text`, `Citekey`, `Added On`, `Integrated` and `Page`. Use the filter to find anything. +4. **Fix Conflicts.** If a Kindle title did not match a Zotero item, `Conflicts` shows one expandable row for that title with up to three suggestions `citekey · title (score%)`. Expand it to inspect individual highlights and positioning problems. Click `Use` on the right candidate, enter a citation key, Zotero key or item ID under `Use Custom`, or choose `Ignore Title`. Tables can be sorted and resized. +5. **Re-import.** After you have fixed one or more titles, click `Re-import with saved overrides`. It reuses the last file, or asks for it. Only new, changed, deferred or removed highlights are processed. Already integrated ones are skipped, so the second run is fast. Use `Retry unresolved attachments and positions` for prior failures without rebuilding successful annotations. Tick `Full re-import from scratch` only if you want to rebuild everything. +6. **Mappings.** See all titles you have approved or ignored, newest first, with date. Use `Delete` to send any entry back to `Conflicts` for rematching. +7. **Settings and Artifacts.** Change `Python` or `Zotero DB` paths and `Save Settings`. Or use `Open` or `Reveal` for any generated file, such as `mismatch-review.md` or `match-overrides.json`. Your choices are saved in `match-overrides.json` in Kanzi's private runtime folder inside your Zotero profile. Back it up or delete an entry to undo a mapping; it is excluded from Git because it contains your document titles. @@ -62,11 +85,11 @@ Both `[o]` and `[orange]` work, case is not sensitive, and space after the brack `Tools` → `Kanzi…` opens the manager. -**Summary.** Shows `clippings`, `unique titles`, `final annotations` and `conflicts`. Also `Match Statuses` and `Plan Statuses`. The run panel shows `Choose My Clippings.txt`, progress, stage and detail, and the `Full re-import` checkbox. +**Summary.** Shows `clippings`, `unique titles`, `final annotations` and `conflicts`. Also `Match Statuses` and `Plan Statuses`. The run panel shows `Choose My Clippings.txt`, progress, stage and detail, and the `Full re-import` checkbox. After positioning, the pending review lets you select exactly what is written to Zotero. **Integrated.** All annotations remembered in `import-plan.final.json`. Columns are `Kindle Title`, `Citekey`, `Highlight Text`, `Added On`, `Integrated` and `Page` with colour swatch. Filter at the top right; results are resizable and paginated. -**Conflicts.** Unresolved titles. Type is `Title match`, `Override suggestion` or `Attachment/position`. `Candidates` shows `N suggestion(s) — top: citekey (score%)`. Each candidate has a `Use` button. Below that is `Use Custom` and `Ignore Title`. Saving shows the `Re-import` bar. If you use `chabot2013` for one Simondon variant, it offers to apply to other variants with the same candidate. +**Conflicts.** One row per unresolved Kindle title, combining title-match and attachment/position statuses. Expand a row to inspect its highlights and positioning problems. `Candidates` shows `N suggestion(s) — top: citekey (score%)`. Each candidate has a `Use` button. Below that is `Use Custom` and `Ignore Title`. Saving shows the `Re-import` bar. Columns are sortable and resizable. **Mappings.** Persistent `match-overrides.json`. Columns are `Kindle Title`, `Resolution`, `Status`, `Count` and `Updated`. Use `Delete` to send it back to `Conflicts`. Filterable. diff --git a/plugin/bootstrap.js b/plugin/bootstrap.js index a157319..816c5de 100644 --- a/plugin/bootstrap.js +++ b/plugin/bootstrap.js @@ -1,4 +1,4 @@ -var KindleZoteroImporter = { +var Kanzi = { id: "kindle-zotero-importer@utkubilen.de", menuItems: [], runtimeProjectDir: "", @@ -16,7 +16,7 @@ var KindleZoteroImporter = { ); this.chromeHandle = addonManagerStartup.registerChrome( Services.io.newURI(this.rootURI + "manifest.json"), - [["content", "kindle-zotero-importer", ""]] + [["content", "kanzi", ""]] ); await Zotero.initializationPromise; await this.ensureRuntime(); @@ -81,7 +81,7 @@ var KindleZoteroImporter = { } const doc = win.document; - if (doc.getElementById("kindle-zotero-importer-manager")) { + if (doc.getElementById("kanzi-manager")) { return; } @@ -92,7 +92,7 @@ var KindleZoteroImporter = { } const managerItem = doc.createXULElement("menuitem"); - managerItem.setAttribute("id", "kindle-zotero-importer-manager"); + managerItem.setAttribute("id", "kanzi-manager"); managerItem.setAttribute("label", "Kanzi..."); managerItem.addEventListener("command", () => this.openManager()); popup.appendChild(managerItem); @@ -136,7 +136,14 @@ var KindleZoteroImporter = { }, getPythonPath() { - return this.getPref("pythonPath", "python3"); + const fallback = this.getDefaultPython(); + const value = this.getPref("pythonPath", fallback); + // Existing Windows installs may have "python3" stored from the old + // Unix-only default; that binary name does not exist on Windows. + if (this.isWindows() && value === "python3") { + return "python"; + } + return value || fallback; }, getZoteroDbPath() { @@ -149,12 +156,115 @@ var KindleZoteroImporter = { return this.getPref("zoteroStorageRoot", dataDir ? PathUtils.join(dataDir, "storage") : ""); }, + isWindows() { + try { + return Services.appinfo.OS === "WINNT"; + } catch (_error) { + return false; + } + }, + + getDefaultPython() { + return this.isWindows() ? "python" : "python3"; + }, + + getEnvironmentVariable(name) { + try { + return Cc["@mozilla.org/process/environment;1"] + .getService(Ci.nsIEnvironment) + .get(name) || ""; + } catch (_error) { + return ""; + } + }, + + joinWindowsPath(base, ...parts) { + if (!base) { + return ""; + } + return [String(base).replace(/[\\/]+$/, ""), ...parts] + .map((part, index) => index ? String(part).replace(/^[\\/]+|[\\/]+$/g, "") : part) + .join("\\"); + }, + + getWindowsPythonCandidates(configuredPath) { + const localAppData = this.getEnvironmentVariable("LOCALAPPDATA"); + const programFiles = this.getEnvironmentVariable("ProgramFiles"); + const programFilesX86 = this.getEnvironmentVariable("ProgramFiles(x86)"); + const userProfile = this.getEnvironmentVariable("USERPROFILE"); + const systemDrive = this.getEnvironmentVariable("SystemDrive") || "C:"; + const versions = ["314", "313", "312", "311"]; + const candidates = [configuredPath]; + for (const version of versions) { + candidates.push( + this.joinWindowsPath(localAppData, "Programs", "Python", `Python${version}`, "python.exe"), + this.joinWindowsPath(programFiles, `Python${version}`, "python.exe"), + this.joinWindowsPath(programFilesX86, `Python${version}-32`, "python.exe"), + this.joinWindowsPath(systemDrive, `Python${version}`, "python.exe") + ); + } + candidates.push( + this.joinWindowsPath(localAppData, "Python", "bin", "python.exe"), + this.joinWindowsPath(userProfile, "scoop", "apps", "python", "current", "python.exe"), + this.joinWindowsPath(userProfile, "miniconda3", "python.exe"), + this.joinWindowsPath(userProfile, "anaconda3", "python.exe") + ); + for (const directory of this.getEnvironmentVariable("PATH").split(";")) { + const cleanDirectory = directory.trim().replace(/^"|"$/g, ""); + if (cleanDirectory) { + candidates.push( + this.joinWindowsPath(cleanDirectory, "python.exe"), + this.joinWindowsPath(cleanDirectory, "python3.exe") + ); + } + } + return [...new Set(candidates.filter(Boolean))]; + }, + + async resolvePythonPath(configuredPath) { + let candidates; + if (this.isWindows()) { + candidates = this.getWindowsPythonCandidates(configuredPath); + } else if (Services.appinfo.OS === "Darwin") { + candidates = [configuredPath]; + candidates.push( + "/opt/homebrew/bin/python3", + "/opt/homebrew/opt/python@3.14/bin/python3.14", + "/opt/homebrew/opt/python@3.13/bin/python3.13", + "/opt/homebrew/opt/python@3.12/bin/python3.12", + "/opt/homebrew/opt/python@3.11/bin/python3.11", + "/usr/local/bin/python3", + "/usr/local/opt/python@3.14/bin/python3.14", + "/usr/local/opt/python@3.13/bin/python3.13", + "/usr/local/opt/python@3.12/bin/python3.12", + "/usr/local/opt/python@3.11/bin/python3.11", + "/usr/bin/python3" + ); + } else { + candidates = [configuredPath, "/usr/bin/python3", "/usr/local/bin/python3"]; + } + for (const candidate of candidates) { + if (!candidate || (!candidate.includes("/") && !candidate.includes("\\"))) continue; + try { + if (await IOUtils.exists(candidate)) return candidate; + } catch (_error) {} + } + return configuredPath || this.getDefaultPython(); + }, + + joinPath(...parts) { + if (typeof PathUtils !== "undefined" && PathUtils.join) { + return PathUtils.join(...parts); + } + return parts.join("/").replaceAll("//", "/"); + }, + getConfigPath() { - return this.getProjectDir() + "/plugin-config.json"; + return this.joinPath(this.getProjectDir(), "plugin-config.json"); }, artifactPath(relativePath) { - return this.getProjectDir() + "/" + relativePath; + return this.joinPath(this.getProjectDir(), ...String(relativePath).split("/")); }, async openArtifact(relativePath) { @@ -177,17 +287,6 @@ var KindleZoteroImporter = { zoteroDbPath: this.getZoteroDbPath(), zoteroStorageRoot: this.getZoteroStorageRoot(), }; - if (settings.pythonPath === "python3") { - const candidates = Services.appinfo.OS === "Darwin" - ? ["/opt/homebrew/bin/python3", "/usr/local/bin/python3", "/usr/bin/python3"] - : ["/usr/bin/python3", "/usr/local/bin/python3"]; - for (const candidate of candidates) { - if (await IOUtils.exists(candidate)) { - settings.pythonPath = candidate; - break; - } - } - } try { const text = await Zotero.File.getContentsAsync(this.getConfigPath()); const parsed = JSON.parse(text); @@ -200,12 +299,13 @@ var KindleZoteroImporter = { if (parsed.colorMap) settings.colourMap = parsed.colorMap; // also check colour-map.json for Python pipeline try { - const cmapText = await Zotero.File.getContentsAsync(this.getProjectDir() + "/colour-map.json"); + const cmapText = await Zotero.File.getContentsAsync(this.joinPath(this.getProjectDir(), "colour-map.json")); settings.colourMap = JSON.parse(cmapText); } catch (_e2) {} } catch (_error) { // The config file is optional; defaults/preferences are enough for first run. } + settings.pythonPath = await this.resolvePythonPath(settings.pythonPath); return settings; }, @@ -267,8 +367,8 @@ var KindleZoteroImporter = { const data = await this.loadManagerData(); const io = { plugin: this, settings, data }; this.managerWindow = this.getMainWindow().openDialog( - "chrome://kindle-zotero-importer/content/manager.html", - "kindle-zotero-importer-manager-window", + "chrome://kanzi/content/manager.html", + "kanzi-manager-window", "chrome,centerscreen,resizable", io ); @@ -276,6 +376,7 @@ var KindleZoteroImporter = { async loadManagerData() { return { + clippings: await this.readJsonArtifact("clippings.json"), matches: await this.readJsonArtifact("matches.json"), generatedOverrides: await this.readJsonArtifact("match-overrides.generated.json"), positionedPlan: await this.readJsonArtifact("import-plan.positioned.json"), @@ -305,11 +406,17 @@ var KindleZoteroImporter = { const index = payload.overrides.findIndex( (entry) => entry.clipping_title === resolution.clipping_title ); + const clippings = await this.readJsonArtifact("clippings.json"); + const sourceCount = this.countClippingsForTitle(clippings, resolution.clipping_title); + const clippingCount = sourceCount || resolution.clipping_count || resolution.review?.clipping_count || 0; const nowIso = new Date().toISOString(); const entry = { clipping_title: resolution.clipping_title, resolution: resolution.resolution, - review: resolution.review || { status: "manager", clipping_count: resolution.clipping_count || 0 }, + review: { + ...(resolution.review || { status: "manager" }), + clipping_count: clippingCount, + }, created_at: nowIso, updated_at: nowIso, }; @@ -329,6 +436,13 @@ var KindleZoteroImporter = { return entry; }, + countClippingsForTitle(payload, clippingTitle) { + return (payload?.clippings || []).reduce( + (count, clipping) => count + (clipping.title === clippingTitle ? 1 : 0), + 0 + ); + }, + async deleteOverride(clippingTitle) { if (!clippingTitle) { throw new Error("Missing clipping title for delete"); @@ -346,6 +460,23 @@ var KindleZoteroImporter = { return true; }, + async discardClip(clippingId) { + if (!clippingId) { + throw new Error("Missing clipping id for discard"); + } + const path = this.artifactPath("import-plan.final.json"); + const plan = JSON.parse(await Zotero.File.getContentsAsync(path)); + const discarded = new Set(plan.discarded_clipping_ids || []); + discarded.add(clippingId); + plan.discarded_clipping_ids = [...discarded].sort(); + // Also drop from deferred so it is never re-queued. + if (Array.isArray(plan.deferred_clipping_ids)) { + plan.deferred_clipping_ids = plan.deferred_clipping_ids.filter((id) => id !== clippingId); + } + await Zotero.File.putContentsAsync(path, JSON.stringify(plan, null, 2) + "\n"); + return true; + }, + async saveSettingsFromManager(newSettings) { const toSave = { projectDir: newSettings.projectDir || this.getProjectDir(), @@ -358,7 +489,7 @@ var KindleZoteroImporter = { // also write colour-map.json for Python pipeline (mirrors plugin-config) try { await Zotero.File.putContentsAsync( - this.getProjectDir() + "/colour-map.json", + this.joinPath(this.getProjectDir(), "colour-map.json"), JSON.stringify(newSettings.colourMap, null, 2) + "\n" ); } catch (_e) {} @@ -376,7 +507,7 @@ var KindleZoteroImporter = { async loadColourMap() { try { - const text = await Zotero.File.getContentsAsync(this.getProjectDir() + "/colour-map.json"); + const text = await Zotero.File.getContentsAsync(this.joinPath(this.getProjectDir(), "colour-map.json")); return JSON.parse(text); } catch (_e) {} try { @@ -416,7 +547,7 @@ var KindleZoteroImporter = { if (result !== fp.returnOK) { return null; } - return fp.file; + return this.filePickerResultToPath(fp.file); } catch (error) { Zotero.logError(error); return this.promptForPath( @@ -427,12 +558,32 @@ var KindleZoteroImporter = { } }, + filePickerResultToPath(file) { + if (!file) { + return null; + } + if (typeof file === "string") { + return file; + } + // Mozilla FilePicker returns an nsIFile; Python needs a native string path. + if (typeof file.path === "string" && file.path) { + return file.path; + } + try { + const str = String(file); + if (str && str !== "[object Object]") { + return str; + } + } catch (_error) {} + return null; + }, + async runPipeline(clippingsPath, settings, isFull = false, retryUnresolved = false) { - const python = settings.pythonPath; + const python = settings.pythonPath || this.getDefaultPython(); const projectDir = settings.projectDir; - const runner = projectDir + "/plugin_runner.py"; - const summaryOutput = projectDir + "/plugin-summary.json"; - const progressOutput = projectDir + "/plugin-progress.json"; + const runner = this.joinPath(projectDir, "plugin_runner.py"); + const summaryOutput = this.joinPath(projectDir, "plugin-summary.json"); + const progressOutput = this.joinPath(projectDir, "plugin-progress.json"); const args = [ runner, "run", @@ -458,12 +609,34 @@ var KindleZoteroImporter = { args.push("--retry-unresolved"); } - await this.exec(python, args); - return JSON.parse(await Zotero.File.getContentsAsync(summaryOutput)); + let execResult = null; + try { + execResult = await this.exec(python, args); + } catch (error) { + const installHint = this.isWindows() + ? `Install Python 3.11 or newer, fully restart Zotero, and let Kanzi detect python.exe. ` + : `Install Python 3.11 or newer and check the Python path in Kanzi Settings. `; + throw new Error( + `Python could not run the Kanzi pipeline (${python}): ${error && error.message ? error.message : error}. ` + + installHint + ); + } + try { + return JSON.parse(await Zotero.File.getContentsAsync(summaryOutput)); + } catch (error) { + const detail = execResult ? ` Python output: ${JSON.stringify(execResult).slice(0, 1000)}.` : ""; + throw new Error( + `Python pipeline did not write ${summaryOutput}.${detail} ` + + `Python: ${python}. Runner: ${runner}. ` + + `Make sure Python 3 is installed and reachable, then check the project directory is writable.` + ); + } }, async runManagedImport(manager) { const clippingsPath = await this.pickFile("Select Kindle My Clippings.txt", "*.txt"); + // NOTE: writing now happens only after the user reviews the pending + // highlights and presses Integrate (see integratePending). if (!clippingsPath) { return; } @@ -474,17 +647,23 @@ var KindleZoteroImporter = { async runManagedImportWithPath(manager, clippingsPath, isFullOverride, retryUnresolvedOverride) { try { + clippingsPath = this.filePickerResultToPath(clippingsPath) || clippingsPath; if (!clippingsPath) { throw new Error("No clippings file selected"); } const settings = await this.getRuntimeSettings(); - const progressPath = settings.projectDir + "/plugin-progress.json"; + const progressPath = this.joinPath(settings.projectDir, "plugin-progress.json"); // Determine full vs incremental from explicit arg or manager checkbox const isFull = typeof isFullOverride === "boolean" ? isFullOverride : (manager.getFullReimport ? manager.getFullReimport() : false); const retryUnresolved = typeof retryUnresolvedOverride === "boolean" ? retryUnresolvedOverride : (manager.getRetryUnresolved ? manager.getRetryUnresolved() : false); manager.beginRun(clippingsPath); + const finalPlanPath = this.joinPath(settings.projectDir, "import-plan.final.json"); + let previousFinalPlan = null; + try { + previousFinalPlan = JSON.parse(await Zotero.File.getContentsAsync(finalPlanPath)); + } catch (_backupError) {} try { await Zotero.File.removeIfExists(progressPath); } catch (_error) {} @@ -511,20 +690,142 @@ var KindleZoteroImporter = { const finalPlan = JSON.parse( await Zotero.File.getContentsAsync(summary.outputs.final_plan) ); - manager.updateProgress(96, "Writing annotations", "Saving annotations in Zotero"); - const results = await this.writeAnnotations(finalPlan, false, (completed, total) => { - const percent = total ? 96 + Math.floor((completed / total) * 4) : 100; - manager.updateProgress(percent, "Writing annotations", `${completed} of ${total} checked`); - }); - await this.reconcileWriterState(summary.outputs.final_plan, finalPlan, results); - const data = await this.loadManagerData(); - await manager.completeRun(summary, results, data); + const pendingCount = (finalPlan.annotations || []).length; + const pendingDeletions = (finalPlan.deletions || []).length; + if (!pendingCount && !pendingDeletions) { + // Nothing new — run the maintenance sweep and finish as before. + manager.updateProgress(96, "Writing annotations", "Checking existing annotations"); + const emptyResults = await this.writeAnnotations(finalPlan, false, (completed, total) => { + const percent = total ? 96 + Math.floor((completed / total) * 4) : 100; + manager.updateProgress(percent, "Writing annotations", `${completed} of ${total} checked`); + }); + await this.reconcileWriterState(summary.outputs.final_plan, finalPlan, emptyResults); + const emptyData = await this.loadManagerData(); + await manager.completeRun(summary, emptyResults, emptyData); + return; + } + // Pause before writing: let the user review and select what to integrate. + const fullDeltaIds = (finalPlan.annotations || []).map((a) => a.clipping_id); + // The pipeline has already written its generated history. Replace it with + // a safe ledger until Integrate is pressed, including if the window closes. + const deferredPlan = this.deferPendingPlan(previousFinalPlan, finalPlan, fullDeltaIds); + await Zotero.File.putContentsAsync( + summary.outputs.final_plan, + JSON.stringify(deferredPlan, null, 2) + "\n" + ); + this._pending = { + summary, + finalPlan, + finalPlanPath: summary.outputs.final_plan, + fullDeltaIds, + previousFinalPlan, + }; + manager.updateProgress( + 90, + "Review pending", + `${pendingCount} highlight(s) ready — select and press Integrate` + ); + manager.showPendingReview(summary, finalPlan); } catch (error) { Zotero.logError(error); manager.failRun(String(error && error.stack ? error.stack : error)); } }, + deferPendingPlan(previousPlan, generatedPlan, deferredIds) { + const plan = previousPlan + ? { ...previousPlan } + : { + ...generatedPlan, + annotations: [], + all_annotations: [], + deletions: [], + annotation_count: 0, + delta_annotation_count: 0, + }; + const deferred = new Set(plan.deferred_clipping_ids || []); + for (const id of deferredIds || []) deferred.add(id); + plan.deferred_clipping_ids = [...deferred].sort(); + return plan; + }, + + // Pure helper (tested): narrow a pipeline plan to the user's selection. + filterPlanForSelection(finalPlan, fullDeltaIds, selectedIds, previousPlan = null) { + const selected = new Set(selectedIds || []); + const unselected = new Set( + (fullDeltaIds || []).filter((id) => !selected.has(id)) + ); + const plan = { ...finalPlan }; + plan.annotations = (finalPlan.annotations || []).filter((a) => + selected.has(a.clipping_id) + ); + if (unselected.size && Array.isArray(finalPlan.all_annotations)) { + plan.all_annotations = finalPlan.all_annotations.filter( + (a) => !unselected.has(a.clipping_id) + ); + const present = new Set(plan.all_annotations.map((a) => a.clipping_id)); + for (const annotation of previousPlan?.all_annotations || previousPlan?.annotations || []) { + if (unselected.has(annotation.clipping_id) && !present.has(annotation.clipping_id)) { + plan.all_annotations.push(annotation); + present.add(annotation.clipping_id); + } + } + } + const deferred = new Set(previousPlan?.deferred_clipping_ids || []); + for (const id of selected) deferred.delete(id); + for (const id of unselected) deferred.add(id); + plan.deferred_clipping_ids = [...deferred].sort(); + plan.delta_annotation_count = plan.annotations.length; + if (Array.isArray(plan.all_annotations)) { + plan.annotation_count = plan.all_annotations.length; + } + return plan; + }, + + async integratePending(manager, selectedIds) { + const pending = this._pending; + if (!pending) { + throw new Error("No pending review to integrate"); + } + try { + const plan = this.filterPlanForSelection( + pending.finalPlan, + pending.fullDeltaIds, + selectedIds, + pending.previousFinalPlan + ); + manager.updateProgress(96, "Writing annotations", "Saving selected annotations in Zotero"); + const results = await this.writeAnnotations(plan, false, (completed, total) => { + const percent = total ? 96 + Math.floor((completed / total) * 4) : 100; + manager.updateProgress(percent, "Writing annotations", `${completed} of ${total} checked`); + }); + await this.reconcileWriterState(pending.finalPlanPath, plan, results); + this._pending = null; + const data = await this.loadManagerData(); + await manager.completeRun(pending.summary, results, data); + } catch (error) { + Zotero.logError(error); + manager.failRun(String(error && error.stack ? error.stack : error)); + } + }, + + async cancelPendingReview(manager) { + // The deferred ledger was written before the review appeared, so closing + // or cancelling never records pending highlights as integrated. + this._pending = null; + manager.hidePendingReview(); + manager.cancelRun("Review cancelled — nothing was written. Run again when ready."); + try { + const data = await this.loadManagerData(); + manager.data = data; + manager.rows = manager.conflictRows(); + manager.refreshRows(true); + manager.refreshTitleOverview(); + manager.refreshIntegrated(true); + manager.refreshMappings(true); + } catch (_reloadError) {} + }, + async exec(command, args) { if (!Zotero.Utilities || !Zotero.Utilities.Internal || !Zotero.Utilities.Internal.exec) { throw new Error("Zotero.Utilities.Internal.exec is unavailable in this Zotero build"); @@ -814,6 +1115,117 @@ var KindleZoteroImporter = { throw new Error("Existing annotation cannot be saved by this Zotero build"); }, + async saveRelatedItem(item) { + const options = { skipDateModifiedUpdate: true }; + if (item.save) { + await item.save(options); + return; + } + if (item.saveTx) { + await item.saveTx(options); + return; + } + throw new Error("Related item cannot be saved by this Zotero build"); + }, + + async addRelatedItemRelations(entries, dryRun, results) { + const pairs = new Map(); + for (const entry of entries || []) { + for (const target of entry.related_items || []) { + const sourceID = Number(entry.parent_item_id); + const targetID = Number(target.item_id); + if (!sourceID || !targetID || sourceID === targetID) continue; + const pairKey = [sourceID, targetID].sort((a, b) => a - b).join(":"); + if (!pairs.has(pairKey)) { + pairs.set(pairKey, { sourceID, targetID, citationKey: target.citation_key || "" }); + } + } + } + + for (const pair of pairs.values()) { + try { + const source = Zotero.Items.get(pair.sourceID); + const target = Zotero.Items.get(pair.targetID); + if (!source || !target) { + throw new Error(`Related item pair not found: ${pair.sourceID} -> ${pair.targetID}`); + } + if (dryRun) { + results.relationsPlanned += 1; + continue; + } + if (!source.addRelatedItem || !target.addRelatedItem) { + throw new Error("This Zotero build does not support addRelatedItem()"); + } + + let changed = false; + const addBothDirections = async () => { + const sourceChanged = source.addRelatedItem(target); + const targetChanged = target.addRelatedItem(source); + if (sourceChanged) await this.saveRelatedItem(source); + if (targetChanged) await this.saveRelatedItem(target); + changed = sourceChanged || targetChanged; + }; + if (Zotero.DB?.executeTransaction) { + await Zotero.DB.executeTransaction(addBothDirections); + } else { + await addBothDirections(); + } + if (changed) results.relationsAdded += 1; + else results.relationsSkipped += 1; + } catch (error) { + Zotero.logError(error); + results.relationFailures.push({ + parent_item_id: pair.sourceID, + related_item_id: pair.targetID, + citation_key: pair.citationKey, + message: String(error), + }); + } + } + }, + + async backfillRelationLinks(entries, dryRun, results) { + // Appends missing `Related: @key ` blocks to pre-existing + // annotations. Cross-library targets can never become Related items, + // so the comment link is the only visible connection for them. + for (const entry of entries || []) { + const targets = entry.related_items || []; + if (!targets.length || !entry.clipping_id || !entry.attachment_item_id) continue; + const block = "Related: " + targets.map( + (target) => `@${target.citation_key} ${target.link || ""}`.trim() + ).join(" · "); + try { + const attachment = Zotero.Items.get(entry.attachment_item_id); + if (!attachment) { + throw new Error(`Attachment not found: ${entry.attachment_item_id}`); + } + const annotations = attachment.getAnnotations ? attachment.getAnnotations() : []; + const tag = `kindle-id:${entry.clipping_id}`; + const existing = (annotations || []).find((annotation) => + this.existingTags(annotation).includes(tag) + ); + if (!existing) continue; + const comment = String(existing.annotationComment || ""); + if (comment.includes(block)) continue; + if (dryRun) { + results.commentLinksPlanned += 1; + continue; + } + existing.annotationComment = comment ? `${comment}\n\n${block}` : block; + await this.saveExistingAnnotation(existing); + results.commentLinksAdded += 1; + } catch (error) { + Zotero.logError(error); + results.relationFailures.push({ + parent_item_id: entry.parent_item_id, + related_item_id: (targets[0] || {}).item_id, + citation_key: (targets[0] || {}).citation_key, + message: `comment link: ${error}`, + }); + } + } + }, + async reconcileWriterState(path, plan, results) { const plannedIds = new Set((plan.annotations || []).map((entry) => entry.clipping_id)); const failedIds = new Set( @@ -861,6 +1273,12 @@ var KindleZoteroImporter = { blockedRecreateMissingTag: 0, recreateDeclined: 0, removedDuplicates: 0, + relationsAdded: 0, + relationsSkipped: 0, + relationsPlanned: 0, + commentLinksAdded: 0, + commentLinksPlanned: 0, + relationFailures: [], deletedForIncremental: 0, failedDeletionIds: [], maintenanceDeletedIds: [], @@ -1045,6 +1463,14 @@ var KindleZoteroImporter = { } } + const currentIDs = new Set((plan.annotations || []).map((entry) => entry.clipping_id)); + const relationEntries = (plan.all_annotations || []).filter( + (entry) => !currentIDs.has(entry.clipping_id) + ); + relationEntries.push(...successfulEntries.values()); + await this.addRelatedItemRelations(relationEntries, dryRun, results); + await this.backfillRelationLinks(relationEntries, dryRun, results); + if (!dryRun && plan.deletions && plan.deletions.length) { const deletionIds = new Set(plan.deletions); const alreadyDeletedIds = new Set(results.maintenanceDeletedIds); @@ -1146,7 +1572,7 @@ function uninstall(_data, _reason) {} async function startup(data, _reason) { try { - await KindleZoteroImporter.startup(data); + await Kanzi.startup(data); } catch (error) { Zotero.logError(error); } @@ -1154,7 +1580,7 @@ async function startup(data, _reason) { function shutdown(_data, _reason) { try { - KindleZoteroImporter.shutdown(); + Kanzi.shutdown(); } catch (error) { Zotero.logError(error); } @@ -1162,7 +1588,7 @@ function shutdown(_data, _reason) { function onMainWindowLoad({ window }) { try { - KindleZoteroImporter.onMainWindowLoad(window); + Kanzi.onMainWindowLoad(window); } catch (error) { Zotero.logError(error); } @@ -1170,7 +1596,7 @@ function onMainWindowLoad({ window }) { function onMainWindowUnload({ window }) { try { - KindleZoteroImporter.onMainWindowUnload(window); + Kanzi.onMainWindowUnload(window); } catch (error) { Zotero.logError(error); } diff --git a/plugin/manager.html b/plugin/manager.html index 69b94dc..4ef281a 100644 --- a/plugin/manager.html +++ b/plugin/manager.html @@ -50,14 +50,34 @@ table { width: 100%; border-collapse: collapse; background: var(--panel); table-layout: fixed; border: 1px solid var(--line); border-radius: var(--radius); overflow: hidden; box-shadow: var(--shadow); } th, td { text-align: left; vertical-align: top; border-bottom: 1px solid var(--line); padding: 10px 12px; font-size: 12.5px; overflow-wrap: anywhere; user-select: text; position: relative; } th { background: var(--panel-soft); color: var(--muted); position: sticky; top: 0; font-weight: 600; font-family: 'JetBrains Mono', monospace; text-transform: uppercase; letter-spacing: .06em; font-size: 10px; border-bottom: 1px solid var(--line); z-index: 1; } - th .resizer { position: absolute; top: 0; right: -3px; width: 6px; height: 100%; cursor: col-resize; user-select: none; } - th .resizer:hover { background: var(--accent); opacity: 0.15; } + th.sortable { cursor: pointer; user-select: none; } + th.sortable:hover { color: var(--text); background: #e9e9e9; } + th .sort-label { pointer-events: none; } + th .sort-arrow { pointer-events: none; color: var(--text); } + th .resizer { position: absolute; top: 0; right: -3px; width: 8px; height: 100%; cursor: col-resize; user-select: none; border-right: 2px solid transparent; } + th:hover .resizer { border-right-color: var(--line-strong); } + th .resizer:hover, th .resizer.dragging { border-right-color: var(--accent); background: rgba(17,17,17,.06); } + .expander { cursor: pointer; border: 1px solid var(--line); background: var(--panel); border-radius: 6px; width: 24px; height: 24px; line-height: 1; font-size: 13px; color: var(--text); padding: 0; } + .expander:hover { background: var(--panel-soft); border-color: var(--line-strong); } + tr.detail-row td { background: var(--panel-soft); border-bottom: 1px solid var(--line-strong); padding: 0; } + .clip-list { display: flex; flex-direction: column; gap: 8px; padding: 10px 12px 12px 38px; } + .clip-item { background: var(--panel); border: 1px solid var(--line); border-radius: var(--radius); padding: 8px 10px; font-size: 12px; box-shadow: var(--shadow); } + .clip-item .clip-head { display: flex; gap: 6px; align-items: center; flex-wrap: wrap; color: var(--muted); font-size: 10.5px; font-family: 'JetBrains Mono', monospace; text-transform: uppercase; letter-spacing: .05em; margin-bottom: 4px; } + .clip-item .clip-text { white-space: pre-wrap; word-break: break-word; line-height: 1.45; } + .clip-item .clip-problem { margin-top: 4px; color: var(--danger); font-size: 11px; } + .status-stack { display: flex; gap: 4px; flex-wrap: wrap; } th:nth-child(1), td:nth-child(1) { width: 10%; } th:nth-child(2), td:nth-child(2) { width: 9%; } th:nth-child(3), td:nth-child(3) { width: 6%; } th:nth-child(4), td:nth-child(4) { width: 24%; } th:nth-child(5), td:nth-child(5) { width: 21%; } th:nth-child(6), td:nth-child(6) { width: 30%; } + #tab-conflicts th:nth-child(1), #tab-conflicts td:nth-child(1) { width: 36px; } + #tab-conflicts th:nth-child(2), #tab-conflicts td:nth-child(2) { width: 13%; } + #tab-conflicts th:nth-child(3), #tab-conflicts td:nth-child(3) { width: 6%; } + #tab-conflicts th:nth-child(4), #tab-conflicts td:nth-child(4) { width: 24%; } + #tab-conflicts th:nth-child(5), #tab-conflicts td:nth-child(5) { width: 22%; } + #tab-conflicts th:nth-child(6), #tab-conflicts td:nth-child(6) { width: 30%; } #tab-integrated table { table-layout: fixed; } #tab-integrated th:nth-child(1), #tab-integrated td:nth-child(1) { width: 18%; } #tab-integrated th:nth-child(2), #tab-integrated td:nth-child(2) { width: 12%; } @@ -65,6 +85,15 @@ #tab-integrated th:nth-child(4), #tab-integrated td:nth-child(4) { width: 11%; } #tab-integrated th:nth-child(5), #tab-integrated td:nth-child(5) { width: 11%; } #tab-integrated th:nth-child(6), #tab-integrated td:nth-child(6) { width: 12%; } + .title-status-wrap { max-height: 520px; overflow: auto; border: 1px solid var(--line); border-radius: var(--radius); } + #title-status-table { border: 0; box-shadow: none; } + #title-status-table th:nth-child(1), #title-status-table td:nth-child(1) { width: 13%; } + #title-status-table th:nth-child(2), #title-status-table td:nth-child(2) { width: 12%; } + #title-status-table th:nth-child(3), #title-status-table td:nth-child(3), + #title-status-table th:nth-child(4), #title-status-table td:nth-child(4), + #title-status-table th:nth-child(5), #title-status-table td:nth-child(5) { width: 8%; } + #title-status-table th:nth-child(6), #title-status-table td:nth-child(6) { width: 34%; } + #title-status-table th:nth-child(7), #title-status-table td:nth-child(7) { width: 17%; } .pill { display: inline-block; background: var(--accent); color: white; border-radius: var(--pill-radius); padding: 3px 8px; font-size: 10px; font-weight: 600; letter-spacing: 0.06em; text-transform: uppercase; border: 1px solid var(--accent); } .row-actions button, .row-actions .row-btn { margin: 2px; padding: 6px 10px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--panel); color: var(--text); cursor: pointer; font-size: 12px; font-family: inherit; transition: all .12s; } .row-actions button:hover, .row-actions .row-btn:hover { border-color: var(--line-strong); background: var(--panel-soft); } @@ -93,6 +122,27 @@ .pagination { display: flex; align-items: center; justify-content: flex-end; gap: 8px; margin: 10px 0; color: var(--muted); font-size: 11px; } .pagination button, .pagination select { width: auto; padding: 5px 8px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--panel); color: var(--text); font: inherit; } .pagination button:disabled { color: var(--faint); background: var(--panel-soft); cursor: not-allowed; } + #pending-review { display: none; background: var(--panel); border: 2px solid var(--accent); border-radius: 14px; padding: 16px 18px; margin: 16px 0; box-shadow: var(--shadow-strong); } + #pending-review h2 { margin-top: 0; } + .pending-toolbar { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; margin: 10px 0; } + .pending-toolbar button { width: auto; padding: 6px 12px; border: 1px solid var(--line); border-radius: var(--radius); background: var(--panel); color: var(--text); cursor: pointer; font: inherit; font-size: 12px; } + .pending-toolbar button:hover { background: var(--panel-soft); border-color: var(--line-strong); } + .pending-group { border: 1px solid var(--line); border-radius: var(--radius); margin: 8px 0; overflow: hidden; } + .pending-group-head { display: flex; gap: 8px; align-items: center; padding: 8px 10px; background: var(--panel-soft); font-size: 12.5px; font-weight: 600; } + .pending-group-head input { width: auto; } + .pending-group-head .cite { font-family: 'JetBrains Mono', monospace; font-size: 11px; color: var(--muted); font-weight: 400; } + .pending-items { display: flex; flex-direction: column; } + .pending-item { display: flex; gap: 8px; align-items: flex-start; padding: 7px 10px 7px 34px; border-top: 1px solid var(--line); font-size: 12px; line-height: 1.45; } + .pending-item input { width: auto; margin-top: 3px; flex-shrink: 0; } + .pending-item .swatch { display: inline-block; width: 10px; height: 10px; border-radius: 2px; border: 1px solid #999; flex-shrink: 0; margin-top: 3px; } + .pending-item .pg { color: var(--muted); font-family: 'JetBrains Mono', monospace; font-size: 10.5px; flex-shrink: 0; } + .pending-item .txt { white-space: pre-wrap; word-break: break-word; } + .pending-footer { display: flex; gap: 8px; align-items: center; margin-top: 12px; flex-wrap: wrap; } + #pending-integrate { width: auto; padding: 10px 18px; border: 1px solid var(--accent); background: var(--accent); color: white; cursor: pointer; font: inherit; font-weight: 700; border-radius: var(--radius); } + #pending-integrate:disabled { opacity: 0.4; cursor: not-allowed; } + #pending-integrate:not(:disabled):hover { background: #1a1a1a; } + #pending-cancel { width: auto; padding: 10px 14px; border: 1px solid var(--line); background: var(--panel); color: var(--text); cursor: pointer; font: inherit; border-radius: var(--radius); } + #pending-cancel:hover { background: var(--panel-soft); } ::selection { background: var(--accent); color: white; }