From 830b27e5109be0f47353667cec5d59e56ff3fb7c Mon Sep 17 00:00:00 2001 From: Utku Bilen Demir <84389167+UtkuBilenDemir@users.noreply.github.com> Date: Wed, 16 Sep 2026 14:44:31 +0200 Subject: [PATCH] 0.6.4: incremental id-based import, Mappings/Integrated, black iA Duo, resizable tables - id-based incremental (new_ids - prev_integrated_ids) with --full flag, kindle-id tag, deletions handling - Mappings tab with delete to return to Conflicts, sorted by updated_at newest first - Integrated tab with Added On / Integrated dates and Citekey - black over white iA Writer Duo theme, candidate-list per-row Use buttons, resizable/sortable tables - Settings editable + save, Artifacts Open/Reveal, Re-import bar with last file reuse - Fix override single-field validation and ignore cleanup --- .gitignore | 6 + README.md | 14 + plugin/README.md | 25 + plugin/bootstrap.js | 1275 +++++++++++++++++++ plugin/manager.html | 1199 +++++++++++++++++ plugin/manifest.json | 16 + plugin/prefs.js | 4 + plugin_runner.py | 17 + scripts/build_canary_plugin.py | 62 + scripts/build_plugin.py | 74 ++ src/kindle_zotero_importer/cli.py | 270 ++++ src/kindle_zotero_importer/epub_position.py | 23 +- src/kindle_zotero_importer/final_plan.py | 22 +- src/kindle_zotero_importer/pdf_position.py | 27 +- 14 files changed, 3013 insertions(+), 21 deletions(-) create mode 100644 plugin/README.md create mode 100644 plugin/bootstrap.js create mode 100644 plugin/manager.html create mode 100644 plugin/manifest.json create mode 100644 plugin/prefs.js create mode 100644 plugin_runner.py create mode 100644 scripts/build_canary_plugin.py create mode 100644 scripts/build_plugin.py diff --git a/.gitignore b/.gitignore index 1f733a9..06dfa7b 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,9 @@ __pycache__/ *.py[cod] .venv/ .pytest_cache/ +.maschine/ +dist/ +plugin-summary.json +plugin-progress.json +plugin-config.json +kindle-import-manager.html diff --git a/README.md b/README.md index af8e82a..3d43919 100644 --- a/README.md +++ b/README.md @@ -100,3 +100,17 @@ PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a P ## Safety Rule Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes. + +## Zotero Plugin Wrapper + +The `plugin/` directory contains a hybrid Zotero plugin with one user-facing Tools action. It lets the user choose `My Clippings.txt`, runs the importer with visible progress, and writes positioned annotations through Zotero's native APIs. Unresolved cases are reported after the import; intermediate JSON artifacts are implementation details. + +Build the plugin package: + +```sh +python scripts/build_plugin.py +``` + +Install `dist/kindle-zotero-importer.xpi` in Zotero via `Tools > Plugins`. Open `Tools > Kindle Zotero Importer...`, choose `My Clippings.txt`, and keep the manager open to see the current stage, percentage, elapsed time, and final annotation counts. + +The XPI manifest targets Zotero `6.999` through `10.0.*`. diff --git a/plugin/README.md b/plugin/README.md new file mode 100644 index 0000000..9b0210f --- /dev/null +++ b/plugin/README.md @@ -0,0 +1,25 @@ +# Kindle Zotero Importer Zotero Plugin + +This is a hybrid Zotero plugin wrapper around the current Python importer. Zotero owns the user interface and writes native annotations through Zotero APIs; the Python project still performs parsing, matching, planning, EPUB positioning, and PDF positioning. + +## Use + +1. Install `kindle-zotero-importer.xpi` in Zotero through `Tools > Plugins`. +2. Open `Tools > Kindle Zotero Importer...`. +3. Select the updated Kindle `My Clippings.txt` in the manager. +4. Follow the current stage, percentage, and elapsed time in the manager. +5. Review created, existing, updated, failed, and unresolved counts when it completes. + +The plugin uses `match-overrides.json` from the project directory as the persistent source of title mappings and ignored titles. Generated JSON artifacts are still written to the project directory for audit and debugging. + +## Build XPI + +From the repository root: + +```sh +python scripts/build_plugin.py +``` + +The output is `dist/kindle-zotero-importer.xpi`. + +The plugin manifest targets Zotero `6.999` through `10.0.*`. diff --git a/plugin/bootstrap.js b/plugin/bootstrap.js new file mode 100644 index 0000000..8121a00 --- /dev/null +++ b/plugin/bootstrap.js @@ -0,0 +1,1275 @@ +var KindleZoteroImporter = { + id: "kindle-zotero-importer@utkubilen.de", + menuItems: [], + defaultProjectDir: + "/home/user/Library/Mobile Documents/iCloud~md~obsidian/Documents/rhizome/06_projects/UTI/kindle-zotero-importer", + defaultClippingsPath: + "/home/user/Library/Mobile Documents/com~apple~CloudDocs/Projects/test/My Clippings.txt", + prefBranch: "extensions.kindleZoteroImporter.", + rootURI: null, + managerWindow: null, + chromeHandle: null, + + async startup(data) { + this.rootURI = data.rootURI; + const addonManagerStartup = Cc["@mozilla.org/addons/addon-manager-startup;1"].getService( + Ci.amIAddonManagerStartup + ); + this.chromeHandle = addonManagerStartup.registerChrome( + Services.io.newURI(this.rootURI + "manifest.json"), + [["content", "kindle-zotero-importer", ""]] + ); + await Zotero.initializationPromise; + this.addToAllWindows(); + }, + + shutdown() { + this.removeMenuItems(); + if (this.managerWindow && !this.managerWindow.closed) { + this.managerWindow.close(); + } + this.managerWindow = null; + if (this.chromeHandle) { + this.chromeHandle.destruct(); + this.chromeHandle = null; + } + }, + + onMainWindowLoad(win) { + this.addMenuItems(win); + }, + + onMainWindowUnload(_win) { + this.menuItems = []; + }, + + addMenuItems(win) { + win = win || Zotero.getMainWindow(); + if (!win || !win.document) { + return; + } + + const doc = win.document; + if (doc.getElementById("kindle-zotero-importer-manager")) { + return; + } + + const popup = doc.getElementById("menu_ToolsPopup"); + if (!popup) { + Zotero.debug("Kindle Zotero Importer: Tools menu not found"); + return; + } + + const managerItem = doc.createXULElement("menuitem"); + managerItem.setAttribute("id", "kindle-zotero-importer-manager"); + managerItem.setAttribute("label", "Kindle Zotero Importer..."); + managerItem.addEventListener("command", () => this.openManager()); + popup.appendChild(managerItem); + this.menuItems.push(managerItem); + + }, + + addToAllWindows() { + const windows = Zotero.getMainWindows ? Zotero.getMainWindows() : [Zotero.getMainWindow()]; + for (const win of windows) { + if (win && win.ZoteroPane) { + this.addMenuItems(win); + } + } + }, + + removeMenuItems() { + for (const item of this.menuItems) { + if (item && item.parentNode) { + item.parentNode.removeChild(item); + } + } + this.menuItems = []; + }, + + getPref(name, fallback) { + try { + return Services.prefs.getCharPref(this.prefBranch + name); + } catch (_error) { + return fallback; + } + }, + + setPref(name, value) { + Services.prefs.setCharPref(this.prefBranch + name, String(value)); + }, + + getProjectDir() { + return this.getPref("projectDir", this.defaultProjectDir); + }, + + getPythonPath() { + return this.getPref("pythonPath", "/opt/homebrew/bin/python3"); + }, + + getZoteroDbPath() { + return this.getPref("zoteroDbPath", "/home/user/Zotero/zotero.sqlite"); + }, + + getZoteroStorageRoot() { + return this.getPref("zoteroStorageRoot", "/home/user/Zotero/storage"); + }, + + getConfigPath() { + return this.getProjectDir() + "/plugin-config.json"; + }, + + artifactPath(relativePath) { + return this.getProjectDir() + "/" + relativePath; + }, + + async openArtifact(relativePath) { + const path = this.artifactPath(relativePath); + try { + await Zotero.File.getContentsAsync(path); + this.revealFile(path); + } catch (_error) { + this.alert( + "Kindle Zotero Importer", + `File not found yet:\n${path}\n\nRun Import Kindle Clippings first to generate review artifacts.` + ); + } + }, + + async getRuntimeSettings() { + const settings = { + projectDir: this.getProjectDir(), + pythonPath: this.getPythonPath(), + zoteroDbPath: this.getZoteroDbPath(), + zoteroStorageRoot: this.getZoteroStorageRoot(), + }; + try { + const text = await Zotero.File.getContentsAsync(this.getConfigPath()); + const parsed = JSON.parse(text); + for (const key of Object.keys(settings)) { + if (parsed[key]) { + settings[key] = parsed[key]; + } + } + } catch (_error) { + // The config file is optional; defaults/preferences are enough for first run. + } + return settings; + }, + + getMainWindow() { + return Zotero.getMainWindow(); + }, + + getPickerWindow() { + return Services.wm.getMostRecentWindow("navigator:browser") || this.getMainWindow(); + }, + + alert(title, message) { + Services.prompt.alert(this.getMainWindow(), title, message); + }, + + notify(message, success) { + try { + const progressWindow = new Zotero.ProgressWindow({ closeOnClick: true }); + progressWindow.changeHeadline("Kindle Zotero Importer"); + const icon = success ? "chrome://zotero/skin/tick.png" : "chrome://zotero/skin/cross.png"; + const item = new progressWindow.ItemProgress(icon, message); + item.setProgress(100); + progressWindow.show(); + progressWindow.startCloseTimer(success ? 5000 : 10000); + return progressWindow; + } catch (_error) { + return null; + } + }, + + createProgress(message) { + try { + const progressWindow = new Zotero.ProgressWindow({ closeOnClick: false }); + progressWindow.changeHeadline("Kindle Zotero Importer"); + progressWindow.progress = new progressWindow.ItemProgress( + "chrome://zotero/skin/treesource-collection.png", + message + ); + progressWindow.progress.setProgress(0); + progressWindow.show(); + return progressWindow; + } catch (_error) { + return null; + } + }, + + promptForPath(title, message, fallback) { + const value = { value: fallback || "" }; + const ok = Services.prompt.prompt(this.getMainWindow(), title, message, value, null, {}); + return ok && value.value ? value.value : null; + }, + + async configure() { + const current = { + projectDir: this.getProjectDir(), + pythonPath: this.getPythonPath(), + zoteroDbPath: this.getZoteroDbPath(), + zoteroStorageRoot: this.getZoteroStorageRoot(), + }; + try { + const configPath = this.getConfigPath(); + await Zotero.File.putContentsAsync(configPath, JSON.stringify(current, null, 2) + "\n"); + this.alert( + "Kindle Zotero Importer Configuration", + `Configuration file written to:\n${configPath}\n\nEdit this file to change paths. The import dialog will show the active settings before running.` + ); + this.revealFile(configPath); + } catch (error) { + Zotero.logError(error); + this.alert("Kindle Zotero Importer Failed", String(error && error.stack ? error.stack : error)); + } + }, + + async openManager() { + if (this.managerWindow && !this.managerWindow.closed) { + this.managerWindow.focus(); + return; + } + const settings = await this.getRuntimeSettings(); + const data = await this.loadManagerData(); + const io = { plugin: this, settings, data }; + this.managerWindow = this.getMainWindow().openDialog( + "chrome://kindle-zotero-importer/content/manager.html", + "kindle-zotero-importer-manager-window", + "chrome,centerscreen,resizable", + io + ); + }, + + async loadManagerData() { + return { + matches: await this.readJsonArtifact("matches.json"), + generatedOverrides: await this.readJsonArtifact("match-overrides.generated.json"), + positionedPlan: await this.readJsonArtifact("import-plan.positioned.json"), + finalPlan: await this.readJsonArtifact("import-plan.final.json"), + summary: await this.readJsonArtifact("plugin-summary.json"), + persistentOverrides: await this.readJsonArtifact("match-overrides.json"), + }; + }, + + async openStaticDashboard() { + const settings = await this.getRuntimeSettings(); + const data = { + matches: await this.readJsonArtifact("matches.json"), + generatedOverrides: await this.readJsonArtifact("match-overrides.generated.json"), + positionedPlan: await this.readJsonArtifact("import-plan.positioned.json"), + finalPlan: await this.readJsonArtifact("import-plan.final.json"), + summary: await this.readJsonArtifact("plugin-summary.json"), + persistentOverrides: await this.readJsonArtifact("match-overrides.json"), + }; + const outputPath = this.artifactPath("kindle-import-manager.html"); + await Zotero.File.putContentsAsync(outputPath, this.buildManagerHtml(settings, data)); + this.openFile(outputPath); + }, + + async loadOverridesFile() { + const path = this.artifactPath("match-overrides.json"); + try { + return JSON.parse(await Zotero.File.getContentsAsync(path)); + } catch (_error) { + return { format: "kindle-zotero-importer.match-overrides.v1", overrides: [] }; + } + }, + + async saveOverride(resolution) { + if (!resolution || !resolution.clipping_title) { + throw new Error("Missing clipping title for override"); + } + const payload = await this.loadOverridesFile(); + if (!Array.isArray(payload.overrides)) { + payload.overrides = []; + } + const index = payload.overrides.findIndex( + (entry) => entry.clipping_title === resolution.clipping_title + ); + const nowIso = new Date().toISOString(); + const entry = { + clipping_title: resolution.clipping_title, + resolution: resolution.resolution, + review: resolution.review || { status: "manager", clipping_count: resolution.clipping_count || 0 }, + created_at: nowIso, + updated_at: nowIso, + }; + if (index >= 0) { + const prev = payload.overrides[index]; + entry.created_at = prev.created_at || prev.updated_at || nowIso; + entry.updated_at = nowIso; + payload.overrides[index] = entry; + } else { + payload.overrides.push(entry); + } + payload.overrides.sort((a, b) => (b.updated_at || b.created_at || "").localeCompare(a.updated_at || a.created_at || "") || a.clipping_title.localeCompare(b.clipping_title)); + await Zotero.File.putContentsAsync( + this.artifactPath("match-overrides.json"), + JSON.stringify(payload, null, 2) + "\n" + ); + return entry; + }, + + async deleteOverride(clippingTitle) { + if (!clippingTitle) { + throw new Error("Missing clipping title for delete"); + } + const payload = await this.loadOverridesFile(); + const before = payload.overrides.length; + payload.overrides = payload.overrides.filter((e) => e.clipping_title !== clippingTitle); + if (payload.overrides.length === before) { + throw new Error(`No override found for: ${clippingTitle}`); + } + await Zotero.File.putContentsAsync( + this.artifactPath("match-overrides.json"), + JSON.stringify(payload, null, 2) + "\n" + ); + return true; + }, + + async saveSettingsFromManager(newSettings) { + const toSave = { + projectDir: newSettings.projectDir || this.getProjectDir(), + pythonPath: newSettings.pythonPath || this.getPythonPath(), + zoteroDbPath: newSettings.zoteroDbPath || this.getZoteroDbPath(), + zoteroStorageRoot: newSettings.zoteroStorageRoot || this.getZoteroStorageRoot(), + }; + for (const [key, value] of Object.entries(toSave)) { + this.setPref(key, value); + } + await Zotero.File.putContentsAsync( + this.getConfigPath(), + JSON.stringify(toSave, null, 2) + "\n" + ); + return toSave; + }, + + async readJsonArtifact(relativePath) { + try { + return JSON.parse(await Zotero.File.getContentsAsync(this.artifactPath(relativePath))); + } catch (_error) { + return null; + } + }, + + escapeHtml(value) { + return String(value ?? "") + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """) + .replaceAll("'", "'"); + }, + + fileHref(path) { + return "file://" + encodeURI(path).replaceAll("#", "%23"); + }, + + artifactLink(label, relativePath) { + const path = this.artifactPath(relativePath); + return `${this.escapeHtml(label)}`; + }, + + countStatuses(items, key = "status") { + const counts = {}; + for (const item of items || []) { + const status = item[key] || "unknown"; + counts[status] = (counts[status] || 0) + 1; + } + return counts; + }, + + statusCards(counts) { + return Object.entries(counts || {}) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([status, count]) => `
${this.escapeHtml(count)}${this.escapeHtml(status)}
`) + .join("\n"); + }, + + conflictRows(data) { + const rows = []; + for (const match of data.matches?.matches || []) { + if (["matched", "ignored"].includes(match.status)) { + continue; + } + const candidates = (match.candidates || []) + .slice(0, 3) + .map((candidate) => `${candidate.citation_key || candidate.key || candidate.item_id}: ${candidate.title || ""}`) + .join("; "); + rows.push({ + type: "title-match", + status: match.status, + title: match.clipping_title, + count: match.clipping_count, + detail: candidates || "No candidates", + snippet: this.overrideSnippet(match.clipping_title, match.clipping_count, match.status, match.candidates?.[0]), + }); + } + + for (const override of data.generatedOverrides?.overrides || []) { + const candidateText = (override.candidates || []) + .slice(0, 3) + .map((candidate) => `${candidate.citation_key || candidate.key || candidate.item_id}: ${candidate.title || ""}`) + .join("; "); + rows.push({ + type: "override-suggestion", + status: override.review?.status || "review", + title: override.clipping_title, + count: override.review?.clipping_count || "", + detail: candidateText || override.notes || "Needs override decision", + snippet: JSON.stringify({ + clipping_title: override.clipping_title, + resolution: override.resolution || { ignore: false, citation_key: "" }, + review: override.review, + }, null, 2), + }); + } + + for (const item of data.positionedPlan?.items || []) { + const status = item.status || "unknown"; + if (["positioned", "ignored-title"].includes(status)) { + continue; + } + rows.push({ + type: "positioning/import-plan", + status, + title: item.clipping?.title || "(missing clipping title)", + count: 1, + detail: (item.problems || []).join("; ") || item.annotation?.text || "Needs review", + snippet: "Fix source data, attachment availability, or matching override; then rerun Import Kindle Clippings.", + }); + } + + return rows; + }, + + overrideSnippet(title, clippingCount, status, candidate) { + const resolution = candidate?.citation_key + ? { ignore: false, citation_key: candidate.citation_key } + : { ignore: true }; + return JSON.stringify({ + clipping_title: title, + resolution, + review: { + status, + clipping_count: clippingCount, + }, + }, null, 2); + }, + + groupedRows(rows) { + const groups = { + "Title Matching": [], + "Override Suggestions": [], + "Attachment / Positioning": [], + }; + for (const row of rows) { + if (row.type === "title-match") { + groups["Title Matching"].push(row); + } else if (row.type === "override-suggestion") { + groups["Override Suggestions"].push(row); + } else { + groups["Attachment / Positioning"].push(row); + } + } + return groups; + }, + + tableRows(rows, limit = 200) { + if (!rows.length) { + return 'No conflicts found in this group.'; + } + return rows.slice(0, limit).map((row) => ` + + ${this.escapeHtml(row.type)} + ${this.escapeHtml(row.status)} + ${this.escapeHtml(row.count)} + ${this.escapeHtml(row.title)} + ${this.escapeHtml(row.detail)} +
${this.escapeHtml(row.snippet || "")}
+ `).join("\n"); + }, + + conflictTables(rows) { + const groups = this.groupedRows(rows); + return Object.entries(groups).map(([label, groupRows]) => ` +
+

${this.escapeHtml(label)} ${this.escapeHtml(groupRows.length)} rows

+ + + ${this.tableRows(groupRows)} +
TypeStatusCountKindle TitleDetails / Candidate DifferencesOverride / Fix Snippet
+
`).join("\n"); + }, + + buildManagerHtml(settings, data) { + const matchCounts = this.countStatuses(data.matches?.matches || []); + const planCounts = data.positionedPlan?.status_counts || this.countStatuses(data.positionedPlan?.items || []); + const finalCount = data.finalPlan?.annotation_count || 0; + const skippedCounts = data.finalPlan?.skipped_counts || {}; + const conflicts = this.conflictRows(data); + const generatedAt = new Date().toLocaleString(); + return ` + + + + Kindle Zotero Importer Manager + + + + +

Kindle Zotero Importer Manager

+
Generated ${this.escapeHtml(generatedAt)} from ${this.escapeHtml(settings.projectDir)}
+ + + +
+ This dashboard is generated from importer artifacts. Copy override snippets into match-overrides.json for now, then rerun Import Kindle Clippings. Direct in-dashboard editing is the next step. +
+ +

Settings

+ + +

Run Import

+
+ Run from Zotero: Tools > Import Kindle Clippings.... The import now shows preflight settings, progress, final counts, existing-comment updates, existing-sort-order repairs, and any guarded delete/recreate confirmations for kindle-import annotations. +
+ +

Import Summary

+
+
${this.escapeHtml(data.summary?.counts?.clippings || "-")}clippings
+
${this.escapeHtml(data.summary?.counts?.unique_titles || "-")}unique titles
+
${this.escapeHtml(finalCount)}final annotations
+
${this.escapeHtml(conflicts.length)}conflict rows
+
+ +

Match Statuses

+
${this.statusCards(matchCounts) || '
No match data yet
'}
+ +

Plan Statuses

+
${this.statusCards(planCounts) || '
No plan data yet
'}
+ +

Final Plan Skips

+
${this.statusCards(skippedCounts) || '
No skipped final annotations
'}
+ +

Artifacts

+ + +

Conflicts and Exceptions

+

Rows include unresolved title matches, generated override suggestions, attachment problems, and positioning failures. Use search to filter titles, statuses, citekeys, and candidate text. Showing first 200 rows per group.

+ + ${this.conflictTables(conflicts)} + +`; + }, + + async importClippings() { + let progressWindow = null; + try { + const clippingsPath = await this.pickFile("Select Kindle My Clippings.txt", "*.txt"); + if (!clippingsPath) { + return; + } + + const settings = await this.getRuntimeSettings(); + const settingsMessage = [ + "Run Kindle import pipeline with these settings?", + "", + `Clippings: ${clippingsPath}`, + `Project directory: ${settings.projectDir}`, + `Python: ${settings.pythonPath}`, + `Zotero DB: ${settings.zoteroDbPath}`, + `Zotero storage: ${settings.zoteroStorageRoot}`, + `Config file: ${this.getConfigPath()}`, + "", + "This first creates/updates review files. You can choose whether to import annotations after the summary.", + ].join("\n"); + if (!Services.prompt.confirm(this.getMainWindow(), "Kindle Zotero Importer", settingsMessage)) { + return; + } + + progressWindow = this.createProgress("Starting Kindle import..."); + if (progressWindow) { + progressWindow.changeHeadline("Kindle Zotero Importer: reading clippings"); + } + const summary = await this.runPipeline(clippingsPath, settings); + if (progressWindow && progressWindow.progress) { + progressWindow.progress.setProgress(100); + progressWindow.changeHeadline("Kindle Zotero Importer: pipeline complete"); + } + const finalPlanPath = summary.outputs.final_plan; + const finalPlan = JSON.parse(await Zotero.File.getContentsAsync(finalPlanPath)); + this.notify( + `Pipeline completed. ${summary.counts.final_annotations || 0} final annotations ready.`, + true + ); + + const decision = this.confirmImport(summary); + if (decision === "review") { + this.revealFile(summary.outputs.mismatch_review); + return; + } + if (decision !== "import") { + return; + } + + if (progressWindow) { + progressWindow.changeHeadline("Kindle Zotero Importer: writing annotations"); + } + const results = await this.writeAnnotations(finalPlan, false); + this.notify( + `Import finished. Created ${results.created}; skipped ${results.skippedExisting}; updated order ${results.updatedSortIndex}; recreated ${results.recreatedForSortIndex}; failed ${results.failed.length}.`, + results.failed.length === 0 + ); + this.alert("Kindle Zotero Importer", this.formatWriterResults(results)); + } catch (error) { + Zotero.logError(error); + this.notify("Import failed. See the error dialog for details.", false); + this.alert("Kindle Zotero Importer Failed", String(error && error.stack ? error.stack : error)); + } finally { + if (progressWindow && progressWindow.startCloseTimer) { + progressWindow.startCloseTimer(1000); + } + } + }, + + async pickFile(title, filter) { + try { + const pickerModule = ChromeUtils.importESModule( + "chrome://zotero/content/modules/filePicker.mjs" + ); + const FilePicker = pickerModule.FilePicker; + if (typeof FilePicker !== "function") { + throw new Error(`Zotero FilePicker module did not export a constructor: ${Object.keys(pickerModule)}`); + } + const fp = new FilePicker(); + fp.init(this.getPickerWindow(), title, fp.modeOpen); + if (filter) { + fp.appendFilter("Kindle clippings", filter); + } + fp.appendFilters(fp.filterAll); + + const result = await fp.show(); + if (result !== fp.returnOK) { + return null; + } + return fp.file; + } catch (error) { + Zotero.logError(error); + return this.promptForPath( + "Kindle Zotero Importer", + "File picker failed. Enter the full path to My Clippings.txt:", + this.defaultClippingsPath + ); + } + }, + + async pickDirectory(title) { + try { + const pickerModule = ChromeUtils.importESModule( + "chrome://zotero/content/modules/filePicker.mjs" + ); + const FilePicker = pickerModule.FilePicker; + if (typeof FilePicker !== "function") { + throw new Error(`Zotero FilePicker module did not export a constructor: ${Object.keys(pickerModule)}`); + } + const fp = new FilePicker(); + fp.init(this.getPickerWindow(), title, fp.modeGetFolder); + + const result = await fp.show(); + if (result !== fp.returnOK) { + return null; + } + return fp.file; + } catch (error) { + Zotero.logError(error); + return this.promptForPath( + "Kindle Zotero Importer", + "Folder picker failed. Enter the full project directory path:", + this.defaultProjectDir + ); + } + }, + + async runPipeline(clippingsPath, settings, isFull = false) { + const python = settings.pythonPath; + const projectDir = settings.projectDir; + const runner = projectDir + "/plugin_runner.py"; + const summaryOutput = projectDir + "/plugin-summary.json"; + const progressOutput = projectDir + "/plugin-progress.json"; + const args = [ + runner, + "run", + clippingsPath, + "--workdir", + projectDir, + "--db", + settings.zoteroDbPath, + "--storage-root", + settings.zoteroStorageRoot, + "--overrides", + "match-overrides.json", + "--summary-output", + summaryOutput, + "--progress-output", + progressOutput, + "--pretty", + ]; + if (isFull) { + args.push("--full"); + } + + await this.exec(python, args); + return JSON.parse(await Zotero.File.getContentsAsync(summaryOutput)); + }, + + async runManagedImport(manager) { + const clippingsPath = await this.pickFile("Select Kindle My Clippings.txt", "*.txt"); + if (!clippingsPath) { + return; + } + const isFull = manager.getFullReimport ? manager.getFullReimport() : false; + await this.runManagedImportWithPath(manager, clippingsPath, isFull); + }, + + async runManagedImportWithPath(manager, clippingsPath, isFullOverride) { + try { + if (!clippingsPath) { + throw new Error("No clippings file selected"); + } + const settings = await this.getRuntimeSettings(); + const progressPath = settings.projectDir + "/plugin-progress.json"; + // Determine full vs incremental from explicit arg or manager checkbox + const isFull = typeof isFullOverride === "boolean" ? isFullOverride : (manager.getFullReimport ? manager.getFullReimport() : false); + manager.beginRun(clippingsPath); + try { + await Zotero.File.removeIfExists(progressPath); + } catch (_error) {} + + let polling = true; + const poll = async () => { + while (polling) { + try { + const progress = JSON.parse(await Zotero.File.getContentsAsync(progressPath)); + manager.updateProgress(progress.percent, progress.stage, progress.detail); + } catch (_error) {} + await Zotero.Promise.delay(500); + } + }; + const pollPromise = poll(); + let summary; + try { + summary = await this.runPipeline(clippingsPath, settings, isFull); + } finally { + polling = false; + await pollPromise; + } + + const finalPlan = JSON.parse( + await Zotero.File.getContentsAsync(summary.outputs.final_plan) + ); + manager.updateProgress(96, "Writing annotations", "Saving annotations in Zotero"); + const results = await this.writeAnnotations(finalPlan, false, (completed, total) => { + const percent = total ? 96 + Math.floor((completed / total) * 4) : 100; + manager.updateProgress(percent, "Writing annotations", `${completed} of ${total} checked`); + }); + const data = await this.loadManagerData(); + manager.completeRun(summary, results, data); + } catch (error) { + Zotero.logError(error); + manager.failRun(String(error && error.stack ? error.stack : error)); + } + }, + + async exec(command, args) { + if (!Zotero.Utilities || !Zotero.Utilities.Internal || !Zotero.Utilities.Internal.exec) { + throw new Error("Zotero.Utilities.Internal.exec is unavailable in this Zotero build"); + } + return Zotero.Utilities.Internal.exec(command, args); + }, + + confirmImport(summary) { + const counts = summary.counts || {}; + const planStatuses = counts.plan_statuses || {}; + const matchStatuses = counts.match_statuses || {}; + const unresolved = + (matchStatuses.unmatched || 0) + + (matchStatuses.ambiguous || 0) + + (planStatuses["matched-title-no-attachment"] || 0) + + (planStatuses["matched-title-ambiguous-attachment"] || 0); + const positioned = planStatuses.positioned || 0; + + const message = [ + "Pipeline completed.", + "", + `Clippings: ${counts.clippings || 0}`, + `Unique Kindle titles: ${counts.unique_titles || 0}`, + `Positioned annotations ready: ${positioned}`, + `Final annotations to check/import: ${counts.final_annotations || 0}`, + `Unresolved or ambiguous cases: ${unresolved}`, + "", + "Import positioned annotations now, open the mismatch review, or cancel?", + ].join("\n"); + + const choice = Services.prompt.confirmEx( + this.getMainWindow(), + "Kindle Zotero Importer", + message, + Services.prompt.BUTTON_POS_0 * Services.prompt.BUTTON_TITLE_IS_STRING + + Services.prompt.BUTTON_POS_1 * Services.prompt.BUTTON_TITLE_IS_STRING + + Services.prompt.BUTTON_POS_2 * Services.prompt.BUTTON_TITLE_CANCEL, + "Import", + "Open Review", + null, + null, + {} + ); + + if (choice === 0) { + return "import"; + } + if (choice === 1) { + return "review"; + } + return "cancel"; + }, + + normalizePosition(position) { + if (!position) { + return ""; + } + if (typeof position === "string") { + try { + return JSON.stringify(JSON.parse(position)); + } catch (_error) { + return position; + } + } + return JSON.stringify(position); + }, + + annotationFingerprint(annotation) { + return `${annotation.text || ""}\u0000${this.normalizePosition(annotation.position)}`; + }, + + existingAnnotationFingerprint(annotation) { + return `${annotation.annotationText || ""}\u0000${this.normalizePosition(annotation.annotationPosition)}`; + }, + + mergeComments(existingComment, newComment) { + const seen = new Set(); + const merged = []; + for (const comment of [existingComment, newComment]) { + for (const part of String(comment || "").split(/\n\s*\n/)) { + const cleaned = part.trim(); + if (!cleaned || seen.has(cleaned)) { + continue; + } + seen.add(cleaned); + merged.push(cleaned); + } + } + return merged.join("\n\n"); + }, + + existingSortIndex(annotation) { + return annotation.annotationSortIndex || annotation._annotationSortIndex || annotation.sortIndex || ""; + }, + + existingTags(annotation) { + try { + if (annotation.getTags) { + return annotation.getTags().map((tag) => tag.tag || tag.name || String(tag)); + } + } catch (_error) { + // Fall through to property checks below. + } + const tags = annotation.annotationTags || annotation.tags || []; + if (!Array.isArray(tags)) { + return []; + } + return tags.map((tag) => tag.tag || tag.name || String(tag)); + }, + + hasKindleImportTag(annotation) { + return this.existingTags(annotation).includes("kindle-import"); + }, + + confirmRecreateForSortIndex(entry, existingAnnotation, oldSortIndex, newSortIndex) { + const tags = this.existingTags(existingAnnotation).join(", ") || "(none)"; + const message = [ + "Zotero could not update an existing annotation's sort order in place.", + "", + "The plugin can delete and recreate this annotation to fix ordering, but only because it has the kindle-import tag.", + "", + `Title: ${entry.clipping_title}`, + `Attachment item: ${entry.attachment_item_id}`, + `Current sortIndex: ${oldSortIndex || "(empty)"}`, + `New sortIndex: ${newSortIndex}`, + `Tags: ${tags}`, + "", + "Text, comment, color, tags, and position will be preserved from the import plan.", + "", + "Delete and recreate this kindle-import annotation?", + ].join("\n"); + return Services.prompt.confirm( + this.getMainWindow(), + "Kindle Zotero Importer: Recreate Annotation?", + message + ); + }, + + async saveExistingAnnotation(existingAnnotation) { + if (existingAnnotation.saveTx) { + await existingAnnotation.saveTx(); + return; + } + if (existingAnnotation.save) { + await existingAnnotation.save(); + return; + } + throw new Error("Existing annotation cannot be saved by this Zotero build"); + }, + + async writeAnnotations(plan, dryRun, onProgress) { + if (plan.format !== "kindle-zotero-importer.zotero-writer-plan.v1") { + throw new Error(`Unsupported plan format: ${plan.format}`); + } + + const results = { + dryRun, + total: plan.annotations.length, + created: 0, + skippedExisting: 0, + updatedComments: 0, + updatedSortIndex: 0, + recreatedForSortIndex: 0, + blockedRecreateMissingTag: 0, + recreateDeclined: 0, + removedDuplicates: 0, + deletedForIncremental: 0, + failed: [], + }; + // Incremental deletions: remove annotations whose clipping_id is no longer present + if (!dryRun && plan.deletions && plan.deletions.length) { + const deletionIds = new Set(plan.deletions); + try { + // Find all annotations with kindle-id tag + const allAnnotations = []; + // Collect via attachments to be safe + const allItems = Zotero.Items.getAll ? await Zotero.Items.getAll() : []; + for (const item of allItems) { + if (item.isAnnotation && item.isAnnotation()) { + allAnnotations.push(item); + } else if (item.getAnnotations) { + const anns = item.getAnnotations(); + for (const a of anns) allAnnotations.push(a); + } + } + // Deduplicate + const seen = new Set(); + for (const ann of allAnnotations) { + if (!ann || seen.has(ann.id)) continue; + seen.add(ann.id); + const tags = this.existingTags(ann); + for (const tag of tags) { + if (tag.startsWith("kindle-id:")) { + const id = tag.substring("kindle-id:".length); + if (deletionIds.has(id)) { + await ann.eraseTx(); + results.deletedForIncremental += 1; + break; + } + } + } + } + } catch (e) { + Zotero.logError(e); + } + } + const existingByAttachment = new Map(); + + const getExistingAnnotations = async (attachment) => { + if (!existingByAttachment.has(attachment.id)) { + const annotations = attachment.getAnnotations ? attachment.getAnnotations() : []; + const byFingerprint = new Map(); + + for (const annotation of annotations) { + const fingerprint = this.existingAnnotationFingerprint(annotation); + const kept = byFingerprint.get(fingerprint); + if (!kept) { + byFingerprint.set(fingerprint, annotation); + continue; + } + + const keptHasComment = Boolean(kept.annotationComment); + const currentHasComment = Boolean(annotation.annotationComment); + const duplicate = keptHasComment || !currentHasComment ? annotation : kept; + const replacement = duplicate === annotation ? kept : annotation; + + byFingerprint.set(fingerprint, replacement); + if (!dryRun) { + await duplicate.eraseTx(); + } + results.removedDuplicates += 1; + } + + existingByAttachment.set(attachment.id, byFingerprint); + } + return existingByAttachment.get(attachment.id); + }; + + let completed = 0; + for (const entry of plan.annotations) { + try { + const attachment = Zotero.Items.get(entry.attachment_item_id); + if (!attachment) { + throw new Error(`Attachment not found: ${entry.attachment_item_id}`); + } + + const annotation = { + key: Zotero.DataObjectUtilities.generateKey(), + ...entry.annotation, + }; + if (!annotation.sortIndex) { + delete annotation.sortIndex; + } + + const existingAnnotations = await getExistingAnnotations(attachment); + const fingerprint = this.annotationFingerprint(annotation); + const existingAnnotation = existingAnnotations.get(fingerprint); + if (existingAnnotation) { + const mergedComment = this.mergeComments( + existingAnnotation.annotationComment, + annotation.comment + ); + let changed = false; + if (mergedComment && existingAnnotation.annotationComment !== mergedComment) { + existingAnnotation.annotationComment = mergedComment; + changed = true; + results.updatedComments += 1; + } + + const oldSortIndex = this.existingSortIndex(existingAnnotation); + const newSortIndex = annotation.sortIndex || ""; + const sortIndexChanged = Boolean(newSortIndex) && oldSortIndex !== newSortIndex; + if (sortIndexChanged) { + existingAnnotation.annotationSortIndex = newSortIndex; + changed = true; + } + + if (changed && !dryRun) { + try { + await this.saveExistingAnnotation(existingAnnotation); + if (sortIndexChanged) { + results.updatedSortIndex += 1; + } + } catch (error) { + if (!sortIndexChanged) { + throw error; + } + if (!this.hasKindleImportTag(existingAnnotation)) { + results.blockedRecreateMissingTag += 1; + throw new Error( + `Could not update sortIndex and existing annotation is not tagged kindle-import: ${error}` + ); + } + if (!this.confirmRecreateForSortIndex(entry, existingAnnotation, oldSortIndex, newSortIndex)) { + results.recreateDeclined += 1; + results.skippedExisting += 1; + continue; + } + await existingAnnotation.eraseTx(); + await Zotero.Annotations.saveFromJSON(attachment, annotation); + existingAnnotations.set(fingerprint, { + annotationComment: annotation.comment || "", + annotationPosition: annotation.position, + annotationSortIndex: annotation.sortIndex || "", + annotationTags: annotation.tags || [], + annotationText: annotation.text || "", + }); + results.recreatedForSortIndex += 1; + results.skippedExisting += 1; + continue; + } + } else if (sortIndexChanged && dryRun) { + results.updatedSortIndex += 1; + } + results.skippedExisting += 1; + continue; + } + + if (!dryRun) { + await Zotero.Annotations.saveFromJSON(attachment, annotation); + } + existingAnnotations.set(fingerprint, { + annotationComment: annotation.comment || "", + annotationPosition: annotation.position, + annotationSortIndex: annotation.sortIndex || "", + annotationTags: annotation.tags || [], + annotationText: annotation.text || "", + }); + results.created += 1; + } catch (error) { + results.failed.push({ + clipping_id: entry.clipping_id, + attachment_item_id: entry.attachment_item_id, + message: String(error), + }); + } finally { + completed += 1; + if (onProgress) { + onProgress(completed, plan.annotations.length); + } + } + } + + return results; + }, + + formatWriterResults(results) { + const lines = [ + "Import finished.", + "", + `Annotations in final plan: ${results.total}`, + `Created: ${results.created}`, + `Skipped existing: ${results.skippedExisting}`, + `Updated comments: ${results.updatedComments}`, + `Updated sort order: ${results.updatedSortIndex}`, + `Recreated for sort order: ${results.recreatedForSortIndex}`, + `Recreate blocked (missing kindle-import tag): ${results.blockedRecreateMissingTag}`, + `Recreate declined: ${results.recreateDeclined}`, + `Removed exact duplicates: ${results.removedDuplicates}`, + `Failed: ${results.failed.length}`, + ]; + if (results.failed.length) { + lines.push("", "First failures:"); + for (const failure of results.failed.slice(0, 5)) { + lines.push(`- ${failure.clipping_id}: ${failure.message}`); + } + } + return lines.join("\n"); + }, + + revealFile(path) { + try { + if (Zotero.File && Zotero.File.reveal) { + Zotero.File.reveal(path); + return; + } + const file = Components.classes["@mozilla.org/file/local;1"].createInstance( + Components.interfaces.nsIFile + ); + file.initWithPath(path); + file.reveal(); + } catch (error) { + this.alert("Kindle Zotero Importer", `Review file written to:\n${path}\n\n${error}`); + } + }, + + openFile(path) { + try { + const file = Components.classes["@mozilla.org/file/local;1"].createInstance( + Components.interfaces.nsIFile + ); + file.initWithPath(path); + if (file.exists() && file.isFile()) { + file.launch(); + return; + } + this.revealFile(path); + } catch (error) { + Zotero.logError(error); + this.revealFile(path); + } + }, +}; + +function install(_data, _reason) {} + +function uninstall(_data, _reason) {} + +async function startup(data, _reason) { + try { + await KindleZoteroImporter.startup(data); + } catch (error) { + Zotero.logError(error); + } +} + +function shutdown(_data, _reason) { + try { + KindleZoteroImporter.shutdown(); + } catch (error) { + Zotero.logError(error); + } +} + +function onMainWindowLoad({ window }) { + try { + KindleZoteroImporter.onMainWindowLoad(window); + } catch (error) { + Zotero.logError(error); + } +} + +function onMainWindowUnload({ window }) { + try { + KindleZoteroImporter.onMainWindowUnload(window); + } catch (error) { + Zotero.logError(error); + } +} diff --git a/plugin/manager.html b/plugin/manager.html new file mode 100644 index 0000000..6e767f6 --- /dev/null +++ b/plugin/manager.html @@ -0,0 +1,1199 @@ + + + + + Kindle Zotero Importer + + + + +
Loading Kindle Zotero Importer...
+ + diff --git a/plugin/manifest.json b/plugin/manifest.json new file mode 100644 index 0000000..92ba78d --- /dev/null +++ b/plugin/manifest.json @@ -0,0 +1,16 @@ +{ + "manifest_version": 2, + "name": "Kindle Zotero Importer", + "version": "0.6.4", + "description": "Import Kindle My Clippings.txt highlights and notes into Zotero as native annotations.", + "homepage_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer", + "author": "Utku Bilen Demir", + "applications": { + "zotero": { + "id": "kindlezoteroimporter@utkubilen.de", + "update_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/latest/download/updates.json", + "strict_min_version": "6.999", + "strict_max_version": "10.0.*" + } + } +} diff --git a/plugin/prefs.js b/plugin/prefs.js new file mode 100644 index 0000000..6fbe1e0 --- /dev/null +++ b/plugin/prefs.js @@ -0,0 +1,4 @@ +pref("extensions.kindleZoteroImporter.projectDir", "/home/user/Library/Mobile Documents/iCloud~md~obsidian/Documents/rhizome/06_projects/UTI/kindle-zotero-importer"); +pref("extensions.kindleZoteroImporter.pythonPath", "/opt/homebrew/bin/python3"); +pref("extensions.kindleZoteroImporter.zoteroDbPath", "/home/user/Zotero/zotero.sqlite"); +pref("extensions.kindleZoteroImporter.zoteroStorageRoot", "/home/user/Zotero/storage"); diff --git a/plugin_runner.py b/plugin_runner.py new file mode 100644 index 0000000..8295962 --- /dev/null +++ b/plugin_runner.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent +SRC = ROOT / "src" + +if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + +from kindle_zotero_importer.cli import main # noqa: E402 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_canary_plugin.py b/scripts/build_canary_plugin.py new file mode 100644 index 0000000..c7e4378 --- /dev/null +++ b/scripts/build_canary_plugin.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import json +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DIST_DIR = ROOT / "dist" +OUTPUT = DIST_DIR / "kindle-zotero-importer-canary.xpi" + + +def main() -> int: + DIST_DIR.mkdir(exist_ok=True) + if OUTPUT.exists(): + OUTPUT.unlink() + + manifest = { + "manifest_version": 2, + "name": "Kindle Zotero Importer Canary", + "version": "0.1.0", + "description": "Minimal install test for Kindle Zotero Importer.", + "homepage_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer", + "author": "Utku Bilen Demir", + "applications": { + "zotero": { + "id": "kindlezoteroimportercanary@utkubilen.de", + "update_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/latest/download/updates-canary.json", + "strict_min_version": "6.999", + "strict_max_version": "9.*", + } + }, + } + bootstrap = """ +function install(data, reason) {} +function uninstall(data, reason) {} +async function startup(data, reason) { + await Zotero.initializationPromise; + Zotero.debug('Kindle Zotero Importer Canary started'); +} +function shutdown(data, reason) {} +""".lstrip() + + prefs = 'pref("extensions.kindleZoteroImporterCanary.enabled", true);\n' + + with zipfile.ZipFile(OUTPUT, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in [ + ("bootstrap.js", bootstrap), + ("manifest.json", json.dumps(manifest, indent=2) + "\n"), + ("prefs.js", prefs), + ]: + info = zipfile.ZipInfo(name) + info.create_system = 3 + info.external_attr = 0o644 << 16 + archive.writestr(info, content, zipfile.ZIP_DEFLATED) + + print(f"Built {OUTPUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_plugin.py b/scripts/build_plugin.py new file mode 100644 index 0000000..3815701 --- /dev/null +++ b/scripts/build_plugin.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import json +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PLUGIN_DIR = ROOT / "plugin" +DIST_DIR = ROOT / "dist" +OUTPUT = DIST_DIR / "kindle-zotero-importer.xpi" + + +def main() -> int: + DIST_DIR.mkdir(exist_ok=True) + if OUTPUT.exists(): + OUTPUT.unlink() + + manifest_path = PLUGIN_DIR / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + zotero_app = manifest.get("applications", {}).get("zotero", {}) + required_manifest_fields = [ + "manifest_version", + "name", + "version", + "description", + ] + missing_manifest_fields = [ + field for field in required_manifest_fields if not manifest.get(field) + ] + missing_zotero_fields = [ + field + for field in ["id", "strict_min_version", "strict_max_version"] + if not zotero_app.get(field) + ] + missing_files = [ + str(path.relative_to(PLUGIN_DIR)) + for path in [PLUGIN_DIR / "manifest.json", PLUGIN_DIR / "bootstrap.js"] + if not path.exists() + ] + if missing_manifest_fields or missing_zotero_fields or missing_files: + problems = [] + if missing_manifest_fields: + problems.append(f"manifest fields: {', '.join(missing_manifest_fields)}") + if missing_zotero_fields: + problems.append( + f"applications.zotero fields: {', '.join(missing_zotero_fields)}" + ) + if missing_files: + problems.append(f"files: {', '.join(missing_files)}") + raise SystemExit("Invalid plugin package; missing " + "; ".join(problems)) + + with zipfile.ZipFile(OUTPUT, "w", zipfile.ZIP_DEFLATED) as archive: + for path in PLUGIN_DIR.rglob("*"): + if path.is_file(): + archive_name = str(path.relative_to(PLUGIN_DIR)) + info = zipfile.ZipInfo(archive_name) + info.create_system = 3 + info.external_attr = 0o644 << 16 + archive.writestr(info, path.read_bytes(), zipfile.ZIP_DEFLATED) + + with zipfile.ZipFile(OUTPUT) as archive: + names = set(archive.namelist()) + if "manifest.json" not in names or "bootstrap.js" not in names: + raise SystemExit( + "Invalid XPI: manifest.json and bootstrap.js must be at archive root" + ) + + print(f"Built {OUTPUT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/kindle_zotero_importer/cli.py b/src/kindle_zotero_importer/cli.py index 86d4896..9897423 100644 --- a/src/kindle_zotero_importer/cli.py +++ b/src/kindle_zotero_importer/cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse import json import sys +from pathlib import Path from .clippings import clippings_to_jsonable, load_clippings from .epub_position import add_epub_positions @@ -141,6 +142,45 @@ def main(argv: list[str] | None = None) -> int: "--output", "-o", help="Write Markdown to this file instead of stdout" ) + run_parser = subparsers.add_parser( + "run", help="Run the full importer pipeline for a Kindle My Clippings.txt file" + ) + run_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt") + run_parser.add_argument( + "--workdir", + default=".", + help="Directory for generated importer artifacts", + ) + run_parser.add_argument( + "--db", default="/home/user/Zotero/zotero.sqlite", help="Path to zotero.sqlite" + ) + run_parser.add_argument( + "--storage-root", + default="/home/user/Zotero/storage", + help="Path to Zotero storage directory for storage: attachments", + ) + run_parser.add_argument( + "--overrides", + default="match-overrides.json", + help="Path to reusable match overrides JSON, relative to workdir unless absolute", + ) + run_parser.add_argument( + "--summary-output", + help="Write pipeline summary JSON to this path instead of stdout", + ) + run_parser.add_argument( + "--progress-output", + help="Write machine-readable pipeline progress JSON to this path", + ) + run_parser.add_argument( + "--pretty", action="store_true", help="Pretty-print JSON artifacts" + ) + run_parser.add_argument( + "--full", + action="store_true", + help="Force full re-import from scratch, ignoring incremental state (processes all clippings)", + ) + args = parser.parse_args(argv) if args.command == "parse": @@ -275,6 +315,236 @@ def main(argv: list[str] | None = None) -> int: sys.stdout.write(text) return 0 + if args.command == "run": + workdir = Path(args.workdir).expanduser().resolve() + workdir.mkdir(parents=True, exist_ok=True) + + progress_path = ( + Path(args.progress_output).expanduser().resolve() + if args.progress_output + else None + ) + + def report_progress(percent: int, stage: str, detail: str) -> None: + if not progress_path: + return + temporary_path = progress_path.with_suffix(progress_path.suffix + ".tmp") + temporary_path.write_text( + json.dumps( + {"percent": percent, "stage": stage, "detail": detail}, + ensure_ascii=False, + ) + + "\n", + encoding="utf-8", + ) + temporary_path.replace(progress_path) + + def artifact(name: str) -> Path: + return workdir / name + + overrides_path = Path(args.overrides).expanduser() + if not overrides_path.is_absolute(): + overrides_path = workdir / overrides_path + + report_progress(2, "Reading clippings", "Parsing My Clippings.txt") + new_clippings_list = load_clippings(args.clippings_file) + new_clippings = clippings_to_jsonable(new_clippings_list) + new_ids = {c["id"] for c in new_clippings.get("clippings", [])} + + # Incremental state: previous clippings and previous integrated ids + is_incremental = not args.full + prev_clippings_ids: set[str] = set() + prev_integrated_ids: set[str] = set() + prev_final_plan = None + if is_incremental: + prev_clippings_path = artifact("clippings.json") + prev_final_path = artifact("import-plan.final.json") + if prev_clippings_path.exists(): + try: + prev_clippings_ids = {c["id"] for c in load_json(str(prev_clippings_path)).get("clippings", [])} + except Exception: + prev_clippings_ids = set() + if prev_final_path.exists(): + try: + prev_final_plan = load_json(str(prev_final_path)) + prev_integrated_ids = {a.get("clipping_id") for a in prev_final_plan.get("annotations", []) if a.get("clipping_id")} + except Exception: + prev_final_plan = None + prev_integrated_ids = set() + # If no previous state, fall back to full + if not prev_clippings_ids and not prev_integrated_ids: + is_incremental = False + + if is_incremental: + # New or previously not integrated (skipped) -> need to (re)process + to_process_ids = set(new_ids - prev_integrated_ids) + # Deletions: previously integrated but now absent (removed or changed text) + deletions_ids = sorted(prev_integrated_ids - new_ids) + # Also re-queue any previously integrated clippings whose title now has an override + # (mapping may have changed, need to move annotation). We need overrides dict for this. + # Load overrides early for this check (already loaded later, but do here for incremental decision) + # We will reload overrides here if not yet loaded + try: + _ov_for_inc = load_overrides(load_json(str(overrides_path))) if overrides_path.exists() else {} + except Exception: + _ov_for_inc = {} + for c in new_clippings_list: + if c.id in prev_integrated_ids and c.title in _ov_for_inc: + to_process_ids.add(c.id) + # Also handle deleted overrides: if a previously integrated title's override was deleted, + # its annotations should be deleted. Detect via previous final plan's titles vs current overrides. + # For incremental, we don't have previous overrides, so we handle deletions via the main deletions list + # (IDs no longer present). For deleted overrides where ID still present but now unmatched, + # the match report will make them not positioned, and they will be in to_process but then skipped; + # we need to ensure they are deleted. We do this by checking if a previously integrated clipping's + # title is now not in overrides and would be unmatched -> treat as to_delete. + # Simplify: any prev_integrated clipping whose title is not in current new match's "matched" will be + # handled as not in to_process_ids? Actually it is still in new_ids, but we re-queued it above only if title in overrides. + # For now, rely on full re-import for deleted-override cleanup. + # Build filtered clippings for this delta run + filtered_list = [c for c in new_clippings_list if c.id in to_process_ids] + # If nothing to do and no deletions, still need to report but skip expensive steps + if not filtered_list and not deletions_ids: + report_progress(15, "Matching titles", "No new highlights — incremental skip") + # Still need to generate artifacts for consistency (empty delta) + zotero_index = build_zotero_index(args.db, args.storage_root) + overrides = None + if overrides_path.exists(): + overrides = load_overrides(load_json(str(overrides_path))) + clippings = new_clippings + matches = build_match_report(clippings, zotero_index, overrides) + override_skeleton = generate_override_skeleton(matches) + plan = {"format": "kindle-zotero-importer.import-plan.v1", "items": []} + positioned_plan = {"format": plan["format"], "items": []} + final_plan = {"format": FINAL_FORMAT, "source_format": positioned_plan.get("format"), "annotation_count": 0, "skipped_counts": {}, "annotations": [], "deletions": [], "is_incremental": True, "incremental_stats": {"new_clippings": len(new_ids), "prev_integrated": len(prev_integrated_ids), "to_process": 0, "deletions": 0}} + mismatch_review = build_mismatch_review(positioned_plan, matches) + # Will write artifacts below with empty delta + else: + report_progress(8, "Indexing Zotero", "Reading library items and attachments (incremental)") + zotero_index = build_zotero_index(args.db, args.storage_root) + overrides = None + if overrides_path.exists(): + overrides = load_overrides(load_json(str(overrides_path))) + # Only the delta clippings go through matching/positioning + delta_clippings = clippings_to_jsonable(filtered_list) + report_progress(15, "Matching titles", f"Matching {len(filtered_list)} new/pending highlights (incremental)") + matches = build_match_report(delta_clippings, zotero_index, overrides) + override_skeleton = generate_override_skeleton(matches) + # For artifact completeness, also build full matches for summary? Use delta matches for now + # But we need full matches for review? Keep delta + report_progress(22, "Building plan", "Selecting attachments (incremental)") + plan = build_import_plan(delta_clippings, zotero_index, matches) + report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files (incremental)") + epub_plan = add_epub_positions(plan) + report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files (incremental)") + positioned_plan = add_pdf_positions(epub_plan) + report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report") + final_plan = build_final_writer_plan(positioned_plan) + # Attach deletions for writer + final_plan["deletions"] = deletions_ids + final_plan["is_incremental"] = True + final_plan["incremental_stats"] = { + "new_clippings": len(new_ids), + "prev_integrated": len(prev_integrated_ids), + "to_process": len(to_process_ids), + "deletions": len(deletions_ids), + } + mismatch_review = build_mismatch_review(positioned_plan, matches) + # For outputs, clippings should be the full new set (for next diff), but plan artifacts are delta + clippings = new_clippings + else: + # Full re-import + report_progress(8, "Indexing Zotero", "Reading library items and attachments") + zotero_index = build_zotero_index(args.db, args.storage_root) + overrides = None + if overrides_path.exists(): + overrides = load_overrides(load_json(str(overrides_path))) + clippings = new_clippings + report_progress(15, "Matching titles", "Matching Kindle titles to Zotero items") + matches = build_match_report(clippings, zotero_index, overrides) + override_skeleton = generate_override_skeleton(matches) + report_progress(22, "Building plan", "Selecting attachments") + plan = build_import_plan(clippings, zotero_index, matches) + report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files") + epub_plan = add_epub_positions(plan) + report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files; this is usually the longest stage") + positioned_plan = add_pdf_positions(epub_plan) + report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report") + final_plan = build_final_writer_plan(positioned_plan) + final_plan["deletions"] = [] + final_plan["is_incremental"] = False + mismatch_review = build_mismatch_review(positioned_plan, matches) + + indent = 2 if args.pretty else None + outputs = { + "clippings": artifact("clippings.json"), + "zotero_index": artifact("zotero-index.json"), + "matches": artifact("matches.json"), + "generated_overrides": artifact("match-overrides.generated.json"), + "import_plan": artifact("import-plan.json"), + "epub_plan": artifact("import-plan.epub.json"), + "positioned_plan": artifact("import-plan.positioned.json"), + "final_plan": artifact("import-plan.final.json"), + "mismatch_review": workdir / "docs" / "mismatch-review.md", + } + outputs["mismatch_review"].parent.mkdir(parents=True, exist_ok=True) + + report_progress(95, "Saving results", "Writing import artifacts") + for key, payload in [ + ("clippings", clippings), + ("zotero_index", zotero_index), + ("matches", matches), + ("generated_overrides", override_skeleton), + ("import_plan", plan), + ("epub_plan", epub_plan), + ("positioned_plan", positioned_plan), + ("final_plan", final_plan), + ]: + with open(outputs[key], "w", encoding="utf-8") as file: + json.dump(payload, file, ensure_ascii=False, indent=indent) + file.write("\n") + + with open(outputs["mismatch_review"], "w", encoding="utf-8") as file: + file.write(mismatch_review) + + status_counts: dict[str, int] = {} + for item in positioned_plan.get("items", []): + status = item.get("status", "unknown") + status_counts[status] = status_counts.get(status, 0) + 1 + + match_counts: dict[str, int] = {} + for match in matches.get("matches", []): + status = match.get("status", "unknown") + match_counts[status] = match_counts.get(status, 0) + 1 + + summary = { + "format": "kindle-zotero-importer.pipeline-summary.v1", + "workdir": str(workdir), + "overrides_path": str(overrides_path), + "outputs": {key: str(path) for key, path in outputs.items()}, + "counts": { + "clippings": len(clippings.get("clippings", [])), + "unique_titles": len(matches.get("matches", [])), + "final_annotations": final_plan.get("annotation_count", 0), + "match_statuses": match_counts, + "plan_statuses": status_counts, + "skipped_final": final_plan.get("skipped_counts", {}), + "is_incremental": final_plan.get("is_incremental", False), + "incremental_stats": final_plan.get("incremental_stats", {}), + "deletions": len(final_plan.get("deletions", [])), + }, + } + json_text = json.dumps(summary, ensure_ascii=False, indent=2) + if args.summary_output: + with open(args.summary_output, "w", encoding="utf-8") as file: + file.write(json_text) + file.write("\n") + else: + sys.stdout.write(json_text) + sys.stdout.write("\n") + report_progress(100, "Pipeline complete", "Ready to import annotations into Zotero") + return 0 + parser.error(f"unknown command: {args.command}") return 2 diff --git a/src/kindle_zotero_importer/epub_position.py b/src/kindle_zotero_importer/epub_position.py index 8639ee3..9c8e2a2 100644 --- a/src/kindle_zotero_importer/epub_position.py +++ b/src/kindle_zotero_importer/epub_position.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from html import unescape from html.parser import HTMLParser import posixpath -import re import zipfile import xml.etree.ElementTree as ET from typing import Any @@ -65,8 +64,8 @@ def _position_item( updated = dict(item) annotation = dict(updated["annotation"]) - annotation["position"] = position - annotation["sortIndex"] = _epub_sort_index(position["value"]) + annotation["position"] = position["selector"] + annotation["sortIndex"] = position["sortIndex"] updated["annotation"] = annotation updated["status"] = "positioned" return updated @@ -97,7 +96,7 @@ def extract_epub_text_nodes(path: str) -> list[TextNode]: return nodes -def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, str] | None: +def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, Any] | None: haystack = " ".join(node.text for node in text_nodes) normalized_haystack, haystack_map = _normalize_with_map(haystack) normalized_quote, _ = _normalize_with_map(quote) @@ -134,9 +133,12 @@ def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, str] | No f"{end_node.cfi_parent_path}/{end_node.text_step}:{end_offset})" ) return { - "type": "FragmentSelector", - "conformsTo": "http://www.idpf.org/epub/linking/cfi/epub-cfi.html", - "value": value, + "selector": { + "type": "FragmentSelector", + "conformsTo": "http://www.idpf.org/epub/linking/cfi/epub-cfi.html", + "value": value, + }, + "sortIndex": f"{spine_step:05d}|{raw_start:08d}", } @@ -257,13 +259,6 @@ def _node_at(nodes: list[TextNode], offset: int) -> tuple[TextNode | None, int]: return None, 0 -def _epub_sort_index(cfi: str) -> str: - numbers = [int(number) for number in re.findall(r"/([0-9]+)", cfi)] - spine = numbers[1] if len(numbers) > 1 else 0 - content = numbers[-1] if numbers else 0 - return f"{spine:05d}|{content:08d}" - - def _with_problem(item: dict[str, Any], problem: str) -> dict[str, Any]: updated = dict(item) updated["status"] = problem diff --git a/src/kindle_zotero_importer/final_plan.py b/src/kindle_zotero_importer/final_plan.py index 682882e..925dcfd 100644 --- a/src/kindle_zotero_importer/final_plan.py +++ b/src/kindle_zotero_importer/final_plan.py @@ -1,5 +1,6 @@ from __future__ import annotations +import datetime from typing import Any @@ -20,10 +21,14 @@ def build_final_writer_plan(positioned_plan: dict[str, Any]) -> dict[str, Any]: skipped.get("positioned-missing-writer-fields", 0) + 1 ) continue + clipping = item["clipping"] annotations.append( { - "clipping_id": item["clipping"]["id"], - "clipping_title": item["clipping"]["title"], + "clipping_id": clipping["id"], + "clipping_title": clipping["title"], + "clipping_added_on": clipping.get("added_on"), + "clipping_added_on_iso": clipping.get("added_on_iso"), + "integrated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), "attachment_item_id": attachment["item_id"], "attachment_key": attachment["key"], "parent_item_id": item["zotero"]["parent_item_id"], @@ -37,11 +42,22 @@ def build_final_writer_plan(positioned_plan: dict[str, Any]) -> dict[str, Any]: "pageLabel": annotation.get("pageLabel") or "", "sortIndex": annotation.get("sortIndex"), "position": annotation["position"], - "tags": annotation.get("tags") or [{"name": "kindle-import"}], + "tags": [ + {"name": "kindle-import"}, + {"name": f"kindle-id:{clipping['id']}"}, + ], }, } ) + annotations.sort( + key=lambda entry: ( + entry.get("attachment_item_id") or 0, + entry["annotation"].get("sortIndex") or "", + entry.get("clipping_id") or "", + ) + ) + return { "format": FINAL_FORMAT, "source_format": positioned_plan.get("format"), diff --git a/src/kindle_zotero_importer/pdf_position.py b/src/kindle_zotero_importer/pdf_position.py index 5bc52b5..e84917c 100644 --- a/src/kindle_zotero_importer/pdf_position.py +++ b/src/kindle_zotero_importer/pdf_position.py @@ -4,12 +4,31 @@ from dataclasses import dataclass import hashlib import os import re +import shutil import subprocess import tempfile import xml.etree.ElementTree as ET from typing import Any +_TOOL_PATHS = { + "pdftotext": ("/home/user/bin/pdftotext", "/opt/homebrew/bin/pdftotext"), + "pdftohtml": ("/home/user/bin/pdftohtml", "/opt/homebrew/bin/pdftohtml"), + "pdfinfo": ("/home/user/bin/pdfinfo", "/opt/homebrew/bin/pdfinfo"), + "qpdf": ("/home/user/bin/qpdf", "/opt/homebrew/bin/qpdf"), +} + + +def _tool(name: str) -> str: + resolved = shutil.which(name) + if resolved: + return resolved + for candidate in _TOOL_PATHS.get(name, ()): + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return name + + @dataclass(frozen=True) class PDFWord: text: str @@ -117,7 +136,7 @@ def _position_item( def extract_pdf_text_pages(path: str) -> list[str]: result = subprocess.run( - ["pdftotext", "-enc", "UTF-8", path, "-"], + [_tool("pdftotext"), "-enc", "UTF-8", path, "-"], check=True, capture_output=True, text=True, @@ -157,7 +176,7 @@ def _decrypted_pdf_path( base_name = os.path.splitext(os.path.basename(path))[0] or "document" output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf") subprocess.run( - ["qpdf", "--decrypt", path, output_path], + [_tool("qpdf"), "--decrypt", path, output_path], check=True, capture_output=True, text=True, @@ -194,7 +213,7 @@ def extract_pdf_page_xml(path: str, page_index: int) -> PDFPageXML: page_number = page_index + 1 result = subprocess.run( [ - "pdftohtml", + _tool("pdftohtml"), "-f", str(page_number), "-l", @@ -302,7 +321,7 @@ def _pdf_page_size( path: str, page_number: int, fallback_width: float, fallback_height: float ) -> tuple[float, float]: result = subprocess.run( - ["pdfinfo", "-f", str(page_number), "-l", str(page_number), path], + [_tool("pdfinfo"), "-f", str(page_number), "-l", str(page_number), path], check=True, capture_output=True, text=True,