1603 lines
57 KiB
JavaScript
1603 lines
57 KiB
JavaScript
var Kanzi = {
|
|
id: "kindle-zotero-importer@utkubilen.de",
|
|
menuItems: [],
|
|
runtimeProjectDir: "",
|
|
defaultProjectDir: "",
|
|
defaultClippingsPath: "",
|
|
prefBranch: "extensions.kindleZoteroImporter.",
|
|
rootURI: null,
|
|
managerWindow: null,
|
|
chromeHandle: null,
|
|
|
|
async startup(data) {
|
|
this.rootURI = data.rootURI;
|
|
const addonManagerStartup = Cc["@mozilla.org/addons/addon-manager-startup;1"].getService(
|
|
Ci.amIAddonManagerStartup
|
|
);
|
|
this.chromeHandle = addonManagerStartup.registerChrome(
|
|
Services.io.newURI(this.rootURI + "manifest.json"),
|
|
[["content", "kanzi", ""]]
|
|
);
|
|
await Zotero.initializationPromise;
|
|
await this.ensureRuntime();
|
|
this.addToAllWindows();
|
|
},
|
|
|
|
async ensureRuntime() {
|
|
const runtimeDir = PathUtils.join(PathUtils.profileDir, "kanzi-runtime");
|
|
const files = [
|
|
"plugin_runner.py",
|
|
"src/kindle_zotero_importer/__init__.py",
|
|
"src/kindle_zotero_importer/__main__.py",
|
|
"src/kindle_zotero_importer/cli.py",
|
|
"src/kindle_zotero_importer/clippings.py",
|
|
"src/kindle_zotero_importer/epub_position.py",
|
|
"src/kindle_zotero_importer/final_plan.py",
|
|
"src/kindle_zotero_importer/import_plan.py",
|
|
"src/kindle_zotero_importer/matcher.py",
|
|
"src/kindle_zotero_importer/mismatch_review.py",
|
|
"src/kindle_zotero_importer/overrides.py",
|
|
"src/kindle_zotero_importer/pdf_position.py",
|
|
"src/kindle_zotero_importer/zotero_index.py",
|
|
];
|
|
await IOUtils.makeDirectory(runtimeDir, { createAncestors: true, ignoreExisting: true });
|
|
for (const relativePath of files) {
|
|
const destination = PathUtils.join(runtimeDir, ...relativePath.split("/"));
|
|
await IOUtils.makeDirectory(PathUtils.parent(destination), {
|
|
createAncestors: true,
|
|
ignoreExisting: true,
|
|
});
|
|
const response = await fetch(this.rootURI + "runtime/" + relativePath);
|
|
if (!response.ok) throw new Error(`Missing Kanzi runtime file: ${relativePath}`);
|
|
await Zotero.File.putContentsAsync(destination, await response.text());
|
|
}
|
|
this.runtimeProjectDir = runtimeDir;
|
|
},
|
|
|
|
shutdown() {
|
|
this.removeMenuItems();
|
|
if (this.managerWindow && !this.managerWindow.closed) {
|
|
this.managerWindow.close();
|
|
}
|
|
this.managerWindow = null;
|
|
if (this.chromeHandle) {
|
|
this.chromeHandle.destruct();
|
|
this.chromeHandle = null;
|
|
}
|
|
},
|
|
|
|
onMainWindowLoad(win) {
|
|
this.addMenuItems(win);
|
|
},
|
|
|
|
onMainWindowUnload(_win) {
|
|
this.menuItems = [];
|
|
},
|
|
|
|
addMenuItems(win) {
|
|
win = win || Zotero.getMainWindow();
|
|
if (!win || !win.document) {
|
|
return;
|
|
}
|
|
|
|
const doc = win.document;
|
|
if (doc.getElementById("kanzi-manager")) {
|
|
return;
|
|
}
|
|
|
|
const popup = doc.getElementById("menu_ToolsPopup");
|
|
if (!popup) {
|
|
Zotero.debug("Kanzi: Tools menu not found");
|
|
return;
|
|
}
|
|
|
|
const managerItem = doc.createXULElement("menuitem");
|
|
managerItem.setAttribute("id", "kanzi-manager");
|
|
managerItem.setAttribute("label", "Kanzi...");
|
|
managerItem.addEventListener("command", () => this.openManager());
|
|
popup.appendChild(managerItem);
|
|
this.menuItems.push(managerItem);
|
|
|
|
},
|
|
|
|
addToAllWindows() {
|
|
const windows = Zotero.getMainWindows ? Zotero.getMainWindows() : [Zotero.getMainWindow()];
|
|
for (const win of windows) {
|
|
if (win && win.ZoteroPane) {
|
|
this.addMenuItems(win);
|
|
}
|
|
}
|
|
},
|
|
|
|
removeMenuItems() {
|
|
for (const item of this.menuItems) {
|
|
if (item && item.parentNode) {
|
|
item.parentNode.removeChild(item);
|
|
}
|
|
}
|
|
this.menuItems = [];
|
|
},
|
|
|
|
getPref(name, fallback) {
|
|
try {
|
|
const value = Services.prefs.getCharPref(this.prefBranch + name);
|
|
return value || fallback;
|
|
} catch (_error) {
|
|
return fallback;
|
|
}
|
|
},
|
|
|
|
setPref(name, value) {
|
|
Services.prefs.setCharPref(this.prefBranch + name, String(value));
|
|
},
|
|
|
|
getProjectDir() {
|
|
return this.getPref("projectDir", this.runtimeProjectDir || this.defaultProjectDir);
|
|
},
|
|
|
|
getPythonPath() {
|
|
const fallback = this.getDefaultPython();
|
|
const value = this.getPref("pythonPath", fallback);
|
|
// Existing Windows installs may have "python3" stored from the old
|
|
// Unix-only default; that binary name does not exist on Windows.
|
|
if (this.isWindows() && value === "python3") {
|
|
return "python";
|
|
}
|
|
return value || fallback;
|
|
},
|
|
|
|
getZoteroDbPath() {
|
|
const dataDir = Zotero.DataDirectory && Zotero.DataDirectory.dir;
|
|
return this.getPref("zoteroDbPath", dataDir ? PathUtils.join(dataDir, "zotero.sqlite") : "");
|
|
},
|
|
|
|
getZoteroStorageRoot() {
|
|
const dataDir = Zotero.DataDirectory && Zotero.DataDirectory.dir;
|
|
return this.getPref("zoteroStorageRoot", dataDir ? PathUtils.join(dataDir, "storage") : "");
|
|
},
|
|
|
|
isWindows() {
|
|
try {
|
|
return Services.appinfo.OS === "WINNT";
|
|
} catch (_error) {
|
|
return false;
|
|
}
|
|
},
|
|
|
|
getDefaultPython() {
|
|
return this.isWindows() ? "python" : "python3";
|
|
},
|
|
|
|
getEnvironmentVariable(name) {
|
|
try {
|
|
return Cc["@mozilla.org/process/environment;1"]
|
|
.getService(Ci.nsIEnvironment)
|
|
.get(name) || "";
|
|
} catch (_error) {
|
|
return "";
|
|
}
|
|
},
|
|
|
|
joinWindowsPath(base, ...parts) {
|
|
if (!base) {
|
|
return "";
|
|
}
|
|
return [String(base).replace(/[\\/]+$/, ""), ...parts]
|
|
.map((part, index) => index ? String(part).replace(/^[\\/]+|[\\/]+$/g, "") : part)
|
|
.join("\\");
|
|
},
|
|
|
|
getWindowsPythonCandidates(configuredPath) {
|
|
const localAppData = this.getEnvironmentVariable("LOCALAPPDATA");
|
|
const programFiles = this.getEnvironmentVariable("ProgramFiles");
|
|
const programFilesX86 = this.getEnvironmentVariable("ProgramFiles(x86)");
|
|
const userProfile = this.getEnvironmentVariable("USERPROFILE");
|
|
const systemDrive = this.getEnvironmentVariable("SystemDrive") || "C:";
|
|
const versions = ["314", "313", "312", "311"];
|
|
const candidates = [configuredPath];
|
|
for (const version of versions) {
|
|
candidates.push(
|
|
this.joinWindowsPath(localAppData, "Programs", "Python", `Python${version}`, "python.exe"),
|
|
this.joinWindowsPath(programFiles, `Python${version}`, "python.exe"),
|
|
this.joinWindowsPath(programFilesX86, `Python${version}-32`, "python.exe"),
|
|
this.joinWindowsPath(systemDrive, `Python${version}`, "python.exe")
|
|
);
|
|
}
|
|
candidates.push(
|
|
this.joinWindowsPath(localAppData, "Python", "bin", "python.exe"),
|
|
this.joinWindowsPath(userProfile, "scoop", "apps", "python", "current", "python.exe"),
|
|
this.joinWindowsPath(userProfile, "miniconda3", "python.exe"),
|
|
this.joinWindowsPath(userProfile, "anaconda3", "python.exe")
|
|
);
|
|
for (const directory of this.getEnvironmentVariable("PATH").split(";")) {
|
|
const cleanDirectory = directory.trim().replace(/^"|"$/g, "");
|
|
if (cleanDirectory) {
|
|
candidates.push(
|
|
this.joinWindowsPath(cleanDirectory, "python.exe"),
|
|
this.joinWindowsPath(cleanDirectory, "python3.exe")
|
|
);
|
|
}
|
|
}
|
|
return [...new Set(candidates.filter(Boolean))];
|
|
},
|
|
|
|
async resolvePythonPath(configuredPath) {
|
|
let candidates;
|
|
if (this.isWindows()) {
|
|
candidates = this.getWindowsPythonCandidates(configuredPath);
|
|
} else if (Services.appinfo.OS === "Darwin") {
|
|
candidates = [configuredPath];
|
|
candidates.push(
|
|
"/opt/homebrew/bin/python3",
|
|
"/opt/homebrew/opt/python@3.14/bin/python3.14",
|
|
"/opt/homebrew/opt/python@3.13/bin/python3.13",
|
|
"/opt/homebrew/opt/python@3.12/bin/python3.12",
|
|
"/opt/homebrew/opt/python@3.11/bin/python3.11",
|
|
"/usr/local/bin/python3",
|
|
"/usr/local/opt/python@3.14/bin/python3.14",
|
|
"/usr/local/opt/python@3.13/bin/python3.13",
|
|
"/usr/local/opt/python@3.12/bin/python3.12",
|
|
"/usr/local/opt/python@3.11/bin/python3.11",
|
|
"/usr/bin/python3"
|
|
);
|
|
} else {
|
|
candidates = [configuredPath, "/usr/bin/python3", "/usr/local/bin/python3"];
|
|
}
|
|
for (const candidate of candidates) {
|
|
if (!candidate || (!candidate.includes("/") && !candidate.includes("\\"))) continue;
|
|
try {
|
|
if (await IOUtils.exists(candidate)) return candidate;
|
|
} catch (_error) {}
|
|
}
|
|
return configuredPath || this.getDefaultPython();
|
|
},
|
|
|
|
joinPath(...parts) {
|
|
if (typeof PathUtils !== "undefined" && PathUtils.join) {
|
|
return PathUtils.join(...parts);
|
|
}
|
|
return parts.join("/").replaceAll("//", "/");
|
|
},
|
|
|
|
getConfigPath() {
|
|
return this.joinPath(this.getProjectDir(), "plugin-config.json");
|
|
},
|
|
|
|
artifactPath(relativePath) {
|
|
return this.joinPath(this.getProjectDir(), ...String(relativePath).split("/"));
|
|
},
|
|
|
|
async openArtifact(relativePath) {
|
|
const path = this.artifactPath(relativePath);
|
|
try {
|
|
await Zotero.File.getContentsAsync(path);
|
|
this.revealFile(path);
|
|
} catch (_error) {
|
|
this.alert(
|
|
"Kanzi",
|
|
`File not found yet:\n${path}\n\nRun Import Kindle Clippings first to generate review artifacts.`
|
|
);
|
|
}
|
|
},
|
|
|
|
async getRuntimeSettings() {
|
|
const settings = {
|
|
projectDir: this.getProjectDir(),
|
|
pythonPath: this.getPythonPath(),
|
|
zoteroDbPath: this.getZoteroDbPath(),
|
|
zoteroStorageRoot: this.getZoteroStorageRoot(),
|
|
};
|
|
try {
|
|
const text = await Zotero.File.getContentsAsync(this.getConfigPath());
|
|
const parsed = JSON.parse(text);
|
|
for (const key of Object.keys(settings)) {
|
|
if (parsed[key]) {
|
|
settings[key] = parsed[key];
|
|
}
|
|
}
|
|
if (parsed.colourMap) settings.colourMap = parsed.colourMap;
|
|
if (parsed.colorMap) settings.colourMap = parsed.colorMap;
|
|
// also check colour-map.json for Python pipeline
|
|
try {
|
|
const cmapText = await Zotero.File.getContentsAsync(this.joinPath(this.getProjectDir(), "colour-map.json"));
|
|
settings.colourMap = JSON.parse(cmapText);
|
|
} catch (_e2) {}
|
|
} catch (_error) {
|
|
// The config file is optional; defaults/preferences are enough for first run.
|
|
}
|
|
settings.pythonPath = await this.resolvePythonPath(settings.pythonPath);
|
|
return settings;
|
|
},
|
|
|
|
getMainWindow() {
|
|
return Zotero.getMainWindow();
|
|
},
|
|
|
|
getPickerWindow() {
|
|
return Services.wm.getMostRecentWindow("navigator:browser") || this.getMainWindow();
|
|
},
|
|
|
|
alert(title, message) {
|
|
Services.prompt.alert(this.getMainWindow(), title, message);
|
|
},
|
|
|
|
notify(message, success) {
|
|
try {
|
|
const progressWindow = new Zotero.ProgressWindow({ closeOnClick: true });
|
|
progressWindow.changeHeadline("Kanzi");
|
|
const icon = success ? "chrome://zotero/skin/tick.png" : "chrome://zotero/skin/cross.png";
|
|
const item = new progressWindow.ItemProgress(icon, message);
|
|
item.setProgress(100);
|
|
progressWindow.show();
|
|
progressWindow.startCloseTimer(success ? 5000 : 10000);
|
|
return progressWindow;
|
|
} catch (_error) {
|
|
return null;
|
|
}
|
|
},
|
|
|
|
createProgress(message) {
|
|
try {
|
|
const progressWindow = new Zotero.ProgressWindow({ closeOnClick: false });
|
|
progressWindow.changeHeadline("Kanzi");
|
|
progressWindow.progress = new progressWindow.ItemProgress(
|
|
"chrome://zotero/skin/treesource-collection.png",
|
|
message
|
|
);
|
|
progressWindow.progress.setProgress(0);
|
|
progressWindow.show();
|
|
return progressWindow;
|
|
} catch (_error) {
|
|
return null;
|
|
}
|
|
},
|
|
|
|
promptForPath(title, message, fallback) {
|
|
const value = { value: fallback || "" };
|
|
const ok = Services.prompt.prompt(this.getMainWindow(), title, message, value, null, {});
|
|
return ok && value.value ? value.value : null;
|
|
},
|
|
|
|
async openManager() {
|
|
if (this.managerWindow && !this.managerWindow.closed) {
|
|
this.managerWindow.focus();
|
|
return;
|
|
}
|
|
const settings = await this.getRuntimeSettings();
|
|
const data = await this.loadManagerData();
|
|
const io = { plugin: this, settings, data };
|
|
this.managerWindow = this.getMainWindow().openDialog(
|
|
"chrome://kanzi/content/manager.html",
|
|
"kanzi-manager-window",
|
|
"chrome,centerscreen,resizable",
|
|
io
|
|
);
|
|
},
|
|
|
|
async loadManagerData() {
|
|
return {
|
|
clippings: await this.readJsonArtifact("clippings.json"),
|
|
matches: await this.readJsonArtifact("matches.json"),
|
|
generatedOverrides: await this.readJsonArtifact("match-overrides.generated.json"),
|
|
positionedPlan: await this.readJsonArtifact("import-plan.positioned.json"),
|
|
finalPlan: await this.readJsonArtifact("import-plan.final.json"),
|
|
summary: await this.readJsonArtifact("plugin-summary.json"),
|
|
persistentOverrides: await this.readJsonArtifact("match-overrides.json"),
|
|
};
|
|
},
|
|
|
|
async loadOverridesFile() {
|
|
const path = this.artifactPath("match-overrides.json");
|
|
try {
|
|
return JSON.parse(await Zotero.File.getContentsAsync(path));
|
|
} catch (_error) {
|
|
return { format: "kindle-zotero-importer.match-overrides.v1", overrides: [] };
|
|
}
|
|
},
|
|
|
|
async saveOverride(resolution) {
|
|
if (!resolution || !resolution.clipping_title) {
|
|
throw new Error("Missing clipping title for override");
|
|
}
|
|
const payload = await this.loadOverridesFile();
|
|
if (!Array.isArray(payload.overrides)) {
|
|
payload.overrides = [];
|
|
}
|
|
const index = payload.overrides.findIndex(
|
|
(entry) => entry.clipping_title === resolution.clipping_title
|
|
);
|
|
const clippings = await this.readJsonArtifact("clippings.json");
|
|
const sourceCount = this.countClippingsForTitle(clippings, resolution.clipping_title);
|
|
const clippingCount = sourceCount || resolution.clipping_count || resolution.review?.clipping_count || 0;
|
|
const nowIso = new Date().toISOString();
|
|
const entry = {
|
|
clipping_title: resolution.clipping_title,
|
|
resolution: resolution.resolution,
|
|
review: {
|
|
...(resolution.review || { status: "manager" }),
|
|
clipping_count: clippingCount,
|
|
},
|
|
created_at: nowIso,
|
|
updated_at: nowIso,
|
|
};
|
|
if (index >= 0) {
|
|
const prev = payload.overrides[index];
|
|
entry.created_at = prev.created_at || prev.updated_at || nowIso;
|
|
entry.updated_at = nowIso;
|
|
payload.overrides[index] = entry;
|
|
} else {
|
|
payload.overrides.push(entry);
|
|
}
|
|
payload.overrides.sort((a, b) => (b.updated_at || b.created_at || "").localeCompare(a.updated_at || a.created_at || "") || a.clipping_title.localeCompare(b.clipping_title));
|
|
await Zotero.File.putContentsAsync(
|
|
this.artifactPath("match-overrides.json"),
|
|
JSON.stringify(payload, null, 2) + "\n"
|
|
);
|
|
return entry;
|
|
},
|
|
|
|
countClippingsForTitle(payload, clippingTitle) {
|
|
return (payload?.clippings || []).reduce(
|
|
(count, clipping) => count + (clipping.title === clippingTitle ? 1 : 0),
|
|
0
|
|
);
|
|
},
|
|
|
|
async deleteOverride(clippingTitle) {
|
|
if (!clippingTitle) {
|
|
throw new Error("Missing clipping title for delete");
|
|
}
|
|
const payload = await this.loadOverridesFile();
|
|
const before = payload.overrides.length;
|
|
payload.overrides = payload.overrides.filter((e) => e.clipping_title !== clippingTitle);
|
|
if (payload.overrides.length === before) {
|
|
throw new Error(`No override found for: ${clippingTitle}`);
|
|
}
|
|
await Zotero.File.putContentsAsync(
|
|
this.artifactPath("match-overrides.json"),
|
|
JSON.stringify(payload, null, 2) + "\n"
|
|
);
|
|
return true;
|
|
},
|
|
|
|
async discardClip(clippingId) {
|
|
if (!clippingId) {
|
|
throw new Error("Missing clipping id for discard");
|
|
}
|
|
const path = this.artifactPath("import-plan.final.json");
|
|
const plan = JSON.parse(await Zotero.File.getContentsAsync(path));
|
|
const discarded = new Set(plan.discarded_clipping_ids || []);
|
|
discarded.add(clippingId);
|
|
plan.discarded_clipping_ids = [...discarded].sort();
|
|
// Also drop from deferred so it is never re-queued.
|
|
if (Array.isArray(plan.deferred_clipping_ids)) {
|
|
plan.deferred_clipping_ids = plan.deferred_clipping_ids.filter((id) => id !== clippingId);
|
|
}
|
|
await Zotero.File.putContentsAsync(path, JSON.stringify(plan, null, 2) + "\n");
|
|
return true;
|
|
},
|
|
|
|
async saveSettingsFromManager(newSettings) {
|
|
const toSave = {
|
|
projectDir: newSettings.projectDir || this.getProjectDir(),
|
|
pythonPath: newSettings.pythonPath || this.getPythonPath(),
|
|
zoteroDbPath: newSettings.zoteroDbPath || this.getZoteroDbPath(),
|
|
zoteroStorageRoot: newSettings.zoteroStorageRoot || this.getZoteroStorageRoot(),
|
|
};
|
|
if (newSettings.colourMap) {
|
|
toSave.colourMap = newSettings.colourMap;
|
|
// also write colour-map.json for Python pipeline (mirrors plugin-config)
|
|
try {
|
|
await Zotero.File.putContentsAsync(
|
|
this.joinPath(this.getProjectDir(), "colour-map.json"),
|
|
JSON.stringify(newSettings.colourMap, null, 2) + "\n"
|
|
);
|
|
} catch (_e) {}
|
|
}
|
|
for (const [key, value] of Object.entries(toSave)) {
|
|
if (key === "colourMap") continue;
|
|
this.setPref(key, value);
|
|
}
|
|
await Zotero.File.putContentsAsync(
|
|
this.getConfigPath(),
|
|
JSON.stringify(toSave, null, 2) + "\n"
|
|
);
|
|
return toSave;
|
|
},
|
|
|
|
async loadColourMap() {
|
|
try {
|
|
const text = await Zotero.File.getContentsAsync(this.joinPath(this.getProjectDir(), "colour-map.json"));
|
|
return JSON.parse(text);
|
|
} catch (_e) {}
|
|
try {
|
|
const text = await Zotero.File.getContentsAsync(this.getConfigPath());
|
|
const parsed = JSON.parse(text);
|
|
if (parsed.colourMap) return parsed.colourMap;
|
|
if (parsed.colorMap) return parsed.colorMap;
|
|
} catch (_e) {}
|
|
return null;
|
|
},
|
|
|
|
async readJsonArtifact(relativePath) {
|
|
try {
|
|
return JSON.parse(await Zotero.File.getContentsAsync(this.artifactPath(relativePath)));
|
|
} catch (_error) {
|
|
return null;
|
|
}
|
|
},
|
|
|
|
async pickFile(title, filter) {
|
|
try {
|
|
const pickerModule = ChromeUtils.importESModule(
|
|
"chrome://zotero/content/modules/filePicker.mjs"
|
|
);
|
|
const FilePicker = pickerModule.FilePicker;
|
|
if (typeof FilePicker !== "function") {
|
|
throw new Error(`Zotero FilePicker module did not export a constructor: ${Object.keys(pickerModule)}`);
|
|
}
|
|
const fp = new FilePicker();
|
|
fp.init(this.getPickerWindow(), title, fp.modeOpen);
|
|
if (filter) {
|
|
fp.appendFilter("Kindle clippings", filter);
|
|
}
|
|
fp.appendFilters(fp.filterAll);
|
|
|
|
const result = await fp.show();
|
|
if (result !== fp.returnOK) {
|
|
return null;
|
|
}
|
|
return this.filePickerResultToPath(fp.file);
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
return this.promptForPath(
|
|
"Kanzi",
|
|
"File picker failed. Enter the full path to My Clippings.txt:",
|
|
this.defaultClippingsPath
|
|
);
|
|
}
|
|
},
|
|
|
|
filePickerResultToPath(file) {
|
|
if (!file) {
|
|
return null;
|
|
}
|
|
if (typeof file === "string") {
|
|
return file;
|
|
}
|
|
// Mozilla FilePicker returns an nsIFile; Python needs a native string path.
|
|
if (typeof file.path === "string" && file.path) {
|
|
return file.path;
|
|
}
|
|
try {
|
|
const str = String(file);
|
|
if (str && str !== "[object Object]") {
|
|
return str;
|
|
}
|
|
} catch (_error) {}
|
|
return null;
|
|
},
|
|
|
|
async runPipeline(clippingsPath, settings, isFull = false, retryUnresolved = false) {
|
|
const python = settings.pythonPath || this.getDefaultPython();
|
|
const projectDir = settings.projectDir;
|
|
const runner = this.joinPath(projectDir, "plugin_runner.py");
|
|
const summaryOutput = this.joinPath(projectDir, "plugin-summary.json");
|
|
const progressOutput = this.joinPath(projectDir, "plugin-progress.json");
|
|
const args = [
|
|
runner,
|
|
"run",
|
|
clippingsPath,
|
|
"--workdir",
|
|
projectDir,
|
|
"--db",
|
|
settings.zoteroDbPath,
|
|
"--storage-root",
|
|
settings.zoteroStorageRoot,
|
|
"--overrides",
|
|
"match-overrides.json",
|
|
"--summary-output",
|
|
summaryOutput,
|
|
"--progress-output",
|
|
progressOutput,
|
|
"--pretty",
|
|
];
|
|
if (isFull) {
|
|
args.push("--full");
|
|
}
|
|
if (retryUnresolved) {
|
|
args.push("--retry-unresolved");
|
|
}
|
|
|
|
let execResult = null;
|
|
try {
|
|
execResult = await this.exec(python, args);
|
|
} catch (error) {
|
|
const installHint = this.isWindows()
|
|
? `Install Python 3.11 or newer, fully restart Zotero, and let Kanzi detect python.exe. `
|
|
: `Install Python 3.11 or newer and check the Python path in Kanzi Settings. `;
|
|
throw new Error(
|
|
`Python could not run the Kanzi pipeline (${python}): ${error && error.message ? error.message : error}. ` +
|
|
installHint
|
|
);
|
|
}
|
|
try {
|
|
return JSON.parse(await Zotero.File.getContentsAsync(summaryOutput));
|
|
} catch (error) {
|
|
const detail = execResult ? ` Python output: ${JSON.stringify(execResult).slice(0, 1000)}.` : "";
|
|
throw new Error(
|
|
`Python pipeline did not write ${summaryOutput}.${detail} ` +
|
|
`Python: ${python}. Runner: ${runner}. ` +
|
|
`Make sure Python 3 is installed and reachable, then check the project directory is writable.`
|
|
);
|
|
}
|
|
},
|
|
|
|
async runManagedImport(manager) {
|
|
const clippingsPath = await this.pickFile("Select Kindle My Clippings.txt", "*.txt");
|
|
// NOTE: writing now happens only after the user reviews the pending
|
|
// highlights and presses Integrate (see integratePending).
|
|
if (!clippingsPath) {
|
|
return;
|
|
}
|
|
const isFull = manager.getFullReimport ? manager.getFullReimport() : false;
|
|
const retryUnresolved = manager.getRetryUnresolved ? manager.getRetryUnresolved() : false;
|
|
await this.runManagedImportWithPath(manager, clippingsPath, isFull, retryUnresolved);
|
|
},
|
|
|
|
async runManagedImportWithPath(manager, clippingsPath, isFullOverride, retryUnresolvedOverride) {
|
|
try {
|
|
clippingsPath = this.filePickerResultToPath(clippingsPath) || clippingsPath;
|
|
if (!clippingsPath) {
|
|
throw new Error("No clippings file selected");
|
|
}
|
|
const settings = await this.getRuntimeSettings();
|
|
const progressPath = this.joinPath(settings.projectDir, "plugin-progress.json");
|
|
// Determine full vs incremental from explicit arg or manager checkbox
|
|
const isFull = typeof isFullOverride === "boolean" ? isFullOverride : (manager.getFullReimport ? manager.getFullReimport() : false);
|
|
const retryUnresolved = typeof retryUnresolvedOverride === "boolean"
|
|
? retryUnresolvedOverride
|
|
: (manager.getRetryUnresolved ? manager.getRetryUnresolved() : false);
|
|
manager.beginRun(clippingsPath);
|
|
const finalPlanPath = this.joinPath(settings.projectDir, "import-plan.final.json");
|
|
let previousFinalPlan = null;
|
|
try {
|
|
previousFinalPlan = JSON.parse(await Zotero.File.getContentsAsync(finalPlanPath));
|
|
} catch (_backupError) {}
|
|
try {
|
|
await Zotero.File.removeIfExists(progressPath);
|
|
} catch (_error) {}
|
|
|
|
let polling = true;
|
|
const poll = async () => {
|
|
while (polling) {
|
|
try {
|
|
const progress = JSON.parse(await Zotero.File.getContentsAsync(progressPath));
|
|
manager.updateProgress(progress.percent, progress.stage, progress.detail);
|
|
} catch (_error) {}
|
|
await Zotero.Promise.delay(500);
|
|
}
|
|
};
|
|
const pollPromise = poll();
|
|
let summary;
|
|
try {
|
|
summary = await this.runPipeline(clippingsPath, settings, isFull, retryUnresolved);
|
|
} finally {
|
|
polling = false;
|
|
await pollPromise;
|
|
}
|
|
|
|
const finalPlan = JSON.parse(
|
|
await Zotero.File.getContentsAsync(summary.outputs.final_plan)
|
|
);
|
|
const pendingCount = (finalPlan.annotations || []).length;
|
|
const pendingDeletions = (finalPlan.deletions || []).length;
|
|
if (!pendingCount && !pendingDeletions) {
|
|
// Nothing new — run the maintenance sweep and finish as before.
|
|
manager.updateProgress(96, "Writing annotations", "Checking existing annotations");
|
|
const emptyResults = await this.writeAnnotations(finalPlan, false, (completed, total) => {
|
|
const percent = total ? 96 + Math.floor((completed / total) * 4) : 100;
|
|
manager.updateProgress(percent, "Writing annotations", `${completed} of ${total} checked`);
|
|
});
|
|
await this.reconcileWriterState(summary.outputs.final_plan, finalPlan, emptyResults);
|
|
const emptyData = await this.loadManagerData();
|
|
await manager.completeRun(summary, emptyResults, emptyData);
|
|
return;
|
|
}
|
|
// Pause before writing: let the user review and select what to integrate.
|
|
const fullDeltaIds = (finalPlan.annotations || []).map((a) => a.clipping_id);
|
|
// The pipeline has already written its generated history. Replace it with
|
|
// a safe ledger until Integrate is pressed, including if the window closes.
|
|
const deferredPlan = this.deferPendingPlan(previousFinalPlan, finalPlan, fullDeltaIds);
|
|
await Zotero.File.putContentsAsync(
|
|
summary.outputs.final_plan,
|
|
JSON.stringify(deferredPlan, null, 2) + "\n"
|
|
);
|
|
this._pending = {
|
|
summary,
|
|
finalPlan,
|
|
finalPlanPath: summary.outputs.final_plan,
|
|
fullDeltaIds,
|
|
previousFinalPlan,
|
|
};
|
|
manager.updateProgress(
|
|
90,
|
|
"Review pending",
|
|
`${pendingCount} highlight(s) ready — select and press Integrate`
|
|
);
|
|
manager.showPendingReview(summary, finalPlan);
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
manager.failRun(String(error && error.stack ? error.stack : error));
|
|
}
|
|
},
|
|
|
|
deferPendingPlan(previousPlan, generatedPlan, deferredIds) {
|
|
const plan = previousPlan
|
|
? { ...previousPlan }
|
|
: {
|
|
...generatedPlan,
|
|
annotations: [],
|
|
all_annotations: [],
|
|
deletions: [],
|
|
annotation_count: 0,
|
|
delta_annotation_count: 0,
|
|
};
|
|
const deferred = new Set(plan.deferred_clipping_ids || []);
|
|
for (const id of deferredIds || []) deferred.add(id);
|
|
plan.deferred_clipping_ids = [...deferred].sort();
|
|
return plan;
|
|
},
|
|
|
|
// Pure helper (tested): narrow a pipeline plan to the user's selection.
|
|
filterPlanForSelection(finalPlan, fullDeltaIds, selectedIds, previousPlan = null) {
|
|
const selected = new Set(selectedIds || []);
|
|
const unselected = new Set(
|
|
(fullDeltaIds || []).filter((id) => !selected.has(id))
|
|
);
|
|
const plan = { ...finalPlan };
|
|
plan.annotations = (finalPlan.annotations || []).filter((a) =>
|
|
selected.has(a.clipping_id)
|
|
);
|
|
if (unselected.size && Array.isArray(finalPlan.all_annotations)) {
|
|
plan.all_annotations = finalPlan.all_annotations.filter(
|
|
(a) => !unselected.has(a.clipping_id)
|
|
);
|
|
const present = new Set(plan.all_annotations.map((a) => a.clipping_id));
|
|
for (const annotation of previousPlan?.all_annotations || previousPlan?.annotations || []) {
|
|
if (unselected.has(annotation.clipping_id) && !present.has(annotation.clipping_id)) {
|
|
plan.all_annotations.push(annotation);
|
|
present.add(annotation.clipping_id);
|
|
}
|
|
}
|
|
}
|
|
const deferred = new Set(previousPlan?.deferred_clipping_ids || []);
|
|
for (const id of selected) deferred.delete(id);
|
|
for (const id of unselected) deferred.add(id);
|
|
plan.deferred_clipping_ids = [...deferred].sort();
|
|
plan.delta_annotation_count = plan.annotations.length;
|
|
if (Array.isArray(plan.all_annotations)) {
|
|
plan.annotation_count = plan.all_annotations.length;
|
|
}
|
|
return plan;
|
|
},
|
|
|
|
async integratePending(manager, selectedIds) {
|
|
const pending = this._pending;
|
|
if (!pending) {
|
|
throw new Error("No pending review to integrate");
|
|
}
|
|
try {
|
|
const plan = this.filterPlanForSelection(
|
|
pending.finalPlan,
|
|
pending.fullDeltaIds,
|
|
selectedIds,
|
|
pending.previousFinalPlan
|
|
);
|
|
manager.updateProgress(96, "Writing annotations", "Saving selected annotations in Zotero");
|
|
const results = await this.writeAnnotations(plan, false, (completed, total) => {
|
|
const percent = total ? 96 + Math.floor((completed / total) * 4) : 100;
|
|
manager.updateProgress(percent, "Writing annotations", `${completed} of ${total} checked`);
|
|
});
|
|
await this.reconcileWriterState(pending.finalPlanPath, plan, results);
|
|
this._pending = null;
|
|
const data = await this.loadManagerData();
|
|
await manager.completeRun(pending.summary, results, data);
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
manager.failRun(String(error && error.stack ? error.stack : error));
|
|
}
|
|
},
|
|
|
|
async cancelPendingReview(manager) {
|
|
// The deferred ledger was written before the review appeared, so closing
|
|
// or cancelling never records pending highlights as integrated.
|
|
this._pending = null;
|
|
manager.hidePendingReview();
|
|
manager.cancelRun("Review cancelled — nothing was written. Run again when ready.");
|
|
try {
|
|
const data = await this.loadManagerData();
|
|
manager.data = data;
|
|
manager.rows = manager.conflictRows();
|
|
manager.refreshRows(true);
|
|
manager.refreshTitleOverview();
|
|
manager.refreshIntegrated(true);
|
|
manager.refreshMappings(true);
|
|
} catch (_reloadError) {}
|
|
},
|
|
|
|
async exec(command, args) {
|
|
if (!Zotero.Utilities || !Zotero.Utilities.Internal || !Zotero.Utilities.Internal.exec) {
|
|
throw new Error("Zotero.Utilities.Internal.exec is unavailable in this Zotero build");
|
|
}
|
|
return Zotero.Utilities.Internal.exec(command, args);
|
|
},
|
|
|
|
confirmImport(summary) {
|
|
const counts = summary.counts || {};
|
|
const planStatuses = counts.plan_statuses || {};
|
|
const matchStatuses = counts.match_statuses || {};
|
|
const unresolved =
|
|
(matchStatuses.unmatched || 0) +
|
|
(matchStatuses.ambiguous || 0) +
|
|
(planStatuses["matched-title-no-attachment"] || 0) +
|
|
(planStatuses["matched-title-ambiguous-attachment"] || 0);
|
|
const positioned = planStatuses.positioned || 0;
|
|
|
|
const message = [
|
|
"Pipeline completed.",
|
|
"",
|
|
`Clippings: ${counts.clippings || 0}`,
|
|
`Unique Kindle titles: ${counts.unique_titles || 0}`,
|
|
`Positioned annotations ready: ${positioned}`,
|
|
`Final annotations to check/import: ${counts.final_annotations || 0}`,
|
|
`Unresolved or ambiguous cases: ${unresolved}`,
|
|
"",
|
|
"Import positioned annotations now, open the mismatch review, or cancel?",
|
|
].join("\n");
|
|
|
|
const choice = Services.prompt.confirmEx(
|
|
this.getMainWindow(),
|
|
"Kanzi",
|
|
message,
|
|
Services.prompt.BUTTON_POS_0 * Services.prompt.BUTTON_TITLE_IS_STRING +
|
|
Services.prompt.BUTTON_POS_1 * Services.prompt.BUTTON_TITLE_IS_STRING +
|
|
Services.prompt.BUTTON_POS_2 * Services.prompt.BUTTON_TITLE_CANCEL,
|
|
"Import",
|
|
"Open Review",
|
|
null,
|
|
null,
|
|
{}
|
|
);
|
|
|
|
if (choice === 0) {
|
|
return "import";
|
|
}
|
|
if (choice === 1) {
|
|
return "review";
|
|
}
|
|
return "cancel";
|
|
},
|
|
|
|
normalizePosition(position) {
|
|
if (!position) {
|
|
return "";
|
|
}
|
|
if (typeof position === "string") {
|
|
try {
|
|
return JSON.stringify(JSON.parse(position));
|
|
} catch (_error) {
|
|
return position;
|
|
}
|
|
}
|
|
return JSON.stringify(position);
|
|
},
|
|
|
|
annotationFingerprint(annotation) {
|
|
return `${annotation.text || ""}\u0000${this.normalizePosition(annotation.position)}`;
|
|
},
|
|
|
|
existingAnnotationFingerprint(annotation) {
|
|
return `${annotation.annotationText || ""}\u0000${this.normalizePosition(annotation.annotationPosition)}`;
|
|
},
|
|
|
|
mergeComments(existingComment, newComment) {
|
|
const seen = new Set();
|
|
const merged = [];
|
|
for (const comment of [existingComment, newComment]) {
|
|
for (const part of String(comment || "").split(/\n\s*\n/)) {
|
|
const cleaned = this.stripColourCode(part);
|
|
if (!cleaned) continue;
|
|
const key = cleaned.toLowerCase();
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
merged.push(cleaned);
|
|
}
|
|
}
|
|
return merged.join("\n\n");
|
|
},
|
|
|
|
stripColourCode(value) {
|
|
return String(value || "")
|
|
.replace(/^\s*\[(?:y|yellow|o|orange|r|red|e|grey|gray|g|green|b|blue|bl|p|purple|m|magenta|pink)\]\s*/i, "")
|
|
.trim();
|
|
},
|
|
|
|
normalizeAnnotationText(value) {
|
|
return String(value || "")
|
|
.toLowerCase()
|
|
.normalize("NFKC")
|
|
.replace(/-\s*\n\s*/g, "")
|
|
.replace(/[^\p{L}\p{N}\s]+/gu, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
},
|
|
|
|
annotationPositionAnchor(annotation) {
|
|
const raw = annotation.annotationPosition || annotation.position;
|
|
let position = raw;
|
|
if (typeof raw === "string") {
|
|
try { position = JSON.parse(raw); } catch (_error) { return raw; }
|
|
}
|
|
if (!position || typeof position !== "object") return "";
|
|
if (position.type === "FragmentSelector" && position.value) {
|
|
const value = String(position.value);
|
|
const finalComma = value.lastIndexOf(",");
|
|
return `epub:${finalComma > 0 ? value.slice(0, finalComma) : value}`;
|
|
}
|
|
if (Number.isInteger(position.pageIndex) && Array.isArray(position.rects) && position.rects.length) {
|
|
const rect = position.rects[0] || [];
|
|
const x = Number(rect[0]);
|
|
const y = Number(rect[1]);
|
|
return `pdf:${position.pageIndex}:${x.toFixed(1)}:${y.toFixed(1)}`;
|
|
}
|
|
return this.normalizePosition(position);
|
|
},
|
|
|
|
areImportedDuplicates(left, right) {
|
|
const leftAnchor = this.annotationPositionAnchor(left);
|
|
const rightAnchor = this.annotationPositionAnchor(right);
|
|
if (!leftAnchor || leftAnchor !== rightAnchor) return false;
|
|
const leftPosition = this.normalizePosition(left.annotationPosition || left.position);
|
|
const rightPosition = this.normalizePosition(right.annotationPosition || right.position);
|
|
if (leftPosition && leftPosition === rightPosition) return true;
|
|
const leftText = this.normalizeAnnotationText(left.annotationText || left.text);
|
|
const rightText = this.normalizeAnnotationText(right.annotationText || right.text);
|
|
if (!leftText || !rightText) return false;
|
|
if (leftText === rightText) return true;
|
|
const shorter = leftText.length < rightText.length ? leftText : rightText;
|
|
const longer = shorter === leftText ? rightText : leftText;
|
|
return shorter.length >= 20 && longer.startsWith(shorter);
|
|
},
|
|
|
|
async collectImportedAnnotations() {
|
|
const annotations = [];
|
|
let allItems = [];
|
|
if (Zotero.DB && Zotero.DB.columnQueryAsync) {
|
|
const ids = await Zotero.DB.columnQueryAsync("SELECT itemID FROM itemAnnotations");
|
|
allItems = Zotero.Items.getAsync ? await Zotero.Items.getAsync(ids) : Zotero.Items.get(ids);
|
|
} else if (Zotero.Items.getAll) {
|
|
allItems = await Zotero.Items.getAll();
|
|
}
|
|
for (const item of allItems || []) {
|
|
if (!item || !item.isAnnotation || !item.isAnnotation()) continue;
|
|
if (this.hasKindleImportTag(item)) annotations.push(item);
|
|
}
|
|
return annotations;
|
|
},
|
|
|
|
async maintainImportedAnnotations(annotations, dryRun, results) {
|
|
if (!results.maintenanceDeletedIds) results.maintenanceDeletedIds = [];
|
|
const byAttachment = new Map();
|
|
for (const annotation of annotations) {
|
|
const parentID = annotation.parentItemID || annotation.parentID;
|
|
if (!byAttachment.has(parentID)) byAttachment.set(parentID, []);
|
|
byAttachment.get(parentID).push(annotation);
|
|
}
|
|
|
|
for (const group of byAttachment.values()) {
|
|
group.sort((a, b) =>
|
|
this.normalizeAnnotationText(b.annotationText).length -
|
|
this.normalizeAnnotationText(a.annotationText).length
|
|
);
|
|
const kept = [];
|
|
for (const annotation of group) {
|
|
const canonicalComment = this.mergeComments(annotation.annotationComment, "");
|
|
if (String(annotation.annotationComment || "") !== canonicalComment) {
|
|
annotation.annotationComment = canonicalComment;
|
|
results.updatedComments += 1;
|
|
if (!dryRun) await this.saveExistingAnnotation(annotation);
|
|
}
|
|
|
|
const duplicateOf = kept.find((candidate) => this.areImportedDuplicates(candidate, annotation));
|
|
if (!duplicateOf) {
|
|
kept.push(annotation);
|
|
continue;
|
|
}
|
|
const mergedComment = this.mergeComments(duplicateOf.annotationComment, annotation.annotationComment);
|
|
if (duplicateOf.annotationComment !== mergedComment) {
|
|
duplicateOf.annotationComment = mergedComment;
|
|
results.updatedComments += 1;
|
|
if (!dryRun) await this.saveExistingAnnotation(duplicateOf);
|
|
}
|
|
if (!dryRun) await annotation.eraseTx();
|
|
const keptClippingIds = new Set(
|
|
this.existingTags(duplicateOf)
|
|
.filter((tag) => tag.startsWith("kindle-id:"))
|
|
.map((tag) => tag.slice("kindle-id:".length))
|
|
);
|
|
for (const tag of this.existingTags(annotation)) {
|
|
const clippingID = tag.startsWith("kindle-id:")
|
|
? tag.slice("kindle-id:".length)
|
|
: "";
|
|
if (clippingID && !keptClippingIds.has(clippingID)) {
|
|
results.maintenanceDeletedIds.push(clippingID);
|
|
}
|
|
}
|
|
results.removedDuplicates += 1;
|
|
}
|
|
}
|
|
},
|
|
|
|
existingSortIndex(annotation) {
|
|
return annotation.annotationSortIndex || annotation._annotationSortIndex || annotation.sortIndex || "";
|
|
},
|
|
|
|
existingTags(annotation) {
|
|
try {
|
|
if (annotation.getTags) {
|
|
return annotation.getTags().map((tag) => tag.tag || tag.name || String(tag));
|
|
}
|
|
} catch (_error) {
|
|
// Fall through to property checks below.
|
|
}
|
|
const tags = annotation.annotationTags || annotation.tags || [];
|
|
if (!Array.isArray(tags)) {
|
|
return [];
|
|
}
|
|
return tags.map((tag) => tag.tag || tag.name || String(tag));
|
|
},
|
|
|
|
hasKindleImportTag(annotation) {
|
|
return this.existingTags(annotation).includes("kindle-import");
|
|
},
|
|
|
|
commentForExistingAnnotation(annotation, newComment, previousImportedComment) {
|
|
if (!this.hasKindleImportTag(annotation) || previousImportedComment === undefined) {
|
|
return this.mergeComments(annotation.annotationComment, newComment);
|
|
}
|
|
const previousParts = new Set(
|
|
String(previousImportedComment || "")
|
|
.split(/\n\s*\n/)
|
|
.map((part) => this.stripColourCode(part).toLowerCase())
|
|
.filter(Boolean)
|
|
);
|
|
const manualParts = String(annotation.annotationComment || "")
|
|
.split(/\n\s*\n/)
|
|
.filter((part) => !previousParts.has(this.stripColourCode(part).toLowerCase()))
|
|
.join("\n\n");
|
|
return this.mergeComments(manualParts, newComment);
|
|
},
|
|
|
|
confirmRecreateForSortIndex(entry, existingAnnotation, oldSortIndex, newSortIndex) {
|
|
const tags = this.existingTags(existingAnnotation).join(", ") || "(none)";
|
|
const message = [
|
|
"Zotero could not update an existing annotation's sort order in place.",
|
|
"",
|
|
"The plugin can delete and recreate this annotation to fix ordering, but only because it has the kindle-import tag.",
|
|
"",
|
|
`Title: ${entry.clipping_title}`,
|
|
`Attachment item: ${entry.attachment_item_id}`,
|
|
`Current sortIndex: ${oldSortIndex || "(empty)"}`,
|
|
`New sortIndex: ${newSortIndex}`,
|
|
`Tags: ${tags}`,
|
|
"",
|
|
"Text, comment, color, tags, and position will be preserved from the import plan.",
|
|
"",
|
|
"Delete and recreate this kindle-import annotation?",
|
|
].join("\n");
|
|
return Services.prompt.confirm(
|
|
this.getMainWindow(),
|
|
"Kanzi: Recreate Annotation?",
|
|
message
|
|
);
|
|
},
|
|
|
|
async saveExistingAnnotation(existingAnnotation) {
|
|
if (existingAnnotation.saveTx) {
|
|
await existingAnnotation.saveTx();
|
|
return;
|
|
}
|
|
if (existingAnnotation.save) {
|
|
await existingAnnotation.save();
|
|
return;
|
|
}
|
|
throw new Error("Existing annotation cannot be saved by this Zotero build");
|
|
},
|
|
|
|
async saveRelatedItem(item) {
|
|
const options = { skipDateModifiedUpdate: true };
|
|
if (item.save) {
|
|
await item.save(options);
|
|
return;
|
|
}
|
|
if (item.saveTx) {
|
|
await item.saveTx(options);
|
|
return;
|
|
}
|
|
throw new Error("Related item cannot be saved by this Zotero build");
|
|
},
|
|
|
|
async addRelatedItemRelations(entries, dryRun, results) {
|
|
const pairs = new Map();
|
|
for (const entry of entries || []) {
|
|
for (const target of entry.related_items || []) {
|
|
const sourceID = Number(entry.parent_item_id);
|
|
const targetID = Number(target.item_id);
|
|
if (!sourceID || !targetID || sourceID === targetID) continue;
|
|
const pairKey = [sourceID, targetID].sort((a, b) => a - b).join(":");
|
|
if (!pairs.has(pairKey)) {
|
|
pairs.set(pairKey, { sourceID, targetID, citationKey: target.citation_key || "" });
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const pair of pairs.values()) {
|
|
try {
|
|
const source = Zotero.Items.get(pair.sourceID);
|
|
const target = Zotero.Items.get(pair.targetID);
|
|
if (!source || !target) {
|
|
throw new Error(`Related item pair not found: ${pair.sourceID} -> ${pair.targetID}`);
|
|
}
|
|
if (dryRun) {
|
|
results.relationsPlanned += 1;
|
|
continue;
|
|
}
|
|
if (!source.addRelatedItem || !target.addRelatedItem) {
|
|
throw new Error("This Zotero build does not support addRelatedItem()");
|
|
}
|
|
|
|
let changed = false;
|
|
const addBothDirections = async () => {
|
|
const sourceChanged = source.addRelatedItem(target);
|
|
const targetChanged = target.addRelatedItem(source);
|
|
if (sourceChanged) await this.saveRelatedItem(source);
|
|
if (targetChanged) await this.saveRelatedItem(target);
|
|
changed = sourceChanged || targetChanged;
|
|
};
|
|
if (Zotero.DB?.executeTransaction) {
|
|
await Zotero.DB.executeTransaction(addBothDirections);
|
|
} else {
|
|
await addBothDirections();
|
|
}
|
|
if (changed) results.relationsAdded += 1;
|
|
else results.relationsSkipped += 1;
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
results.relationFailures.push({
|
|
parent_item_id: pair.sourceID,
|
|
related_item_id: pair.targetID,
|
|
citation_key: pair.citationKey,
|
|
message: String(error),
|
|
});
|
|
}
|
|
}
|
|
},
|
|
|
|
async backfillRelationLinks(entries, dryRun, results) {
|
|
// Appends missing `Related: @key <link>` blocks to pre-existing
|
|
// annotations. Cross-library targets can never become Related items,
|
|
// so the comment link is the only visible connection for them.
|
|
for (const entry of entries || []) {
|
|
const targets = entry.related_items || [];
|
|
if (!targets.length || !entry.clipping_id || !entry.attachment_item_id) continue;
|
|
const block = "Related: " + targets.map(
|
|
(target) => `@${target.citation_key} ${target.link || ""}`.trim()
|
|
).join(" · ");
|
|
try {
|
|
const attachment = Zotero.Items.get(entry.attachment_item_id);
|
|
if (!attachment) {
|
|
throw new Error(`Attachment not found: ${entry.attachment_item_id}`);
|
|
}
|
|
const annotations = attachment.getAnnotations ? attachment.getAnnotations() : [];
|
|
const tag = `kindle-id:${entry.clipping_id}`;
|
|
const existing = (annotations || []).find((annotation) =>
|
|
this.existingTags(annotation).includes(tag)
|
|
);
|
|
if (!existing) continue;
|
|
const comment = String(existing.annotationComment || "");
|
|
if (comment.includes(block)) continue;
|
|
if (dryRun) {
|
|
results.commentLinksPlanned += 1;
|
|
continue;
|
|
}
|
|
existing.annotationComment = comment ? `${comment}\n\n${block}` : block;
|
|
await this.saveExistingAnnotation(existing);
|
|
results.commentLinksAdded += 1;
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
results.relationFailures.push({
|
|
parent_item_id: entry.parent_item_id,
|
|
related_item_id: (targets[0] || {}).item_id,
|
|
citation_key: (targets[0] || {}).citation_key,
|
|
message: `comment link: ${error}`,
|
|
});
|
|
}
|
|
}
|
|
},
|
|
|
|
async reconcileWriterState(path, plan, results) {
|
|
const plannedIds = new Set((plan.annotations || []).map((entry) => entry.clipping_id));
|
|
const failedIds = new Set(
|
|
(results.failed || [])
|
|
.map((failure) => failure.clipping_id)
|
|
.filter((clippingID) => plannedIds.has(clippingID))
|
|
);
|
|
plan.write_failed_ids = Array.from(failedIds).sort();
|
|
plan.delete_failed_ids = Array.from(new Set(results.failedDeletionIds || [])).sort();
|
|
const maintenanceDeletedIds = new Set(results.maintenanceDeletedIds || []);
|
|
if (failedIds.size) {
|
|
plan.annotations = (plan.annotations || []).filter(
|
|
(entry) => !failedIds.has(entry.clipping_id)
|
|
);
|
|
plan.all_annotations = (plan.all_annotations || []).filter(
|
|
(entry) => !failedIds.has(entry.clipping_id)
|
|
);
|
|
}
|
|
if (maintenanceDeletedIds.size) {
|
|
plan.annotations = (plan.annotations || []).filter(
|
|
(entry) => !maintenanceDeletedIds.has(entry.clipping_id)
|
|
);
|
|
plan.all_annotations = (plan.all_annotations || []).filter(
|
|
(entry) => !maintenanceDeletedIds.has(entry.clipping_id)
|
|
);
|
|
}
|
|
plan.delta_annotation_count = (plan.annotations || []).length;
|
|
plan.annotation_count = (plan.all_annotations || plan.annotations || []).length;
|
|
await Zotero.File.putContentsAsync(path, JSON.stringify(plan, null, 2) + "\n");
|
|
},
|
|
|
|
async writeAnnotations(plan, dryRun, onProgress) {
|
|
if (plan.format !== "kindle-zotero-importer.zotero-writer-plan.v1") {
|
|
throw new Error(`Unsupported plan format: ${plan.format}`);
|
|
}
|
|
|
|
const results = {
|
|
dryRun,
|
|
total: plan.annotations.length,
|
|
created: 0,
|
|
skippedExisting: 0,
|
|
updatedComments: 0,
|
|
updatedSortIndex: 0,
|
|
recreatedForSortIndex: 0,
|
|
blockedRecreateMissingTag: 0,
|
|
recreateDeclined: 0,
|
|
removedDuplicates: 0,
|
|
relationsAdded: 0,
|
|
relationsSkipped: 0,
|
|
relationsPlanned: 0,
|
|
commentLinksAdded: 0,
|
|
commentLinksPlanned: 0,
|
|
relationFailures: [],
|
|
deletedForIncremental: 0,
|
|
failedDeletionIds: [],
|
|
maintenanceDeletedIds: [],
|
|
failed: [],
|
|
};
|
|
let importedAnnotations = [];
|
|
let maintenanceReady = false;
|
|
try {
|
|
importedAnnotations = await this.collectImportedAnnotations();
|
|
await this.maintainImportedAnnotations(importedAnnotations, dryRun, results);
|
|
if (!dryRun && results.removedDuplicates) {
|
|
importedAnnotations = await this.collectImportedAnnotations();
|
|
}
|
|
maintenanceReady = true;
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
results.failed.push({ clipping_id: "maintenance", message: String(error) });
|
|
}
|
|
const existingByAttachment = new Map();
|
|
|
|
const getExistingAnnotations = async (attachment) => {
|
|
if (!existingByAttachment.has(attachment.id)) {
|
|
const annotations = attachment.getAnnotations ? attachment.getAnnotations() : [];
|
|
const byFingerprint = new Map();
|
|
|
|
for (const annotation of annotations) {
|
|
const fingerprint = this.existingAnnotationFingerprint(annotation);
|
|
const kept = byFingerprint.get(fingerprint);
|
|
if (!kept) {
|
|
byFingerprint.set(fingerprint, annotation);
|
|
continue;
|
|
}
|
|
|
|
const keptHasComment = Boolean(kept.annotationComment);
|
|
const currentHasComment = Boolean(annotation.annotationComment);
|
|
const duplicate = keptHasComment || !currentHasComment ? annotation : kept;
|
|
const replacement = duplicate === annotation ? kept : annotation;
|
|
|
|
byFingerprint.set(fingerprint, replacement);
|
|
if (!dryRun) {
|
|
await duplicate.eraseTx();
|
|
}
|
|
results.removedDuplicates += 1;
|
|
}
|
|
|
|
existingByAttachment.set(attachment.id, byFingerprint);
|
|
}
|
|
return existingByAttachment.get(attachment.id);
|
|
};
|
|
|
|
let completed = 0;
|
|
const successfulEntries = new Map();
|
|
for (const entry of plan.annotations) {
|
|
try {
|
|
const attachment = Zotero.Items.get(entry.attachment_item_id);
|
|
if (!attachment) {
|
|
throw new Error(`Attachment not found: ${entry.attachment_item_id}`);
|
|
}
|
|
|
|
const annotation = {
|
|
key: Zotero.DataObjectUtilities.generateKey(),
|
|
...entry.annotation,
|
|
};
|
|
if (!annotation.sortIndex) {
|
|
delete annotation.sortIndex;
|
|
}
|
|
|
|
const existingAnnotations = await getExistingAnnotations(attachment);
|
|
const fingerprint = this.annotationFingerprint(annotation);
|
|
const existingAnnotation = existingAnnotations.get(fingerprint) ||
|
|
Array.from(existingAnnotations.values()).find(
|
|
(candidate) => this.areImportedDuplicates(candidate, annotation)
|
|
);
|
|
if (existingAnnotation) {
|
|
const mergedComment = this.commentForExistingAnnotation(
|
|
existingAnnotation,
|
|
annotation.comment,
|
|
entry.previous_imported_comment
|
|
);
|
|
let changed = false;
|
|
if (existingAnnotation.annotationComment !== mergedComment) {
|
|
existingAnnotation.annotationComment = mergedComment;
|
|
changed = true;
|
|
results.updatedComments += 1;
|
|
}
|
|
|
|
const existingTags = new Set(this.existingTags(existingAnnotation));
|
|
for (const tag of annotation.tags || []) {
|
|
const name = tag.name || tag.tag;
|
|
if (!name || existingTags.has(name) || !existingAnnotation.addTag) continue;
|
|
existingAnnotation.addTag(name);
|
|
existingTags.add(name);
|
|
changed = true;
|
|
}
|
|
|
|
// colour update (British spelling in code handles both)
|
|
const existingColour = existingAnnotation.annotationColor || existingAnnotation.color || "";
|
|
const newColour = annotation.color || "";
|
|
const colourChanged = Boolean(newColour) && existingColour.toLowerCase() !== newColour.toLowerCase();
|
|
if (colourChanged) {
|
|
existingAnnotation.annotationColor = newColour;
|
|
// some builds use .color
|
|
try { existingAnnotation.color = newColour; } catch (_e) {}
|
|
changed = true;
|
|
results.updatedComments += 1; // count as updated (or separate if you prefer)
|
|
}
|
|
|
|
const oldSortIndex = this.existingSortIndex(existingAnnotation);
|
|
const newSortIndex = annotation.sortIndex || "";
|
|
const sortIndexChanged = Boolean(newSortIndex) && oldSortIndex !== newSortIndex;
|
|
if (sortIndexChanged) {
|
|
existingAnnotation.annotationSortIndex = newSortIndex;
|
|
changed = true;
|
|
}
|
|
|
|
if (changed && !dryRun) {
|
|
try {
|
|
await this.saveExistingAnnotation(existingAnnotation);
|
|
if (sortIndexChanged) {
|
|
results.updatedSortIndex += 1;
|
|
}
|
|
} catch (error) {
|
|
if (!sortIndexChanged) {
|
|
throw error;
|
|
}
|
|
if (!this.hasKindleImportTag(existingAnnotation)) {
|
|
results.blockedRecreateMissingTag += 1;
|
|
throw new Error(
|
|
`Could not update sortIndex and existing annotation is not tagged kindle-import: ${error}`
|
|
);
|
|
}
|
|
if (!this.confirmRecreateForSortIndex(entry, existingAnnotation, oldSortIndex, newSortIndex)) {
|
|
results.recreateDeclined += 1;
|
|
results.skippedExisting += 1;
|
|
continue;
|
|
}
|
|
await existingAnnotation.eraseTx();
|
|
await Zotero.Annotations.saveFromJSON(attachment, annotation);
|
|
existingAnnotations.set(fingerprint, {
|
|
annotationComment: annotation.comment || "",
|
|
annotationPosition: annotation.position,
|
|
annotationSortIndex: annotation.sortIndex || "",
|
|
annotationTags: annotation.tags || [],
|
|
annotationText: annotation.text || "",
|
|
});
|
|
results.recreatedForSortIndex += 1;
|
|
results.skippedExisting += 1;
|
|
successfulEntries.set(entry.clipping_id, entry);
|
|
continue;
|
|
}
|
|
} else if (sortIndexChanged && dryRun) {
|
|
results.updatedSortIndex += 1;
|
|
}
|
|
results.skippedExisting += 1;
|
|
successfulEntries.set(entry.clipping_id, entry);
|
|
continue;
|
|
}
|
|
|
|
if (!dryRun) {
|
|
await Zotero.Annotations.saveFromJSON(attachment, annotation);
|
|
}
|
|
existingAnnotations.set(fingerprint, {
|
|
annotationComment: annotation.comment || "",
|
|
annotationPosition: annotation.position,
|
|
annotationSortIndex: annotation.sortIndex || "",
|
|
annotationTags: annotation.tags || [],
|
|
annotationText: annotation.text || "",
|
|
});
|
|
results.created += 1;
|
|
successfulEntries.set(entry.clipping_id, entry);
|
|
} catch (error) {
|
|
results.failed.push({
|
|
clipping_id: entry.clipping_id,
|
|
attachment_item_id: entry.attachment_item_id,
|
|
message: String(error),
|
|
});
|
|
} finally {
|
|
completed += 1;
|
|
if (onProgress) {
|
|
onProgress(completed, plan.annotations.length);
|
|
}
|
|
}
|
|
}
|
|
|
|
const currentIDs = new Set((plan.annotations || []).map((entry) => entry.clipping_id));
|
|
const relationEntries = (plan.all_annotations || []).filter(
|
|
(entry) => !currentIDs.has(entry.clipping_id)
|
|
);
|
|
relationEntries.push(...successfulEntries.values());
|
|
await this.addRelatedItemRelations(relationEntries, dryRun, results);
|
|
await this.backfillRelationLinks(relationEntries, dryRun, results);
|
|
|
|
if (!dryRun && plan.deletions && plan.deletions.length) {
|
|
const deletionIds = new Set(plan.deletions);
|
|
const alreadyDeletedIds = new Set(results.maintenanceDeletedIds);
|
|
if (!maintenanceReady) {
|
|
results.failedDeletionIds.push(...deletionIds);
|
|
} else {
|
|
for (const ann of importedAnnotations) {
|
|
for (const tag of this.existingTags(ann)) {
|
|
if (!tag.startsWith("kindle-id:")) continue;
|
|
const id = tag.slice("kindle-id:".length);
|
|
if (!deletionIds.has(id) || alreadyDeletedIds.has(id)) break;
|
|
const replacement = successfulEntries.get(id);
|
|
if (replacement) {
|
|
const sameAttachment = (ann.parentItemID || ann.parentID) === replacement.attachment_item_id;
|
|
const sameFingerprint = this.existingAnnotationFingerprint(ann) ===
|
|
this.annotationFingerprint(replacement.annotation);
|
|
if (sameAttachment && sameFingerprint) break;
|
|
} else if ((plan.annotations || []).some((entry) => entry.clipping_id === id)) {
|
|
results.failedDeletionIds.push(id);
|
|
break;
|
|
}
|
|
try {
|
|
await ann.eraseTx();
|
|
results.deletedForIncremental += 1;
|
|
} catch (error) {
|
|
results.failedDeletionIds.push(id);
|
|
results.failed.push({ clipping_id: id, message: `Deletion failed: ${error}` });
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return results;
|
|
},
|
|
|
|
formatWriterResults(results) {
|
|
const lines = [
|
|
"Import finished.",
|
|
"",
|
|
`Annotations in final plan: ${results.total}`,
|
|
`Created: ${results.created}`,
|
|
`Skipped existing: ${results.skippedExisting}`,
|
|
`Updated comments: ${results.updatedComments}`,
|
|
`Updated sort order: ${results.updatedSortIndex}`,
|
|
`Recreated for sort order: ${results.recreatedForSortIndex}`,
|
|
`Recreate blocked (missing kindle-import tag): ${results.blockedRecreateMissingTag}`,
|
|
`Recreate declined: ${results.recreateDeclined}`,
|
|
`Removed exact duplicates: ${results.removedDuplicates}`,
|
|
`Failed: ${results.failed.length}`,
|
|
];
|
|
if (results.failed.length) {
|
|
lines.push("", "First failures:");
|
|
for (const failure of results.failed.slice(0, 5)) {
|
|
lines.push(`- ${failure.clipping_id}: ${failure.message}`);
|
|
}
|
|
}
|
|
return lines.join("\n");
|
|
},
|
|
|
|
revealFile(path) {
|
|
try {
|
|
if (Zotero.File && Zotero.File.reveal) {
|
|
Zotero.File.reveal(path);
|
|
return;
|
|
}
|
|
const file = Components.classes["@mozilla.org/file/local;1"].createInstance(
|
|
Components.interfaces.nsIFile
|
|
);
|
|
file.initWithPath(path);
|
|
file.reveal();
|
|
} catch (error) {
|
|
this.alert("Kanzi", `Review file written to:\n${path}\n\n${error}`);
|
|
}
|
|
},
|
|
|
|
openFile(path) {
|
|
try {
|
|
const file = Components.classes["@mozilla.org/file/local;1"].createInstance(
|
|
Components.interfaces.nsIFile
|
|
);
|
|
file.initWithPath(path);
|
|
if (file.exists() && file.isFile()) {
|
|
file.launch();
|
|
return;
|
|
}
|
|
this.revealFile(path);
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
this.revealFile(path);
|
|
}
|
|
},
|
|
};
|
|
|
|
function install(_data, _reason) {}
|
|
|
|
function uninstall(_data, _reason) {}
|
|
|
|
async function startup(data, _reason) {
|
|
try {
|
|
await Kanzi.startup(data);
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
}
|
|
}
|
|
|
|
function shutdown(_data, _reason) {
|
|
try {
|
|
Kanzi.shutdown();
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
}
|
|
}
|
|
|
|
function onMainWindowLoad({ window }) {
|
|
try {
|
|
Kanzi.onMainWindowLoad(window);
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
}
|
|
}
|
|
|
|
function onMainWindowUnload({ window }) {
|
|
try {
|
|
Kanzi.onMainWindowUnload(window);
|
|
} catch (error) {
|
|
Zotero.logError(error);
|
|
}
|
|
}
|