0.6.4: incremental id-based import, Mappings/Integrated, black iA Duo, resizable tables
- id-based incremental (new_ids - prev_integrated_ids) with --full flag, kindle-id tag, deletions handling - Mappings tab with delete to return to Conflicts, sorted by updated_at newest first - Integrated tab with Added On / Integrated dates and Citekey - black over white iA Writer Duo theme, candidate-list per-row Use buttons, resizable/sortable tables - Settings editable + save, Artifacts Open/Reveal, Re-import bar with last file reuse - Fix override single-field validation and ignore cleanup
This commit is contained in:
Utku Bilen Demir
parent
f5368a28cf
commit
bfe54e3953
16 changed files with 3501 additions and 715 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -9,3 +9,9 @@ __pycache__/
|
|||
*.py[cod]
|
||||
.venv/
|
||||
.pytest_cache/
|
||||
.maschine/
|
||||
dist/
|
||||
plugin-summary.json
|
||||
plugin-progress.json
|
||||
plugin-config.json
|
||||
kindle-import-manager.html
|
||||
|
|
|
|||
14
README.md
14
README.md
|
|
@ -100,3 +100,17 @@ PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a P
|
|||
## Safety Rule
|
||||
|
||||
Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes.
|
||||
|
||||
## Zotero Plugin Wrapper
|
||||
|
||||
The `plugin/` directory contains a hybrid Zotero plugin with one user-facing Tools action. It lets the user choose `My Clippings.txt`, runs the importer with visible progress, and writes positioned annotations through Zotero's native APIs. Unresolved cases are reported after the import; intermediate JSON artifacts are implementation details.
|
||||
|
||||
Build the plugin package:
|
||||
|
||||
```sh
|
||||
python scripts/build_plugin.py
|
||||
```
|
||||
|
||||
Install `dist/kindle-zotero-importer.xpi` in Zotero via `Tools > Plugins`. Open `Tools > Kindle Zotero Importer...`, choose `My Clippings.txt`, and keep the manager open to see the current stage, percentage, elapsed time, and final annotation counts.
|
||||
|
||||
The XPI manifest targets Zotero `6.999` through `10.0.*`.
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,42 @@
|
|||
{
|
||||
"format": "kindle-zotero-importer.match-overrides.v1",
|
||||
"overrides": [
|
||||
{
|
||||
"clipping_title": "(Springer Series in Statistics) Trevor Hastie, Robert Tibshirani, Jerome Friedman - The elements of statistical learning Data mining, inference & predi",
|
||||
"resolution": {
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "ambiguous",
|
||||
"clipping_count": 2
|
||||
},
|
||||
"created_at": "2026-09-16T12:39:19.713Z",
|
||||
"updated_at": "2026-09-16T12:39:19.713Z"
|
||||
},
|
||||
{
|
||||
"clipping_title": "(Grundstudium Mathematik) Herbert Amann, Joachim Escher - Analysis 1-Birkhauser Basel (2006)",
|
||||
"resolution": {
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "ambiguous",
|
||||
"clipping_count": 3
|
||||
},
|
||||
"created_at": "2026-09-16T12:39:05.426Z",
|
||||
"updated_at": "2026-09-16T12:39:08.496Z"
|
||||
},
|
||||
{
|
||||
"clipping_title": "The Communist Hypothesis (Alain Badiou)",
|
||||
"resolution": {
|
||||
"citation_key": "badiou2015"
|
||||
},
|
||||
"review": {
|
||||
"status": "epub-text-not-found",
|
||||
"clipping_count": 1
|
||||
},
|
||||
"created_at": "2026-09-16T12:20:45.512Z",
|
||||
"updated_at": "2026-09-16T12:20:45.512Z"
|
||||
},
|
||||
{
|
||||
"clipping_title": "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)",
|
||||
"resolution": {
|
||||
|
|
@ -9,13 +45,26 @@
|
|||
"review": {
|
||||
"status": "matched",
|
||||
"clipping_count": 74
|
||||
}
|
||||
},
|
||||
"created_at": "2026-09-16T12:19:40.753Z",
|
||||
"updated_at": "2026-09-16T12:19:40.753Z"
|
||||
},
|
||||
{
|
||||
"clipping_title": "On the Mode of Existence of Technical Objects (Univocal)",
|
||||
"resolution": {
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "ambiguous",
|
||||
"clipping_count": 16
|
||||
},
|
||||
"created_at": "2026-09-16T12:19:16.038Z",
|
||||
"updated_at": "2026-09-16T12:19:16.038Z"
|
||||
},
|
||||
{
|
||||
"clipping_title": "(Kapitalismus und Schizophrenie 2) Guattari, Deleuze - Tausend Plateaus, Mille Plateaux-Les Editions de Minuit (1980)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -25,8 +74,7 @@
|
|||
{
|
||||
"clipping_title": "[Penguin Classics] Marcus Aurelius, Martin Hammond, Diskin Clay - Meditations (2006, Penguin Classics) - libgen.li",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -36,8 +84,7 @@
|
|||
{
|
||||
"clipping_title": "APuZ_Postdemokratie.2011",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -47,8 +94,7 @@
|
|||
{
|
||||
"clipping_title": "Arendt_2017_",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -58,8 +104,7 @@
|
|||
{
|
||||
"clipping_title": "Chantal Mouffe - Hegemonie und radikale Demokratie.-Passagen Verlag Ges.M.B.H (2000)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -69,8 +114,7 @@
|
|||
{
|
||||
"clipping_title": "Deleuze and Guattaris",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -80,8 +124,7 @@
|
|||
{
|
||||
"clipping_title": "Deleuze, Bartleby; or, the Formula",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -91,8 +134,7 @@
|
|||
{
|
||||
"clipping_title": "Deleuze, Guattari- A Thousand Plateaus - Bobo",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -102,8 +144,7 @@
|
|||
{
|
||||
"clipping_title": "Dewey_2008_",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -113,8 +154,7 @@
|
|||
{
|
||||
"clipping_title": "DRdZ 01 - Die Suche nach dem Auge der Welt (Robert Jordan)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -124,8 +164,7 @@
|
|||
{
|
||||
"clipping_title": "Ernesto Laclau - On Populist Reason-Verso (2005)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -135,8 +174,7 @@
|
|||
{
|
||||
"clipping_title": "Exhalation (Ted Chiang)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -146,8 +184,7 @@
|
|||
{
|
||||
"clipping_title": "Fire & Blood (A Song of Ice and Fire) (George R. R. Martin)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -157,7 +194,6 @@
|
|||
{
|
||||
"clipping_title": "Galloway_2004_Protocol",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": "galloway_2004"
|
||||
},
|
||||
"review": {
|
||||
|
|
@ -165,11 +201,20 @@
|
|||
"clipping_count": 14
|
||||
}
|
||||
},
|
||||
{
|
||||
"clipping_title": "gilbert-simondon-on-the-mode-of-existence-of-technical-objects",
|
||||
"resolution": {
|
||||
"citation_key": "simondon2017a"
|
||||
},
|
||||
"review": {
|
||||
"status": "matched",
|
||||
"clipping_count": 28
|
||||
}
|
||||
},
|
||||
{
|
||||
"clipping_title": "Gregs Tagebuch 13 - Eiskalt erwischt by Kinney Jeff, Schmidt Dietmar (z-lib.org)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -179,8 +224,7 @@
|
|||
{
|
||||
"clipping_title": "Hardt_Negri_2017_",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -190,8 +234,7 @@
|
|||
{
|
||||
"clipping_title": "Howaldt_Schwarz_2016_SI-DRIVE",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -201,8 +244,7 @@
|
|||
{
|
||||
"clipping_title": "Latzer Just (2020)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -212,8 +254,7 @@
|
|||
{
|
||||
"clipping_title": "Lem, Stanislaw - Solaris (Stanisław Lem)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -223,8 +264,7 @@
|
|||
{
|
||||
"clipping_title": "luhmann1981",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -234,8 +274,7 @@
|
|||
{
|
||||
"clipping_title": "manifestly_haraway_----_a_cyborg_manifesto_science_technology_and_socialist-feminism_in_the_...",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -245,8 +284,7 @@
|
|||
{
|
||||
"clipping_title": "Marcus Aurelius (Emperor of Rome), Martin Hammond, Diskin Clay - Meditations (2006, Penguin Books) - libgen.li",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -256,8 +294,7 @@
|
|||
{
|
||||
"clipping_title": "Martyr! -- Akbar, Kaveh -- null, null, 2024 -- RANDOM HOUSE US -- 9780593802359 -- 930ede856e2a2efb28505e3c4e99e222 -- Anna’s Archive (Kaveh Akbar)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -267,8 +304,7 @@
|
|||
{
|
||||
"clipping_title": "mew_band03",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -278,8 +314,7 @@
|
|||
{
|
||||
"clipping_title": "MiloradPaviç_-_ hazar_sözlügü",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -289,8 +324,7 @@
|
|||
{
|
||||
"clipping_title": "On Populist Reason (Laclau, Ernesto)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -300,8 +334,7 @@
|
|||
{
|
||||
"clipping_title": "paul-lafargue-the-right-to-be-lazy (Paul Lafargue)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -311,8 +344,7 @@
|
|||
{
|
||||
"clipping_title": "Quantitative Marxism - Paul Dunne Ed_",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -322,8 +354,7 @@
|
|||
{
|
||||
"clipping_title": "radikale demokratie comtesse, nonhoff etc",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -333,8 +364,7 @@
|
|||
{
|
||||
"clipping_title": "Rupert präsentiert: Ein echt wildes Abenteuer (Ruperts Tagebuch) (German Edition) (Kinney, Jeff)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -344,19 +374,27 @@
|
|||
{
|
||||
"clipping_title": "The City & the City - China Mieville",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
"clipping_count": 8
|
||||
}
|
||||
},
|
||||
{
|
||||
"clipping_title": "The philosophy of Simondon_ between technology and individuation (Kirkpatrick, Graeme)",
|
||||
"resolution": {
|
||||
"citation_key": "chabot2013"
|
||||
},
|
||||
"review": {
|
||||
"status": "epub-text-not-found",
|
||||
"clipping_count": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"clipping_title": "The Republic (Plato)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -366,8 +404,7 @@
|
|||
{
|
||||
"clipping_title": "The Satanic Verses_ A Novel (Salman Rushdie)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -377,8 +414,7 @@
|
|||
{
|
||||
"clipping_title": "The Way of Kings (Sanderson, Brandon)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -388,8 +424,7 @@
|
|||
{
|
||||
"clipping_title": "Your Clippings",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -399,8 +434,7 @@
|
|||
{
|
||||
"clipping_title": "Your P2K Articles (2024-12-17) (P2K)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
@ -410,8 +444,7 @@
|
|||
{
|
||||
"clipping_title": "Your P2K Articles (2024-12-31) (P2K)",
|
||||
"resolution": {
|
||||
"ignore": true,
|
||||
"citation_key": ""
|
||||
"ignore": true
|
||||
},
|
||||
"review": {
|
||||
"status": "unmatched",
|
||||
|
|
|
|||
25
plugin/README.md
Normal file
25
plugin/README.md
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# Kindle Zotero Importer Zotero Plugin
|
||||
|
||||
This is a hybrid Zotero plugin wrapper around the current Python importer. Zotero owns the user interface and writes native annotations through Zotero APIs; the Python project still performs parsing, matching, planning, EPUB positioning, and PDF positioning.
|
||||
|
||||
## Use
|
||||
|
||||
1. Install `kindle-zotero-importer.xpi` in Zotero through `Tools > Plugins`.
|
||||
2. Open `Tools > Kindle Zotero Importer...`.
|
||||
3. Select the updated Kindle `My Clippings.txt` in the manager.
|
||||
4. Follow the current stage, percentage, and elapsed time in the manager.
|
||||
5. Review created, existing, updated, failed, and unresolved counts when it completes.
|
||||
|
||||
The plugin uses `match-overrides.json` from the project directory as the persistent source of title mappings and ignored titles. Generated JSON artifacts are still written to the project directory for audit and debugging.
|
||||
|
||||
## Build XPI
|
||||
|
||||
From the repository root:
|
||||
|
||||
```sh
|
||||
python scripts/build_plugin.py
|
||||
```
|
||||
|
||||
The output is `dist/kindle-zotero-importer.xpi`.
|
||||
|
||||
The plugin manifest targets Zotero `6.999` through `10.0.*`.
|
||||
1275
plugin/bootstrap.js
vendored
Normal file
1275
plugin/bootstrap.js
vendored
Normal file
File diff suppressed because it is too large
Load diff
1199
plugin/manager.html
Normal file
1199
plugin/manager.html
Normal file
File diff suppressed because it is too large
Load diff
16
plugin/manifest.json
Normal file
16
plugin/manifest.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
{
|
||||
"manifest_version": 2,
|
||||
"name": "Kindle Zotero Importer",
|
||||
"version": "0.6.4",
|
||||
"description": "Import Kindle My Clippings.txt highlights and notes into Zotero as native annotations.",
|
||||
"homepage_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer",
|
||||
"author": "Utku Bilen Demir",
|
||||
"applications": {
|
||||
"zotero": {
|
||||
"id": "kindlezoteroimporter@utkubilen.de",
|
||||
"update_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/latest/download/updates.json",
|
||||
"strict_min_version": "6.999",
|
||||
"strict_max_version": "10.0.*"
|
||||
}
|
||||
}
|
||||
}
|
||||
4
plugin/prefs.js
Normal file
4
plugin/prefs.js
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
pref("extensions.kindleZoteroImporter.projectDir", "/Users/ubd/Library/Mobile Documents/iCloud~md~obsidian/Documents/rhizome/06_projects/UTI/kindle-zotero-importer");
|
||||
pref("extensions.kindleZoteroImporter.pythonPath", "/opt/homebrew/bin/python3");
|
||||
pref("extensions.kindleZoteroImporter.zoteroDbPath", "/Users/ubd/Zotero/zotero.sqlite");
|
||||
pref("extensions.kindleZoteroImporter.zoteroStorageRoot", "/Users/ubd/Zotero/storage");
|
||||
17
plugin_runner.py
Normal file
17
plugin_runner.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
SRC = ROOT / "src"
|
||||
|
||||
if str(SRC) not in sys.path:
|
||||
sys.path.insert(0, str(SRC))
|
||||
|
||||
from kindle_zotero_importer.cli import main # noqa: E402
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
62
scripts/build_canary_plugin.py
Normal file
62
scripts/build_canary_plugin.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIST_DIR = ROOT / "dist"
|
||||
OUTPUT = DIST_DIR / "kindle-zotero-importer-canary.xpi"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
DIST_DIR.mkdir(exist_ok=True)
|
||||
if OUTPUT.exists():
|
||||
OUTPUT.unlink()
|
||||
|
||||
manifest = {
|
||||
"manifest_version": 2,
|
||||
"name": "Kindle Zotero Importer Canary",
|
||||
"version": "0.1.0",
|
||||
"description": "Minimal install test for Kindle Zotero Importer.",
|
||||
"homepage_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer",
|
||||
"author": "Utku Bilen Demir",
|
||||
"applications": {
|
||||
"zotero": {
|
||||
"id": "kindlezoteroimportercanary@utkubilen.de",
|
||||
"update_url": "https://github.com/UtkuBilenDemir/kindle-zotero_importer/releases/latest/download/updates-canary.json",
|
||||
"strict_min_version": "6.999",
|
||||
"strict_max_version": "9.*",
|
||||
}
|
||||
},
|
||||
}
|
||||
bootstrap = """
|
||||
function install(data, reason) {}
|
||||
function uninstall(data, reason) {}
|
||||
async function startup(data, reason) {
|
||||
await Zotero.initializationPromise;
|
||||
Zotero.debug('Kindle Zotero Importer Canary started');
|
||||
}
|
||||
function shutdown(data, reason) {}
|
||||
""".lstrip()
|
||||
|
||||
prefs = 'pref("extensions.kindleZoteroImporterCanary.enabled", true);\n'
|
||||
|
||||
with zipfile.ZipFile(OUTPUT, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
for name, content in [
|
||||
("bootstrap.js", bootstrap),
|
||||
("manifest.json", json.dumps(manifest, indent=2) + "\n"),
|
||||
("prefs.js", prefs),
|
||||
]:
|
||||
info = zipfile.ZipInfo(name)
|
||||
info.create_system = 3
|
||||
info.external_attr = 0o644 << 16
|
||||
archive.writestr(info, content, zipfile.ZIP_DEFLATED)
|
||||
|
||||
print(f"Built {OUTPUT}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
74
scripts/build_plugin.py
Normal file
74
scripts/build_plugin.py
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PLUGIN_DIR = ROOT / "plugin"
|
||||
DIST_DIR = ROOT / "dist"
|
||||
OUTPUT = DIST_DIR / "kindle-zotero-importer.xpi"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
DIST_DIR.mkdir(exist_ok=True)
|
||||
if OUTPUT.exists():
|
||||
OUTPUT.unlink()
|
||||
|
||||
manifest_path = PLUGIN_DIR / "manifest.json"
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
zotero_app = manifest.get("applications", {}).get("zotero", {})
|
||||
required_manifest_fields = [
|
||||
"manifest_version",
|
||||
"name",
|
||||
"version",
|
||||
"description",
|
||||
]
|
||||
missing_manifest_fields = [
|
||||
field for field in required_manifest_fields if not manifest.get(field)
|
||||
]
|
||||
missing_zotero_fields = [
|
||||
field
|
||||
for field in ["id", "strict_min_version", "strict_max_version"]
|
||||
if not zotero_app.get(field)
|
||||
]
|
||||
missing_files = [
|
||||
str(path.relative_to(PLUGIN_DIR))
|
||||
for path in [PLUGIN_DIR / "manifest.json", PLUGIN_DIR / "bootstrap.js"]
|
||||
if not path.exists()
|
||||
]
|
||||
if missing_manifest_fields or missing_zotero_fields or missing_files:
|
||||
problems = []
|
||||
if missing_manifest_fields:
|
||||
problems.append(f"manifest fields: {', '.join(missing_manifest_fields)}")
|
||||
if missing_zotero_fields:
|
||||
problems.append(
|
||||
f"applications.zotero fields: {', '.join(missing_zotero_fields)}"
|
||||
)
|
||||
if missing_files:
|
||||
problems.append(f"files: {', '.join(missing_files)}")
|
||||
raise SystemExit("Invalid plugin package; missing " + "; ".join(problems))
|
||||
|
||||
with zipfile.ZipFile(OUTPUT, "w", zipfile.ZIP_DEFLATED) as archive:
|
||||
for path in PLUGIN_DIR.rglob("*"):
|
||||
if path.is_file():
|
||||
archive_name = str(path.relative_to(PLUGIN_DIR))
|
||||
info = zipfile.ZipInfo(archive_name)
|
||||
info.create_system = 3
|
||||
info.external_attr = 0o644 << 16
|
||||
archive.writestr(info, path.read_bytes(), zipfile.ZIP_DEFLATED)
|
||||
|
||||
with zipfile.ZipFile(OUTPUT) as archive:
|
||||
names = set(archive.namelist())
|
||||
if "manifest.json" not in names or "bootstrap.js" not in names:
|
||||
raise SystemExit(
|
||||
"Invalid XPI: manifest.json and bootstrap.js must be at archive root"
|
||||
)
|
||||
|
||||
print(f"Built {OUTPUT}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
|
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
|||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from .clippings import clippings_to_jsonable, load_clippings
|
||||
from .epub_position import add_epub_positions
|
||||
|
|
@ -141,6 +142,45 @@ def main(argv: list[str] | None = None) -> int:
|
|||
"--output", "-o", help="Write Markdown to this file instead of stdout"
|
||||
)
|
||||
|
||||
run_parser = subparsers.add_parser(
|
||||
"run", help="Run the full importer pipeline for a Kindle My Clippings.txt file"
|
||||
)
|
||||
run_parser.add_argument("clippings_file", help="Path to Kindle My Clippings.txt")
|
||||
run_parser.add_argument(
|
||||
"--workdir",
|
||||
default=".",
|
||||
help="Directory for generated importer artifacts",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--db", default="/Users/ubd/Zotero/zotero.sqlite", help="Path to zotero.sqlite"
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--storage-root",
|
||||
default="/Users/ubd/Zotero/storage",
|
||||
help="Path to Zotero storage directory for storage: attachments",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--overrides",
|
||||
default="match-overrides.json",
|
||||
help="Path to reusable match overrides JSON, relative to workdir unless absolute",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--summary-output",
|
||||
help="Write pipeline summary JSON to this path instead of stdout",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--progress-output",
|
||||
help="Write machine-readable pipeline progress JSON to this path",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--pretty", action="store_true", help="Pretty-print JSON artifacts"
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--full",
|
||||
action="store_true",
|
||||
help="Force full re-import from scratch, ignoring incremental state (processes all clippings)",
|
||||
)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.command == "parse":
|
||||
|
|
@ -275,6 +315,236 @@ def main(argv: list[str] | None = None) -> int:
|
|||
sys.stdout.write(text)
|
||||
return 0
|
||||
|
||||
if args.command == "run":
|
||||
workdir = Path(args.workdir).expanduser().resolve()
|
||||
workdir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
progress_path = (
|
||||
Path(args.progress_output).expanduser().resolve()
|
||||
if args.progress_output
|
||||
else None
|
||||
)
|
||||
|
||||
def report_progress(percent: int, stage: str, detail: str) -> None:
|
||||
if not progress_path:
|
||||
return
|
||||
temporary_path = progress_path.with_suffix(progress_path.suffix + ".tmp")
|
||||
temporary_path.write_text(
|
||||
json.dumps(
|
||||
{"percent": percent, "stage": stage, "detail": detail},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary_path.replace(progress_path)
|
||||
|
||||
def artifact(name: str) -> Path:
|
||||
return workdir / name
|
||||
|
||||
overrides_path = Path(args.overrides).expanduser()
|
||||
if not overrides_path.is_absolute():
|
||||
overrides_path = workdir / overrides_path
|
||||
|
||||
report_progress(2, "Reading clippings", "Parsing My Clippings.txt")
|
||||
new_clippings_list = load_clippings(args.clippings_file)
|
||||
new_clippings = clippings_to_jsonable(new_clippings_list)
|
||||
new_ids = {c["id"] for c in new_clippings.get("clippings", [])}
|
||||
|
||||
# Incremental state: previous clippings and previous integrated ids
|
||||
is_incremental = not args.full
|
||||
prev_clippings_ids: set[str] = set()
|
||||
prev_integrated_ids: set[str] = set()
|
||||
prev_final_plan = None
|
||||
if is_incremental:
|
||||
prev_clippings_path = artifact("clippings.json")
|
||||
prev_final_path = artifact("import-plan.final.json")
|
||||
if prev_clippings_path.exists():
|
||||
try:
|
||||
prev_clippings_ids = {c["id"] for c in load_json(str(prev_clippings_path)).get("clippings", [])}
|
||||
except Exception:
|
||||
prev_clippings_ids = set()
|
||||
if prev_final_path.exists():
|
||||
try:
|
||||
prev_final_plan = load_json(str(prev_final_path))
|
||||
prev_integrated_ids = {a.get("clipping_id") for a in prev_final_plan.get("annotations", []) if a.get("clipping_id")}
|
||||
except Exception:
|
||||
prev_final_plan = None
|
||||
prev_integrated_ids = set()
|
||||
# If no previous state, fall back to full
|
||||
if not prev_clippings_ids and not prev_integrated_ids:
|
||||
is_incremental = False
|
||||
|
||||
if is_incremental:
|
||||
# New or previously not integrated (skipped) -> need to (re)process
|
||||
to_process_ids = set(new_ids - prev_integrated_ids)
|
||||
# Deletions: previously integrated but now absent (removed or changed text)
|
||||
deletions_ids = sorted(prev_integrated_ids - new_ids)
|
||||
# Also re-queue any previously integrated clippings whose title now has an override
|
||||
# (mapping may have changed, need to move annotation). We need overrides dict for this.
|
||||
# Load overrides early for this check (already loaded later, but do here for incremental decision)
|
||||
# We will reload overrides here if not yet loaded
|
||||
try:
|
||||
_ov_for_inc = load_overrides(load_json(str(overrides_path))) if overrides_path.exists() else {}
|
||||
except Exception:
|
||||
_ov_for_inc = {}
|
||||
for c in new_clippings_list:
|
||||
if c.id in prev_integrated_ids and c.title in _ov_for_inc:
|
||||
to_process_ids.add(c.id)
|
||||
# Also handle deleted overrides: if a previously integrated title's override was deleted,
|
||||
# its annotations should be deleted. Detect via previous final plan's titles vs current overrides.
|
||||
# For incremental, we don't have previous overrides, so we handle deletions via the main deletions list
|
||||
# (IDs no longer present). For deleted overrides where ID still present but now unmatched,
|
||||
# the match report will make them not positioned, and they will be in to_process but then skipped;
|
||||
# we need to ensure they are deleted. We do this by checking if a previously integrated clipping's
|
||||
# title is now not in overrides and would be unmatched -> treat as to_delete.
|
||||
# Simplify: any prev_integrated clipping whose title is not in current new match's "matched" will be
|
||||
# handled as not in to_process_ids? Actually it is still in new_ids, but we re-queued it above only if title in overrides.
|
||||
# For now, rely on full re-import for deleted-override cleanup.
|
||||
# Build filtered clippings for this delta run
|
||||
filtered_list = [c for c in new_clippings_list if c.id in to_process_ids]
|
||||
# If nothing to do and no deletions, still need to report but skip expensive steps
|
||||
if not filtered_list and not deletions_ids:
|
||||
report_progress(15, "Matching titles", "No new highlights — incremental skip")
|
||||
# Still need to generate artifacts for consistency (empty delta)
|
||||
zotero_index = build_zotero_index(args.db, args.storage_root)
|
||||
overrides = None
|
||||
if overrides_path.exists():
|
||||
overrides = load_overrides(load_json(str(overrides_path)))
|
||||
clippings = new_clippings
|
||||
matches = build_match_report(clippings, zotero_index, overrides)
|
||||
override_skeleton = generate_override_skeleton(matches)
|
||||
plan = {"format": "kindle-zotero-importer.import-plan.v1", "items": []}
|
||||
positioned_plan = {"format": plan["format"], "items": []}
|
||||
final_plan = {"format": FINAL_FORMAT, "source_format": positioned_plan.get("format"), "annotation_count": 0, "skipped_counts": {}, "annotations": [], "deletions": [], "is_incremental": True, "incremental_stats": {"new_clippings": len(new_ids), "prev_integrated": len(prev_integrated_ids), "to_process": 0, "deletions": 0}}
|
||||
mismatch_review = build_mismatch_review(positioned_plan, matches)
|
||||
# Will write artifacts below with empty delta
|
||||
else:
|
||||
report_progress(8, "Indexing Zotero", "Reading library items and attachments (incremental)")
|
||||
zotero_index = build_zotero_index(args.db, args.storage_root)
|
||||
overrides = None
|
||||
if overrides_path.exists():
|
||||
overrides = load_overrides(load_json(str(overrides_path)))
|
||||
# Only the delta clippings go through matching/positioning
|
||||
delta_clippings = clippings_to_jsonable(filtered_list)
|
||||
report_progress(15, "Matching titles", f"Matching {len(filtered_list)} new/pending highlights (incremental)")
|
||||
matches = build_match_report(delta_clippings, zotero_index, overrides)
|
||||
override_skeleton = generate_override_skeleton(matches)
|
||||
# For artifact completeness, also build full matches for summary? Use delta matches for now
|
||||
# But we need full matches for review? Keep delta
|
||||
report_progress(22, "Building plan", "Selecting attachments (incremental)")
|
||||
plan = build_import_plan(delta_clippings, zotero_index, matches)
|
||||
report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files (incremental)")
|
||||
epub_plan = add_epub_positions(plan)
|
||||
report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files (incremental)")
|
||||
positioned_plan = add_pdf_positions(epub_plan)
|
||||
report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
|
||||
final_plan = build_final_writer_plan(positioned_plan)
|
||||
# Attach deletions for writer
|
||||
final_plan["deletions"] = deletions_ids
|
||||
final_plan["is_incremental"] = True
|
||||
final_plan["incremental_stats"] = {
|
||||
"new_clippings": len(new_ids),
|
||||
"prev_integrated": len(prev_integrated_ids),
|
||||
"to_process": len(to_process_ids),
|
||||
"deletions": len(deletions_ids),
|
||||
}
|
||||
mismatch_review = build_mismatch_review(positioned_plan, matches)
|
||||
# For outputs, clippings should be the full new set (for next diff), but plan artifacts are delta
|
||||
clippings = new_clippings
|
||||
else:
|
||||
# Full re-import
|
||||
report_progress(8, "Indexing Zotero", "Reading library items and attachments")
|
||||
zotero_index = build_zotero_index(args.db, args.storage_root)
|
||||
overrides = None
|
||||
if overrides_path.exists():
|
||||
overrides = load_overrides(load_json(str(overrides_path)))
|
||||
clippings = new_clippings
|
||||
report_progress(15, "Matching titles", "Matching Kindle titles to Zotero items")
|
||||
matches = build_match_report(clippings, zotero_index, overrides)
|
||||
override_skeleton = generate_override_skeleton(matches)
|
||||
report_progress(22, "Building plan", "Selecting attachments")
|
||||
plan = build_import_plan(clippings, zotero_index, matches)
|
||||
report_progress(30, "Positioning EPUB highlights", "Locating highlights in EPUB files")
|
||||
epub_plan = add_epub_positions(plan)
|
||||
report_progress(55, "Positioning PDF highlights", "Locating highlights in PDF files; this is usually the longest stage")
|
||||
positioned_plan = add_pdf_positions(epub_plan)
|
||||
report_progress(90, "Finalizing", "Preparing Zotero annotations and mismatch report")
|
||||
final_plan = build_final_writer_plan(positioned_plan)
|
||||
final_plan["deletions"] = []
|
||||
final_plan["is_incremental"] = False
|
||||
mismatch_review = build_mismatch_review(positioned_plan, matches)
|
||||
|
||||
indent = 2 if args.pretty else None
|
||||
outputs = {
|
||||
"clippings": artifact("clippings.json"),
|
||||
"zotero_index": artifact("zotero-index.json"),
|
||||
"matches": artifact("matches.json"),
|
||||
"generated_overrides": artifact("match-overrides.generated.json"),
|
||||
"import_plan": artifact("import-plan.json"),
|
||||
"epub_plan": artifact("import-plan.epub.json"),
|
||||
"positioned_plan": artifact("import-plan.positioned.json"),
|
||||
"final_plan": artifact("import-plan.final.json"),
|
||||
"mismatch_review": workdir / "docs" / "mismatch-review.md",
|
||||
}
|
||||
outputs["mismatch_review"].parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
report_progress(95, "Saving results", "Writing import artifacts")
|
||||
for key, payload in [
|
||||
("clippings", clippings),
|
||||
("zotero_index", zotero_index),
|
||||
("matches", matches),
|
||||
("generated_overrides", override_skeleton),
|
||||
("import_plan", plan),
|
||||
("epub_plan", epub_plan),
|
||||
("positioned_plan", positioned_plan),
|
||||
("final_plan", final_plan),
|
||||
]:
|
||||
with open(outputs[key], "w", encoding="utf-8") as file:
|
||||
json.dump(payload, file, ensure_ascii=False, indent=indent)
|
||||
file.write("\n")
|
||||
|
||||
with open(outputs["mismatch_review"], "w", encoding="utf-8") as file:
|
||||
file.write(mismatch_review)
|
||||
|
||||
status_counts: dict[str, int] = {}
|
||||
for item in positioned_plan.get("items", []):
|
||||
status = item.get("status", "unknown")
|
||||
status_counts[status] = status_counts.get(status, 0) + 1
|
||||
|
||||
match_counts: dict[str, int] = {}
|
||||
for match in matches.get("matches", []):
|
||||
status = match.get("status", "unknown")
|
||||
match_counts[status] = match_counts.get(status, 0) + 1
|
||||
|
||||
summary = {
|
||||
"format": "kindle-zotero-importer.pipeline-summary.v1",
|
||||
"workdir": str(workdir),
|
||||
"overrides_path": str(overrides_path),
|
||||
"outputs": {key: str(path) for key, path in outputs.items()},
|
||||
"counts": {
|
||||
"clippings": len(clippings.get("clippings", [])),
|
||||
"unique_titles": len(matches.get("matches", [])),
|
||||
"final_annotations": final_plan.get("annotation_count", 0),
|
||||
"match_statuses": match_counts,
|
||||
"plan_statuses": status_counts,
|
||||
"skipped_final": final_plan.get("skipped_counts", {}),
|
||||
"is_incremental": final_plan.get("is_incremental", False),
|
||||
"incremental_stats": final_plan.get("incremental_stats", {}),
|
||||
"deletions": len(final_plan.get("deletions", [])),
|
||||
},
|
||||
}
|
||||
json_text = json.dumps(summary, ensure_ascii=False, indent=2)
|
||||
if args.summary_output:
|
||||
with open(args.summary_output, "w", encoding="utf-8") as file:
|
||||
file.write(json_text)
|
||||
file.write("\n")
|
||||
else:
|
||||
sys.stdout.write(json_text)
|
||||
sys.stdout.write("\n")
|
||||
report_progress(100, "Pipeline complete", "Ready to import annotations into Zotero")
|
||||
return 0
|
||||
|
||||
parser.error(f"unknown command: {args.command}")
|
||||
return 2
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from dataclasses import dataclass
|
|||
from html import unescape
|
||||
from html.parser import HTMLParser
|
||||
import posixpath
|
||||
import re
|
||||
import zipfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
|
@ -65,8 +64,8 @@ def _position_item(
|
|||
|
||||
updated = dict(item)
|
||||
annotation = dict(updated["annotation"])
|
||||
annotation["position"] = position
|
||||
annotation["sortIndex"] = _epub_sort_index(position["value"])
|
||||
annotation["position"] = position["selector"]
|
||||
annotation["sortIndex"] = position["sortIndex"]
|
||||
updated["annotation"] = annotation
|
||||
updated["status"] = "positioned"
|
||||
return updated
|
||||
|
|
@ -97,7 +96,7 @@ def extract_epub_text_nodes(path: str) -> list[TextNode]:
|
|||
return nodes
|
||||
|
||||
|
||||
def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, str] | None:
|
||||
def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, Any] | None:
|
||||
haystack = " ".join(node.text for node in text_nodes)
|
||||
normalized_haystack, haystack_map = _normalize_with_map(haystack)
|
||||
normalized_quote, _ = _normalize_with_map(quote)
|
||||
|
|
@ -134,9 +133,12 @@ def find_epub_cfi(text_nodes: list[TextNode], quote: str) -> dict[str, str] | No
|
|||
f"{end_node.cfi_parent_path}/{end_node.text_step}:{end_offset})"
|
||||
)
|
||||
return {
|
||||
"selector": {
|
||||
"type": "FragmentSelector",
|
||||
"conformsTo": "http://www.idpf.org/epub/linking/cfi/epub-cfi.html",
|
||||
"value": value,
|
||||
},
|
||||
"sortIndex": f"{spine_step:05d}|{raw_start:08d}",
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -257,13 +259,6 @@ def _node_at(nodes: list[TextNode], offset: int) -> tuple[TextNode | None, int]:
|
|||
return None, 0
|
||||
|
||||
|
||||
def _epub_sort_index(cfi: str) -> str:
|
||||
numbers = [int(number) for number in re.findall(r"/([0-9]+)", cfi)]
|
||||
spine = numbers[1] if len(numbers) > 1 else 0
|
||||
content = numbers[-1] if numbers else 0
|
||||
return f"{spine:05d}|{content:08d}"
|
||||
|
||||
|
||||
def _with_problem(item: dict[str, Any], problem: str) -> dict[str, Any]:
|
||||
updated = dict(item)
|
||||
updated["status"] = problem
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
from typing import Any
|
||||
|
||||
|
||||
|
|
@ -20,10 +21,14 @@ def build_final_writer_plan(positioned_plan: dict[str, Any]) -> dict[str, Any]:
|
|||
skipped.get("positioned-missing-writer-fields", 0) + 1
|
||||
)
|
||||
continue
|
||||
clipping = item["clipping"]
|
||||
annotations.append(
|
||||
{
|
||||
"clipping_id": item["clipping"]["id"],
|
||||
"clipping_title": item["clipping"]["title"],
|
||||
"clipping_id": clipping["id"],
|
||||
"clipping_title": clipping["title"],
|
||||
"clipping_added_on": clipping.get("added_on"),
|
||||
"clipping_added_on_iso": clipping.get("added_on_iso"),
|
||||
"integrated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"attachment_item_id": attachment["item_id"],
|
||||
"attachment_key": attachment["key"],
|
||||
"parent_item_id": item["zotero"]["parent_item_id"],
|
||||
|
|
@ -37,11 +42,22 @@ def build_final_writer_plan(positioned_plan: dict[str, Any]) -> dict[str, Any]:
|
|||
"pageLabel": annotation.get("pageLabel") or "",
|
||||
"sortIndex": annotation.get("sortIndex"),
|
||||
"position": annotation["position"],
|
||||
"tags": annotation.get("tags") or [{"name": "kindle-import"}],
|
||||
"tags": [
|
||||
{"name": "kindle-import"},
|
||||
{"name": f"kindle-id:{clipping['id']}"},
|
||||
],
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
annotations.sort(
|
||||
key=lambda entry: (
|
||||
entry.get("attachment_item_id") or 0,
|
||||
entry["annotation"].get("sortIndex") or "",
|
||||
entry.get("clipping_id") or "",
|
||||
)
|
||||
)
|
||||
|
||||
return {
|
||||
"format": FINAL_FORMAT,
|
||||
"source_format": positioned_plan.get("format"),
|
||||
|
|
|
|||
|
|
@ -4,12 +4,31 @@ from dataclasses import dataclass
|
|||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Any
|
||||
|
||||
|
||||
_TOOL_PATHS = {
|
||||
"pdftotext": ("/Users/ubd/bin/pdftotext", "/opt/homebrew/bin/pdftotext"),
|
||||
"pdftohtml": ("/Users/ubd/bin/pdftohtml", "/opt/homebrew/bin/pdftohtml"),
|
||||
"pdfinfo": ("/Users/ubd/bin/pdfinfo", "/opt/homebrew/bin/pdfinfo"),
|
||||
"qpdf": ("/Users/ubd/bin/qpdf", "/opt/homebrew/bin/qpdf"),
|
||||
}
|
||||
|
||||
|
||||
def _tool(name: str) -> str:
|
||||
resolved = shutil.which(name)
|
||||
if resolved:
|
||||
return resolved
|
||||
for candidate in _TOOL_PATHS.get(name, ()):
|
||||
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
|
||||
return candidate
|
||||
return name
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PDFWord:
|
||||
text: str
|
||||
|
|
@ -117,7 +136,7 @@ def _position_item(
|
|||
|
||||
def extract_pdf_text_pages(path: str) -> list[str]:
|
||||
result = subprocess.run(
|
||||
["pdftotext", "-enc", "UTF-8", path, "-"],
|
||||
[_tool("pdftotext"), "-enc", "UTF-8", path, "-"],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
|
@ -157,7 +176,7 @@ def _decrypted_pdf_path(
|
|||
base_name = os.path.splitext(os.path.basename(path))[0] or "document"
|
||||
output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf")
|
||||
subprocess.run(
|
||||
["qpdf", "--decrypt", path, output_path],
|
||||
[_tool("qpdf"), "--decrypt", path, output_path],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
|
@ -194,7 +213,7 @@ def extract_pdf_page_xml(path: str, page_index: int) -> PDFPageXML:
|
|||
page_number = page_index + 1
|
||||
result = subprocess.run(
|
||||
[
|
||||
"pdftohtml",
|
||||
_tool("pdftohtml"),
|
||||
"-f",
|
||||
str(page_number),
|
||||
"-l",
|
||||
|
|
@ -302,7 +321,7 @@ def _pdf_page_size(
|
|||
path: str, page_number: int, fallback_width: float, fallback_height: float
|
||||
) -> tuple[float, float]:
|
||||
result = subprocess.run(
|
||||
["pdfinfo", "-f", str(page_number), "-l", str(page_number), path],
|
||||
[_tool("pdfinfo"), "-f", str(page_number), "-l", str(page_number), path],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue