From b713faa5831550803c4378f7dfd583db9f9a037d Mon Sep 17 00:00:00 2001 From: Utku Bilen Demir <84389167+UtkuBilenDemir@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:51:15 +0200 Subject: [PATCH] Improve override review and PDF positioning --- README.md | 15 + docs/mismatch-review.md | 260 +++++++---- maschine/maschine_log_20260527.md | 22 + match-overrides.json | 411 ++++++++++++++++++ src/kindle_zotero_importer/__main__.py | 6 + src/kindle_zotero_importer/import_plan.py | 5 + src/kindle_zotero_importer/matcher.py | 8 +- src/kindle_zotero_importer/mismatch_review.py | 22 +- src/kindle_zotero_importer/overrides.py | 83 +++- src/kindle_zotero_importer/pdf_position.py | 77 +++- 10 files changed, 823 insertions(+), 86 deletions(-) create mode 100644 src/kindle_zotero_importer/__main__.py diff --git a/README.md b/README.md index 8133499..af8e82a 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Generate reusable manual overrides for ambiguous/unmatched titles: python -m kindle_zotero_importer generate-overrides matches.json --output match-overrides.generated.json --pretty ``` +Generated overrides are designed for batch editing: unresolved titles are sorted by clipping count, unmatched titles default to `ignore: true`, and already-entered overrides are kept at the end as an audit trail. Replace `ignore` with a confirmed `citation_key`, `zotero_key`, or `zotero_item_id` to import a title instead of discarding it. + Apply reviewed overrides: ```sh @@ -38,6 +40,17 @@ python -m kindle_zotero_importer match clippings.json zotero-index.json --overri Keep confirmed mappings in `match-overrides.json`. The `match-overrides.generated.json` file is only a disposable review skeleton and can be regenerated from the current `matches.json` at any time. +To permanently skip a Kindle title that should not be imported, add an ignore override: + +```json +{ + "clipping_title": "Example Kindle Title", + "resolution": { + "ignore": true + } +} +``` + If a matched Zotero item has multiple usable PDF/EPUB attachments, add the selected attachment to the same override entry: ```json @@ -82,6 +95,8 @@ python -m kindle_zotero_importer position-pdf import-plan.epub.json --output imp python -m kindle_zotero_importer finalize import-plan.positioned.json --output import-plan.final.json --pretty ``` +PDF positioning uses Poppler tools (`pdftotext`, `pdftohtml`, `pdfinfo`). If a PDF permits viewing but blocks text copying, the importer retries extraction through a temporary `qpdf --decrypt` copy. The Zotero attachment itself is not modified. + ## Safety Rule Do not write directly to `zotero.sqlite`. Use read-only SQLite access for indexing and Zotero's JavaScript API for writes. diff --git a/docs/mismatch-review.md b/docs/mismatch-review.md index 02e2f55..310b1eb 100644 --- a/docs/mismatch-review.md +++ b/docs/mismatch-review.md @@ -41,31 +41,10 @@ Candidates: ## Unmatched Kindle Titles Status: `unmatched-title` -Clippings: 636 -Unique titles: 42 +Clippings: 534 +Unique titles: 40 Action: Add one confirmed mapping to match-overrides.json using citation_key, zotero_key, or zotero_item_id. -### WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku) - -```yaml -clipping_title: "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)" -status: unmatched-title -clipping_count: 69 -kinds: {highlight: 69} -citation_key: "" -zotero_item_id: null -zotero_key: "burroughs1992" -zotero_title: "" -attachment_count: null -expected_attachment_type: "pdf-or-epub" -match_score: null -match_reason: "" -``` - -- Example: highlight page `8` loc `56-57`: The addict in the street who must have junk to live is the one irreplaceable factor in the junk equation. When there are no more addicts to buy junk there will be no junk trafic. A -- Example: highlight page `15` loc `141-142`: Paregoric Babes of the World Unite. We have nothing to lose but Our Pushers. And THEY are NOT NECESSARY. -- Example: highlight page `19` loc `185-185`: "Well," I said, tapping my arm, "duty calls. As one judge said to another: 'Be just and if you can't be just, be arbitrary.' " - ### The Satanic Verses_ A Novel (Salman Rushdie) ```yaml @@ -73,7 +52,7 @@ clipping_title: "The Satanic Verses_ A Novel (Salman Rushdie)" status: unmatched-title clipping_count: 54 kinds: {highlight: 54} -citation_key: "" +citation_key: "rushdie2011" zotero_item_id: null zotero_key: "" zotero_title: "" @@ -179,35 +158,6 @@ match_reason: "" - Example: highlight page `None` loc `7257-7257`: gebaut in Zeitalter von Legenden, man sagt. Da sein eine - Example: highlight page `None` loc `7257-7258`: gebaut in Zeitalter von Legenden, man sagt. Da sein eine Wand mit Fries, der zeigt Tiere, die kein lebender -### Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou (Iain MacKenzie) - -```yaml -clipping_title: "Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou (Iain MacKenzie)" -status: unmatched-title -clipping_count: 33 -kinds: {highlight: 33} -citation_key: "mackenzie2018" -zotero_item_id: 46710 -zotero_key: "QA46VR8D" -zotero_title: "Resistance and the politics of truth: Foucault, Deleuze, Badiou" -attachment_count: 2 -expected_attachment_type: "pdf-or-epub" -match_score: 0.92 -match_reason: "title-contained-in-clipping-title" -override_entry: - clipping_title: "Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou (Iain MacKenzie)" - resolution: - citation_key: "mackenzie2018" -``` - -Candidates: -- citation_key: `mackenzie2018`, zotero_item_id: `46710`, zotero_key: `QA46VR8D`, attachments: `2`, score: `0.92`, title: Resistance and the politics of truth: Foucault, Deleuze, Badiou -- citation_key: `deleuze1988`, zotero_item_id: `118`, zotero_key: `D4DFMW8L`, attachments: `1`, score: `0.92`, title: Foucault - -- Example: highlight page `189` loc `2324-2325`: All of which amounts to a defence of the idea of critique and its importance in establishing practices of resistance: without an immanent critique of one’s situation there is no ho -- Example: highlight page `189` loc `2329-2330`: The previous chapters have addressed the first of these problems in establishing a general idea of immanent critique as a creative practice; -- Example: highlight page `191` loc `2349-2349`: (because the learner is the thinker of pure difference), - ### Marcus Aurelius (Emperor of Rome), Martin Hammond, Diskin Clay - Meditations (2006, Penguin Books) - libgen.li ```yaml @@ -715,7 +665,49 @@ match_reason: "" - Example: highlight page `None` loc `532-533`: Then, I said, no science or art considers or enjoins the interest of the stronger or superior, but only the interest of the subject and weaker? - Example: highlight page `None` loc `550-552`: First of all, in private contracts: wherever the unjust is the partner of the just you will find that, when the partnership is dissolved, the unjust man has always more and the jus -_Showing first 30 of 42 unique titles._ +### The Way of Kings (Sanderson, Brandon) + +```yaml +clipping_title: "The Way of Kings (Sanderson, Brandon)" +status: unmatched-title +clipping_count: 3 +kinds: {highlight: 3} +citation_key: "" +zotero_item_id: null +zotero_key: "" +zotero_title: "" +attachment_count: null +expected_attachment_type: "epub-preferred" +match_score: null +match_reason: "" +``` + +- Example: highlight page `None` loc `369-370`: For if you were going to assassinate a man, he was entitled to see you coming. +- Example: highlight page `None` loc `366-370`: Today, that included wearing white. Loose white trousers tied at the waist with a rope, and over them a filmy shirt with long sleeves, open at the front. White clothing for a kille +- Example: highlight page `None` loc `407-408`: As Truthless, there was only one life he was forbidden to take. And that was his own. + +### Your P2K Articles (2024-12-17) (P2K) + +```yaml +clipping_title: "Your P2K Articles (2024-12-17) (P2K)" +status: unmatched-title +clipping_count: 3 +kinds: {highlight: 3} +citation_key: "" +zotero_item_id: null +zotero_key: "" +zotero_title: "" +attachment_count: null +expected_attachment_type: "pdf-or-epub" +match_score: null +match_reason: "" +``` + +- Example: highlight page `11` loc `85-85`: Captagon, +- Example: highlight page `12` loc `90-92`: particular importance here is Hezbollah, whose elite units played a key part in fighting the jihadi opposition in Syria. But most of these have been transferred to Lebanon to fight +- Example: highlight page `15` loc `114-117`: Nobody knows if Iran and the regime would have been weakened without the recent Israeli attacks in Syria, which have allowed us to return and free the lands and the country,’ a man + +_Showing first 30 of 40 unique titles._ ## Matched Attachment But Missing File Path @@ -800,8 +792,8 @@ Candidates: ## Attachment Found But Text Position Failed Status: `epub-text-not-found` -Clippings: 181 -Unique titles: 11 +Clippings: 183 +Unique titles: 12 Action: Needs looser EPUB text matching or manual review. ### Dostoyevsky and Taluy_2020_Karamazov kardeşler @@ -986,8 +978,8 @@ zotero_key: "MR7YD8FW" zotero_title: "Deleuze and Guattari's A thousand plateaus: a reader's guide" attachment_count: 1 expected_attachment_type: "epub" -match_score: 0.92 -match_reason: "title-contained-in-clipping-title" +match_score: 1.0 +match_reason: "citationKey" attachment_choices: - attachment_item_id: 45062 attachment_key: "DVSTKU2W" @@ -1003,7 +995,7 @@ override_entry: ``` Candidates: -- citation_key: `holland2013`, zotero_item_id: `45058`, zotero_key: `MR7YD8FW`, attachments: `1`, score: `0.92`, title: Deleuze and Guattari's A thousand plateaus: a reader's guide +- citation_key: `holland2013`, zotero_item_id: `45058`, zotero_key: `MR7YD8FW`, attachments: `1`, score: `1.0`, title: Deleuze and Guattari's A thousand plateaus: a reader's guide - Example: highlight page `41` loc `520-522`: debt; whereas Marx considered production to be primary, Deleuze & Guattari draw here on Nietzsche more than Marx, and instead consider debt to be the primary organizing element in - Example: highlight page `41` loc `513-516`: The economic instance, in other words, is a Problematic virtual structure (how to produce means of life to assure survival) to which various societies (or a given society at variou @@ -1058,8 +1050,8 @@ zotero_key: "95BBKLYJ" zotero_title: "Three novels" attachment_count: 1 expected_attachment_type: "epub" -match_score: 0.92 -match_reason: "title-contained-in-clipping-title" +match_score: 1.0 +match_reason: "citationKey" attachment_choices: - attachment_item_id: 42882 attachment_key: "SK7YDIFX" @@ -1075,7 +1067,7 @@ override_entry: ``` Candidates: -- citation_key: `beckett2009`, zotero_item_id: `42879`, zotero_key: `95BBKLYJ`, attachments: `1`, score: `0.92`, title: Three novels +- citation_key: `beckett2009`, zotero_item_id: `42879`, zotero_key: `95BBKLYJ`, attachments: `1`, score: `1.0`, title: Three novels - Example: highlight page `518` loc `6641-6647`: But when it falters? And when it stops? But it falters every instant, it stops every instant! Yes, but when it stops for a good few moments, a good few moments (what are a good few - Example: highlight page `517` loc `6633-6641`: @@ -1125,6 +1117,47 @@ Candidates: - Example: highlight page `551` loc `7237-7248`: The choice is between one of two poles, the paranoiac counterescape that motivates all the conformist, reactionary, and fascisizing investments, and the schizophrenic escape conver - Example: highlight page `552` loc `7256-7258`: the neurotic needs a global object in relation to which the partial objects can be determined as a lack, and inversely,38 But on a more general level, the statistical transformatio +### Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou (Iain MacKenzie) + +```yaml +clipping_title: "Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou (Iain MacKenzie)" +status: epub-text-not-found +clipping_count: 2 +kinds: {highlight: 2} +citation_key: "mackenzie2018" +zotero_item_id: 46710 +zotero_key: "QA46VR8D" +zotero_title: "Resistance and the politics of truth: Foucault, Deleuze, Badiou" +attachment_count: 2 +expected_attachment_type: "pdf-or-epub" +match_score: 1.0 +match_reason: "citationKey" +attachment_choices: + - attachment_item_id: 46712 + attachment_key: "QPJNPMKP" + attachment_title: "Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou.epub" + content_type: "application/epub+zip" + path: "/Users/ubd/Library/Mobile Documents/com~apple~CloudDocs/03_Academia/ZoteroFiles/Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou.epub" + - attachment_item_id: 46714 + attachment_key: "KE7XRV35" + attachment_title: "Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou.pdf" + content_type: "application/pdf" + path: "/Users/ubd/Library/Mobile Documents/com~apple~CloudDocs/03_Academia/ZoteroFiles/Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou.pdf" +override_entry: + clipping_title: "Mackenzie_2018_Resistance and the politics of truth Foucault, Deleuze, Badiou (Iain MacKenzie)" + resolution: + citation_key: "mackenzie2018" + attachment_key: "PASTE_SELECTED_ATTACHMENT_KEY" + attachment_item_id: null +``` + +Candidates: +- citation_key: `mackenzie2018`, zotero_item_id: `46710`, zotero_key: `QA46VR8D`, attachments: `2`, score: `1.0`, title: Resistance and the politics of truth: Foucault, Deleuze, Badiou +- citation_key: `deleuze1988`, zotero_item_id: `118`, zotero_key: `D4DFMW8L`, attachments: `1`, score: `0.92`, title: Foucault + +- Example: highlight page `198` loc `2449-2455`: Rouvroy puts it succinctly when she says that algorithmically produced knowledge is no longer produced by humans about the world, rather it is ‘produced from the digital world’ (20 +- Example: highlight page `203` loc `2501-2505`: In Chaosmosis, Guattari turns to art and aesthetics to understand how we might challenge the emergent forms of control that shape contemporary liberal capitalist democracies. He no + ### Buchner_2004_Lenz (Georg Buchner) ```yaml @@ -1138,8 +1171,8 @@ zotero_key: "AA8YM2SD" zotero_title: "Lenz" attachment_count: 1 expected_attachment_type: "epub" -match_score: 0.92 -match_reason: "title-contained-in-clipping-title" +match_score: 1.0 +match_reason: "citationKey" attachment_choices: - attachment_item_id: 42872 attachment_key: "8ZMLY422" @@ -1155,7 +1188,7 @@ override_entry: ``` Candidates: -- citation_key: `buchner2004`, zotero_item_id: `42869`, zotero_key: `AA8YM2SD`, attachments: `1`, score: `0.92`, title: Lenz +- citation_key: `buchner2004`, zotero_item_id: `42869`, zotero_key: `AA8YM2SD`, attachments: `1`, score: `1.0`, title: Lenz - Example: highlight page `14` loc `140-143`: Laß in mir die heil’gen Schmerzen, Tiefe Bronnen ganz aufbrechen; Leiden sei all’mein Gewinst, Leiden sei mein Gottesdienst. @@ -1235,8 +1268,8 @@ Candidates: ## PDF Text Not Found Status: `pdf-text-not-found` -Clippings: 25 -Unique titles: 4 +Clippings: 26 +Unique titles: 5 Action: Needs looser PDF text matching/OCR/page-offset handling. ### (Undergraduate texts in mathematics) Sheldon Axler - Linear Algebra Done Right-Springer (1997) @@ -1288,8 +1321,8 @@ zotero_key: "MBGL8DAK" zotero_title: "Deleuze and Guattari's Anti-Oedipus: a reader's guide" attachment_count: 1 expected_attachment_type: "pdf" -match_score: 0.92 -match_reason: "title-contained-in-clipping-title" +match_score: 1.0 +match_reason: "citationKey" attachment_choices: - attachment_item_id: 40764 attachment_key: "VXSNTSKV" @@ -1305,7 +1338,7 @@ override_entry: ``` Candidates: -- citation_key: `buchanan2008`, zotero_item_id: `380`, zotero_key: `MBGL8DAK`, attachments: `1`, score: `0.92`, title: Deleuze and Guattari's Anti-Oedipus: a reader's guide +- citation_key: `buchanan2008`, zotero_item_id: `380`, zotero_key: `MBGL8DAK`, attachments: `1`, score: `1.0`, title: Deleuze and Guattari's Anti-Oedipus: a reader's guide - Example: highlight page `67-67` loc `None`: The first paralogism of psychoanalysis, the paralogism of extrapolation, is this: psychoanalysis converts a detachable partial object - e.g., the penis - into a detached complete o - Example: highlight page `101-101` loc `None`: The voice is the voice of the alliance, the marked body is the body of affiliation, and the appreciative eye enables the declension of the two. Primitive inscription should not be @@ -1381,11 +1414,45 @@ Candidates: - Example: highlight page `132-132` loc `None`: Adamın suratı kızarmış, gözleri öfkeden yine büyümüş­ tü. O an hâkim olan sessizliği Kibar bozdu: “Dimitrakopulo ve Gel Ey Esmerim.” “Ne?” “Senin o altınından kalan son kuruşla Daz +### WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku) + +```yaml +clipping_title: "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)" +status: pdf-text-not-found +clipping_count: 1 +kinds: {highlight: 1} +citation_key: "burroughs1992" +zotero_item_id: 39116 +zotero_key: "SGPP6ATY" +zotero_title: "Naked lunch" +attachment_count: 1 +expected_attachment_type: "pdf" +match_score: 1.0 +match_reason: "override-citation-key:burroughs1992" +attachment_choices: + - attachment_item_id: 39118 + attachment_key: "TN5ANVDD" + attachment_title: "Burroughs_1992_.pdf" + content_type: "application/pdf" + path: "/Users/ubd/Library/Mobile Documents/com~apple~CloudDocs/03_Academia/ZoteroFiles/Burroughs_1992_.pdf" +override_entry: + clipping_title: "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)" + resolution: + citation_key: "burroughs1992" + attachment_key: "PASTE_SELECTED_ATTACHMENT_KEY" + attachment_item_id: null +``` + +Candidates: +- citation_key: `burroughs1992`, zotero_item_id: `39116`, zotero_key: `SGPP6ATY`, attachments: `1`, score: `1.0`, title: Naked lunch + +- Example: highlight page `163` loc `1840-1848`: A. J. is agitating for the destruction of Israel: "With all this feeling against the West a chap has a spot of bother scoring for the young Arab amenities.... The situation is litt + ## PDF Rectangles Not Found Status: `pdf-rects-not-found` -Clippings: 147 -Unique titles: 3 +Clippings: 157 +Unique titles: 4 Action: Needs improved PDF rectangle recovery after text/page match. ### Buchanan_2008_Deleuze and Guattari's Anti-Oedipus a reader's guide @@ -1401,8 +1468,8 @@ zotero_key: "MBGL8DAK" zotero_title: "Deleuze and Guattari's Anti-Oedipus: a reader's guide" attachment_count: 1 expected_attachment_type: "pdf" -match_score: 0.92 -match_reason: "title-contained-in-clipping-title" +match_score: 1.0 +match_reason: "citationKey" attachment_choices: - attachment_item_id: 40764 attachment_key: "VXSNTSKV" @@ -1418,7 +1485,7 @@ override_entry: ``` Candidates: -- citation_key: `buchanan2008`, zotero_item_id: `380`, zotero_key: `MBGL8DAK`, attachments: `1`, score: `0.92`, title: Deleuze and Guattari's Anti-Oedipus: a reader's guide +- citation_key: `buchanan2008`, zotero_item_id: `380`, zotero_key: `MBGL8DAK`, attachments: `1`, score: `1.0`, title: Deleuze and Guattari's Anti-Oedipus: a reader's guide - Example: highlight page `8-8` loc `None`: They were particularly opposed to the Left Bank Maoism that blossomed that spring like the proverbial hundredflowers.They rejected the idea that May '68 constituted a first or prel - Example: highlight page `8-8` loc `None`: -Oedipus is a May '68 book providing it is understood that Deleuze and Guattari were not 'soixante-huitards' or May 68ers and that their view of those events was quite different fr @@ -1460,6 +1527,42 @@ Candidates: - Example: highlight page `28-28` loc `None`: Mürettebatın en genci olan Hamamcı idi bu, hem bâkir hem de habaza olduğu için, kovanlarında kolayca kaysınlar diye baş taraf torpidolarını gresle ona yağlatırlardı - Example: highlight page `76-76` loc `None`: Bunu duyunca Kumandan o anda yerinden hışımla fırlayarak erbaş yatak yerinin fosfor bronzu kaportasını açıp içeri atladığında, geçen nice seneden sonra yine ilk kez, korkunun ta ke +### WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku) + +```yaml +clipping_title: "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)" +status: pdf-rects-not-found +clipping_count: 10 +kinds: {highlight: 10} +citation_key: "burroughs1992" +zotero_item_id: 39116 +zotero_key: "SGPP6ATY" +zotero_title: "Naked lunch" +attachment_count: 1 +expected_attachment_type: "pdf" +match_score: 1.0 +match_reason: "override-citation-key:burroughs1992" +attachment_choices: + - attachment_item_id: 39118 + attachment_key: "TN5ANVDD" + attachment_title: "Burroughs_1992_.pdf" + content_type: "application/pdf" + path: "/Users/ubd/Library/Mobile Documents/com~apple~CloudDocs/03_Academia/ZoteroFiles/Burroughs_1992_.pdf" +override_entry: + clipping_title: "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)" + resolution: + citation_key: "burroughs1992" + attachment_key: "PASTE_SELECTED_ATTACHMENT_KEY" + attachment_item_id: null +``` + +Candidates: +- citation_key: `burroughs1992`, zotero_item_id: `39116`, zotero_key: `SGPP6ATY`, attachments: `1`, score: `1.0`, title: Naked lunch + +- Example: highlight page `8` loc `56-57`: The addict in the street who must have junk to live is the one irreplaceable factor in the junk equation. When there are no more addicts to buy junk there will be no junk trafic. A +- Example: highlight page `49` loc `539-541`: A group of P.R.'s -Partially Reconditioned -- have surrounded some homosexual tourists with horrible knowing smiles showing the Nordic skull beneath in double exposure. "What do yo +- Example: highlight page `67` loc `760-762`: The Guard is a sharp dresser, since he has nothing to do and saves all his pay to buy fine clothes and changes three times a day in front of an enormous magnifying mirror. He has a + ### Bartleby, the Scrivener (Melville, Herman) ```yaml @@ -1493,3 +1596,10 @@ Candidates: - citation_key: `melville2006`, zotero_item_id: `41142`, zotero_key: `YF4FV42C`, attachments: `1`, score: `0.98`, title: Bartleby the scrivener - Example: highlight page `None` loc `250-251`: sudden spasmodic passions with him. For it was exceeding difficult + +## PDF Position Errors + +Status: `pdf-position-error:` +Clippings: 0 +Unique titles: 0 +Action: Inspect the PDF extraction command error, file validity, encryption, or permissions. diff --git a/maschine/maschine_log_20260527.md b/maschine/maschine_log_20260527.md index 87bb97f..f528a86 100644 --- a/maschine/maschine_log_20260527.md +++ b/maschine/maschine_log_20260527.md @@ -9,3 +9,25 @@ project: "[[kindle-zotero-importer]]" - Created initial Git commit `a38ac33` (`Initial Kindle Zotero importer baseline`) including source, docs, overrides, Zotero writer, and the maschine log. - Extracted mismatch-review worklist from `docs/mismatch-review.md`: remaining work covers missing Coeckelbergh attachment, unmatched Kindle title mappings, missing PDF paths, EPUB positioning failures, PDF text lookup failures, and PDF rectangle recovery failures. + + +- Recovered stalled session state: inspected dirty repo, existing maschine log, README, core importer modules, and generated mismatch-review diff. +- Completed cleanup for ignore overrides: `ignore: true` now rejects all other resolution fields, ignored matches are excluded from generated override skeletons, ignored titles flow to `ignored-title` import-plan status, and README documents permanent skips. +- Verified recovery with `PYTHONPATH=src python -m compileall -q src`, match/plan/review CLI smoke checks using generated temp outputs under `/var/folders/dz/bzxsk9tj3clf0cs7vg6r7h_80000gn/T/opencode/`, and an ignore override validation smoke test. +- Current uncommitted work remains in README, docs/mismatch-review.md, importer modules, and new `src/kindle_zotero_importer/__main__.py`; no commit was made. + + +- Diagnosed Burroughs import failure: citekey override `burroughs1992` matched successfully to Zotero item 39116 / key SGPP6ATY / PDF `/Users/ubd/Library/Mobile Documents/com~apple~CloudDocs/03_Academia/ZoteroFiles/Burroughs_1992_.pdf`; all 69 planned highlights failed at PDF positioning because `pdftotext` exits with permission error on an encrypted PDF where copying text is disallowed. Final writer plan therefore contained 0 Burroughs annotations. + +- Implemented PDF permission-encryption fallback in src/kindle_zotero_importer/pdf_position.py: when Poppler text extraction reports copying is not allowed, the positioning step creates a temporary qpdf --decrypt copy and retries extraction/geometry against that copy without modifying the Zotero attachment. +- Regenerated import-plan.positioned.json, import-plan.final.json, and docs/mismatch-review.md after the fallback. Burroughs now has 58 positioned/final annotations out of 69 planned highlights; 10 remain pdf-rects-not-found and 1 remains pdf-text-not-found. Final writer plan annotation_count is now 859. + +- Reviewed current mismatch-review structure for discard workflow. Remaining unmatched titles can be permanently skipped by adding `ignore: true` entries to match-overrides.json rather than editing docs/mismatch-review.md directly. + +- Populated match-overrides.json with all 37 currently unmatched titles as editable `ignore: true` entries, preserving the Burroughs citation-key mapping. Validated overrides with a temp match run: 37 ignored, 24 matched, 6 ambiguous. Did not regenerate main pipeline so entries can be edited first. + +- Changed override skeleton workflow so generated overrides always include review metadata with status and clipping_count. Unmatched generated entries now default to ignore: true for batch editing; ambiguous entries keep candidate resolution fields. Added the same review metadata to current match-overrides.json and regenerated match-overrides.generated.json. + +- Refined generated override review ordering: unresolved entries sort first by clipping_count descending; already-entered overrides are retained at the end as an audit trail. Regenerated matches.json with current overrides and regenerated match-overrides.generated.json; current match statuses are 37 ignored, 24 matched, 6 ambiguous. + +- Updated override editing workflow so ignored/unmatched entries include a blank citation_key field by default. Blank citation_key values are ignored by the loader, so entries remain ignored until edited; generated audit entries now preserve the blank citation_key for ignored items. \ No newline at end of file diff --git a/match-overrides.json b/match-overrides.json index 9e2a8e4..fcfdaa4 100644 --- a/match-overrides.json +++ b/match-overrides.json @@ -5,6 +5,417 @@ "clipping_title": "WilliamS.Burroughs-Nakedlunch-GrovePress(2001) (Utku)", "resolution": { "citation_key": "burroughs1992" + }, + "review": { + "status": "matched", + "clipping_count": 74 + } + }, + { + "clipping_title": "(Kapitalismus und Schizophrenie 2) Guattari, Deleuze - Tausend Plateaus, Mille Plateaux-Les Editions de Minuit (1980)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 27 + } + }, + { + "clipping_title": "[Penguin Classics] Marcus Aurelius, Martin Hammond, Diskin Clay - Meditations (2006, Penguin Classics) - libgen.li", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 1 + } + }, + { + "clipping_title": "APuZ_Postdemokratie.2011", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 2 + } + }, + { + "clipping_title": "Arendt_2017_", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 22 + } + }, + { + "clipping_title": "Chantal Mouffe - Hegemonie und radikale Demokratie.-Passagen Verlag Ges.M.B.H (2000)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 8 + } + }, + { + "clipping_title": "Deleuze and Guattaris", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 6 + } + }, + { + "clipping_title": "Deleuze, Bartleby; or, the Formula", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 3 + } + }, + { + "clipping_title": "Deleuze, Guattari- A Thousand Plateaus - Bobo", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 4 + } + }, + { + "clipping_title": "Dewey_2008_", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 1 + } + }, + { + "clipping_title": "DRdZ 01 - Die Suche nach dem Auge der Welt (Robert Jordan)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 33 + } + }, + { + "clipping_title": "Ernesto Laclau - On Populist Reason-Verso (2005)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 6 + } + }, + { + "clipping_title": "Exhalation (Ted Chiang)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 15 + } + }, + { + "clipping_title": "Fire & Blood (A Song of Ice and Fire) (George R. R. Martin)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 15 + } + }, + { + "clipping_title": "Galloway_2004_Protocol", + "resolution": { + "ignore": true, + "citation_key": "galloway_2004" + }, + "review": { + "status": "unmatched", + "clipping_count": 14 + } + }, + { + "clipping_title": "Gregs Tagebuch 13 - Eiskalt erwischt by Kinney Jeff, Schmidt Dietmar (z-lib.org)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 10 + } + }, + { + "clipping_title": "Hardt_Negri_2017_", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 16 + } + }, + { + "clipping_title": "Howaldt_Schwarz_2016_SI-DRIVE", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 4 + } + }, + { + "clipping_title": "Latzer Just (2020)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 7 + } + }, + { + "clipping_title": "Lem, Stanislaw - Solaris (Stanisław Lem)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 7 + } + }, + { + "clipping_title": "luhmann1981", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 1 + } + }, + { + "clipping_title": "manifestly_haraway_----_a_cyborg_manifesto_science_technology_and_socialist-feminism_in_the_...", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 1 + } + }, + { + "clipping_title": "Marcus Aurelius (Emperor of Rome), Martin Hammond, Diskin Clay - Meditations (2006, Penguin Books) - libgen.li", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 36 + } + }, + { + "clipping_title": "Martyr! -- Akbar, Kaveh -- null, null, 2024 -- RANDOM HOUSE US -- 9780593802359 -- 930ede856e2a2efb28505e3c4e99e222 -- Anna’s Archive (Kaveh Akbar)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 32 + } + }, + { + "clipping_title": "mew_band03", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 40 + } + }, + { + "clipping_title": "MiloradPaviç_-_ hazar_sözlügü", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 43 + } + }, + { + "clipping_title": "On Populist Reason (Laclau, Ernesto)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 35 + } + }, + { + "clipping_title": "paul-lafargue-the-right-to-be-lazy (Paul Lafargue)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 23 + } + }, + { + "clipping_title": "Quantitative Marxism - Paul Dunne Ed_", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 2 + } + }, + { + "clipping_title": "radikale demokratie comtesse, nonhoff etc", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 2 + } + }, + { + "clipping_title": "Rupert präsentiert: Ein echt wildes Abenteuer (Ruperts Tagebuch) (German Edition) (Kinney, Jeff)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 7 + } + }, + { + "clipping_title": "The City & the City - China Mieville", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 8 + } + }, + { + "clipping_title": "The Republic (Plato)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 3 + } + }, + { + "clipping_title": "The Satanic Verses_ A Novel (Salman Rushdie)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 58 + } + }, + { + "clipping_title": "The Way of Kings (Sanderson, Brandon)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 3 + } + }, + { + "clipping_title": "Your Clippings", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 2 + } + }, + { + "clipping_title": "Your P2K Articles (2024-12-17) (P2K)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 4 + } + }, + { + "clipping_title": "Your P2K Articles (2024-12-31) (P2K)", + "resolution": { + "ignore": true, + "citation_key": "" + }, + "review": { + "status": "unmatched", + "clipping_count": 2 } } ] diff --git a/src/kindle_zotero_importer/__main__.py b/src/kindle_zotero_importer/__main__.py new file mode 100644 index 0000000..42fbee7 --- /dev/null +++ b/src/kindle_zotero_importer/__main__.py @@ -0,0 +1,6 @@ +from __future__ import annotations + +from .cli import main + + +raise SystemExit(main()) diff --git a/src/kindle_zotero_importer/import_plan.py b/src/kindle_zotero_importer/import_plan.py index bff5685..258ce20 100644 --- a/src/kindle_zotero_importer/import_plan.py +++ b/src/kindle_zotero_importer/import_plan.py @@ -58,6 +58,11 @@ def _plan_clipping( "problems": [], } match = matches_by_title.get(clipping["title"]) + if match and match.get("status") == "ignored": + base["status"] = "ignored-title" + base["match"] = None + base["problems"] = ["ignored by match override"] + return base if not match or match.get("status") != "matched" or not match.get("candidates"): return base diff --git a/src/kindle_zotero_importer/matcher.py b/src/kindle_zotero_importer/matcher.py index 20b9069..2689f4a 100644 --- a/src/kindle_zotero_importer/matcher.py +++ b/src/kindle_zotero_importer/matcher.py @@ -27,6 +27,7 @@ class TitleMatch: clipping_title: str clipping_count: int candidates: list[MatchCandidate] + status_override: str | None = None def load_json(path: str) -> dict[str, Any]: @@ -60,6 +61,7 @@ def build_match_report( "matched_title_count": status_counts["matched"], "ambiguous_title_count": status_counts["ambiguous"], "unmatched_title_count": status_counts["unmatched"], + "ignored_title_count": status_counts["ignored"], "matches": [ { "clipping_title": match.clipping_title, @@ -89,6 +91,8 @@ def _match_title( override: dict[str, Any] | None = None, ) -> TitleMatch: if override: + if override.get("ignore") is True: + return TitleMatch(clipping_title, count, [], "ignored") item, reason = resolve_override(clipping_title, override, items) if item: return TitleMatch(clipping_title, count, [_candidate(item, 1.0, reason)]) @@ -193,13 +197,15 @@ def _normalize_title(value: str) -> str: def _status_counts(matches: list[TitleMatch]) -> dict[str, int]: - counts = {"matched": 0, "ambiguous": 0, "unmatched": 0} + counts = {"matched": 0, "ambiguous": 0, "unmatched": 0, "ignored": 0} for match in matches: counts[_match_status(match)] += 1 return counts def _match_status(match: TitleMatch) -> str: + if match.status_override: + return match.status_override if not match.candidates: return "unmatched" if len(match.candidates) == 1: diff --git a/src/kindle_zotero_importer/mismatch_review.py b/src/kindle_zotero_importer/mismatch_review.py index bfa2104..2a7056d 100644 --- a/src/kindle_zotero_importer/mismatch_review.py +++ b/src/kindle_zotero_importer/mismatch_review.py @@ -36,6 +36,11 @@ SECTIONS = [ "pdf-rects-not-found", "Needs improved PDF rectangle recovery after text/page match.", ), + ( + "PDF Position Errors", + "pdf-position-error:", + "Inspect the PDF extraction command error, file validity, encryption, or permissions.", + ), ] @@ -57,12 +62,13 @@ def build_mismatch_review( "", ] for heading, status, instruction in SECTIONS: - summaries = _title_summaries(by_status.get(status, []), match_by_title) + section_items = _items_for_status(by_status, status) + summaries = _title_summaries(section_items, match_by_title) lines += [ f"## {heading}", "", f"Status: `{status}`", - f"Clippings: {len(by_status.get(status, []))}", + f"Clippings: {len(section_items)}", f"Unique titles: {len(summaries)}", f"Action: {instruction}", "", @@ -75,6 +81,18 @@ def build_mismatch_review( return "\n".join(lines).rstrip() + "\n" +def _items_for_status( + by_status: dict[str, list[dict[str, Any]]], status: str +) -> list[dict[str, Any]]: + if status.endswith(":"): + items: list[dict[str, Any]] = [] + for item_status, status_items in by_status.items(): + if item_status.startswith(status): + items.extend(status_items) + return items + return by_status.get(status, []) + + def _title_summaries( items: list[dict[str, Any]], match_by_title: dict[str, dict[str, Any]] ) -> list[dict[str, Any]]: diff --git a/src/kindle_zotero_importer/overrides.py b/src/kindle_zotero_importer/overrides.py index 437d53a..f6dec7b 100644 --- a/src/kindle_zotero_importer/overrides.py +++ b/src/kindle_zotero_importer/overrides.py @@ -24,6 +24,25 @@ def load_overrides(payload: dict[str, Any]) -> dict[str, dict[str, Any]]: resolution = _clean_resolution(entry.get("resolution") or {}) if not resolution: continue + if resolution.get("ignore") is True: + conflicting_fields = [ + key + for key in ( + "citation_key", + "zotero_key", + "zotero_item_id", + "attachment_key", + "attachment_item_id", + ) + if key in resolution + ] + if conflicting_fields: + raise OverrideError( + f"ignore override for {clipping_title!r} cannot contain other resolution fields" + ) + overrides[clipping_title] = resolution + continue + item_fields = [ key for key in ("citation_key", "zotero_key", "zotero_item_id") @@ -70,20 +89,71 @@ def resolve_override( def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]: overrides = [] - for match in match_report.get("matches", []): - if match.get("status") == "matched": + matches = match_report.get("matches", []) + review_matches = [ + match for match in matches if match.get("status") not in {"matched", "ignored"} + ] + resolved_overrides = [ + match + for match in matches + if match.get("status") in {"matched", "ignored"} and match.get("override") + ] + sorted_matches = [ + *sorted(review_matches, key=_review_sort_key), + *sorted(resolved_overrides, key=_review_sort_key), + ] + + for match in sorted_matches: + if match.get("status") == "unmatched": + overrides.append( + { + "clipping_title": match["clipping_title"], + "review": { + "status": match.get("status"), + "clipping_count": match.get("clipping_count"), + }, + "resolution": { + "ignore": True, + "citation_key": "", + }, + "notes": "Default is to discard unmatched titles. To import, change ignore to false and fill citation_key, or replace it with zotero_key/zotero_item_id.", + "candidates": match.get("candidates", []), + } + ) + continue + if match.get("status") in {"matched", "ignored"}: + resolution = dict(match.get("override") or {}) + if resolution.get("ignore") is True: + resolution.setdefault("citation_key", "") + overrides.append( + { + "clipping_title": match["clipping_title"], + "review": { + "status": match.get("status"), + "clipping_count": match.get("clipping_count"), + }, + "resolution": resolution, + "notes": "Already resolved by match-overrides.json. Kept at the end for review/audit.", + "candidates": match.get("candidates", []), + } + ) continue overrides.append( { "clipping_title": match["clipping_title"], + "review": { + "status": match.get("status"), + "clipping_count": match.get("clipping_count"), + }, "resolution": { + "ignore": False, "citation_key": "", "zotero_key": "", "zotero_item_id": None, "attachment_key": "", "attachment_item_id": None, }, - "notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments.", + "notes": "Fill exactly one item resolution field. Prefer citation_key when available. Optionally add attachment_key or attachment_item_id for ambiguous attachments. Use ignore: true for works that should not be imported.", "candidates": match.get("candidates", []), } ) @@ -91,9 +161,14 @@ def generate_override_skeleton(match_report: dict[str, Any]) -> dict[str, Any]: return {"format": OVERRIDES_FORMAT, "overrides": overrides} +def _review_sort_key(match: dict[str, Any]) -> tuple[int, str]: + return -int(match.get("clipping_count") or 0), match["clipping_title"].casefold() + + def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]: cleaned: dict[str, Any] = {} for key in ( + "ignore", "citation_key", "zotero_key", "zotero_item_id", @@ -105,6 +180,8 @@ def _clean_resolution(resolution: dict[str, Any]) -> dict[str, Any]: continue if isinstance(value, str) and not value.strip(): continue + if key == "ignore" and value is not True: + continue cleaned[key] = value.strip() if isinstance(value, str) else value return cleaned diff --git a/src/kindle_zotero_importer/pdf_position.py b/src/kindle_zotero_importer/pdf_position.py index 8dc5bda..5bc52b5 100644 --- a/src/kindle_zotero_importer/pdf_position.py +++ b/src/kindle_zotero_importer/pdf_position.py @@ -1,8 +1,11 @@ from __future__ import annotations from dataclasses import dataclass +import hashlib +import os import re import subprocess +import tempfile import xml.etree.ElementTree as ET from typing import Any @@ -30,9 +33,20 @@ def add_pdf_positions(plan: dict[str, Any]) -> dict[str, Any]: text_cache: dict[str, list[str]] = {} xml_cache: dict[tuple[str, int], PDFPageXML] = {} size_cache: dict[tuple[str, int], tuple[float, float]] = {} + decrypted_cache: dict[str, str] = {} positioned = [] - for item in plan["items"]: - positioned.append(_position_item(item, text_cache, xml_cache, size_cache)) + with tempfile.TemporaryDirectory(prefix="kindle-zotero-pdf-") as temp_dir: + for item in plan["items"]: + positioned.append( + _position_item( + item, + text_cache, + xml_cache, + size_cache, + decrypted_cache, + temp_dir, + ) + ) status_counts: dict[str, int] = {} for item in positioned: @@ -49,6 +63,8 @@ def _position_item( text_cache: dict[str, list[str]], xml_cache: dict[tuple[str, int], PDFPageXML], size_cache: dict[tuple[str, int], tuple[float, float]], + decrypted_cache: dict[str, str], + temp_dir: str, ) -> dict[str, Any]: if item.get("status") != "ready-for-positioning": return item @@ -63,16 +79,21 @@ def _position_item( return _with_problem(item, "pdf-position-missing-path") try: - text_pages = text_cache.setdefault(path, extract_pdf_text_pages(path)) + text_pages, extraction_path = _text_pages_for_path( + path, text_cache, decrypted_cache, temp_dir + ) page_index = find_pdf_text_page( text_pages, item["clipping"]["text"], item["clipping"].get("page") ) if page_index is None: return _with_problem(item, "pdf-text-not-found") page_xml = xml_cache.setdefault( - (path, page_index), extract_pdf_page_xml(path, page_index) + (extraction_path, page_index), + extract_pdf_page_xml(extraction_path, page_index), + ) + position = find_pdf_rects( + extraction_path, page_xml, item["clipping"]["text"], size_cache ) - position = find_pdf_rects(path, page_xml, item["clipping"]["text"], size_cache) except Exception as error: # noqa: BLE001 - preserve failure in plan, do not abort batch return _with_problem(item, f"pdf-position-error:{error}") @@ -104,6 +125,52 @@ def extract_pdf_text_pages(path: str) -> list[str]: return result.stdout.split("\f") +def _text_pages_for_path( + path: str, + text_cache: dict[str, list[str]], + decrypted_cache: dict[str, str], + temp_dir: str, +) -> tuple[list[str], str]: + if path in text_cache: + return text_cache[path], decrypted_cache.get(path, path) + + try: + text_cache[path] = extract_pdf_text_pages(path) + return text_cache[path], path + except subprocess.CalledProcessError as error: + if not _is_pdf_permission_error(error): + raise + + decrypted_path = _decrypted_pdf_path(path, decrypted_cache, temp_dir) + text_cache[path] = extract_pdf_text_pages(decrypted_path) + return text_cache[path], decrypted_path + + +def _decrypted_pdf_path( + path: str, decrypted_cache: dict[str, str], temp_dir: str +) -> str: + cached = decrypted_cache.get(path) + if cached: + return cached + + digest = hashlib.sha256(path.encode("utf-8")).hexdigest()[:16] + base_name = os.path.splitext(os.path.basename(path))[0] or "document" + output_path = os.path.join(temp_dir, f"{base_name}.{digest}.decrypted.pdf") + subprocess.run( + ["qpdf", "--decrypt", path, output_path], + check=True, + capture_output=True, + text=True, + ) + decrypted_cache[path] = output_path + return output_path + + +def _is_pdf_permission_error(error: subprocess.CalledProcessError) -> bool: + output = "\n".join(part for part in (error.stdout, error.stderr) if part) + return "Permission Error" in output and "Copying of text" in output + + def find_pdf_text_page( pages: list[str], quote: str, kindle_page: str | None = None ) -> int | None: