357 lines
13 KiB
Python
357 lines
13 KiB
Python
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
|
|
from kindle_zotero_importer import cli
|
|
from kindle_zotero_importer.cli import (
|
|
_add_previous_imported_comments,
|
|
_changed_note_clipping_ids,
|
|
_changed_override_titles,
|
|
_clipping_ids_for_titles,
|
|
_deferred_clipping_ids,
|
|
_deletion_ids,
|
|
_discarded_clipping_ids,
|
|
_merge_annotation_history,
|
|
_merge_positioned_history,
|
|
_unresolved_clipping_ids,
|
|
)
|
|
from kindle_zotero_importer.clippings import clippings_to_jsonable, load_clippings
|
|
from kindle_zotero_importer import pdf_position
|
|
|
|
def test_incremental_ids(tmp_path: Path):
|
|
# simulate previous final with one integrated id
|
|
prev_final = {"annotations":[{"clipping_id":"aaa"}]}
|
|
(tmp_path/"import-plan.final.json").write_text(json.dumps(prev_final))
|
|
prev_clipp = {"clippings":[{"id":"aaa","title":"T"},{"id":"bbb","title":"U"}]}
|
|
(tmp_path/"clippings.json").write_text(json.dumps(prev_clipp))
|
|
|
|
new_ids = {"aaa","bbb","ccc"} # ccc is new
|
|
prev_integrated = {"aaa"}
|
|
to_process = new_ids - prev_integrated
|
|
assert to_process == {"bbb","ccc"}
|
|
deletions = prev_integrated - new_ids
|
|
assert deletions == set() # none removed
|
|
# changed: aaa text changed -> new id ddd, old aaa no longer in new -> deletion
|
|
new_ids2 = {"ddd","bbb","ccc"}
|
|
assert (prev_integrated - new_ids2) == {"aaa"}
|
|
|
|
|
|
def test_merge_annotation_history_preserves_old_and_replaces_changed():
|
|
old_a = {"clipping_id": "a", "attachment_item_id": 2, "annotation": {"sortIndex": "2"}}
|
|
old_b = {"clipping_id": "b", "attachment_item_id": 1, "annotation": {"sortIndex": "1"}}
|
|
new_a = {"clipping_id": "a", "attachment_item_id": 2, "annotation": {"sortIndex": "3"}}
|
|
|
|
merged = _merge_annotation_history(
|
|
{"all_annotations": [old_a, old_b]}, [new_a], []
|
|
)
|
|
|
|
assert merged == [old_b, new_a]
|
|
|
|
|
|
def test_merge_annotation_history_removes_deletions():
|
|
previous = {
|
|
"annotations": [
|
|
{"clipping_id": "a", "attachment_item_id": 1, "annotation": {}},
|
|
{"clipping_id": "b", "attachment_item_id": 1, "annotation": {}},
|
|
]
|
|
}
|
|
|
|
merged = _merge_annotation_history(previous, [], ["a"])
|
|
|
|
assert [annotation["clipping_id"] for annotation in merged] == ["b"]
|
|
|
|
|
|
def test_full_history_retains_current_unresolved_annotation():
|
|
previous = {
|
|
"all_annotations": [
|
|
{"clipping_id": "unresolved", "attachment_item_id": 1, "annotation": {}},
|
|
{"clipping_id": "removed", "attachment_item_id": 1, "annotation": {}},
|
|
]
|
|
}
|
|
|
|
merged = _merge_annotation_history(previous, [], ["removed"])
|
|
|
|
assert [annotation["clipping_id"] for annotation in merged] == ["unresolved"]
|
|
|
|
|
|
def test_deletions_include_removed_and_previous_failures():
|
|
assert _deletion_ids(
|
|
{"kept", "removed"},
|
|
{"kept"},
|
|
{"delete_failed_ids": ["retry"]},
|
|
) == {"removed", "retry"}
|
|
|
|
|
|
def test_deferred_clippings_are_reprocessed_while_they_remain_in_source():
|
|
previous = {"deferred_clipping_ids": ["retry", "removed"]}
|
|
|
|
assert _deferred_clipping_ids(previous, {"retry", "current"}) == {"retry"}
|
|
|
|
|
|
def test_discarded_clips_never_requeue_and_skips_are_not_unresolved():
|
|
previous = {"discarded_clipping_ids": ["gone", "removed"]}
|
|
assert _discarded_clipping_ids(previous, {"gone", "current"}) == {"gone"}
|
|
|
|
plan = {
|
|
"items": [
|
|
{"clipping": {"id": "note"}, "status": "pdf-position-skipped-non-highlight"},
|
|
{"clipping": {"id": "fail"}, "status": "pdf-text-not-found"},
|
|
{"clipping": {"id": "done"}, "status": "positioned"},
|
|
]
|
|
}
|
|
assert _unresolved_clipping_ids(plan, {"note", "fail", "done"}) == {"fail"}
|
|
|
|
|
|
def test_alternate_attachment_retry_prefers_pdf_when_epub_misses():
|
|
from kindle_zotero_importer.cli import _reattempt_failed_with_alternate_attachments
|
|
from kindle_zotero_importer.epub_position import add_epub_positions
|
|
from kindle_zotero_importer.import_plan import build_import_plan
|
|
from kindle_zotero_importer.pdf_position import add_pdf_positions
|
|
|
|
fixtures = Path(__file__).parent / "fixtures" / "windows-smoke"
|
|
epub_path = str(fixtures / "kanzi-dummy.epub")
|
|
pdf_path = str(fixtures / "kanzi-dummy.pdf")
|
|
quote = "This is a test highlight long enough to import for PDF."
|
|
clippings_payload = {
|
|
"format": "kindle-zotero-importer.clippings.v1",
|
|
"clippings": [
|
|
{
|
|
"id": "q1",
|
|
"title": "Dummy Book",
|
|
"kind": "highlight",
|
|
"text": quote,
|
|
"page": None,
|
|
"location": "100",
|
|
}
|
|
],
|
|
}
|
|
zotero_index = {
|
|
"format": "index",
|
|
"items": [
|
|
{
|
|
"item_id": 1,
|
|
"key": "ABC12345",
|
|
"title": "Dummy Book",
|
|
"fields": {},
|
|
"attachments": [
|
|
{
|
|
"item_id": 11,
|
|
"key": "EPUBKEY1",
|
|
"title": "epub",
|
|
"content_type": "application/epub+zip",
|
|
"path": epub_path,
|
|
"resolved_path": epub_path,
|
|
},
|
|
{
|
|
"item_id": 12,
|
|
"key": "PDFKEY123",
|
|
"title": "pdf",
|
|
"content_type": "application/pdf",
|
|
"path": pdf_path,
|
|
"resolved_path": pdf_path,
|
|
},
|
|
],
|
|
}
|
|
],
|
|
}
|
|
match_report = {
|
|
"format": "matches",
|
|
"matches": [
|
|
{
|
|
"clipping_title": "Dummy Book",
|
|
"status": "matched",
|
|
"candidates": [{"item_id": 1}],
|
|
}
|
|
],
|
|
}
|
|
plan = build_import_plan(clippings_payload, zotero_index, match_report, None)
|
|
assert plan["items"][0]["zotero"]["attachment"]["content_type"] == "application/epub+zip"
|
|
positioned = add_pdf_positions(add_epub_positions(plan))
|
|
assert positioned["items"][0]["status"] == "positioned"
|
|
assert "partial-text-match" in positioned["items"][0]["problems"]
|
|
rescued = _reattempt_failed_with_alternate_attachments(positioned)
|
|
assert rescued["items"][0]["status"] == "positioned"
|
|
assert "positioned-via-alternate-attachment" in rescued["items"][0]["problems"]
|
|
assert "partial-text-match" not in rescued["items"][0]["problems"]
|
|
|
|
|
|
def test_changed_override_requeues_every_clipping_for_exact_title():
|
|
previous = {"Shared title": {"citation_key": "old"}}
|
|
current = {"Shared title": {"citation_key": "new"}}
|
|
clippings = [
|
|
{"id": "first", "title": "Shared title"},
|
|
{"id": "second", "title": "Shared title"},
|
|
{"id": "third", "title": "Different title"},
|
|
]
|
|
|
|
changed_titles = _changed_override_titles(previous, current)
|
|
|
|
assert changed_titles == {"Shared title"}
|
|
assert _clipping_ids_for_titles(clippings, changed_titles) == {"first", "second"}
|
|
|
|
|
|
def test_previous_imported_comment_is_attached_to_reprocessed_entry():
|
|
current = [{"clipping_id": "a", "annotation": {"comment": "new"}}]
|
|
previous = {
|
|
"all_annotations": [
|
|
{"clipping_id": "a", "annotation": {"comment": "old"}}
|
|
]
|
|
}
|
|
|
|
_add_previous_imported_comments(current, previous)
|
|
|
|
assert current[0]["previous_imported_comment"] == "old"
|
|
|
|
|
|
def test_retry_unresolved_excludes_positioned_ignored_and_removed_entries():
|
|
previous = {
|
|
"items": [
|
|
{"clipping": {"id": "retry"}, "status": "pdf-text-not-found"},
|
|
{"clipping": {"id": "done"}, "status": "positioned"},
|
|
{"clipping": {"id": "ignored"}, "status": "ignored-title"},
|
|
{"clipping": {"id": "removed"}, "status": "unmatched-title"},
|
|
]
|
|
}
|
|
|
|
assert _unresolved_clipping_ids(previous, {"retry", "done", "ignored"}) == {
|
|
"retry"
|
|
}
|
|
|
|
|
|
def test_changed_note_ids_detect_add_replace_and_remove():
|
|
previous = [
|
|
{"clipping_id": "added", "note_ids": []},
|
|
{"clipping_id": "replaced", "note_ids": ["old"]},
|
|
{"clipping_id": "removed", "note_ids": ["old"]},
|
|
{"clipping_id": "same", "note_ids": ["same-note"]},
|
|
]
|
|
current = [
|
|
{"id": "added", "note_ids": ["new"]},
|
|
{"id": "replaced", "note_ids": ["new"]},
|
|
{"id": "removed"},
|
|
{"id": "same", "note_ids": ["same-note"]},
|
|
]
|
|
|
|
assert _changed_note_clipping_ids(current, previous) == {
|
|
"added",
|
|
"replaced",
|
|
"removed",
|
|
}
|
|
|
|
|
|
def test_positioned_history_preserves_unresolved_and_replaces_current():
|
|
previous = {
|
|
"items": [
|
|
{"clipping": {"id": "keep"}, "status": "pdf-text-not-found"},
|
|
{"clipping": {"id": "replace"}, "status": "epub-text-not-found"},
|
|
{"clipping": {"id": "removed"}, "status": "unmatched-title"},
|
|
]
|
|
}
|
|
current = {
|
|
"format": "plan",
|
|
"items": [{"clipping": {"id": "replace"}, "status": "positioned"}],
|
|
}
|
|
|
|
merged = _merge_positioned_history(previous, current, {"keep", "replace"})
|
|
|
|
assert {item["clipping"]["id"]: item["status"] for item in merged["items"]} == {
|
|
"keep": "pdf-text-not-found",
|
|
"replace": "positioned",
|
|
}
|
|
assert merged["status_counts"] == {"pdf-text-not-found": 1, "positioned": 1}
|
|
|
|
|
|
def test_pdf_position_reports_missing_tools():
|
|
original = pdf_position._missing_required_tools
|
|
pdf_position._missing_required_tools = lambda: ["pdftotext", "pdfinfo"]
|
|
try:
|
|
result = pdf_position.add_pdf_positions(
|
|
{
|
|
"items": [
|
|
{
|
|
"status": "ready-for-positioning",
|
|
"clipping": {"kind": "highlight", "text": "highlight"},
|
|
"zotero": {
|
|
"attachment": {"content_type": "application/pdf"}
|
|
},
|
|
}
|
|
]
|
|
}
|
|
)
|
|
finally:
|
|
pdf_position._missing_required_tools = original
|
|
|
|
assert result["items"][0]["status"] == "pdf-tools-missing:pdftotext,pdfinfo"
|
|
|
|
|
|
def test_pdf_position_preserves_unresolved_item_without_zotero_match():
|
|
item = {
|
|
"status": "unmatched-title",
|
|
"clipping": {"kind": "highlight", "text": "highlight"},
|
|
"zotero": None,
|
|
}
|
|
|
|
result = pdf_position.add_pdf_positions({"items": [item]})
|
|
|
|
assert result["items"] == [item]
|
|
|
|
|
|
def test_noop_incremental_writes_complete_artifacts():
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
workdir = Path(temporary)
|
|
source = workdir / "My Clippings.txt"
|
|
source.write_text(
|
|
"Book\n- Your Highlight at location 10-11 | Added on Tuesday, 18 May 2021 14:18:13\n\n"
|
|
"A highlight long enough to import\n==========\n",
|
|
encoding="utf-8",
|
|
)
|
|
clippings = clippings_to_jsonable(load_clippings(str(source)))
|
|
clipping_id = clippings["clippings"][0]["id"]
|
|
(workdir / "clippings.json").write_text(json.dumps(clippings), encoding="utf-8")
|
|
annotation = {
|
|
"clipping_id": clipping_id,
|
|
"attachment_item_id": 1,
|
|
"annotation": {"sortIndex": "1"},
|
|
}
|
|
(workdir / "import-plan.final.json").write_text(
|
|
json.dumps({"annotations": [annotation], "all_annotations": [annotation], "overrides_snapshot": {}}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
originals = (
|
|
cli.build_zotero_index,
|
|
cli.build_match_report,
|
|
cli.generate_override_skeleton,
|
|
cli.build_mismatch_review,
|
|
)
|
|
cli.build_zotero_index = lambda *_args: {"format": "index", "items": []}
|
|
cli.build_match_report = lambda *_args: {"format": "matches", "matches": []}
|
|
cli.generate_override_skeleton = lambda *_args: {"format": "overrides", "overrides": []}
|
|
cli.build_mismatch_review = lambda *_args: "review\n"
|
|
try:
|
|
result = cli.main(
|
|
[
|
|
"run",
|
|
str(source),
|
|
"--workdir",
|
|
str(workdir),
|
|
"--db",
|
|
str(workdir / "zotero.sqlite"),
|
|
"--storage-root",
|
|
str(workdir / "storage"),
|
|
]
|
|
)
|
|
finally:
|
|
(
|
|
cli.build_zotero_index,
|
|
cli.build_match_report,
|
|
cli.generate_override_skeleton,
|
|
cli.build_mismatch_review,
|
|
) = originals
|
|
|
|
assert result == 0
|
|
final_plan = json.loads((workdir / "import-plan.final.json").read_text())
|
|
assert final_plan["annotations"] == []
|
|
assert final_plan["all_annotations"] == [{**annotation, "related_items": []}]
|
|
assert final_plan["delta_annotation_count"] == 0
|
|
assert (workdir / "import-plan.epub.json").exists()
|