from __future__ import annotations from datetime import datetime, timezone import hashlib import json import os from pathlib import Path import re import secrets import sqlite3 import unicodedata from fastapi import FastAPI, HTTPException, Request from fastapi.responses import HTMLResponse DB = Path(os.environ.get("KANZI_ANNOTATION_DB", Path(__file__).parent / "annotations.db")) MAX_PAGE_SIZE = 200 MAX_REQUEST_BYTES = 2_000_000 MAX_BATCH_SIZE = 5_000 app = FastAPI(title="Kanzi Annotation Collector") def _normalise(text: str) -> str: value = unicodedata.normalize("NFKC", text or "").lower() value = re.sub(r"-\s*\n\s*", "", value) value = re.sub(r"[\u2010-\u2015\u2018\u2019\u201c\u201d]", "'", value) value = re.sub(r"[^a-z0-9]+", " ", value) return re.sub(r"\s+", " ", value).strip() def _hash_norm(text: str) -> str: return hashlib.sha256(_normalise(text).encode()).hexdigest()[:16] def _fuzzy_hash(text: str) -> str: norm = _normalise(text) if len(norm) < 3: return _hash_norm(text) trigrams = {norm[index : index + 3] for index in range(len(norm) - 2)} words = sorted(norm.split()) word_trigrams = { " ".join(words[index : index + 3]) for index in range(max(1, len(words) - 2)) } return hashlib.sha256(" ".join(sorted(trigrams | word_trigrams)).encode()).hexdigest()[:16] def _connect() -> sqlite3.Connection: DB.parent.mkdir(parents=True, exist_ok=True) connection = sqlite3.connect(DB) connection.row_factory = sqlite3.Row connection.execute("PRAGMA busy_timeout = 5000") connection.execute("PRAGMA journal_mode = WAL") return connection def init_db() -> None: with _connect() as connection: connection.execute( """CREATE TABLE IF NOT EXISTS highlights ( id INTEGER PRIMARY KEY AUTOINCREMENT, anon_id TEXT, citation_key TEXT, doi TEXT, isbn TEXT, issn TEXT, publication_title TEXT, clipping_id_hash TEXT, norm_hash TEXT, fuzzy_hash TEXT, text TEXT, color TEXT, has_comment INTEGER, comment TEXT, comment_hash TEXT, clipping_added_on_iso TEXT, integrated_at TEXT, received_at TEXT, raw TEXT )""" ) existing_columns = { row["name"] for row in connection.execute("PRAGMA table_info(highlights)") } required_columns = { "doi": "TEXT", "isbn": "TEXT", "issn": "TEXT", "publication_title": "TEXT", "clipping_id_hash": "TEXT", "norm_hash": "TEXT", "fuzzy_hash": "TEXT", "text": "TEXT", "comment": "TEXT", "comment_hash": "TEXT", "clipping_added_on_iso": "TEXT", "integrated_at": "TEXT", "received_at": "TEXT", "raw": "TEXT", } for name, column_type in required_columns.items(): if name not in existing_columns: connection.execute(f"ALTER TABLE highlights ADD COLUMN {name} {column_type}") connection.execute( """CREATE TABLE IF NOT EXISTS raw_posts ( id INTEGER PRIMARY KEY AUTOINCREMENT, anon_id TEXT, received_at TEXT, body TEXT )""" ) connection.execute( """CREATE TABLE IF NOT EXISTS installations ( anon_id TEXT PRIMARY KEY, token_hash TEXT UNIQUE NOT NULL, created_at TEXT NOT NULL )""" ) # Older collector versions appended every retry. Retain only the newest # record for each installation/clipping before enforcing idempotency. connection.execute( """DELETE FROM highlights WHERE clipping_id_hash IS NOT NULL AND id NOT IN ( SELECT MAX(id) FROM highlights WHERE clipping_id_hash IS NOT NULL GROUP BY anon_id, clipping_id_hash )""" ) connection.execute( """CREATE UNIQUE INDEX IF NOT EXISTS highlights_identity ON highlights(anon_id, clipping_id_hash) WHERE clipping_id_hash IS NOT NULL""" ) init_db() HTML = """ Kanzi annotations

Kanzi annotations

Private pseudonymous collector. Raw highlight text and comments are stored.
-highlights
-citekeys
-installations
-received today

Recent

anoncitekeyDOI / ISBNtextcolourreceived

Top citekeys

citekeycount

Common highlights

Same publication and normalised text across at least two installations.
DOI / ISBNtextnorminstallationscitekeys
""" @app.get("/", response_class=HTMLResponse) def root() -> str: return HTML @app.get("/health") def health() -> dict[str, str]: return {"status": "ok"} def _token_hash(token: str) -> str: return hashlib.sha256(token.encode()).hexdigest() @app.post("/register") def register() -> dict[str, str]: token = secrets.token_urlsafe(32) anon_id = secrets.token_hex(16) with _connect() as connection: connection.execute( "INSERT INTO installations (anon_id, token_hash, created_at) VALUES (?, ?, ?)", (anon_id, _token_hash(token), datetime.now(timezone.utc).isoformat()), ) return {"anonId": anon_id, "token": token} def _authenticated_anon_id(request: Request) -> str: authorization = request.headers.get("authorization", "") scheme, separator, token = authorization.partition(" ") if not separator or scheme.lower() != "bearer" or not token: raise HTTPException(status_code=401, detail="Missing collector credential") with _connect() as connection: row = connection.execute( "SELECT anon_id FROM installations WHERE token_hash = ?", (_token_hash(token),), ).fetchone() if not row: raise HTTPException(status_code=401, detail="Invalid collector credential") return str(row["anon_id"]) @app.get("/stats") def stats() -> dict[str, int]: today = datetime.now(timezone.utc).date().isoformat() with _connect() as connection: row = connection.execute( """SELECT COUNT(*) AS total_highlights, COUNT(DISTINCT citation_key) AS unique_citekeys, COUNT(DISTINCT anon_id) AS unique_anons, SUM(CASE WHEN substr(received_at, 1, 10) = ? THEN 1 ELSE 0 END) AS today FROM highlights""", (today,), ).fetchone() return {key: int(row[key] or 0) for key in row.keys()} @app.get("/recent") def recent(limit: int = 50, offset: int = 0) -> dict: limit = max(1, min(limit, MAX_PAGE_SIZE)) offset = max(0, offset) with _connect() as connection: total = connection.execute("SELECT COUNT(*) FROM highlights").fetchone()[0] rows = connection.execute( """SELECT anon_id, citation_key, doi, isbn, text, clipping_id_hash, norm_hash, fuzzy_hash, color, has_comment, received_at FROM highlights ORDER BY id DESC LIMIT ? OFFSET ?""", (limit, offset), ).fetchall() return {"recent": [dict(row) for row in rows], "total": total} @app.get("/top") def top(limit: int = 20) -> dict: limit = max(1, min(limit, 100)) with _connect() as connection: rows = connection.execute( """SELECT citation_key, COUNT(*) AS count FROM highlights WHERE citation_key IS NOT NULL AND citation_key != '' GROUP BY citation_key ORDER BY count DESC, citation_key LIMIT ?""", (limit,), ).fetchall() return {"top": [dict(row) for row in rows]} @app.get("/common") def common(limit: int = 50, offset: int = 0) -> dict: limit = max(1, min(limit, MAX_PAGE_SIZE)) offset = max(0, offset) group_sql = """FROM highlights WHERE norm_hash IS NOT NULL AND norm_hash != '' AND COALESCE(NULLIF(doi, ''), NULLIF(isbn, ''), NULLIF(citation_key, '')) IS NOT NULL GROUP BY COALESCE(NULLIF(doi, ''), NULLIF(isbn, ''), citation_key), norm_hash HAVING COUNT(DISTINCT anon_id) > 1""" with _connect() as connection: total = connection.execute(f"SELECT COUNT(*) FROM (SELECT 1 {group_sql})").fetchone()[0] rows = connection.execute( f"""SELECT COALESCE(NULLIF(doi, ''), NULLIF(isbn, ''), citation_key) AS publication, norm_hash, COUNT(DISTINCT anon_id) AS installations, GROUP_CONCAT(DISTINCT citation_key) AS citekeys, MAX(text) AS example_text {group_sql} ORDER BY installations DESC, publication, norm_hash LIMIT ? OFFSET ?""", (limit, offset), ).fetchall() return {"common": [dict(row) for row in rows], "total": total} @app.post("/highlights") async def post_highlights(request: Request) -> dict: anon = _authenticated_anon_id(request) content_length = request.headers.get("content-length") if content_length: try: if int(content_length) > MAX_REQUEST_BYTES: raise HTTPException(status_code=413, detail="Request body is too large") except ValueError: raise HTTPException(status_code=400, detail="Invalid Content-Length") body = await request.body() if len(body) > MAX_REQUEST_BYTES: raise HTTPException(status_code=413, detail="Request body is too large") try: payload = json.loads(body) except (json.JSONDecodeError, UnicodeDecodeError) as error: raise HTTPException(status_code=400, detail="Invalid JSON") from error if not isinstance(payload, dict): raise HTTPException(status_code=422, detail="Payload must be an object") highlights = payload.get("data") or payload.get("highlights") or [] if isinstance(highlights, dict): highlights = [highlights] if not isinstance(highlights, list): raise HTTPException(status_code=422, detail="data must be a list") if len(highlights) > MAX_BATCH_SIZE: raise HTTPException(status_code=413, detail="Too many highlights in one request") now = datetime.now(timezone.utc).isoformat() inserted = updated = 0 with _connect() as connection: connection.execute("BEGIN IMMEDIATE") for highlight in highlights: if not isinstance(highlight, dict): continue text = str(highlight.get("text") or "")[:2000] clipping_hash = str(highlight.get("clipping_id_hash") or "")[:128] or None values = ( anon, str(highlight.get("citation_key") or "")[:512] or None, str(highlight.get("doi") or "")[:512] or None, str(highlight.get("isbn") or "")[:512] or None, str(highlight.get("issn") or "")[:512] or None, str(highlight.get("publication_title") or "")[:1000] or None, clipping_hash, highlight.get("norm_hash") or _hash_norm(text), highlight.get("fuzzy_hash") or _fuzzy_hash(text), text, str(highlight.get("color") or "")[:32] or None, 1 if highlight.get("has_comment") else 0, str(highlight.get("comment") or "")[:2000], str(highlight.get("comment_hash") or "")[:256] or None, str(highlight.get("clipping_added_on_iso") or "")[:64] or None, str(highlight.get("integrated_at") or "")[:64] or None, now, json.dumps(highlight, ensure_ascii=False)[:8000], ) existing = None if clipping_hash: legacy_hash = clipping_hash[:8] existing = connection.execute( """SELECT id FROM highlights WHERE anon_id = ? AND clipping_id_hash IN (?, ?) ORDER BY clipping_id_hash = ? DESC LIMIT 1""", (anon, clipping_hash, legacy_hash, clipping_hash), ).fetchone() if existing: connection.execute( """UPDATE highlights SET citation_key=?, doi=?, isbn=?, issn=?, publication_title=?, clipping_id_hash=?, norm_hash=?, fuzzy_hash=?, text=?, color=?, has_comment=?, comment=?, comment_hash=?, clipping_added_on_iso=?, integrated_at=?, received_at=?, raw=? WHERE id=?""", (*values[1:7], *values[7:], existing["id"]), ) updated += 1 else: connection.execute( """INSERT INTO highlights ( anon_id, citation_key, doi, isbn, issn, publication_title, clipping_id_hash, norm_hash, fuzzy_hash, text, color, has_comment, comment, comment_hash, clipping_added_on_iso, integrated_at, received_at, raw ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", values, ) inserted += 1 return { "status": "ok", "received": len(highlights), "inserted": inserted, "updated": updated, "anon": anon, }