diff --git a/.env.example b/.env.example index 016e166..1ba9409 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,9 @@ # Create with: cp .env.example .env ; then replace the placeholder. # Generate: python3 -c "import secrets; print(secrets.token_urlsafe(32))" API_TOKEN=REPLACE_WITH_A_RANDOM_32+_CHARACTER_TOKEN +# Login for the browser viewer at /; replace both values with long random secrets. +VIEWER_USERNAME=admin +VIEWER_PASSWORD=REPLACE_WITH_ANOTHER_RANDOM_32+_CHARACTER_SECRET # Optional. Leave empty to keep all data only in SQLite. DISCORD_WEBHOOK_URL= DATA_DIR=/data diff --git a/README.md b/README.md index bced324..13e1c42 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ docker compose logs -f curl http://127.0.0.1:8080/healthz ``` -Privzeto posluša samo na `127.0.0.1:8080`. Za dostop iz drugih naprav uporabi VPN (Tailscale/WireGuard) ali HTTPS reverse proxy z avtentikacijo. Ne odpiraj porta neposredno v internet. +Privzeto posluša samo na `127.0.0.1:8080`. Za dostop iz drugih naprav uporabi VPN (Tailscale/WireGuard) ali HTTPS reverse proxy. Ne odpiraj porta neposredno v internet. ## API @@ -34,6 +34,16 @@ Privzeto posluša samo na `127.0.0.1:8080`. Za dostop iz drugih naprav uporabi V Odgovor `202 Accepted` pomeni, da je paket zapisan v `./data/history.sqlite3`. Podvojeni obiski se ne podvojijo. +## Spletni pregledovalnik + +Odpri `http://127.0.0.1:8080/` in se prijavi z `VIEWER_USERNAME` ter `VIEWER_PASSWORD` iz `.env`. Pregledovalnik omogoča: + +- filter od/do (čas je UTC), +- filter po napravi, +- iskanje po domeni, naslovu ali URL-ju, +- seznam do 500 zadnjih obiskov, +- pregled unikatnih domen s številom obiskov in zadnjim obiskom. + ## Discord (neobvezno) V `.env` nastavi `DISCORD_WEBHOOK_URL`. Najprej se zapis shrani v SQLite, nato se poskusi poslati Discordu. Neuspeh Discorda ne izbriše podatkov. diff --git a/app/main.py b/app/main.py index 1163f4d..dedb202 100644 --- a/app/main.py +++ b/app/main.py @@ -1,15 +1,19 @@ import asyncio -import json +import base64 +import hmac import logging import os import sqlite3 from contextlib import closing from datetime import datetime, timezone +from html import escape from pathlib import Path -from typing import Literal +from typing import Annotated +from urllib.parse import urlparse import httpx -from fastapi import FastAPI, Header, HTTPException, Request, status +from fastapi import FastAPI, Header, HTTPException, Query, Request, status +from fastapi.responses import HTMLResponse from pydantic import BaseModel, Field, HttpUrl logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s") @@ -18,6 +22,8 @@ logger = logging.getLogger("history-collector") DATA_DIR = Path(os.getenv("DATA_DIR", "/data")) DB_PATH = DATA_DIR / "history.sqlite3" API_TOKEN = os.getenv("API_TOKEN", "") +VIEWER_USERNAME = os.getenv("VIEWER_USERNAME", "admin") +VIEWER_PASSWORD = os.getenv("VIEWER_PASSWORD", "") DISCORD_WEBHOOK_URL = os.getenv("DISCORD_WEBHOOK_URL", "") MAX_VISITS_PER_REQUEST = int(os.getenv("MAX_VISITS_PER_REQUEST", "100")) @@ -33,47 +39,62 @@ class VisitBatch(BaseModel): app = FastAPI(title="Local History Collector", docs_url=None, redoc_url=None) +def domain_from_url(url: str) -> str: + return (urlparse(url).hostname or "").lower() + def connect(): DATA_DIR.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(DB_PATH) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.execute("""CREATE TABLE IF NOT EXISTS visits ( - id INTEGER PRIMARY KEY, - device_id TEXT NOT NULL, - url TEXT NOT NULL, - title TEXT NOT NULL DEFAULT '', - visit_time TEXT NOT NULL, - transition TEXT NOT NULL, - received_at TEXT NOT NULL, - UNIQUE(device_id, url, visit_time) - )""") + id INTEGER PRIMARY KEY, device_id TEXT NOT NULL, url TEXT NOT NULL, + domain TEXT NOT NULL DEFAULT '', title TEXT NOT NULL DEFAULT '', + visit_time TEXT NOT NULL, transition TEXT NOT NULL, received_at TEXT NOT NULL, + UNIQUE(device_id, url, visit_time))""") + # Migration for databases created by collector v1. + columns = {row[1] for row in conn.execute("PRAGMA table_info(visits)")} + if "domain" not in columns: + conn.execute("ALTER TABLE visits ADD COLUMN domain TEXT NOT NULL DEFAULT ''") + missing = conn.execute("SELECT id, url FROM visits WHERE domain = ''").fetchall() + conn.executemany("UPDATE visits SET domain = ? WHERE id = ?", [(domain_from_url(row["url"]), row["id"]) for row in missing]) conn.execute("CREATE INDEX IF NOT EXISTS idx_visits_device_time ON visits(device_id, visit_time DESC)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_visits_domain ON visits(domain)") + conn.commit() return conn @app.on_event("startup") def startup(): if len(API_TOKEN) < 24: raise RuntimeError("API_TOKEN must contain at least 24 characters") + if len(VIEWER_PASSWORD) < 12: + raise RuntimeError("VIEWER_PASSWORD must contain at least 12 characters") with closing(connect()): pass - def require_token(authorization: str | None): - expected = f"Bearer {API_TOKEN}" - if authorization != expected: + if not hmac.compare_digest(authorization or "", f"Bearer {API_TOKEN}"): raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API token") +def require_viewer(request: Request): + header = request.headers.get("authorization", "") + try: + scheme, encoded = header.split(" ", 1) + username, password = base64.b64decode(encoded).decode("utf-8").split(":", 1) + except Exception: + scheme = username = password = "" + if scheme.lower() != "basic" or not (hmac.compare_digest(username, VIEWER_USERNAME) and hmac.compare_digest(password, VIEWER_PASSWORD)): + raise HTTPException(status_code=401, detail="Authentication required", headers={"WWW-Authenticate": 'Basic realm="History Collector"'}) def save_batch(batch: VisitBatch) -> int: received_at = datetime.now(timezone.utc).isoformat() - rows = [(batch.device_id, str(v.url), v.title, v.visit_time.astimezone(timezone.utc).isoformat(), - v.transition, received_at) for v in batch.visits] + rows = [(batch.device_id, str(v.url), domain_from_url(str(v.url)), v.title, + v.visit_time.astimezone(timezone.utc).isoformat(), v.transition, received_at) for v in batch.visits] with closing(connect()) as conn: before = conn.total_changes conn.executemany("""INSERT OR IGNORE INTO visits - (device_id, url, title, visit_time, transition, received_at) - VALUES (?, ?, ?, ?, ?, ?)""", rows) + (device_id, url, domain, title, visit_time, transition, received_at) + VALUES (?, ?, ?, ?, ?, ?, ?)""", rows) inserted = conn.total_changes - before conn.commit() return inserted @@ -82,14 +103,12 @@ async def post_discord(batch: VisitBatch): if not DISCORD_WEBHOOK_URL: return lines = [f"• {v.visit_time.astimezone(timezone.utc):%Y-%m-%d %H:%M UTC} — {v.title or '(brez naslova)'}\n <{v.url}>" for v in batch.visits] - content = f"**Zgodovina brskanja · {batch.device_id}**\n" + "\n".join(lines) - # Discord accepts message content up to 2,000 characters; split safely by visit. - chunks, current = [], "**Zgodovina brskanja · %s**\n" % batch.device_id + chunks, current = [], f"**Zgodovina brskanja · {batch.device_id}**\n" for line in lines: candidate = current + line + "\n" if len(candidate) > 1900 and current.strip(): chunks.append(current) - current = "**Zgodovina brskanja · %s (nadaljevanje)**\n%s\n" % (batch.device_id, line) + current = f"**Zgodovina brskanja · {batch.device_id} (nadaljevanje)**\n{line}\n" else: current = candidate if current.strip(): chunks.append(current) @@ -99,7 +118,6 @@ async def post_discord(batch: VisitBatch): response = await client.post(DISCORD_WEBHOOK_URL, json={"content": content}) response.raise_for_status() except Exception: - # The visits are already safely persisted; don't reject them because Discord is unavailable. logger.exception("Discord relay failed") @app.get("/healthz") @@ -107,8 +125,37 @@ def healthz(): return {"ok": True} @app.post("/v1/visits", status_code=status.HTTP_202_ACCEPTED) -async def receive_visits(batch: VisitBatch, authorization: str | None = Header(default=None)): +async def receive_visits(batch: VisitBatch, authorization: Annotated[str | None, Header()] = None): require_token(authorization) stored = save_batch(batch) asyncio.create_task(post_discord(batch)) return {"accepted": len(batch.visits), "stored": stored} + +def iso_from_local(value: str | None) -> str | None: + if not value: + return None + try: + return datetime.fromisoformat(value).replace(tzinfo=timezone.utc).isoformat() + except ValueError: + return None + +def viewer_page(rows, domains, devices, start, end, device, search): + opts = [''] + [f'' for d in devices] + visit_rows = "".join(f"{escape(r['visit_time'].replace('+00:00', 'Z'))}{escape(r['device_id'])}{escape(r['domain'])}{escape(r['title'])}{escape(r['url'])}" for r in rows) or 'Ni zadetkov.' + domain_rows = "".join(f"{escape(r['domain'])}{r['visits']}{escape(r['last_seen'].replace('+00:00', 'Z'))}" for r in domains) or 'Ni zadetkov.' + return f'''Zgodovina brskanja

Zgodovina brskanja

Obiski (zadnjih 500)

{visit_rows}
Čas (UTC)NapravaDomenaNaslovURL

Unikatne domene

{domain_rows}
DomenaObiskiZadnji obisk
''' + +@app.get("/", response_class=HTMLResponse) +def viewer(request: Request, start: str | None = Query(None), end: str | None = Query(None), device: str | None = Query(None), q: str | None = Query(None)): + require_viewer(request) + filters, params = [], [] + if value := iso_from_local(start): filters.append("visit_time >= ?"); params.append(value) + if value := iso_from_local(end): filters.append("visit_time <= ?"); params.append(value) + if device: filters.append("device_id = ?"); params.append(device) + if q: filters.append("(domain LIKE ? OR title LIKE ? OR url LIKE ?)"); params.extend([f"%{q}%"] * 3) + where = (" WHERE " + " AND ".join(filters)) if filters else "" + with closing(connect()) as conn: + rows = conn.execute(f"SELECT visit_time, device_id, domain, title, url FROM visits{where} ORDER BY visit_time DESC LIMIT 500", params).fetchall() + domains = conn.execute(f"SELECT domain, COUNT(*) visits, MAX(visit_time) last_seen FROM visits{where} GROUP BY domain ORDER BY visits DESC, last_seen DESC LIMIT 200", params).fetchall() + devices = [r[0] for r in conn.execute("SELECT DISTINCT device_id FROM visits ORDER BY device_id")] + return viewer_page(rows, domains, devices, start, end, device, q)