Initial history collector implementation
This commit is contained in:
commit
6f2b238988
7 files changed
+196
No files matched your search
@@ -0,0 +1,8 @@
|
|||||||
|
# Create with: cp .env.example .env ; then replace the placeholder.
|
||||||
|
# Generate: python3 -c "import secrets; print(secrets.token_urlsafe(32))"
|
||||||
|
API_TOKEN=REPLACE_WITH_A_RANDOM_32+_CHARACTER_TOKEN
|
||||||
|
# Optional. Leave empty to keep all data only in SQLite.
|
||||||
|
DISCORD_WEBHOOK_URL=
|
||||||
|
DATA_DIR=/data
|
||||||
|
MAX_VISITS_PER_REQUEST=100
|
||||||
|
LOG_LEVEL=INFO
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
.env
|
||||||
|
data/
|
||||||
|
__pycache__/
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
FROM python:3.13-slim
|
||||||
|
WORKDIR /app
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
COPY app ./app
|
||||||
|
# SQLite data is mounted at /data; the container remains read-only except for that mount.
|
||||||
|
# It runs as root so a freshly-created local bind mount is writable without manual chown.
|
||||||
|
EXPOSE 8080
|
||||||
|
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers"]
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
# History Collector
|
||||||
|
|
||||||
|
Python/FastAPI strežnik za lastno Chrome razširitev: sprejema obiske, jih trajno shrani v SQLite in jih lahko opcijsko pošlje v zaseben Discord kanal prek webhooka.
|
||||||
|
|
||||||
|
> URL-ji so lahko občutljivi. Uporabi samo na napravah, ki jih upravljaš, z jasnim obvestilom uporabnikom. Strežnika ne izpostavljaj neposredno internetu brez HTTPS in dodatne zaščite.
|
||||||
|
|
||||||
|
## Zagon
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp .env.example .env
|
||||||
|
# V .env nastavi dolg, naključen API_TOKEN.
|
||||||
|
docker compose up -d --build
|
||||||
|
docker compose logs -f
|
||||||
|
curl http://127.0.0.1:8080/healthz
|
||||||
|
```
|
||||||
|
|
||||||
|
Privzeto posluša samo na `127.0.0.1:8080`. Za dostop iz drugih naprav uporabi VPN (Tailscale/WireGuard) ali HTTPS reverse proxy z avtentikacijo. Ne odpiraj porta neposredno v internet.
|
||||||
|
|
||||||
|
## API
|
||||||
|
|
||||||
|
`POST /v1/visits`, z glavo `Authorization: Bearer TVOJ_API_TOKEN`:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"device_id": "otrok-prenosnik",
|
||||||
|
"visits": [{
|
||||||
|
"url": "https://example.com/clanek",
|
||||||
|
"title": "Primer",
|
||||||
|
"visit_time": "2026-10-06T18:50:00Z",
|
||||||
|
"transition": "observed"
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Odgovor `202 Accepted` pomeni, da je paket zapisan v `./data/history.sqlite3`. Podvojeni obiski se ne podvojijo.
|
||||||
|
|
||||||
|
## Discord (neobvezno)
|
||||||
|
|
||||||
|
V `.env` nastavi `DISCORD_WEBHOOK_URL`. Najprej se zapis shrani v SQLite, nato se poskusi poslati Discordu. Neuspeh Discorda ne izbriše podatkov.
|
||||||
|
|
||||||
|
## Nastavitev razširitve
|
||||||
|
|
||||||
|
V `extension/config.js` nastavi `collectorUrl`, `apiToken` in `deviceId`. Token naj ostane v tej lokalni datoteki in naj ne konča v repozitoriju. `manifest.json` vsebuje široko dovoljenje za HTTP/HTTPS gostitelje zato, da lahko izbereš lasten API naslov; če želiš strožje dovoljenje, zamenjaj `host_permissions` samo s svojim HTTPS naslovom.
|
||||||
+114
@@ -0,0 +1,114 @@
|
|||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import sqlite3
|
||||||
|
from contextlib import closing
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Literal
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
from fastapi import FastAPI, Header, HTTPException, Request, status
|
||||||
|
from pydantic import BaseModel, Field, HttpUrl
|
||||||
|
|
||||||
|
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
logger = logging.getLogger("history-collector")
|
||||||
|
|
||||||
|
DATA_DIR = Path(os.getenv("DATA_DIR", "/data"))
|
||||||
|
DB_PATH = DATA_DIR / "history.sqlite3"
|
||||||
|
API_TOKEN = os.getenv("API_TOKEN", "")
|
||||||
|
DISCORD_WEBHOOK_URL = os.getenv("DISCORD_WEBHOOK_URL", "")
|
||||||
|
MAX_VISITS_PER_REQUEST = int(os.getenv("MAX_VISITS_PER_REQUEST", "100"))
|
||||||
|
|
||||||
|
class Visit(BaseModel):
|
||||||
|
url: HttpUrl
|
||||||
|
title: str = Field(default="", max_length=1000)
|
||||||
|
visit_time: datetime
|
||||||
|
transition: str = Field(default="observed", max_length=100)
|
||||||
|
|
||||||
|
class VisitBatch(BaseModel):
|
||||||
|
device_id: str = Field(min_length=1, max_length=128, pattern=r"^[a-zA-Z0-9._-]+$")
|
||||||
|
visits: list[Visit] = Field(min_length=1, max_length=MAX_VISITS_PER_REQUEST)
|
||||||
|
|
||||||
|
app = FastAPI(title="Local History Collector", docs_url=None, redoc_url=None)
|
||||||
|
|
||||||
|
def connect():
|
||||||
|
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
conn = sqlite3.connect(DB_PATH)
|
||||||
|
conn.row_factory = sqlite3.Row
|
||||||
|
conn.execute("PRAGMA journal_mode=WAL")
|
||||||
|
conn.execute("""CREATE TABLE IF NOT EXISTS visits (
|
||||||
|
id INTEGER PRIMARY KEY,
|
||||||
|
device_id TEXT NOT NULL,
|
||||||
|
url TEXT NOT NULL,
|
||||||
|
title TEXT NOT NULL DEFAULT '',
|
||||||
|
visit_time TEXT NOT NULL,
|
||||||
|
transition TEXT NOT NULL,
|
||||||
|
received_at TEXT NOT NULL,
|
||||||
|
UNIQUE(device_id, url, visit_time)
|
||||||
|
)""")
|
||||||
|
conn.execute("CREATE INDEX IF NOT EXISTS idx_visits_device_time ON visits(device_id, visit_time DESC)")
|
||||||
|
return conn
|
||||||
|
|
||||||
|
@app.on_event("startup")
|
||||||
|
def startup():
|
||||||
|
if len(API_TOKEN) < 24:
|
||||||
|
raise RuntimeError("API_TOKEN must contain at least 24 characters")
|
||||||
|
with closing(connect()):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def require_token(authorization: str | None):
|
||||||
|
expected = f"Bearer {API_TOKEN}"
|
||||||
|
if authorization != expected:
|
||||||
|
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API token")
|
||||||
|
|
||||||
|
|
||||||
|
def save_batch(batch: VisitBatch) -> int:
|
||||||
|
received_at = datetime.now(timezone.utc).isoformat()
|
||||||
|
rows = [(batch.device_id, str(v.url), v.title, v.visit_time.astimezone(timezone.utc).isoformat(),
|
||||||
|
v.transition, received_at) for v in batch.visits]
|
||||||
|
with closing(connect()) as conn:
|
||||||
|
before = conn.total_changes
|
||||||
|
conn.executemany("""INSERT OR IGNORE INTO visits
|
||||||
|
(device_id, url, title, visit_time, transition, received_at)
|
||||||
|
VALUES (?, ?, ?, ?, ?, ?)""", rows)
|
||||||
|
inserted = conn.total_changes - before
|
||||||
|
conn.commit()
|
||||||
|
return inserted
|
||||||
|
|
||||||
|
async def post_discord(batch: VisitBatch):
|
||||||
|
if not DISCORD_WEBHOOK_URL:
|
||||||
|
return
|
||||||
|
lines = [f"• {v.visit_time.astimezone(timezone.utc):%Y-%m-%d %H:%M UTC} — {v.title or '(brez naslova)'}\n <{v.url}>" for v in batch.visits]
|
||||||
|
content = f"**Zgodovina brskanja · {batch.device_id}**\n" + "\n".join(lines)
|
||||||
|
# Discord accepts message content up to 2,000 characters; split safely by visit.
|
||||||
|
chunks, current = [], "**Zgodovina brskanja · %s**\n" % batch.device_id
|
||||||
|
for line in lines:
|
||||||
|
candidate = current + line + "\n"
|
||||||
|
if len(candidate) > 1900 and current.strip():
|
||||||
|
chunks.append(current)
|
||||||
|
current = "**Zgodovina brskanja · %s (nadaljevanje)**\n%s\n" % (batch.device_id, line)
|
||||||
|
else:
|
||||||
|
current = candidate
|
||||||
|
if current.strip(): chunks.append(current)
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=10) as client:
|
||||||
|
for content in chunks:
|
||||||
|
response = await client.post(DISCORD_WEBHOOK_URL, json={"content": content})
|
||||||
|
response.raise_for_status()
|
||||||
|
except Exception:
|
||||||
|
# The visits are already safely persisted; don't reject them because Discord is unavailable.
|
||||||
|
logger.exception("Discord relay failed")
|
||||||
|
|
||||||
|
@app.get("/healthz")
|
||||||
|
def healthz():
|
||||||
|
return {"ok": True}
|
||||||
|
|
||||||
|
@app.post("/v1/visits", status_code=status.HTTP_202_ACCEPTED)
|
||||||
|
async def receive_visits(batch: VisitBatch, authorization: str | None = Header(default=None)):
|
||||||
|
require_token(authorization)
|
||||||
|
stored = save_batch(batch)
|
||||||
|
asyncio.create_task(post_discord(batch))
|
||||||
|
return {"accepted": len(batch.visits), "stored": stored}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
services:
|
||||||
|
history-collector:
|
||||||
|
build: .
|
||||||
|
container_name: history-collector
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file: .env
|
||||||
|
ports:
|
||||||
|
# Keep it bound to localhost unless you deliberately put it behind a reverse proxy/VPN.
|
||||||
|
- "127.0.0.1:8080:8080"
|
||||||
|
volumes:
|
||||||
|
- ./data:/data
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
fastapi==0.115.12
|
||||||
|
uvicorn[standard]==0.34.2
|
||||||
|
httpx==0.28.1
|
||||||
Reference in new issue
Block a user