Initial history collector implementation
This commit is contained in:
commit
6f2b238988
7 files changed
+196
No files matched your search
@@ -0,0 +1,8 @@
|
||||
# Create with: cp .env.example .env ; then replace the placeholder.
|
||||
# Generate: python3 -c "import secrets; print(secrets.token_urlsafe(32))"
|
||||
API_TOKEN=REPLACE_WITH_A_RANDOM_32+_CHARACTER_TOKEN
|
||||
# Optional. Leave empty to keep all data only in SQLite.
|
||||
DISCORD_WEBHOOK_URL=
|
||||
DATA_DIR=/data
|
||||
MAX_VISITS_PER_REQUEST=100
|
||||
LOG_LEVEL=INFO
|
||||
@@ -0,0 +1,3 @@
|
||||
.env
|
||||
data/
|
||||
__pycache__/
|
||||
@@ -0,0 +1,9 @@
|
||||
FROM python:3.13-slim
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY app ./app
|
||||
# SQLite data is mounted at /data; the container remains read-only except for that mount.
|
||||
# It runs as root so a freshly-created local bind mount is writable without manual chown.
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--proxy-headers"]
|
||||
@@ -0,0 +1,43 @@
|
||||
# History Collector
|
||||
|
||||
Python/FastAPI strežnik za lastno Chrome razširitev: sprejema obiske, jih trajno shrani v SQLite in jih lahko opcijsko pošlje v zaseben Discord kanal prek webhooka.
|
||||
|
||||
> URL-ji so lahko občutljivi. Uporabi samo na napravah, ki jih upravljaš, z jasnim obvestilom uporabnikom. Strežnika ne izpostavljaj neposredno internetu brez HTTPS in dodatne zaščite.
|
||||
|
||||
## Zagon
|
||||
|
||||
```bash
|
||||
cp .env.example .env
|
||||
# V .env nastavi dolg, naključen API_TOKEN.
|
||||
docker compose up -d --build
|
||||
docker compose logs -f
|
||||
curl http://127.0.0.1:8080/healthz
|
||||
```
|
||||
|
||||
Privzeto posluša samo na `127.0.0.1:8080`. Za dostop iz drugih naprav uporabi VPN (Tailscale/WireGuard) ali HTTPS reverse proxy z avtentikacijo. Ne odpiraj porta neposredno v internet.
|
||||
|
||||
## API
|
||||
|
||||
`POST /v1/visits`, z glavo `Authorization: Bearer TVOJ_API_TOKEN`:
|
||||
|
||||
```json
|
||||
{
|
||||
"device_id": "otrok-prenosnik",
|
||||
"visits": [{
|
||||
"url": "https://example.com/clanek",
|
||||
"title": "Primer",
|
||||
"visit_time": "2026-10-06T18:50:00Z",
|
||||
"transition": "observed"
|
||||
}]
|
||||
}
|
||||
```
|
||||
|
||||
Odgovor `202 Accepted` pomeni, da je paket zapisan v `./data/history.sqlite3`. Podvojeni obiski se ne podvojijo.
|
||||
|
||||
## Discord (neobvezno)
|
||||
|
||||
V `.env` nastavi `DISCORD_WEBHOOK_URL`. Najprej se zapis shrani v SQLite, nato se poskusi poslati Discordu. Neuspeh Discorda ne izbriše podatkov.
|
||||
|
||||
## Nastavitev razširitve
|
||||
|
||||
V `extension/config.js` nastavi `collectorUrl`, `apiToken` in `deviceId`. Token naj ostane v tej lokalni datoteki in naj ne konča v repozitoriju. `manifest.json` vsebuje široko dovoljenje za HTTP/HTTPS gostitelje zato, da lahko izbereš lasten API naslov; če želiš strožje dovoljenje, zamenjaj `host_permissions` samo s svojim HTTPS naslovom.
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sqlite3
|
||||
from contextlib import closing
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
import httpx
|
||||
from fastapi import FastAPI, Header, HTTPException, Request, status
|
||||
from pydantic import BaseModel, Field, HttpUrl
|
||||
|
||||
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger("history-collector")
|
||||
|
||||
DATA_DIR = Path(os.getenv("DATA_DIR", "/data"))
|
||||
DB_PATH = DATA_DIR / "history.sqlite3"
|
||||
API_TOKEN = os.getenv("API_TOKEN", "")
|
||||
DISCORD_WEBHOOK_URL = os.getenv("DISCORD_WEBHOOK_URL", "")
|
||||
MAX_VISITS_PER_REQUEST = int(os.getenv("MAX_VISITS_PER_REQUEST", "100"))
|
||||
|
||||
class Visit(BaseModel):
|
||||
url: HttpUrl
|
||||
title: str = Field(default="", max_length=1000)
|
||||
visit_time: datetime
|
||||
transition: str = Field(default="observed", max_length=100)
|
||||
|
||||
class VisitBatch(BaseModel):
|
||||
device_id: str = Field(min_length=1, max_length=128, pattern=r"^[a-zA-Z0-9._-]+$")
|
||||
visits: list[Visit] = Field(min_length=1, max_length=MAX_VISITS_PER_REQUEST)
|
||||
|
||||
app = FastAPI(title="Local History Collector", docs_url=None, redoc_url=None)
|
||||
|
||||
def connect():
|
||||
DATA_DIR.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(DB_PATH)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS visits (
|
||||
id INTEGER PRIMARY KEY,
|
||||
device_id TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
title TEXT NOT NULL DEFAULT '',
|
||||
visit_time TEXT NOT NULL,
|
||||
transition TEXT NOT NULL,
|
||||
received_at TEXT NOT NULL,
|
||||
UNIQUE(device_id, url, visit_time)
|
||||
)""")
|
||||
conn.execute("CREATE INDEX IF NOT EXISTS idx_visits_device_time ON visits(device_id, visit_time DESC)")
|
||||
return conn
|
||||
|
||||
@app.on_event("startup")
|
||||
def startup():
|
||||
if len(API_TOKEN) < 24:
|
||||
raise RuntimeError("API_TOKEN must contain at least 24 characters")
|
||||
with closing(connect()):
|
||||
pass
|
||||
|
||||
|
||||
def require_token(authorization: str | None):
|
||||
expected = f"Bearer {API_TOKEN}"
|
||||
if authorization != expected:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid API token")
|
||||
|
||||
|
||||
def save_batch(batch: VisitBatch) -> int:
|
||||
received_at = datetime.now(timezone.utc).isoformat()
|
||||
rows = [(batch.device_id, str(v.url), v.title, v.visit_time.astimezone(timezone.utc).isoformat(),
|
||||
v.transition, received_at) for v in batch.visits]
|
||||
with closing(connect()) as conn:
|
||||
before = conn.total_changes
|
||||
conn.executemany("""INSERT OR IGNORE INTO visits
|
||||
(device_id, url, title, visit_time, transition, received_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""", rows)
|
||||
inserted = conn.total_changes - before
|
||||
conn.commit()
|
||||
return inserted
|
||||
|
||||
async def post_discord(batch: VisitBatch):
|
||||
if not DISCORD_WEBHOOK_URL:
|
||||
return
|
||||
lines = [f"• {v.visit_time.astimezone(timezone.utc):%Y-%m-%d %H:%M UTC} — {v.title or '(brez naslova)'}\n <{v.url}>" for v in batch.visits]
|
||||
content = f"**Zgodovina brskanja · {batch.device_id}**\n" + "\n".join(lines)
|
||||
# Discord accepts message content up to 2,000 characters; split safely by visit.
|
||||
chunks, current = [], "**Zgodovina brskanja · %s**\n" % batch.device_id
|
||||
for line in lines:
|
||||
candidate = current + line + "\n"
|
||||
if len(candidate) > 1900 and current.strip():
|
||||
chunks.append(current)
|
||||
current = "**Zgodovina brskanja · %s (nadaljevanje)**\n%s\n" % (batch.device_id, line)
|
||||
else:
|
||||
current = candidate
|
||||
if current.strip(): chunks.append(current)
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10) as client:
|
||||
for content in chunks:
|
||||
response = await client.post(DISCORD_WEBHOOK_URL, json={"content": content})
|
||||
response.raise_for_status()
|
||||
except Exception:
|
||||
# The visits are already safely persisted; don't reject them because Discord is unavailable.
|
||||
logger.exception("Discord relay failed")
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz():
|
||||
return {"ok": True}
|
||||
|
||||
@app.post("/v1/visits", status_code=status.HTTP_202_ACCEPTED)
|
||||
async def receive_visits(batch: VisitBatch, authorization: str | None = Header(default=None)):
|
||||
require_token(authorization)
|
||||
stored = save_batch(batch)
|
||||
asyncio.create_task(post_discord(batch))
|
||||
return {"accepted": len(batch.visits), "stored": stored}
|
||||
@@ -0,0 +1,16 @@
|
||||
services:
|
||||
history-collector:
|
||||
build: .
|
||||
container_name: history-collector
|
||||
restart: unless-stopped
|
||||
env_file: .env
|
||||
ports:
|
||||
# Keep it bound to localhost unless you deliberately put it behind a reverse proxy/VPN.
|
||||
- "127.0.0.1:8080:8080"
|
||||
volumes:
|
||||
- ./data:/data
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp
|
||||
@@ -0,0 +1,3 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn[standard]==0.34.2
|
||||
httpx==0.28.1
|
||||
Reference in new issue
Block a user