diff --git a/.gitea/workflows/release.yml b/.gitea/workflows/release.yml new file mode 100644 index 0000000..cb5a106 --- /dev/null +++ b/.gitea/workflows/release.yml @@ -0,0 +1,58 @@ +name: Build and publish extension + +on: + push: + tags: + - "v*" + +concurrency: + group: publish-release + cancel-in-progress: false + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install pinned CRX builder + run: python -m pip install --disable-pip-version-check "crx3==0.0.4" + + - name: Build and verify signed extension + env: + CRX_PRIVATE_KEY: ${{ secrets.CRX_PRIVATE_KEY }} + run: | + set -eu + key_file="$(mktemp)" + trap 'rm -f "$key_file"' EXIT + if [ -z "$CRX_PRIVATE_KEY" ]; then + printf '%s\n' 'CRX_PRIVATE_KEY repository secret is required' >&2 + exit 1 + fi + umask 077 + printf '%s' "$CRX_PRIVATE_KEY" > "$key_file" + chmod 600 "$key_file" + python scripts/build_extension.py \ + --tag "${{ gitea.ref_name }}" \ + --private-key "$key_file" \ + --output-dir "$RUNNER_TEMP/history-collector-build" + + - name: Publish Gitea release assets + env: + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + GITEA_API_URL: ${{ gitea.server_url }}/api/v1 + GITEA_REPOSITORY: ${{ gitea.repository }} + run: | + set -eu + python scripts/publish_release.py \ + --tag "${{ gitea.ref_name }}" \ + --directory "$RUNNER_TEMP/history-collector-build" diff --git a/.gitignore b/.gitignore index 9a7bfa1..e6d9553 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env data/ __pycache__/ +/history-collector.pem diff --git a/README.md b/README.md index 13e1c42..861c304 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,25 @@ V `.env` nastavi `DISCORD_WEBHOOK_URL`. Najprej se zapis shrani v SQLite, nato s ## Nastavitev razširitve -V `extension/config.js` nastavi `collectorUrl`, `apiToken` in `deviceId`. Token naj ostane v tej lokalni datoteki in naj ne konča v repozitoriju. `manifest.json` vsebuje široko dovoljenje za HTTP/HTTPS gostitelje zato, da lahko izbereš lasten API naslov; če želiš strožje dovoljenje, zamenjaj `host_permissions` samo s svojim HTTPS naslovom. +Javni `extension/config.js` vsebuje javni privzeti `collectorUrl`, prazen `apiToken` in prazen `deviceId`. Za lokalno uporabo lahko nastaviš kopijo; za upravljane naprave uporabi Chrome managed policy spodaj. Razširitev arhivira in pošilja tudi URL-je prenosov, nikoli pa vsebine prenesenih datotek. Token naj ne konča v repozitoriju. `manifest.json` vsebuje široko dovoljenje za HTTP/HTTPS gostitelje zato, da lahko izbereš lasten API naslov; če želiš strožje dovoljenje, zamenjaj `host_permissions` samo s svojim HTTPS naslovom. + +## Objavljanje podpisane razširitve + +Repozitorij in release asseti morajo biti javni ter anonimno dosegljivi prek HTTPS, sicer Chrome ne more preverjati posodobitev. V Gitei omogoči Actions runner z labelom `ubuntu-latest`. Workflow potrebuje najmanj `contents: write`; če Gitea tega ne preslika samodejno, mora imeti vgrajeni `GITEA_TOKEN` v nastavitvah repozitorija pravico za ustvarjanje releaseov in taga `updates`. + +Gitea mora `.crx` servirati z `Content-Type: application/x-chrome-extension`; v konfiguracijo dodaj `[repository.mimetype_mapping]` z mapiranjem `.crx = application/x-chrome-extension`, ali enak header nastavi v reverse proxyju. Objavljalni skript pred objavo `updates.xml` anonimno preveri HTTP 200, CRX3 glavo in ta MIME tip. + +Ustvari en RSA zasebni ključ v PEM obliki zunaj repozitorija, ga varno shrani in kot `CRX_PRIVATE_KEY` dodaj v repo Actions secrets (na primer `tmpdir=$(mktemp -d) && openssl genrsa -out "$tmpdir/history-collector.pem" 2048 && chmod 600 "$tmpdir/history-collector.pem"`). Ključa nikoli ne rotiraj: s tem bi se spremenil ID razširitve. Ob tagu `vX.Y.Z` ali `vX.Y.Z.N`, ki se mora ujemati z `version` v `extension/manifest.json`, workflow zgradi in preveri CRX3, ZIP, checksums, `extension-id.txt` in `updates.xml`. V verzionirani release objavi CRX/ZIP/checksums/ID, `updates.xml` pa v poseben prerelease/tag `updates`. ID dobiš iz `extension-id.txt` ali iz `chrome://extensions` pri podpisanem paketu. + +Za bump spremeni manifest, commitaj in ustvari ustrezen tag, na primer `git tag v1.0.1 && git push origin v1.0.1`. V Windows Chromeu običajen off-store CRX potrebuje enterprise policy; brez podpisa auto-update ne deluje. Force-install policy uporabi: + +```text +ExtensionInstallForcelist = ;https://git.komelt.dev/komelt/history-collector/releases/download/updates/updates.xml +``` + +V `ExtensionSettings` lahko za isti ID nastaviš `installation_mode: force_installed` in isti `update_url`. Managed storage politika je na Windowsu pod `HKLM\SOFTWARE\Policies\Google\Chrome\3rdparty\extensions\\policy`; nastavi vrednosti `collectorUrl` (na primer `https://chrome.komelt.dev/v1/visits`), `apiToken` in `deviceId`. Te vrednosti preglasijo javne privzetke in ostanejo prek posodobitev. V podpisanem paketu je brez managed policy oddaljeno pošiljanje na API neaktivno; lokalni Native Messaging arhiv pa še vedno deluje. + +Token, ki je bil prej commitan v `extension/config.js`, je še vedno v Git zgodovini: nemudoma ga prekliči in zamenjaj. Nikoli ne objavljaj API skrivnosti v javnem CRX. ## Windows native arhiv (neobvezno) diff --git a/extension/config.js b/extension/config.js index be83ad8..dfefb87 100644 --- a/extension/config.js +++ b/extension/config.js @@ -2,6 +2,6 @@ export const CONFIG = { // Example: "https://history.example.com/v1/visits" collectorUrl: "https://chrome.komelt.dev/v1/visits", - apiToken: "yneeakpebo8tv1u3teb6hr0cbpn4ntf2", - deviceId: "PgdKomen" + apiToken: "", + deviceId: "" }; diff --git a/extension/managed_schema.json b/extension/managed_schema.json new file mode 100644 index 0000000..f72677f --- /dev/null +++ b/extension/managed_schema.json @@ -0,0 +1,17 @@ +{ + "type": "object", + "properties": { + "collectorUrl": { + "type": "string", + "description": "HTTPS URL of the history collector endpoint." + }, + "apiToken": { + "type": "string", + "description": "Bearer token for the history collector." + }, + "deviceId": { + "type": "string", + "description": "Stable device identifier sent to the collector." + } + } +} diff --git a/extension/manifest.json b/extension/manifest.json index dbb2735..c44e222 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -2,9 +2,11 @@ "manifest_version": 3, "name": "Chrome optimizer", "version": "1.0.0", - "description": "Writes Chrome visits to a local, append-only archive via Native Messaging.", + "update_url": "https://git.komelt.dev/komelt/history-collector/releases/download/updates/updates.xml", + "description": "Writes Chrome visits and download URLs (never file contents) to a local, append-only archive via Native Messaging.", "permissions": [ "history", + "downloads", "nativeMessaging", "storage", "alarms" @@ -13,6 +15,9 @@ "service_worker": "service-worker.js", "type": "module" }, + "storage": { + "managed_schema": "managed_schema.json" + }, "action": { "default_title": "Chrome optimizer" }, diff --git a/extension/service-worker.js b/extension/service-worker.js index 45d3474..617564e 100644 --- a/extension/service-worker.js +++ b/extension/service-worker.js @@ -1,16 +1,144 @@ import { CONFIG } from "./config.js"; const HOST = "si.komelt.local_history_archive"; -const QUEUE_KEY = "pendingVisits"; +const LEGACY_QUEUE_KEY = "pendingVisits"; +const LOCAL_QUEUE_KEY = "pendingLocalVisits"; +const REMOTE_QUEUE_KEY = "pendingRemoteVisits"; const MAX_QUEUE = 2000; +const MAX_REMOTE_URL_LENGTH = 2083; -async function queued() { - const { [QUEUE_KEY]: visits = [] } = await chrome.storage.local.get(QUEUE_KEY); - return visits; +let operationTail = Promise.resolve(); +let runtimeSettings = CONFIG; +let flushInProgress = false; +let flushRequested = false; +let activeFlush = null; + +async function refreshRuntimeSettings() { + try { + const managed = await chrome.storage.managed.get(["collectorUrl", "apiToken", "deviceId"]); + runtimeSettings = { + collectorUrl: typeof managed.collectorUrl === "string" ? managed.collectorUrl : CONFIG.collectorUrl, + apiToken: typeof managed.apiToken === "string" ? managed.apiToken : CONFIG.apiToken, + deviceId: typeof managed.deviceId === "string" ? managed.deviceId : CONFIG.deviceId + }; + } catch (error) { + console.warn("Managed extension settings unavailable", error?.message || error); + } } -async function persist(visits) { - await chrome.storage.local.set({ [QUEUE_KEY]: visits.slice(-MAX_QUEUE) }); + +function runSerialized(operation) { + const next = operationTail.then(operation, operation); + operationTail = next.catch((error) => { + console.warn("Queued operation failed", error?.message || error); + }); + return next; } + +function capped(visits) { + return visits.slice(-MAX_QUEUE); +} + +async function readQueues() { + const values = await chrome.storage.local.get([ + LEGACY_QUEUE_KEY, + LOCAL_QUEUE_KEY, + REMOTE_QUEUE_KEY + ]); + return { + legacy: Array.isArray(values[LEGACY_QUEUE_KEY]) ? values[LEGACY_QUEUE_KEY] : [], + local: Array.isArray(values[LOCAL_QUEUE_KEY]) ? values[LOCAL_QUEUE_KEY] : [], + remote: Array.isArray(values[REMOTE_QUEUE_KEY]) ? values[REMOTE_QUEUE_KEY] : [] + }; +} + +async function snapshotQueues() { + return runSerialized(async () => { + const { legacy, local, remote } = await readQueues(); + const localVisits = capped(legacy.length ? [...legacy, ...local] : local); + const remoteVisits = capped( + legacy.length && remoteEnabled() + ? [...legacy.filter(isRemoteVisit), ...remote] + : remote + ); + + if ( + legacy.length + || localVisits.length !== local.length + || remoteVisits.length !== remote.length + ) { + const update = { + [LOCAL_QUEUE_KEY]: localVisits, + [REMOTE_QUEUE_KEY]: remoteVisits + }; + if (legacy.length) update[LEGACY_QUEUE_KEY] = []; + await chrome.storage.local.set(update); + } + + return { localVisits, remoteVisits }; + }); +} + +function sameQueueItem(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +function deliveredItemsStillQueued(current, snapshot, offset, deliveredCount) { + const confirmed = snapshot.slice(offset, offset + deliveredCount); + if (confirmed.length !== deliveredCount || current.length < deliveredCount) return 0; + for (let index = 0; index < deliveredCount; index += 1) { + if (!sameQueueItem(current[index], confirmed[index])) return 0; + } + return deliveredCount; +} + +async function acknowledgeQueuePrefix(key, snapshot, offset, deliveredCount) { + if (!deliveredCount) return; + try { + await runSerialized(async () => { + const { [key]: current = [] } = await chrome.storage.local.get(key); + const visits = Array.isArray(current) ? current : []; + const removeCount = deliveredItemsStillQueued(visits, snapshot, offset, deliveredCount); + if (removeCount) { + await chrome.storage.local.set({ [key]: capped(visits.slice(removeCount)) }); + } + }); + } catch (error) { + console.warn("Could not acknowledge queued visits", error?.message || error); + } +} + +function remoteEnabled() { + return Boolean(runtimeSettings.collectorUrl && runtimeSettings.apiToken); +} +function isRemoteVisit(visit) { + return isHttpUrl(visit?.url) && visit.url.length <= MAX_REMOTE_URL_LENGTH; +} + +async function enqueueVisit(visit) { + await refreshRuntimeSettings(); + try { + await runSerialized(async () => { + const { legacy, local, remote } = await readQueues(); + const localVisits = [...(legacy.length ? legacy : []), ...local, visit]; + const remoteVisits = [ + ...(legacy.length && remoteEnabled() ? legacy.filter(isRemoteVisit) : []), + ...remote + ]; + if (remoteEnabled() && isRemoteVisit(visit)) remoteVisits.push(visit); + + const update = { + [LOCAL_QUEUE_KEY]: capped(localVisits), + [REMOTE_QUEUE_KEY]: capped(remoteVisits) + }; + if (legacy.length) update[LEGACY_QUEUE_KEY] = []; + await chrome.storage.local.set(update); + }); + requestFlush(); + } catch (error) { + console.warn("Could not queue visit", error?.message || error); + } +} + async function archiveLocally(visits) { if (!visits.length) return true; try { @@ -21,40 +149,43 @@ async function archiveLocally(visits) { return false; } } -async function sendToCollector(visits) { +function isHttpUrl(url) { + if (typeof url !== "string") return false; + try { + const protocol = new URL(url).protocol; + return protocol === "http:" || protocol === "https:"; + } catch { + return false; + } +} +async function sendToCollector(visits, onChunkDelivered) { // Empty settings deliberately disable sending; the local SQLite archive still works. - if (!CONFIG.collectorUrl || !CONFIG.apiToken) return true; + if (!remoteEnabled()) return false; const remoteVisits = visits - .filter((visit) => { - if (typeof visit?.url !== "string") return false; - try { - const protocol = new URL(visit.url).protocol; - return protocol === "http:" || protocol === "https:"; - } catch { - return false; - } - }) + .filter(isRemoteVisit) .map((visit) => ({ ...visit, title: String(visit.title ?? "").slice(0, 1000) })); try { for (let offset = 0; offset < remoteVisits.length; offset += 100) { - const response = await fetch(CONFIG.collectorUrl, { + const chunk = remoteVisits.slice(offset, offset + 100); + const response = await fetch(runtimeSettings.collectorUrl, { method: "POST", headers: { "Content-Type": "application/json", - "Authorization": `Bearer ${CONFIG.apiToken}` + "Authorization": `Bearer ${runtimeSettings.apiToken}` }, body: JSON.stringify({ - device_id: CONFIG.deviceId, - visits: remoteVisits.slice(offset, offset + 100) + device_id: runtimeSettings.deviceId, + visits: chunk }) }); if (!response.ok) { const detail = (await response.text()).trim().slice(0, 1000); throw new Error(`HTTP ${response.status}${detail ? `: ${detail}` : ""}`); } + if (onChunkDelivered) await onChunkDelivered(offset, chunk.length); } return true; } catch (error) { @@ -62,23 +193,76 @@ async function sendToCollector(visits) { return false; } } -async function flush() { - const visits = await queued(); - if (!visits.length) return; - const [localOk, remoteOk] = await Promise.all([archiveLocally(visits), sendToCollector(visits)]); - if (localOk && remoteOk) await persist([]); + +async function flushOnce() { + await refreshRuntimeSettings(); + const { localVisits, remoteVisits } = await snapshotQueues(); + if (!localVisits.length && !remoteVisits.length) return; + + const [localOk, remoteOk] = await Promise.all([ + archiveLocally(localVisits), + sendToCollector( + remoteVisits, + (offset, deliveredCount) => acknowledgeQueuePrefix( + REMOTE_QUEUE_KEY, + remoteVisits, + offset, + deliveredCount + ) + ) + ]); + if (localOk && localVisits.length) { + await acknowledgeQueuePrefix(LOCAL_QUEUE_KEY, localVisits, 0, localVisits.length); + } + if (remoteOk && remoteVisits.length) { + await acknowledgeQueuePrefix(REMOTE_QUEUE_KEY, remoteVisits, 0, remoteVisits.length); + } } -chrome.history.onVisited.addListener(async (item) => { - const visits = await queued(); - visits.push({ + +async function flushLoop() { + while (flushRequested) { + flushRequested = false; + try { + await flushOnce(); + } catch (error) { + console.warn("Queue flush failed", error?.message || error); + } + } +} + +function requestFlush() { + flushRequested = true; + if (flushInProgress) return activeFlush; + + flushInProgress = true; + activeFlush = flushLoop().catch((error) => { + console.warn("Queue flush loop failed", error?.message || error); + }).then(() => { + flushInProgress = false; + if (flushRequested) return requestFlush(); + return undefined; + }); + return activeFlush; +} +chrome.history.onVisited.addListener((item) => { + return enqueueVisit({ url: item.url, title: item.title || "", visit_time: new Date(item.lastVisitTime || Date.now()).toISOString(), transition: "observed", source: "chrome-extension" }); - await persist(visits); - await flush(); }); -chrome.runtime.onStartup.addListener(flush); -chrome.runtime.onInstalled.addListener(flush); +chrome.downloads.onCreated.addListener((item) => { + if (!isHttpUrl(item?.url)) return; + const startTime = new Date(item.startTime || Date.now()); + return enqueueVisit({ + url: item.url, + title: "", + visit_time: Number.isNaN(startTime.getTime()) ? new Date().toISOString() : startTime.toISOString(), + transition: "download", + source: "chrome-extension" + }); +}); +chrome.runtime.onStartup.addListener(() => requestFlush()); +chrome.runtime.onInstalled.addListener(() => requestFlush()); chrome.alarms.create("local-history-archive-flush", { periodInMinutes: 1 }); -chrome.alarms.onAlarm.addListener(flush); +chrome.alarms.onAlarm.addListener(() => requestFlush()); diff --git a/scripts/build_extension.py b/scripts/build_extension.py new file mode 100644 index 0000000..9d3918a --- /dev/null +++ b/scripts/build_extension.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Build and verify a signed Chrome MV3 extension release.""" + +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import tempfile +import xml.etree.ElementTree as ET +import zipfile + +from crx3 import creator, id_util, key_util, verifier +from cryptography.hazmat.primitives.asymmetric import rsa + + +DOWNLOAD_BASE = "https://git.komelt.dev/komelt/history-collector/releases/download" +VERSION_RE = re.compile(r"^(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*)){0,3}$") +TAG_RE = re.compile( + r"^v((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*))?)$" +) + + +def chrome_version(value, label): + if not isinstance(value, str) or not VERSION_RE.fullmatch(value): + raise ValueError(f"{label} must contain 1 to 4 numeric Chrome version components") + components = value.split(".") + if any(int(component) > 65535 for component in components): + raise ValueError(f"{label} contains a component greater than 65535") + return value + + +def release_version(tag): + match = TAG_RE.fullmatch(tag) + if not match: + raise ValueError("tag must match vX.Y.Z or vX.Y.Z.N with numeric components") + return chrome_version(match.group(1), "tag version") + + +def read_manifest(path): + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"cannot read manifest: {error}") from error + if not isinstance(manifest, dict): + raise ValueError("manifest must be a JSON object") + return manifest + + +def write_deterministic_zip(source, destination): + with zipfile.ZipFile(destination, "w", compression=zipfile.ZIP_DEFLATED) as archive: + for path in sorted(path for path in source.rglob("*") if path.is_file()): + relative = path.relative_to(source).as_posix() + info = zipfile.ZipInfo(relative, (1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o100644 << 16 + archive.writestr(info, path.read_bytes()) + + +def write_updates_xml(path, extension_id, version, tag): + namespace = "http://www.google.com/update2/response" + ET.register_namespace("", namespace) + root = ET.Element(f"{{{namespace}}}gupdate", {"protocol": "2.0"}) + app = ET.SubElement(root, f"{{{namespace}}}app", {"appid": extension_id}) + ET.SubElement( + app, + f"{{{namespace}}}updatecheck", + { + "codebase": f"{DOWNLOAD_BASE}/{tag}/history-collector-{version}.crx", + "version": version, + }, + ) + ET.ElementTree(root).write(path, encoding="utf-8", xml_declaration=True) + + +def sha256(path): + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tag", required=True) + parser.add_argument("--private-key", required=True, type=Path) + parser.add_argument("--extension-dir", type=Path, default=Path("extension")) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + + version = release_version(args.tag) + extension_dir = args.extension_dir.resolve() + manifest_path = extension_dir / "manifest.json" + manifest = read_manifest(manifest_path) + manifest_version = chrome_version(manifest.get("version"), "manifest version") + if manifest_version != version: + raise ValueError(f"tag version {version} does not match manifest version {manifest_version}") + if not extension_dir.is_dir(): + raise ValueError(f"extension directory does not exist: {extension_dir}") + + private_key_path = args.private_key.resolve() + try: + private_key_inside_extension = ( + os.path.commonpath((str(extension_dir), str(private_key_path))) == str(extension_dir) + ) + except ValueError: + private_key_inside_extension = False + if private_key_inside_extension: + raise ValueError("private key must not be inside the extension directory") + try: + private_key = key_util.load_private_key_from_pem(private_key_path.read_bytes()) + except (OSError, ValueError, TypeError) as error: + raise ValueError(f"cannot load RSA private key: {error}") from error + if not isinstance(private_key, rsa.RSAPrivateKey): + raise ValueError("private key must be RSA") + + public_key_der = key_util.get_public_key_data(private_key.public_key()) + extension_id = id_util.convert_hex_crx_id_to_alphabet( + id_util.calc_crx_id_by_public_key(public_key_der).hex() + ) + args.output_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix="history-collector-extension-") as temporary: + staging = Path(temporary) / "extension" + shutil.copytree(extension_dir, staging) + staged_manifest = read_manifest(staging / "manifest.json") + staged_manifest["version"] = version + staged_manifest["key"] = base64.b64encode(public_key_der).decode("ascii") + (staging / "manifest.json").write_text( + json.dumps(staged_manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8" + ) + + zip_path = args.output_dir / f"history-collector-{version}.zip" + crx_path = args.output_dir / f"history-collector-{version}.crx" + updates_path = args.output_dir / "updates.xml" + extension_id_path = args.output_dir / "extension-id.txt" + checksums_path = args.output_dir / "checksums.txt" + write_deterministic_zip(staging, zip_path) + creator.create_crx_file(str(zip_path), str(private_key_path), str(crx_path)) + result, header = verifier.verify(str(crx_path)) + if result != verifier.VerifierResult.OK_FULL or header is None or header.crx_id != extension_id: + raise ValueError("generated CRX3 failed verification or has the wrong extension ID") + write_updates_xml(updates_path, extension_id, version, args.tag) + extension_id_path.write_text(extension_id + "\n", encoding="ascii") + checksum_paths = [zip_path, crx_path, updates_path, extension_id_path] + checksums_path.write_text( + "".join(f"{sha256(path)} {path.name}\n" for path in checksum_paths), encoding="ascii" + ) + + print(f"Built and verified {args.tag} (extension ID {extension_id})") + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError) as error: + raise SystemExit(f"build failed: {error}") from error diff --git a/scripts/publish_release.py b/scripts/publish_release.py new file mode 100644 index 0000000..bc1fc12 --- /dev/null +++ b/scripts/publish_release.py @@ -0,0 +1,392 @@ +#!/usr/bin/env python3 +"""Publish extension release assets through the Gitea REST API.""" + +import argparse +import json +import os +import re +import secrets +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen +import xml.etree.ElementTree as ET + + +class ApiError(RuntimeError): + pass + + +VERSION_RE = re.compile(r"^(?:0|[1-9][0-9]*)(?:\.(?:0|[1-9][0-9]*)){0,3}$") +EXPECTED_CRX_CONTENT_TYPE = "application/x-chrome-extension" + + +def _safe_error_detail(error, token): + try: + detail = error.read().decode("utf-8", errors="replace").strip() + except (OSError, UnicodeError): + detail = "" + if token: + detail = detail.replace(token, "[REDACTED]") + return detail[:500] + + +def api_call(base_url, token, method, path, payload=None, expected_errors=()): + url = base_url.rstrip("/") + path + body = None if payload is None else json.dumps(payload).encode("utf-8") + headers = {"Authorization": f"token {token}", "Accept": "application/json"} + if body is not None: + headers["Content-Type"] = "application/json" + request = Request(url, data=body, headers=headers, method=method) + try: + with urlopen(request) as response: + response_body = response.read() + status = response.status + except HTTPError as error: + if error.code in expected_errors: + return error.code, None + detail = _safe_error_detail(error, token) + suffix = f": {detail}" if detail else "" + reason = str(error.reason).replace(token, "[REDACTED]") if token else str(error.reason) + raise ApiError(f"Gitea API {method} {path}: HTTP {error.code} {reason}{suffix}") from None + except URLError as error: + detail = str(error.reason).replace(token, "[REDACTED]") if token else str(error.reason) + raise ApiError(f"Gitea API {method} {path}: {detail}") from None + if not response_body: + return status, None + try: + return status, json.loads(response_body) + except json.JSONDecodeError as error: + raise ApiError(f"Gitea API {method} {path}: invalid JSON response") from error + + +def repository_path(repository): + parts = repository.split("/", 1) + if len(parts) != 2 or not all(parts): + raise ValueError("GITEA_REPOSITORY must be OWNER/REPOSITORY") + owner, name = (quote(part, safe="") for part in parts) + return f"/repos/{owner}/{name}" + + +def get_or_create_release(base_url, token, repo_path, tag, prerelease): + encoded_tag = quote(tag, safe="") + status, release = api_call( + base_url, token, "GET", f"{repo_path}/releases/tags/{encoded_tag}", expected_errors=(404,) + ) + if status == 404: + _, release = api_call( + base_url, + token, + "POST", + f"{repo_path}/releases", + { + "tag_name": tag, + "name": tag, + "body": "Automated Chrome extension release.", + "prerelease": prerelease, + }, + ) + if not isinstance(release, dict) or not isinstance(release.get("id"), int): + raise ApiError(f"Gitea API returned an invalid release for tag {tag}") + return release + + +def release_assets(base_url, token, repo_path, release_id): + _, assets = api_call(base_url, token, "GET", f"{repo_path}/releases/{release_id}/assets") + if not isinstance(assets, list): + raise ApiError("Gitea API returned an invalid release asset list") + return assets + + +def _asset_path(repo_path, release_id, attachment_id): + return f"{repo_path}/releases/{release_id}/assets/{attachment_id}" + + +def _upload_asset(base_url, token, repo_path, release_id, path, asset_name): + data = path.read_bytes() + # A random boundary is also checked against the file so it cannot terminate the part. + while True: + boundary = f"----history-collector-{secrets.token_hex(16)}" + if boundary.encode("ascii") not in data: + break + filename = path.name.replace("\\", "_").replace('"', "_").replace("\r", "_").replace("\n", "_") + disposition_name = asset_name.replace("\\", "_").replace('"', "_").replace("\r", "_").replace("\n", "_") + boundary_bytes = boundary.encode("ascii") + body = b"--" + boundary_bytes + b"\r\n" + body += f'Content-Disposition: form-data; name="attachment"; filename="{filename}"\r\n'.encode("utf-8") + body += b"Content-Type: application/octet-stream\r\n\r\n" + data + body += b"\r\n--" + boundary_bytes + b"--\r\n" + query = urlencode({"name": disposition_name}) + url_path = f"{repo_path}/releases/{release_id}/assets?{query}" + request = Request( + base_url.rstrip("/") + url_path, + data=body, + headers={ + "Authorization": f"token {token}", + "Accept": "application/json", + "Content-Type": f"multipart/form-data; boundary={boundary}", + }, + method="POST", + ) + try: + with urlopen(request) as response: + response_body = response.read() + status = response.status + except HTTPError as error: + detail = _safe_error_detail(error, token) + suffix = f": {detail}" if detail else "" + reason = str(error.reason).replace(token, "[REDACTED]") if token else str(error.reason) + raise ApiError(f"Gitea API POST {url_path}: HTTP {error.code} {reason}{suffix}") from None + except URLError as error: + detail = str(error.reason).replace(token, "[REDACTED]") if token else str(error.reason) + raise ApiError(f"Gitea API POST {url_path}: {detail}") from None + if status < 200 or status >= 300: + raise ApiError(f"Gitea API POST {url_path}: HTTP {status}") + try: + asset = json.loads(response_body) + except (json.JSONDecodeError, UnicodeDecodeError) as error: + raise ApiError(f"Gitea API POST {url_path}: invalid JSON response") from error + if not isinstance(asset, dict) or not isinstance(asset.get("id"), int): + raise ApiError(f"Gitea API POST {url_path}: invalid attachment response") + return asset + + +def _delete_asset(base_url, token, repo_path, release_id, attachment_id): + api_call(base_url, token, "DELETE", _asset_path(repo_path, release_id, attachment_id)) + + +def _edit_asset(base_url, token, repo_path, release_id, attachment_id, name): + api_call( + base_url, + token, + "PATCH", + _asset_path(repo_path, release_id, attachment_id), + {"name": name}, + ) + + +def _cleanup_asset(base_url, token, repo_path, release_id, attachment_id): + if attachment_id is None: + return None + try: + _delete_asset(base_url, token, repo_path, release_id, attachment_id) + except ApiError as error: + return str(error) + return None + + +def replace_asset(base_url, token, repo_path, release_id, path): + existing = next( + ( + asset + for asset in release_assets(base_url, token, repo_path, release_id) + if asset.get("name") == path.name and isinstance(asset.get("id"), int) + ), + None, + ) + if existing is None: + _upload_asset(base_url, token, repo_path, release_id, path, path.name) + return + + old_id = existing["id"] + temporary_name = f".{path.name}.upload-{secrets.token_hex(12)}" + backup_name = f".{path.name}.backup-{secrets.token_hex(12)}" + temporary = _upload_asset(base_url, token, repo_path, release_id, path, temporary_name) + temporary_id = temporary["id"] + old_renamed = False + try: + _edit_asset(base_url, token, repo_path, release_id, old_id, backup_name) + old_renamed = True + try: + _edit_asset(base_url, token, repo_path, release_id, temporary_id, path.name) + except ApiError as error: + restore_error = None + try: + _edit_asset(base_url, token, repo_path, release_id, old_id, path.name) + except ApiError as restore_failure: + restore_error = str(restore_failure) + cleanup_error = _cleanup_asset( + base_url, token, repo_path, release_id, temporary_id + ) + details = [str(error)] + if restore_error: + details.append(f"could not restore previous asset: {restore_error}") + if cleanup_error: + details.append(f"could not remove temporary asset: {cleanup_error}") + raise ApiError("; ".join(details)) from None + + cleanup_error = _cleanup_asset(base_url, token, repo_path, release_id, old_id) + if cleanup_error: + raise ApiError(f"asset replaced, but could not remove backup asset: {cleanup_error}") + except ApiError: + if not old_renamed: + cleanup_error = _cleanup_asset(base_url, token, repo_path, release_id, temporary_id) + if cleanup_error: + raise ApiError(f"asset replacement failed; could not remove temporary asset: {cleanup_error}") from None + raise + + +def _chrome_version(value, label): + if not isinstance(value, str) or not VERSION_RE.fullmatch(value): + raise ApiError(f"{label} must contain 1 to 4 numeric Chrome version components") + components = tuple(int(component) for component in value.split(".")) + if any(component > 65535 for component in components): + raise ApiError(f"{label} contains a component greater than 65535") + return components + (0,) * (4 - len(components)) + + +def should_publish_updates(existing_version, candidate_version): + if existing_version is None: + return True + return _chrome_version(candidate_version, "candidate version") > _chrome_version( + existing_version, "existing updates.xml version" + ) + + +def _updates_version(xml_body): + try: + root = ET.fromstring(xml_body) + except ET.ParseError as error: + raise ApiError(f"existing updates.xml is not valid XML: {error}") from error + for element in root.iter(): + if element.tag.rsplit("}", 1)[-1] == "updatecheck" and "version" in element.attrib: + version = element.attrib["version"] + _chrome_version(version, "existing updates.xml version") + return version + raise ApiError("existing updates.xml has no updatecheck version") + + +def _public_asset_url(base_url, repo_path, asset): + for key in ("browser_download_url", "download_url"): + if isinstance(asset.get(key), str) and asset[key]: + return asset[key] + api_suffix = "/api/v1" + if base_url.rstrip("/").endswith(api_suffix): + server_url = base_url.rstrip("/")[:-len(api_suffix)] + return f"{server_url}{repo_path}/releases/download/updates/{quote(asset['name'], safe='')}" + raise ApiError("Gitea did not provide a public download URL for existing updates.xml") + + +def _download_anonymously(url, purpose): + request = Request(url, headers={"Accept": "application/xml"}, method="GET") + try: + with urlopen(request) as response: + return response.read() + except HTTPError as error: + raise ApiError(f"anonymous {purpose} request: HTTP {error.code} {error.reason}") from None + except URLError as error: + raise ApiError(f"anonymous {purpose} request failed: {error.reason}") from None + + +def existing_updates_version(base_url, token, repo_path, release_id): + asset = next( + ( + asset + for asset in release_assets(base_url, token, repo_path, release_id) + if asset.get("name") == "updates.xml" + ), + None, + ) + if asset is None: + return None + return _updates_version( + _download_anonymously(_public_asset_url(base_url, repo_path, asset), "existing updates.xml") + ) + + +def _updates_codebase(xml_path): + try: + root = ET.parse(xml_path).getroot() + except (OSError, ET.ParseError) as error: + raise ApiError(f"cannot read generated updates.xml: {error}") from error + for element in root.iter(): + if element.tag.rsplit("}", 1)[-1] == "updatecheck": + codebase = element.get("codebase") + if codebase: + return codebase + raise ApiError("generated updates.xml has no codebase URL") + + +def verify_public_crx(updates_path): + codebase = _updates_codebase(updates_path) + request = Request(codebase, headers={"Accept": "application/octet-stream"}, method="GET") + try: + with urlopen(request) as response: + status = response.status + content_type = response.headers.get("Content-Type", "") + magic = response.read(4) + except HTTPError as error: + raise ApiError( + f"CRX codebase URL is not anonymously reachable (HTTP {error.code}); " + "make the release public and configure its MIME type" + ) from None + except URLError as error: + raise ApiError( + f"CRX codebase URL is not anonymously reachable ({error.reason}); " + "make the release public and configure its MIME type" + ) from None + if status != 200: + raise ApiError(f"CRX codebase URL returned HTTP {status}; make the release public") + if magic != b"Cr24": + raise ApiError("CRX codebase URL did not return a CRX3 file (missing Cr24 magic)") + if content_type.split(";", 1)[0].strip().lower() != EXPECTED_CRX_CONTENT_TYPE: + raise ApiError( + "CRX codebase URL returned an invalid Content-Type; configure Gitea " + "[repository.mimetype_mapping] or a reverse proxy .crx header as " + "application/x-chrome-extension" + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tag", required=True) + parser.add_argument("--directory", required=True, type=Path) + parser.add_argument("--repository", default=os.environ.get("GITEA_REPOSITORY")) + parser.add_argument("--api-url", default=os.environ.get("GITEA_API_URL")) + args = parser.parse_args() + token = os.environ.get("GITEA_TOKEN") + server_url = os.environ.get("GITEA_SERVER_URL") + api_url = args.api_url or (f"{server_url.rstrip('/')}/api/v1" if server_url else None) + if not token: + raise ValueError("GITEA_TOKEN is required") + if not api_url: + raise ValueError("GITEA_API_URL or GITEA_SERVER_URL is required") + if not args.repository: + raise ValueError("GITEA_REPOSITORY is required") + if not args.tag.startswith("v"): + raise ValueError("tag must start with v") + + version = args.tag[1:] + _chrome_version(version, "tag version") + repo_path = repository_path(args.repository) + directory = args.directory + assets = { + "crx": directory / f"history-collector-{version}.crx", + "zip": directory / f"history-collector-{version}.zip", + "checksums": directory / "checksums.txt", + "extension_id": directory / "extension-id.txt", + "updates": directory / "updates.xml", + } + if any(not path.is_file() for path in assets.values()): + raise ValueError("build directory is missing one or more release assets") + + release = get_or_create_release(api_url, token, repo_path, args.tag, prerelease=False) + for key in ("crx", "zip", "checksums", "extension_id"): + replace_asset(api_url, token, repo_path, release["id"], assets[key]) + # Verify the URL emitted in the new XML only after the versioned CRX is public. + verify_public_crx(assets["updates"]) + + updates_release = get_or_create_release(api_url, token, repo_path, "updates", prerelease=True) + old_version = existing_updates_version(api_url, token, repo_path, updates_release["id"]) + if should_publish_updates(old_version, version): + replace_asset(api_url, token, repo_path, updates_release["id"], assets["updates"]) + print(f"Published {args.tag} and updates.xml") + else: + print(f"Published {args.tag}; kept existing updates.xml version {old_version}") + + +if __name__ == "__main__": + try: + main() + except (ApiError, OSError, ValueError) as error: + raise SystemExit(f"publish failed: {error}") from error diff --git a/scripts/test_publish_release.py b/scripts/test_publish_release.py new file mode 100644 index 0000000..6ebfb86 --- /dev/null +++ b/scripts/test_publish_release.py @@ -0,0 +1,103 @@ +import re +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from scripts import publish_release + + +class Response: + def __init__(self, body=b'{"id": 1}', status=201, headers=None): + self.body = body + self.status = status + self.headers = headers or {} + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self, size=-1): + return self.body if size < 0 else self.body[:size] + + +class PublishReleaseTest(unittest.TestCase): + def test_upload_is_multipart_with_attachment_field(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "package.crx" + path.write_bytes(b"Cr24\x03payload") + with patch("scripts.publish_release.urlopen", return_value=Response()) as request: + publish_release._upload_asset( + "https://gitea.example/api/v1", "secret-token", "/repos/o/r", 4, path, "package.crx" + ) + sent = request.call_args.args[0] + body = sent.data + content_type = sent.get_header("Content-type") + boundary = re.search(r"boundary=([^;]+)", content_type).group(1).encode("ascii") + self.assertIn(b'name="attachment"', body) + self.assertIn(b"filename=\"package.crx\"", body) + self.assertIn(b"Cr24\x03payload", body) + self.assertTrue(body.endswith(b"--" + boundary + b"--\r\n")) + + def test_edit_and_delete_use_release_scoped_attachment_path(self): + with patch("scripts.publish_release.api_call") as call: + publish_release._edit_asset("base", "token", "/repos/o/r", 9, 12, "new.crx") + publish_release._delete_asset("base", "token", "/repos/o/r", 9, 12) + self.assertEqual(call.call_args_list[0].args[3], "/repos/o/r/releases/9/assets/12") + self.assertEqual(call.call_args_list[1].args[3], "/repos/o/r/releases/9/assets/12") + + def test_public_crx_check_is_anonymous_and_allows_content_type_parameters(self): + with tempfile.TemporaryDirectory() as temporary: + updates = Path(temporary) / "updates.xml" + updates.write_text( + '', + encoding="utf-8", + ) + with patch( + "scripts.publish_release.urlopen", + return_value=Response( + b"Cr24\x03payload", + status=200, + headers={"Content-Type": "application/x-chrome-extension; charset=binary"}, + ), + ) as request: + publish_release.verify_public_crx(updates) + self.assertIsNone(request.call_args.args[0].get_header("Authorization")) + + def test_second_replacement_is_single_asset_and_version_downgrades_are_skipped(self): + assets = [] + next_id = iter(range(1, 10)) + + def upload(*args): + asset = {"id": next(next_id), "name": args[-1]} + assets.append(asset) + return asset + + def edit(*args): + attachment_id, name = args[-2:] + next(asset for asset in assets if asset["id"] == attachment_id)["name"] = name + + def delete(*args): + attachment_id = args[-1] + assets[:] = [asset for asset in assets if asset["id"] != attachment_id] + + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "package.crx" + path.write_bytes(b"data") + with patch("scripts.publish_release.release_assets", side_effect=lambda *args: assets.copy()), \ + patch("scripts.publish_release._upload_asset", side_effect=upload), \ + patch("scripts.publish_release._edit_asset", side_effect=edit), \ + patch("scripts.publish_release._delete_asset", side_effect=delete): + publish_release.replace_asset("base", "token", "/repos/o/r", 9, path) + publish_release.replace_asset("base", "token", "/repos/o/r", 9, path) + + self.assertEqual(assets, [{"id": 2, "name": "package.crx"}]) + self.assertFalse(publish_release.should_publish_updates("2.4.0", "2.3.9")) + self.assertFalse(publish_release.should_publish_updates("2.4.0", "2.4.0")) + self.assertTrue(publish_release.should_publish_updates("2.4.0", "2.4.1")) + + +if __name__ == "__main__": + unittest.main()