feat: measure Schaffa usage and catch errors with self-hosted Umami and GlitchTip

.env.example

@@ -35,6 +35,15 @@ CLAMGATE_PUBLIC_KEY_ID=
 CLAMGATE_TIMEOUT_MS=3600000
 CLAMGATE_GUIDE_TIMEOUT_MS=120000
 
+# Optional telemetry. Leave collection variables empty for development/previews.
+UMAMI_URL=
+UMAMI_WEBSITE_ID=
+GLITCHTIP_DSN=
+SCHAFFA_ENVIRONMENT=
+# Defaults to schaffa@<package.json version>; keep uploads and runtime aligned.
+SENTRY_RELEASE=
+GLITCHTIP_TRACES_SAMPLE_RATE=0.1
+
 # Supply these through the deployment's secret manager at runtime.
 SCHAFFA_TOKEN_PEPPER=
 SCHAFFA_BOOTSTRAP_TOKEN=

.github/workflows/publish.yml

@@ -14,6 +14,55 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  sourcemaps:
+    needs: publish
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout release tag
+        uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6
+      - name: Set up Node
+        uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6
+        with:
+          node-version: 24
+          package-manager-cache: false
+      - name: Enable pnpm
+        run: corepack enable
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+      - name: Build and inject deterministic debug IDs
+        run: pnpm build:server && pnpm exec sentry-cli sourcemaps inject dist
+      - name: Log in to GHCR
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: printf '%s' "$GH_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+      - name: Verify debug IDs against the published image
+        env:
+          RELEASE_IMAGE: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}
+        run: |
+          set -euo pipefail
+          image_dist="$(mktemp -d)"
+          container=""
+          trap 'if [ -n "$container" ]; then docker rm -v "$container"; fi; rm -rf "$image_dist"' EXIT
+          docker pull "$RELEASE_IMAGE"
+          container="$(docker create "$RELEASE_IMAGE")"
+          docker cp "$container:/app/dist/." "$image_dist/"
+          bash scripts/ci-check-debug-ids.sh dist "$image_dist"
+      - name: Upload server maps and finalize release
+        env:
+          SENTRY_URL: https://glitchtip.heerlab.com
+          SENTRY_ORG: beasty-rabbit
+          SENTRY_PROJECT: schaffa
+          SENTRY_AUTH_TOKEN: ${{ secrets.GLITCHTIP_AUTH_TOKEN }}
+        run: |
+          export SENTRY_RELEASE="schaffa@$(node -p 'require("./package.json").version')"
+          pnpm exec sentry-cli releases new "$SENTRY_RELEASE"
+          pnpm exec sentry-cli sourcemaps upload --release "$SENTRY_RELEASE" dist
+          pnpm exec sentry-cli releases finalize "$SENTRY_RELEASE"
+
   arc-smoke:
     if: github.event_name == 'workflow_dispatch'
     runs-on: arc-schaffa

.gitignore

@@ -6,3 +6,5 @@ data/
 .DS_Store
 *.log
 coverage/
+public/vendor/
+.test-dist/

Dockerfile

@@ -7,7 +7,10 @@ RUN pnpm install --frozen-lockfile
 COPY tsconfig.json tsconfig.build.json biome.json ./
 COPY src ./src
 COPY public ./public
-RUN pnpm build:server && CI=true pnpm prune --prod
+COPY scripts/build-telemetry.mjs ./scripts/build-telemetry.mjs
+RUN pnpm build:server && pnpm exec sentry-cli sourcemaps inject dist \
+    && find dist -type f -name '*.js.map' -delete \
+    && CI=true pnpm prune --prod
 
 FROM node:24-alpine@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 AS production-dependencies
 WORKDIR /app
@@ -35,4 +38,4 @@ RUN apk add --upgrade --no-cache 'libcrypto3>=3.5.8-r0' 'libssl3>=3.5.8-r0' \
 USER node
 EXPOSE 3000
 VOLUME ["/data"]
-CMD ["node", "dist/server.js"]
+CMD ["node", "--import", "./dist/instrumentation.js", "dist/server.js"]

README.md

@@ -14,6 +14,12 @@ npx schaffa upload file.html
 
 The command prints a public URL. Deployment, recording, and API details live in `docs/`.
 
+Optional self-hosted Umami analytics, replay and heatmaps require consent on
+Schaffa's own public UI. GlitchTip reports scrubbed application errors. Published
+HTML stays byte-identical and receives no tracking scripts. Configuration and
+retention requirements are in [deployment](docs/deployment.md); `/privacy`
+explains processing and provides Cookie settings.
+
 - Source: [github.com/beastyrabbit/schaffa](https://github.com/beastyrabbit/schaffa)
 - Image: `ghcr.io/beastyrabbit/schaffa:<version>` (`linux/amd64`)
 - License: MIT

biome.json

@@ -9,7 +9,14 @@
   },
   "files": {
     "ignoreUnknown": true,
-    "includes": ["**", "!!**/dist", "!!data", "!!node_modules", "!!coverage"]
+    "includes": [
+      "**",
+      "!!**/dist",
+      "!!data",
+      "!!node_modules",
+      "!!coverage",
+      "!!.claude/settings.json"
+    ]
   },
   "formatter": {
     "enabled": true,

docs/deployment.md

@@ -46,6 +46,7 @@ Set `SCHAFFA_BASE_URL=https://schaffa.dev`. Keep Pangolin authentication enabled
 - `auth/*`
 - `shoo/*`
 - `skills`
+- `privacy`
 - `skills/*`
 - `skills/*/*`
 - `llm.txt`
@@ -61,7 +62,7 @@ Set `SCHAFFA_BASE_URL=https://schaffa.dev`. Keep Pangolin authentication enabled
 
 Pangolin matches each path segment separately, so the additional patterns cover the landing page, OpenAPI metadata, page versions, `/raw`, API operations containing an ID or slug, and the Shoo user login flow. Requests to `/admin` therefore continue to Pangolin authentication. API clients, user accounts, public pages, and files remain directly reachable on the same hostname. Pangolin evaluates rules by priority; do not add a broader bypass rule that also matches `/admin`.
 
-The four legal pages are self-contained server-rendered HTML with inline CSS and no JavaScript or asset dependencies. Add only those four exact bypass paths for them, without wildcards. Each sends both a noindex meta tag and an X-Robots-Tag header.
+The four legal pages are self-contained server-rendered HTML with inline CSS. Optional telemetry adds first-party consent and error-reporting assets; the legal text remains readable without JavaScript. Add only those four exact bypass paths for them, without wildcards. Each sends both a noindex meta tag and an X-Robots-Tag header.
 
 The health check should normally stay on the private backend at `/healthz`; it does not need a public Pangolin route.
 
@@ -96,6 +97,83 @@ The admin UI provides immediate page/file takedown, a publishing lockdown, an in
 
 The user dashboard uses Shoo for Google OAuth/PKCE and stores its own HMAC-hashed, seven-day HttpOnly sessions. The admin can independently disable new local signups, disable all logins (which also clears active sessions), and delete users. User deletion revokes their agent tokens and sessions but deliberately leaves already published content available for a separate, auditable admin takedown. As of this integration Shoo labels itself an early work in progress; keep `SHOO_BASE_URL` and `SHOO_ISSUER` configurable and review Shoo's release/security status before a production rollout.
 
+## Optional analytics and error reporting
+
+Set `UMAMI_URL` to the self-hosted collection origin and `UMAMI_WEBSITE_ID` to
+its website UUID. Set `GLITCHTIP_DSN` to the public project DSN. Leave all three
+unset for local development and preview builds; no collection SDK then starts.
+`SCHAFFA_ENVIRONMENT` defaults to `production` with `NODE_ENV=production`, otherwise
+`development`. `SENTRY_RELEASE` defaults to `schaffa@<package.json version>`;
+the release upload job uses that version, so keep the default in production.
+`GLITCHTIP_TRACES_SAMPLE_RATE` defaults to `0.1` and accepts values from 0 to 1.
+A nightly kub-homelab job deletes Umami analytics data after 25 months, session
+replays and heatmap recordings after 30 days, and GlitchTip error events after
+90 days. The bilingual privacy policy states these periods. The legal pages and
+controller details remain specific to schaffa.dev.
+The privacy text needs human legal review before rollout.
+
+Own public UI loads Umami only after analytics consent. Normal page loads cover
+navigation because Schaffa is a multi-page application. The deployed Umami 3
+tracker supports `data-website-id`, `data-domains`, `data-exclude-search`,
+`data-exclude-hash`, `data-before-send`, `data-performance`, and `data-do-not-track`;
+these were checked against its deployed `script.js`. Outbound links are collected
+by a small first-party handler because that script has no outbound-link attribute.
+The recorder only accepts website and host attributes. It fetches website settings
+from `/api/websites/<website-id>/recorder`, which must be publicly reachable with
+appropriate CORS. Enable replay and heatmaps in those settings and select strict
+masking. All inputs are masked even in moderate mode; rrweb's `rr-block` class
+locally excludes account content, identity and token displays. No recorder loads
+on account pages, token-reveal pages or URLs containing queries or fragments.
+Consent withdrawal clears the in-memory Umami session and navigates with a GET to
+remove recorder observers and suppress its unload flush. The only storage Schaffa's
+telemetry writes is `cc_cookie`, after a banner choice, expiring after six months
+(182 days). Umami reads, but Schaffa
+does not write, the optional `umami.disabled` localStorage preference.
+
+Published HTML, interactive runs, raw versions, files and guide viewers receive
+no telemetry injection. Their existing CSPs are unchanged. Successful GET 200
+responses count as aggregate server views; HEAD, partial responses, 304, scan
+status endpoints and obvious bots do not. Published presentations use the same
+static HTML transport as pages and are classified by Marp's `data-marpit-svg`
+marker. Server events contain public route paths and
+bounded categorical properties only, with no visitor headers or identifiers.
+The fixed User-Agent is `Schaffa/1.0 (aggregate publication metrics)`. Upstream
+Umami's `/api/send` uses `isbot` and accepts this product/version User-Agent;
+the service sees the application's egress IP, never the visitor's. The transport
+allows eight concurrent requests, a 1.5-second timeout, no redirects and no retries.
+Failures and capacity drops are counted. `NODE_DEBUG=schaffa:telemetry` enables
+failure diagnostics without payloads or visitor data. A slow collector cannot block serving.
+
+GlitchTip loads lazily on own UI, including admin, with an early bounded error
+queue. Admin never loads Umami or the banner. Sentry SDK 11 replaced
+`sendDefaultPii` with `dataCollection`; user info, cookies, headers, bodies, queries,
+database values, AI inputs/outputs and local variables are explicitly disabled.
+Browser integrations are an explicit allowlist without BrowserSession, replay,
+feedback or persistent trace linking. `linkPreviousTrace: 'off'` prevents
+sessionStorage writes. Error, breadcrumb and trace payloads are scrubbed again.
+Static transaction envelopes are used for GlitchTip compatibility.
+`POST /admin/api/telemetry-test` requires the existing admin cookie and emits
+`Schaffa GlitchTip test error — safe to ignore` for operator verification.
+
+The official Sentry npm package does not ship its CDN bundles. An esbuild step
+creates a minified SDK asset with tracing and without replay or feedback, plus
+the browser-safe scrubber shared with the server. The UI scripts remain unbundled
+and unminified. Runtime serves these assets and CookieConsent's npm dist JS/CSS
+under content-hashed, immutable first-party URLs. The browser SDK is a build
+dependency; its generated asset stays in the runtime image. No vendor maps are exposed. Docker injects server
+debug IDs offline after tsc, then deletes all `dist/**/*.js.map` files before
+copying `/app/dist` into the runtime image. The SDK uses the retained debug IDs;
+runtime does not need local maps. Do not enable Node's `--enable-source-maps`.
+
+After image publishing, the `ubuntu-latest` tag job rebuilds the server with maps
+and injects debug IDs. It authenticates to GHCR with its job token, extracts
+`/app/dist` from `ghcr.io/<repository>:sha-<commit>`, and fails before upload if
+the image's set of debug IDs differs from the rebuild's or either set is empty.
+It then creates `schaffa@<version>`, uploads the rebuilt maps, and finalizes the
+release through `https://glitchtip.heerlab.com`. Only the required upload and
+release routes bypass SSO; do not run `set-commits`. The upload token is the
+existing `GLITCHTIP_AUTH_TOKEN` repository secret and never enters the image.
+
 ## Docker Compose
 
 The included [compose.yaml](../compose.yaml) runs only Schaffa and binds its HTTP port to loopback. Provision the trusted ClamGate public key before starting:

package.json

@@ -8,19 +8,19 @@
   },
   "scripts": {
     "build": "pnpm build:server && pnpm --filter schaffa build",
-    "build:server": "tsc -p tsconfig.build.json",
+    "build:server": "node scripts/build-telemetry.mjs && tsc -p tsconfig.build.json",
     "check": "pnpm typecheck && pnpm lint && pnpm test && pnpm security",
     "dev": "scripts/dev.sh",
-    "dev:server": "tsx watch src/server.ts",
+    "dev:server": "node scripts/build-telemetry.mjs && tsx watch src/server.ts",
     "format": "biome format --write .",
     "lint": "biome check .",
     "static:lint": "eslint .",
     "pack:cli": "node scripts/pack-cli.mjs",
     "security": "gitleaks dir . --redact --no-banner",
-    "start": "node dist/server.js",
+    "start": "node --import ./dist/instrumentation.js dist/server.js",
     "test": "pnpm test:server && pnpm test:release && pnpm --filter schaffa test",
     "test:release": "node --test test/*.test.mjs",
-    "test:server": "pnpm --filter schaffa build && tsx --test test/**/*.test.ts",
+    "test:server": "pnpm build:server && pnpm --filter schaffa build && tsx --test test/**/*.test.ts",
     "token": "tsx scripts/token.ts",
     "typecheck": "tsc --noEmit && pnpm --filter schaffa exec tsc --noEmit -p tsconfig.json"
   },
@@ -36,19 +36,24 @@
     "@fastify/formbody": "^9.0.0",
     "@fastify/multipart": "^10.1.4",
     "@scalar/fastify-api-reference": "^1.64.1",
+    "@sentry/node": "^11.8.0",
     "fastify": "^5.12.3",
     "jose": "^6.2.8",
     "mime-types": "^3.0.2",
     "parse5": "^8.0.1",
-    "sharp": "^0.35.5"
+    "sharp": "^0.35.5",
+    "vanilla-cookieconsent": "^3.1.0"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.5.7",
     "@eslint/js": "^10.0.1",
+    "@sentry/browser": "^11.8.0",
+    "@sentry/cli": "^3.8.0",
     "@types/mime-types": "^3.0.1",
     "@types/node": "^26.2.0",
     "@typescript/native": "npm:typescript@^7.0.2",
     "ajv": "^8.20.0",
+    "esbuild": "^0.28.2",
     "eslint": "^10.10.0",
     "eslint-plugin-security": "^4.0.1",
     "eslint-plugin-sonarjs": "^4.2.0",