diff --git a/.gitignore b/.gitignore index b864dd6..6f74fd7 100644 --- a/.gitignore +++ b/.gitignore @@ -53,6 +53,9 @@ coverage tmp temp +# Local Cloudflare / wrangler state (never commit) +.wrangler/ + # Vercel .vercel diff --git a/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite b/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite deleted file mode 100644 index 0b58f0d..0000000 Binary files a/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite and /dev/null differ diff --git a/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite-shm b/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite-shm deleted file mode 100644 index 773c8f0..0000000 Binary files a/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite-shm and /dev/null differ diff --git a/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite-wal b/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite-wal deleted file mode 100644 index d6d596b..0000000 Binary files a/.wrangler/state/v3/cache/miniflare-CacheObject/metadata.sqlite-wal and /dev/null differ diff --git a/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite b/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite deleted file mode 100644 index 0b58f0d..0000000 Binary files a/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite and /dev/null differ diff --git a/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite-shm b/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite-shm deleted file mode 100644 index 365a22d..0000000 Binary files a/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite-shm and /dev/null differ diff --git a/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite-wal b/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite-wal deleted file mode 100644 index 1c93864..0000000 Binary files a/.wrangler/state/v3/r2/miniflare-R2BucketObject/6aebfa59b93b510d66af95a59523858d2cd323ace4129f7e9212e16f463e54b6.sqlite-wal and /dev/null differ diff --git a/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite b/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite deleted file mode 100644 index 0b58f0d..0000000 Binary files a/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite and /dev/null differ diff --git a/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite-shm b/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite-shm deleted file mode 100644 index 277a592..0000000 Binary files a/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite-shm and /dev/null differ diff --git a/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite-wal b/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite-wal deleted file mode 100644 index b84d9b9..0000000 Binary files a/.wrangler/state/v3/r2/miniflare-R2BucketObject/metadata.sqlite-wal and /dev/null differ diff --git a/apps/web/src/routes/$slug.tsx b/apps/web/src/routes/$slug.tsx index 5ae855d..e77d553 100644 --- a/apps/web/src/routes/$slug.tsx +++ b/apps/web/src/routes/$slug.tsx @@ -8,12 +8,23 @@ import { SiteViewer } from "@/components/site-viewer"; import { Mark } from "@/components/wordmark"; export const Route = createFileRoute("/$slug")({ + validateSearch: (search: Record) => ({ + t: typeof search.t === "string" ? search.t : "", + }), component: ViewerPage, }); function ViewerPage() { const { slug } = Route.useParams(); - const site = useQuery(api.sites.getBySlug, { slug }); + const { t: token } = Route.useSearch(); + // Edit-token holders may open a private page in the browser, matching what + // the skill promises. Strangers without a token still get a not-found page. + const siteByToken = useQuery( + api.sites.getBySlugForManager, + token ? { slug, editToken: token } : "skip", + ); + const sitePublic = useQuery(api.sites.getBySlug, token ? "skip" : { slug }); + const site = token ? siteByToken : sitePublic; // A published page belongs to whoever published it, so there is no app chrome // here at all: no nav, no spinner flash, just the document and a small badge. diff --git a/apps/web/src/routes/app.tsx b/apps/web/src/routes/app.tsx index e1f0617..2ff2370 100644 --- a/apps/web/src/routes/app.tsx +++ b/apps/web/src/routes/app.tsx @@ -87,6 +87,7 @@ function MySites() { Open diff --git a/apps/web/src/routes/manage.$slug.tsx b/apps/web/src/routes/manage.$slug.tsx index d7d3ae7..d4d9d1b 100644 --- a/apps/web/src/routes/manage.$slug.tsx +++ b/apps/web/src/routes/manage.$slug.tsx @@ -25,7 +25,14 @@ function ManagePage() { const { t: token } = Route.useSearch(); const navigate = useNavigate(); const { isAuthenticated } = useConvexAuth(); - const site = useQuery(api.sites.getBySlug, { slug }); + // Prefer the edit-token query when `?t=` is present so private pages stay + // manageable without a session — that is what manageUrl from deploy returns. + const siteByToken = useQuery( + api.sites.getBySlugForManager, + token ? { slug, editToken: token } : "skip", + ); + const siteBySession = useQuery(api.sites.getBySlug, token ? "skip" : { slug }); + const site = token ? siteByToken : siteBySession; const claim = useMutation(api.sites.claim); const setVisibility = useMutation(api.sites.setVisibility); const [busy, setBusy] = useState(false); diff --git a/nonexistent b/nonexistent deleted file mode 100644 index e69de29..0000000 diff --git a/packages/backend/convex/crons.ts b/packages/backend/convex/crons.ts index 6f95778..9a873ea 100644 --- a/packages/backend/convex/crons.ts +++ b/packages/backend/convex/crons.ts @@ -5,7 +5,9 @@ import { internal } from "./_generated/api"; const crons = cronJobs(); // Daily sweep that deletes expired images (7d) and expired sites (30d anon / -// 90d claimed), including their R2 objects and timeline history. +// 90d claimed), including their R2 objects and timeline history. The mutation +// self-schedules while a batch is saturated so a backlog does not wait another +// day; failed R2 deletes are retried without dropping ledger keys. crons.daily( "cleanup expired sites and images", { hourUTC: 8, minuteUTC: 17 }, diff --git a/packages/backend/convex/healthCheck.ts b/packages/backend/convex/healthCheck.ts index b38d944..ca8f610 100644 --- a/packages/backend/convex/healthCheck.ts +++ b/packages/backend/convex/healthCheck.ts @@ -1,6 +1,7 @@ import { query } from "./_generated/server"; export const get = query({ + args: {}, handler: async () => { return "OK"; }, diff --git a/packages/backend/convex/http.ts b/packages/backend/convex/http.ts index 79418b2..23a04f0 100644 --- a/packages/backend/convex/http.ts +++ b/packages/backend/convex/http.ts @@ -154,6 +154,20 @@ function versionKey(slug: string, kind: "markdown" | "html"): string { return `sites/${slug}/${crypto.randomUUID()}.${kind === "html" ? "html" : "md"}`; } +/** Best-effort rollback when metadata write fails after an R2 store. */ +async function discardR2Key(ctx: ActionCtx, key: string): Promise { + try { + await r2.deleteObject(ctx, key); + } catch { + // Leave the orphan rather than fail the caller twice; cleanup cannot see + // keys that were never ledgered, so this is the only chance we get. + } +} + +function errorMessage(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} + async function uniqueSlug(ctx: ActionCtx): Promise { // Every site gets an unguessable random UUID. Collisions are astronomically // unlikely, but the existence check is cheap insurance. @@ -280,8 +294,12 @@ async function opDeploy(ctx: ActionCtx, rateKey: string, body: unknown, caller: now, expiresAt, }); - } catch { - return { error: "Slug just became unavailable — please retry.", status: 409 }; + } catch (err) { + await discardR2Key(ctx, key); + if (errorMessage(err).includes("slug-taken")) { + return { error: "Slug just became unavailable — please retry.", status: 409 }; + } + return { error: "Failed to record the new site. Please retry.", status: 500 }; } const app = appBase(); @@ -313,15 +331,21 @@ async function opUpdate(ctx: ActionCtx, slug: string, token: string | null, body const key = versionKey(slug, parsed.kind); await r2.store(ctx, parsed.bytes, { key, type: parsed.contentType, cacheControl: VERSION_CACHE }); - await ctx.runMutation(internal.sites.recordUpdate, { - slug, - kind: parsed.kind, - title: parsed.title, - key, - contentType: parsed.contentType, - byteSize: parsed.bytes.byteLength, - visibility: parsed.visibility, - }); + try { + await ctx.runMutation(internal.sites.recordUpdate, { + slug, + kind: parsed.kind, + title: parsed.title, + key, + contentType: parsed.contentType, + byteSize: parsed.bytes.byteLength, + visibility: parsed.visibility, + }); + } catch (err) { + await discardR2Key(ctx, key); + if (errorMessage(err).includes("not-found")) return GONE; + return { error: "Failed to record the update. Please retry.", status: 500 }; + } return { ok: true, ...(await publicStatus(ctx, slug)) }; } @@ -526,15 +550,22 @@ const sitesPost = httpAction(async (ctx, request) => { cacheControl: "public, max-age=604800, immutable", }); const now = Date.now(); - const { imageId } = await ctx.runMutation(internal.sites.recordImage, { - siteId: gate.auth.siteId, - slug, - key, - contentType, - byteSize: bytes.byteLength, - now, - expiresAt: now + RETENTION.imageMs, - }); + let imageId: Id<"siteImages">; + try { + const recorded = await ctx.runMutation(internal.sites.recordImage, { + siteId: gate.auth.siteId, + slug, + key, + contentType, + byteSize: bytes.byteLength, + now, + expiresAt: now + RETENTION.imageMs, + }); + imageId = recorded.imageId; + } catch { + await discardR2Key(ctx, key); + return fail(500, "Failed to record the image. Please retry."); + } return json( { ok: true, @@ -640,6 +671,15 @@ const authorizationServerMetadata = httpAction(async () => /** RFC 7591 dynamic client registration. Public clients, PKCE required. */ const oauthRegister = httpAction(async (ctx, request) => { + const limit = await rateLimiter.limit(ctx, "oauthRegister", { key: clientIp(request) }); + if (!limit.ok) { + return json( + { error: "temporarily_unavailable", error_description: "Rate limit exceeded." }, + 429, + retryHeader(limit.retryAfter), + ); + } + let body: Record; try { body = (await request.json()) as Record; @@ -686,6 +726,15 @@ function formValue(form: URLSearchParams, key: string): string { } const oauthToken = httpAction(async (ctx, request) => { + const limit = await rateLimiter.limit(ctx, "oauthToken", { key: clientIp(request) }); + if (!limit.ok) { + return json( + { error: "temporarily_unavailable", error_description: "Rate limit exceeded." }, + 429, + retryHeader(limit.retryAfter), + ); + } + const form = new URLSearchParams(await request.text()); const grantType = formValue(form, "grant_type"); const clientId = formValue(form, "client_id"); diff --git a/packages/backend/convex/oauth.ts b/packages/backend/convex/oauth.ts index cba3626..f52ef45 100644 --- a/packages/backend/convex/oauth.ts +++ b/packages/backend/convex/oauth.ts @@ -14,6 +14,7 @@ import { v } from "convex/values"; +import { internal } from "./_generated/api"; import { internalMutation, internalQuery, mutation, query } from "./_generated/server"; import { authComponent } from "./auth"; import { generateEditToken, sha256Hex } from "./lib"; @@ -221,27 +222,35 @@ export const consumeRefreshToken = internalMutation({ }, }); -/** Drop expired codes and tokens. Called by the daily cron. */ +/** Drop expired codes and tokens. Called by the daily cron; self-schedules + * while a batch is full so a busy day cannot leave a permanent backlog. */ export const cleanupExpiredGrants = internalMutation({ args: {}, handler: async (ctx) => { const now = Date.now(); + const TOKEN_BATCH = 200; + const CODE_BATCH = 200; + let saturated = false; + const tokens = await ctx.db .query("oauthTokens") .withIndex("by_expiresAt", (q) => q.lt("expiresAt", now)) - .take(200); + .take(TOKEN_BATCH); for (const row of tokens) await ctx.db.delete(row._id); + if (tokens.length === TOKEN_BATCH) saturated = true; - // Codes live five minutes, so a small sweep is always enough. - const codes = await ctx.db.query("oauthCodes").take(100); - let removed = 0; - for (const row of codes) { - if (row.expiresAt <= now) { - await ctx.db.delete(row._id); - removed++; - } + const codes = await ctx.db + .query("oauthCodes") + .withIndex("by_expiresAt", (q) => q.lt("expiresAt", now)) + .take(CODE_BATCH); + for (const row of codes) await ctx.db.delete(row._id); + if (codes.length === CODE_BATCH) saturated = true; + + if (saturated) { + await ctx.scheduler.runAfter(0, internal.oauth.cleanupExpiredGrants, {}); } - return { tokens: tokens.length, codes: removed }; + + return { tokens: tokens.length, codes: codes.length, saturated }; }, }); diff --git a/packages/backend/convex/rateLimiter.ts b/packages/backend/convex/rateLimiter.ts index fb77e5d..b5cafa0 100644 --- a/packages/backend/convex/rateLimiter.ts +++ b/packages/backend/convex/rateLimiter.ts @@ -1,4 +1,4 @@ -import { DAY, HOUR, RateLimiter } from "@convex-dev/rate-limiter"; +import { DAY, HOUR, MINUTE, RateLimiter } from "@convex-dev/rate-limiter"; import { components } from "./_generated/api"; @@ -15,4 +15,8 @@ export const rateLimiter = new RateLimiter(components.rateLimiter, { createSiteAuthed: { kind: "token bucket", rate: 1000, period: DAY, capacity: 100 }, updateSite: { kind: "token bucket", rate: 120, period: HOUR, capacity: 20 }, uploadImage: { kind: "token bucket", rate: 40, period: DAY, capacity: 10 }, + // Dynamic client registration is unauthenticated; keep the oauthClients table + // from growing without bound under a spam loop. + oauthRegister: { kind: "token bucket", rate: 20, period: HOUR, capacity: 5 }, + oauthToken: { kind: "token bucket", rate: 60, period: MINUTE, capacity: 20 }, }); diff --git a/packages/backend/convex/schema.ts b/packages/backend/convex/schema.ts index 89486c4..f1baf58 100644 --- a/packages/backend/convex/schema.ts +++ b/packages/backend/convex/schema.ts @@ -43,7 +43,8 @@ export default defineSchema({ }) .index("by_slug", ["slug"]) .index("by_expiresAt", ["expiresAt"]) - .index("by_owner", ["ownerSubject"]), + .index("by_owner", ["ownerSubject"]) + .index("by_owner_updated", ["ownerSubject", "updatedAt"]), // Long-lived keys that let an agent act as a signed-in account over the HTTP // API. Only the SHA-256 hash is stored; the raw key is shown once at creation. @@ -88,7 +89,9 @@ export default defineSchema({ anonymous: v.boolean(), resource: v.optional(v.string()), expiresAt: v.number(), - }).index("by_hash", ["codeHash"]), + }) + .index("by_hash", ["codeHash"]) + .index("by_expiresAt", ["expiresAt"]), // Issued tokens, stored only as hashes. `audience` is checked on every call so // a token minted for something else cannot be replayed here. diff --git a/packages/backend/convex/sites.ts b/packages/backend/convex/sites.ts index 0074d5c..01ca49e 100644 --- a/packages/backend/convex/sites.ts +++ b/packages/backend/convex/sites.ts @@ -1,5 +1,6 @@ import { v } from "convex/values"; +import { internal } from "./_generated/api"; import type { Doc, Id } from "./_generated/dataModel"; import { type MutationCtx, @@ -10,7 +11,7 @@ import { query, } from "./_generated/server"; import { authComponent } from "./auth"; -import { generateEditToken, mintViewToken, sha256Hex, siteExpiry } from "./lib"; +import { LIMITS, generateEditToken, mintViewToken, sha256Hex, siteExpiry } from "./lib"; import { r2 } from "./r2"; import { siteKind, siteVisibility } from "./schema"; import { MAX_VERSIONS, type SiteVersion, timeline } from "./timeline"; @@ -19,9 +20,18 @@ import { MAX_VERSIONS, type SiteVersion, timeline } from "./timeline"; * cheap while still draining any backlog over successive updates. */ const PRUNE_BATCH = 5; +/** Bounded drain sizes so one mutation stays well inside Convex budgets. */ +const PURGE_OBJECT_BATCH = 10; +const PURGE_IMAGE_BATCH = 10; +const CLEANUP_IMAGE_BATCH = 50; +const CLEANUP_SITE_BATCH = 5; + /** Enough keys to separate machines or agents, few enough to stay reviewable. */ const MAX_KEYS_PER_ACCOUNT = 10; +/** Soft cap on the account sites list; the UI is a recent-sites panel, not an archive browser. */ +const LIST_MINE_LIMIT = 100; + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -63,6 +73,20 @@ async function trackObject(ctx: MutationCtx, siteId: Id<"sites">, key: string): await ctx.db.insert("siteObjects", { siteId, key, createdAt: Date.now() }); } +/** + * Delete an R2 object. Returns true when the key is gone (deleted or already + * absent). On a transient failure, returns false so the caller keeps the ledger + * row and retries later — never forget a key we still owe a delete for. + */ +async function tryDeleteR2(ctx: MutationCtx, key: string): Promise { + try { + await r2.deleteObject(ctx, key); + return true; + } catch { + return false; + } +} + /** Drop content objects that fell out of the retained history. The timeline keeps * MAX_VERSIONS nodes, so anything older can never be reached by undo again. One * update adds one object, so trimming a small batch per update holds steady. */ @@ -76,12 +100,7 @@ async function pruneObjects(ctx: MutationCtx, siteId: Id<"sites">): Promise): Promise { - const tracked = await ctx.db +/** + * One bounded step of destroying a site's R2 blobs + metadata. + * + * Ledger rows are removed only after a successful R2 delete. The site row and + * timeline scope stay until every blob is gone, so a failed delete can be + * retried without losing the key. + * + * - `done` — site document removed + * - `more` — more ledger rows to process (chain immediately) + * - `retry` — R2 failed for at least one key (back off before retrying) + */ +async function drainSite(ctx: MutationCtx, site: Doc<"sites">): Promise<"done" | "more" | "retry"> { + let failed = false; + + const objects = await ctx.db .query("siteObjects") .withIndex("by_site", (q) => q.eq("siteId", site._id)) - .collect(); + .take(PURGE_OBJECT_BATCH); + for (const row of objects) { + if (await tryDeleteR2(ctx, row.key)) await ctx.db.delete(row._id); + else failed = true; + } + if (objects.length === PURGE_OBJECT_BATCH) return failed ? "retry" : "more"; + const images = await ctx.db .query("siteImages") .withIndex("by_site", (q) => q.eq("siteId", site._id)) - .collect(); + .take(PURGE_IMAGE_BATCH); + for (const img of images) { + if (await tryDeleteR2(ctx, img.key)) await ctx.db.delete(img._id); + else failed = true; + } + if (images.length === PURGE_IMAGE_BATCH) return failed ? "retry" : "more"; - // currentKey is included for sites deployed before the ledger existed. - const keys = new Set([site.currentKey]); - for (const row of tracked) keys.add(row.key); + // Anything still on the ledger failed this round — keep the site for retry. + const leftoverObject = await ctx.db + .query("siteObjects") + .withIndex("by_site", (q) => q.eq("siteId", site._id)) + .take(1); + if (leftoverObject.length > 0) return "retry"; + const leftoverImage = await ctx.db + .query("siteImages") + .withIndex("by_site", (q) => q.eq("siteId", site._id)) + .take(1); + if (leftoverImage.length > 0) return "retry"; + + // Keys that predate the ledger, or the live pointer, still live on the site + // row / timeline. Delete them before dropping the only place we remember them. + const legacyKeys = new Set([site.currentKey]); for (const node of await timeline.listNodes(ctx, site.scope)) { - if (isSiteVersion(node.document)) keys.add(node.document.key); + if (isSiteVersion(node.document)) legacyKeys.add(node.document.key); } - for (const img of images) keys.add(img.key); - - for (const key of keys) { - try { - await r2.deleteObject(ctx, key); - } catch { - // Object may already be gone; deleting metadata below is what matters. - } + for (const key of legacyKeys) { + if (!(await tryDeleteR2(ctx, key))) return "retry"; } - for (const row of tracked) await ctx.db.delete(row._id); - for (const img of images) await ctx.db.delete(img._id); + await timeline.deleteScope(ctx, site.scope); await ctx.db.delete(site._id); + return "done"; +} + +/** Make a site unreadable immediately, then drain what this mutation can. */ +async function beginSiteDeletion( + ctx: MutationCtx, + site: Doc<"sites">, +): Promise<"done" | "more" | "retry"> { + const now = Date.now(); + if (site.expiresAt > now) { + await ctx.db.patch(site._id, { expiresAt: now, updatedAt: now }); + site = { ...site, expiresAt: now, updatedAt: now }; + } + return await drainSite(ctx, site); +} + +function scheduleDrain(ctx: MutationCtx, siteId: Id<"sites">, status: "more" | "retry") { + const delay = status === "more" ? 0 : 60_000; + return ctx.scheduler.runAfter(delay, internal.sites.drainSiteById, { siteId }); } // --------------------------------------------------------------------------- // Public queries (used by the web app) // --------------------------------------------------------------------------- +/** Shared viewer/manage payload once the caller is allowed to see the site. */ +async function sitePayload(ctx: QueryCtx, site: Doc<"sites">) { + const visibility = visibilityOf(site); + const status = await timeline.status(ctx, site.scope); + const siteUrl = process.env.CONVEX_SITE_URL ?? ""; + return { + slug: site.slug, + kind: site.kind, + title: site.title ?? null, + byteSize: site.byteSize, + hasImages: site.hasImages, + visibility, + owned: site.ownerSubject !== undefined, + createdAt: site.createdAt, + updatedAt: site.updatedAt, + expiresAt: site.expiresAt, + canUndo: status.canUndo, + canRedo: status.canRedo, + version: status.position, + versions: status.length, + // Versioned so the URL changes on every deploy/update/undo, busting any + // browser cache and reloading the viewer's