fix: bound docs build concurrency to avoid Vercel OOM

.github/workflows/docs-checks.yml

@@ -16,6 +16,7 @@ jobs:
           cache: npm
       - run: npm ci --no-audit --no-fund
       - run: npm run test:social-cards
+      - run: npm run test:build
       - run: npm run test:lens
   writing-style:
     runs-on: ubuntu-latest

README.md

@@ -21,6 +21,8 @@ npm run build
 
 Static output goes to `build/`.
 
+The build renders up to four pages and optimizes up to two images at a time to fit the 8 GB Vercel build machine. Set `DOCUSAURUS_SSR_CONCURRENCY` to override page-rendering concurrency when running `npm run build` on a different machine.
+
 ## Deploy
 
 Deploys are handled automatically by Vercel on push to `main`.

package.json

@@ -6,7 +6,7 @@
     "docusaurus": "docusaurus",
     "start": "docusaurus start",
     "typecheck": "tsc -p .",
-    "build": "docusaurus build",
+    "build": "node scripts/build.cjs",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
@@ -20,6 +20,7 @@
     "lint:docs": "python3 scripts/check-docs.py docs",
     "generate:social-card": "node scripts/social-card/index.cjs",
     "test:social-cards": "node --test scripts/social-card/social-cards.test.cjs",
+    "test:build": "node --test scripts/optimize-images.test.cjs",
     "sync:lens": "node scripts/sync-lens-docs.mjs",
     "test:lens": "node --test scripts/sync-lens-docs.test.mjs"
   },

plugins/optimize-images.js

@@ -3,6 +3,7 @@ const fs = require('fs');
 const path = require('path');
 
 const QUALITY = 75;
+const CONCURRENCY = 2;
 const EXTENSIONS = new Set(['.png', '.jpg', '.jpeg']);
 
 function walk(dir) {
@@ -46,8 +47,12 @@ module.exports = function optimizeImagesPlugin() {
     async postBuild({ outDir }) {
       const files = walk(outDir);
       if (!files.length) return;
+      console.log(`[optimize-images] Optimizing ${files.length} images, ${CONCURRENCY} at a time`);
       let saved = 0;
-      await Promise.all(files.map(async (f) => { saved += await optimizeFile(f); }));
+      for (let offset = 0; offset < files.length; offset += CONCURRENCY) {
+        const savings = await Promise.all(files.slice(offset, offset + CONCURRENCY).map(optimizeFile));
+        saved += savings.reduce((total, bytes) => total + bytes, 0);
+      }
       const mb = (saved / 1024 / 1024).toFixed(1);
       console.log(`\n[optimize-images] Compressed ${files.length} images, saved ${mb} MB`);
     },

scripts/build.cjs

@@ -0,0 +1,14 @@
+const {spawnSync} = require('node:child_process');
+
+// Leave room for image processing on the 8 GB production build machine.
+const env = {DOCUSAURUS_SSR_CONCURRENCY: '4', ...process.env};
+console.log(`[build] Rendering up to ${env.DOCUSAURUS_SSR_CONCURRENCY} pages at a time`);
+
+const result = spawnSync(process.execPath, [
+  require.resolve('@docusaurus/core/bin/docusaurus.mjs'),
+  'build',
+  ...process.argv.slice(2),
+], {env, stdio: 'inherit'});
+
+if (result.error) throw result.error;
+process.exit(result.status ?? 1);

scripts/optimize-images.test.cjs

@@ -0,0 +1,78 @@
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const vm = require('node:vm');
+const {test} = require('node:test');
+
+function loadPlugin(sharp) {
+  const module = {exports: {}};
+  vm.runInNewContext(fs.readFileSync(path.join(__dirname, '../plugins/optimize-images.js'), 'utf8'), {
+    module,
+    require: (name) => name === 'sharp' ? sharp : require(name),
+    console: {log() {}},
+  });
+  return module.exports();
+}
+
+function fixture(t) {
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'docs-images-'));
+  t.after(() => fs.rmSync(dir, {recursive: true, force: true}));
+  return dir;
+}
+
+test('bounds active image pipelines as the image collection grows', async (t) => {
+  const dir = fixture(t);
+  fs.mkdirSync(path.join(dir, 'nested'));
+  const images = Array.from({length: 101}, (_, i) => path.join(dir, i % 2 ? 'nested' : '', `${i}.png`));
+  for (const file of images) fs.writeFileSync(file, 'original image');
+  fs.writeFileSync(path.join(dir, 'untouched.svg'), 'vector image');
+  let active = 0;
+  let peak = 0;
+  let completed = 0;
+  const plugin = loadPlugin(() => {
+    active += 1;
+    peak = Math.max(peak, active);
+    return {
+      png() { return this; },
+      async toFile(output) {
+        await new Promise(setImmediate);
+        fs.writeFileSync(output, 'small');
+        active -= 1;
+        completed += 1;
+      },
+    };
+  });
+  await plugin.postBuild({outDir: dir});
+  assert.ok(peak <= 2, `Opened ${peak} image pipelines simultaneously`);
+  assert.equal(active, 0);
+  assert.equal(completed, images.length);
+  for (const file of images) {
+    assert.equal(fs.readFileSync(file, 'utf8'), 'small');
+    assert.equal(fs.existsSync(`${file}.opt`), false);
+  }
+  assert.equal(fs.readFileSync(path.join(dir, 'untouched.svg'), 'utf8'), 'vector image');
+});
+
+test('preserves originals and continues after failures or larger output', async (t) => {
+  const dir = fixture(t);
+  for (const name of ['broken.png', 'larger.jpg', 'valid.jpeg']) {
+    fs.writeFileSync(path.join(dir, name), 'original');
+  }
+  const plugin = loadPlugin((input) => ({
+    png() { return this; },
+    jpeg() { return this; },
+    async toFile(output) {
+      if (input.endsWith('broken.png')) {
+        fs.writeFileSync(output, 'partial');
+        throw new Error('Could not decode image');
+      }
+      fs.writeFileSync(output, input.endsWith('larger.jpg') ? 'larger than original' : 'small');
+    },
+  }));
+  await plugin.postBuild({outDir: dir});
+  assert.equal(fs.readFileSync(path.join(dir, 'broken.png'), 'utf8'), 'original');
+  assert.equal(fs.readFileSync(path.join(dir, 'larger.jpg'), 'utf8'), 'original');
+  assert.equal(fs.readFileSync(path.join(dir, 'valid.jpeg'), 'utf8'), 'small');
+  assert.equal(fs.readdirSync(dir).some((name) => name.endsWith('.opt')), false);
+});