From db10d9cfbfe3736461ea479b5c7eaafedde78b9d Mon Sep 17 00:00:00 2001 From: Ricardo Date: Mon, 9 Mar 2026 17:37:17 +0100 Subject: [PATCH] fix(og): batch spawning to prevent OOM during watcher rebuilds Full OG regeneration (2,350 images) OOM-kills when the Eleventy watcher is running (~1.8 GB RSS), leaving only ~1.2 GB headroom in the 3 GB container. WASM native memory from Satori/Resvg grows beyond what GC can reclaim within a single process. Solution: spawn og-cli in batches of 100 images. Each invocation exits after its batch, fully releasing all WASM native memory. Exit code 2 signals "more work remains" and the spawner re-loops. Peak memory per batch stays under ~500 MB regardless of total image count. Also seed newManifest from existing manifest so unscanned entries survive batch writes. Confab-Link: http://localhost:8080/sessions/edb1b7b0-da66-4486-bd9c-d1cfa7553b88 --- eleventy.config.js | 49 +++++++++++++++++++++++++++++++++------------- lib/og-cli.js | 19 ++++++++++++++---- lib/og.js | 18 +++++++++++++++-- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/eleventy.config.js b/eleventy.config.js index 615871e..ff134e0 100644 --- a/eleventy.config.js +++ b/eleventy.config.js @@ -1114,25 +1114,46 @@ export default function (eleventyConfig) { return digests; }); - // Generate OpenGraph images for posts without photos - // Runs on every build (including watcher rebuilds) — manifest caching makes it fast - // for incremental: only new posts without an OG image get generated (~200ms each) + // Generate OpenGraph images for posts without photos. + // Uses batch spawning: each invocation generates up to BATCH_SIZE images then exits, + // fully releasing WASM native memory (Satori Yoga + Resvg Rust) between batches. + // Exit code 2 = batch complete, more work remains → re-spawn. + // Manifest caching makes incremental builds fast (only new posts get generated). eleventyConfig.on("eleventy.before", () => { const contentDir = resolve(__dirname, "content"); const cacheDir = resolve(__dirname, ".cache"); const siteName = process.env.SITE_NAME || "My IndieWeb Blog"; + const BATCH_SIZE = 100; + let totalGenerated = 0; + let batch = 0; try { - execFileSync(process.execPath, [ - "--max-old-space-size=512", - "--expose-gc", - resolve(__dirname, "lib", "og-cli.js"), - contentDir, - cacheDir, - siteName, - ], { - stdio: "inherit", - env: { ...process.env, NODE_OPTIONS: "" }, - }); + // eslint-disable-next-line no-constant-condition + while (true) { + batch++; + try { + execFileSync(process.execPath, [ + "--max-old-space-size=512", + "--expose-gc", + resolve(__dirname, "lib", "og-cli.js"), + contentDir, + cacheDir, + siteName, + String(BATCH_SIZE), + ], { + stdio: "inherit", + env: { ...process.env, NODE_OPTIONS: "" }, + }); + // Exit code 0 = all done + break; + } catch (err) { + if (err.status === 2) { + // Exit code 2 = batch complete, more images remain + totalGenerated += BATCH_SIZE; + continue; + } + throw err; + } + } // Sync new OG images to output directory. // During incremental builds, .cache/og is in watchIgnores so Eleventy's diff --git a/lib/og-cli.js b/lib/og-cli.js index 421b5b0..e80c961 100644 --- a/lib/og-cli.js +++ b/lib/og-cli.js @@ -4,16 +4,27 @@ * CLI entry point for OG image generation. * Runs as a separate process to isolate memory from Eleventy. * - * Usage: node lib/og-cli.js + * Usage: node lib/og-cli.js [batchSize] + * + * batchSize: Max images to generate per invocation (0 = unlimited). + * When set, exits after generating that many images so the caller + * can re-spawn (releasing all WASM native memory between batches). + * Exit code 2 = batch complete, more work remains. */ import { generateOgImages } from "./og.js"; -const [contentDir, cacheDir, siteName] = process.argv.slice(2); +const [contentDir, cacheDir, siteName, batchSizeStr] = process.argv.slice(2); if (!contentDir || !cacheDir || !siteName) { - console.error("[og] Usage: node og-cli.js "); + console.error("[og] Usage: node og-cli.js [batchSize]"); process.exit(1); } -await generateOgImages(contentDir, cacheDir, siteName); +const batchSize = parseInt(batchSizeStr, 10) || 0; +const result = await generateOgImages(contentDir, cacheDir, siteName, batchSize); + +// Exit code 2 signals "batch complete, more images remain" +if (result?.hasMore) { + process.exit(2); +} diff --git a/lib/og.js b/lib/og.js index 7b9d394..439a2fb 100644 --- a/lib/og.js +++ b/lib/og.js @@ -278,8 +278,10 @@ function scanContentFiles(contentDir) { * @param {string} contentDir - Path to content/ directory * @param {string} cacheDir - Path to .cache/ directory * @param {string} siteName - Site name for the card + * @param {number} batchSize - Max images to generate (0 = unlimited) + * @returns {{ hasMore: boolean }} Whether more images need generation */ -export async function generateOgImages(contentDir, cacheDir, siteName) { +export async function generateOgImages(contentDir, cacheDir, siteName, batchSize = 0) { const ogDir = join(cacheDir, "og"); mkdirSync(ogDir, { recursive: true }); @@ -296,7 +298,8 @@ export async function generateOgImages(contentDir, cacheDir, siteName) { let generated = 0; let skipped = 0; - const newManifest = {}; + // Seed with existing manifest so unscanned entries survive batch writes + const newManifest = { ...manifest }; const SAVE_INTERVAL = 10; // GC every 5 images to keep WASM native memory bounded. // Satori (Yoga WASM) + Resvg (Rust WASM) allocate ~50-100 MB native memory @@ -352,14 +355,25 @@ export async function generateOgImages(contentDir, cacheDir, siteName) { const rss = process.memoryUsage().rss; if (rss > peakRss) peakRss = rss; } + + // Batch limit: stop after N images so the caller can re-spawn + // (fully releasing WASM native memory between batches) + if (batchSize > 0 && generated >= batchSize) { + break; + } } + const hasMore = batchSize > 0 && generated >= batchSize; + if (hasGC) global.gc(); writeFileSync(manifestPath, JSON.stringify(newManifest, null, 2)); const mem = process.memoryUsage(); if (mem.rss > peakRss) peakRss = mem.rss; console.log( `[og] Generated ${generated} images, skipped ${skipped} (cached or have photos)` + + (hasMore ? ` [batch, more remain]` : ``) + ` | RSS: ${(mem.rss / 1024 / 1024).toFixed(0)} MB, peak: ${(peakRss / 1024 / 1024).toFixed(0)} MB, heap: ${(mem.heapUsed / 1024 / 1024).toFixed(0)} MB`, ); + + return { hasMore }; }