From e0cbf8121e945e22e3e175ff23247acc901544dd Mon Sep 17 00:00:00 2001 From: Ricardo Date: Fri, 20 Feb 2026 12:20:28 +0100 Subject: [PATCH] fix: unfurl shortcode concurrency control and failure caching - Increase timeout from 10s to 20s - Cache failures for 1 day (avoids retrying every build) - Add concurrency limiter (max 5 parallel requests) - Refactor into renderCard/renderFallbackLink helpers --- lib/unfurl-shortcode.js | 145 +++++++++++++++++++++++++--------------- 1 file changed, 92 insertions(+), 53 deletions(-) diff --git a/lib/unfurl-shortcode.js b/lib/unfurl-shortcode.js index 0f34739..9173fec 100644 --- a/lib/unfurl-shortcode.js +++ b/lib/unfurl-shortcode.js @@ -5,6 +5,29 @@ import { createHash } from "crypto"; const CACHE_DIR = resolve(import.meta.dirname, "..", ".cache", "unfurl"); const CACHE_DURATION_MS = 7 * 24 * 60 * 60 * 1000; // 1 week +const FAILURE_CACHE_MS = 24 * 60 * 60 * 1000; // 1 day for failed fetches + +// Concurrency limiter — prevents overwhelming outbound network +let activeRequests = 0; +const MAX_CONCURRENT = 5; +const queue = []; + +function runNext() { + if (queue.length === 0 || activeRequests >= MAX_CONCURRENT) return; + activeRequests++; + const { resolve: res, fn } = queue.shift(); + fn().then(res).finally(() => { + activeRequests--; + runNext(); + }); +} + +function throttled(fn) { + return new Promise((res) => { + queue.push({ resolve: res, fn }); + runNext(); + }); +} function getCachePath(url) { const hash = createHash("md5").update(url).digest("hex"); @@ -13,22 +36,24 @@ function getCachePath(url) { function readCache(url) { const path = getCachePath(url); - if (!existsSync(path)) return null; + if (!existsSync(path)) return undefined; // undefined = not cached try { const data = JSON.parse(readFileSync(path, "utf-8")); - if (Date.now() - data.cachedAt < CACHE_DURATION_MS) { - return data.metadata; + const age = Date.now() - data.cachedAt; + const ttl = data.failed ? FAILURE_CACHE_MS : CACHE_DURATION_MS; + if (age < ttl) { + return data; // return full cache entry (includes .failed flag) } } catch { // Corrupt cache file, ignore } - return null; + return undefined; } -function writeCache(url, metadata) { +function writeCache(url, metadata, failed = false) { mkdirSync(CACHE_DIR, { recursive: true }); const path = getCachePath(url); - writeFileSync(path, JSON.stringify({ cachedAt: Date.now(), metadata })); + writeFileSync(path, JSON.stringify({ cachedAt: Date.now(), metadata, failed })); } function extractDomain(url) { @@ -48,59 +73,38 @@ function escapeHtml(str) { .replace(/"/g, """); } -/** - * Register the {% unfurl "URL" %} shortcode on an Eleventy config. - */ -export default function registerUnfurlShortcode(eleventyConfig) { - eleventyConfig.addAsyncShortcode("unfurl", async function (url) { - if (!url) return ""; +function renderFallbackLink(url) { + const domain = escapeHtml(extractDomain(url)); + return `${domain}`; +} - // Check cache first - let metadata = readCache(url); +function renderCard(url, metadata) { + const og = metadata.open_graph || {}; + const tc = metadata.twitter_card || {}; - if (!metadata) { - try { - metadata = await unfurl(url, { timeout: 10000 }); - writeCache(url, metadata); - } catch (err) { - console.warn(`[unfurl] Failed to fetch ${url}: ${err.message}`); - // Fallback: plain link - const domain = escapeHtml(extractDomain(url)); - return `${domain}`; - } - } + const title = og.title || tc.title || metadata.title || extractDomain(url); + const description = og.description || tc.description || metadata.description || ""; + const image = og.images?.[0]?.url || tc.images?.[0]?.url || null; + const favicon = metadata.favicon || null; + const domain = extractDomain(url); - const og = metadata.open_graph || {}; - const tc = metadata.twitter_card || {}; + const maxDesc = 160; + const desc = description.length > maxDesc + ? description.slice(0, maxDesc).trim() + "\u2026" + : description; - const title = og.title || tc.title || metadata.title || extractDomain(url); - const description = og.description || tc.description || metadata.description || ""; - const image = - og.images?.[0]?.url || - tc.images?.[0]?.url || - null; - const favicon = metadata.favicon || null; - const domain = extractDomain(url); + const imgHtml = image + ? `
+ +
` + : ""; - // Truncate description - const maxDesc = 160; - const desc = description.length > maxDesc - ? description.slice(0, maxDesc).trim() + "\u2026" - : description; + const faviconHtml = favicon + ? `` + : ""; - // Build card HTML - const imgHtml = image - ? `
- -
` - : ""; - - const faviconHtml = favicon - ? `` - : ""; - - return `
+ return `

${escapeHtml(title)}

@@ -110,5 +114,40 @@ export default function registerUnfurlShortcode(eleventyConfig) { ${imgHtml}
`; +} + +/** + * Register the {% unfurl "URL" %} shortcode on an Eleventy config. + */ +export default function registerUnfurlShortcode(eleventyConfig) { + eleventyConfig.addAsyncShortcode("unfurl", async function (url) { + if (!url) return ""; + + // Check cache first (returns undefined if not cached) + const cached = readCache(url); + if (cached !== undefined) { + // Cached failure → render fallback link + if (cached.failed) return renderFallbackLink(url); + // Cached success → render card + return renderCard(url, cached.metadata); + } + + // Not cached — fetch with concurrency limiting + const metadata = await throttled(async () => { + try { + return await unfurl(url, { timeout: 20000 }); + } catch (err) { + console.warn(`[unfurl] Failed to fetch ${url}: ${err.message}`); + return null; + } + }); + + if (!metadata) { + writeCache(url, null, true); // cache failure for 1 day + return renderFallbackLink(url); + } + + writeCache(url, metadata, false); + return renderCard(url, metadata); }); }