svemagie.net/indiekit-server
1
0
Fork 0
Code Issues 1 Actions Packages Projects Releases Activity

perf: reduce microsub BATCH_CONCURRENCY 5→3 to smooth CPU spikes
Deploy Indiekit Server / deploy (push) Successful in 1m20s
Details

Browse Source 
Feed polling was causing 20-35% CPU and 187ms p99 event loop lag during
batch cycles. 9 active feeds at 16-32min tiers drive ~18 fetches/hour;
concurrent XML parsing at 5-way concurrency was the bottleneck.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Sven
2026-04-12 22:13:13 +02:00
parent 015438839b
commit c034bcbe35
2 changed files with 63 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package.json
+2 -2
View File
@@ -5,8 +5,8 @@
"main": "index.js",
"scripts": {
"preinstall": "node scripts/setup-gitea-url-rewrite.mjs",
"postinstall": "xattr -w com.apple.fileprovider.ignore#P 1 node_modules 2>/dev/null || true && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-put-fallback.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-reset-stale.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-bluesky-og-own-post-title.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-ap-oauth-token-expiry-fix.mjs && node scripts/patch-ap-startup-gate-bypass.mjs && node scripts/patch-ap-federation-infra.mjs && node scripts/patch-ap-syndication.mjs && node scripts/patch-ap-mastodon-statuses.mjs && node scripts/patch-ap-mastodon-accounts.mjs && node scripts/patch-ap-mastodon-notifications.mjs && node scripts/patch-ap-mastodon-misc.mjs && node scripts/patch-micropub-category-from-posts.mjs && node scripts/patch-tag-input-autocomplete.mjs",
"serve": "export NODE_ENV=${NODE_ENV:-production} INDIEKIT_DEBUG=${INDIEKIT_DEBUG:-0} && node scripts/preflight-production-security.mjs && node scripts/preflight-mongo-connection.mjs && node scripts/preflight-activitypub-rsa-key.mjs && node scripts/preflight-activitypub-profile-urls.mjs && node scripts/preflight-startup-gate.mjs && node scripts/patch-ap-startup-gate-bypass.mjs && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-put-fallback.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-microsub-reader-ap-dispatch.mjs && node scripts/patch-microsub-compose-draft-guard.mjs && node scripts/patch-microsub-no-bookmark-autofollow.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-reset-stale.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-bluesky-og-own-post-title.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-ap-oauth-token-expiry-fix.mjs && node scripts/patch-ap-federation-infra.mjs && node scripts/patch-ap-syndication.mjs && node scripts/patch-ap-mastodon-statuses.mjs && node scripts/patch-ap-mastodon-accounts.mjs && node scripts/patch-ap-mastodon-notifications.mjs && node scripts/patch-ap-mastodon-misc.mjs && node scripts/patch-micropub-category-from-posts.mjs && node scripts/patch-tag-input-autocomplete.mjs && node --max-old-space-size=1024 --require ./metrics-shim.cjs node_modules/@indiekit/indiekit/bin/cli.js serve --config indiekit.config.mjs",
"postinstall": "xattr -w com.apple.fileprovider.ignore#P 1 node_modules 2>/dev/null || true && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-put-fallback.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-reset-stale.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-bluesky-og-own-post-title.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-ap-oauth-token-expiry-fix.mjs && node scripts/patch-ap-startup-gate-bypass.mjs && node scripts/patch-ap-federation-infra.mjs && node scripts/patch-ap-syndication.mjs && node scripts/patch-ap-mastodon-statuses.mjs && node scripts/patch-ap-mastodon-accounts.mjs && node scripts/patch-ap-mastodon-notifications.mjs && node scripts/patch-ap-mastodon-misc.mjs && node scripts/patch-micropub-category-from-posts.mjs && node scripts/patch-tag-input-autocomplete.mjs && node scripts/patch-microsub-batch-concurrency.mjs",
"serve": "export NODE_ENV=${NODE_ENV:-production} INDIEKIT_DEBUG=${INDIEKIT_DEBUG:-0} && node scripts/preflight-production-security.mjs && node scripts/preflight-mongo-connection.mjs && node scripts/preflight-activitypub-rsa-key.mjs && node scripts/preflight-activitypub-profile-urls.mjs && node scripts/preflight-startup-gate.mjs && node scripts/patch-ap-startup-gate-bypass.mjs && node scripts/patch-lightningcss.mjs && node scripts/patch-endpoint-media-scope.mjs && node scripts/patch-endpoint-media-sharp-runtime.mjs && node scripts/patch-frontend-sharp-runtime.mjs && node scripts/patch-endpoint-files-upload-route.mjs && node scripts/patch-endpoint-files-upload-locales.mjs && node scripts/patch-endpoint-activitypub-locales.mjs && node scripts/patch-endpoint-homepage-locales.mjs && node scripts/patch-endpoint-homepage-identity-defaults.mjs && node scripts/patch-federation-unlisted-guards.mjs && node scripts/patch-endpoint-micropub-where-note-visibility.mjs && node scripts/patch-endpoint-podroll-opml-upload.mjs && node scripts/patch-frontend-serviceworker-file.mjs && node scripts/patch-endpoint-comments-locales.mjs && node scripts/patch-endpoint-posts-locales.mjs && node scripts/patch-endpoint-conversations-locales.mjs && node scripts/patch-conversations-collection-guards.mjs && node scripts/patch-indiekit-routes-rate-limits.mjs && node scripts/patch-indiekit-error-production-stack.mjs && node scripts/patch-indieauth-devmode-guard.mjs && node scripts/patch-listening-endpoint-runtime-guards.mjs && node scripts/patch-endpoint-github-changelog-categories.mjs && node scripts/patch-endpoint-github-contributions-log.mjs && node scripts/patch-store-github-error-message.mjs && node scripts/patch-store-github-update-fallback.mjs && node scripts/patch-store-github-gitea-methods.mjs && node scripts/patch-store-github-put-fallback.mjs && node scripts/patch-store-github-content-type.mjs && node scripts/patch-microsub-reader-ap-dispatch.mjs && node scripts/patch-microsub-compose-draft-guard.mjs && node scripts/patch-microsub-no-bookmark-autofollow.mjs && node scripts/patch-microsub-batch-concurrency.mjs && node scripts/patch-endpoint-blogroll-feeds-alias.mjs && node scripts/patch-endpoint-posts-uid-lookup.mjs && node scripts/patch-conversations-bluesky-self-filter.mjs && node scripts/patch-conversations-bluesky-cursor-fix.mjs && node scripts/patch-endpoint-micropub-source-filter.mjs && node scripts/patch-syndicate-force-checked-default.mjs && node scripts/patch-syndicate-normalize-syndication-array.mjs && node scripts/patch-endpoint-posts-fetch-diagnostic.mjs && node scripts/patch-micropub-fetch-internal-url.mjs && node scripts/patch-micropub-session-token.mjs && node scripts/patch-indiekit-endpoint-urls-protocol.mjs && node scripts/patch-webmention-sender-hentry-syntax.mjs && node scripts/patch-webmention-sender-retry.mjs && node scripts/patch-webmention-sender-livefetch.mjs && node scripts/patch-webmention-sender-reset-stale.mjs && node scripts/patch-webmention-sender-empty-details.mjs && node scripts/patch-bluesky-syndicator-internal-url.mjs && node scripts/patch-bluesky-syndicator-media-type-guard.mjs && node scripts/patch-bluesky-og-own-post-title.mjs && node scripts/patch-bluesky-syndicator-delete.mjs && node scripts/patch-micropub-delete-propagation.mjs && node scripts/patch-ap-oauth-token-expiry-fix.mjs && node scripts/patch-ap-federation-infra.mjs && node scripts/patch-ap-syndication.mjs && node scripts/patch-ap-mastodon-statuses.mjs && node scripts/patch-ap-mastodon-accounts.mjs && node scripts/patch-ap-mastodon-notifications.mjs && node scripts/patch-ap-mastodon-misc.mjs && node scripts/patch-micropub-category-from-posts.mjs && node scripts/patch-tag-input-autocomplete.mjs && node --max-old-space-size=1024 --require ./metrics-shim.cjs node_modules/@indiekit/indiekit/bin/cli.js serve --config indiekit.config.mjs",
"test": "echo \"Error: no test specified\" && exit 1"
},
"keywords": [],
scripts/patch-microsub-batch-concurrency.mjs
+61
View File
@@ -0,0 +1,61 @@
/**
* Patch: reduce microsub feed polling batch concurrency from 5 to 3.
*
* With 50 subscribed feeds and 9 at low tiers (16–32 min refresh), concurrent
* XML/JSON parsing drives p99 event loop lag to ~187ms and sustained CPU to
* 20–35%. Reducing BATCH_CONCURRENCY from 5 to 3 smooths the spikes without
* affecting throughput (scheduler runs every 60s; typical cycle processes 1–4
* feeds anyway).
*/
import { access, readFile, writeFile } from "node:fs/promises";
const MARKER = "// [patch] microsub-batch-concurrency";
const OLD_SNIPPET = `const BATCH_CONCURRENCY = 5; // Process 5 feeds at a time`;
const NEW_SNIPPET = `const BATCH_CONCURRENCY = 3; // Process 3 feeds at a time ${MARKER}`;
const candidates = [
"node_modules/@rmdes/indiekit-endpoint-microsub/lib/polling/scheduler.js",
"node_modules/@indiekit/indiekit/node_modules/@rmdes/indiekit-endpoint-microsub/lib/polling/scheduler.js",
];
async function exists(filePath) {
try {
await access(filePath);
return true;
} catch {
return false;
}
}
let applied = 0;
let skipped = 0;
for (const filePath of candidates) {
if (!(await exists(filePath))) continue;
const src = await readFile(filePath, "utf8");
if (src.includes(MARKER)) {
skipped++;
continue;
}
if (!src.includes(OLD_SNIPPET)) {
console.warn(
`[patch] microsub-batch-concurrency: upstream format changed in ${filePath}`,
);
continue;
}
await writeFile(filePath, src.replace(OLD_SNIPPET, NEW_SNIPPET), "utf8");
applied++;
}
if (applied > 0) {
console.log(
`[patch] microsub-batch-concurrency: applied to ${applied} file(s)`,
);
} else if (skipped > 0) {
console.log(`[patch] microsub-batch-concurrency: already applied`);
}