fix: payload too large error and add migration logging

- Add express.urlencoded({ limit: '5mb' }) to migration POST route
  to handle large CSV files (default 100KB was too small)
- Add per-handle progress logging to console for monitoring imports
- Log failed handles with reasons (WebFinger failure, no AP link, etc.)
- Show failed handles in the UI result notification
- Use error notification type when all imports fail
This commit is contained in:
Ricardo
2026-02-19 10:14:23 +01:00
parent fec4b1f242
commit de00d3a16c
5 changed files with 79 additions and 21 deletions
+1 -1
View File
@@ -179,7 +179,7 @@ export default class ActivityPubEndpoint {
router.get("/admin/following", followingController(mp)); router.get("/admin/following", followingController(mp));
router.get("/admin/activities", activitiesController(mp)); router.get("/admin/activities", activitiesController(mp));
router.get("/admin/migrate", migrateGetController(mp, this.options)); router.get("/admin/migrate", migrateGetController(mp, this.options));
router.post("/admin/migrate", migratePostController(mp, this.options)); router.post("/admin/migrate", express.urlencoded({ extended: true, limit: "5mb" }), migratePostController(mp, this.options));
return router; return router;
} }
+23 -6
View File
@@ -63,11 +63,12 @@ export function migratePostController(mountPath, pluginOptions) {
text: response.locals.__("activitypub.migrate.errorNoFile"), text: response.locals.__("activitypub.migrate.errorNoFile"),
}; };
} else { } else {
let followingResult = { imported: 0, failed: 0 }; let followingResult = { imported: 0, failed: 0, errors: [] };
let followersResult = { imported: 0, failed: 0 }; let followersResult = { imported: 0, failed: 0, errors: [] };
if (importFollowing && followingCollection) { if (importFollowing && followingCollection) {
const handles = parseMastodonFollowingCsv(fileContent); const handles = parseMastodonFollowingCsv(fileContent);
console.log(`[ActivityPub] Migration: parsed ${handles.length} following handles from CSV`);
followingResult = await bulkImportFollowing( followingResult = await bulkImportFollowing(
handles, handles,
followingCollection, followingCollection,
@@ -76,6 +77,7 @@ export function migratePostController(mountPath, pluginOptions) {
if (importFollowers && followersCollection) { if (importFollowers && followersCollection) {
const entries = parseMastodonFollowersList(fileContent); const entries = parseMastodonFollowersList(fileContent);
console.log(`[ActivityPub] Migration: parsed ${entries.length} follower entries from CSV`);
followersResult = await bulkImportFollowers( followersResult = await bulkImportFollowers(
entries, entries,
followersCollection, followersCollection,
@@ -84,13 +86,28 @@ export function migratePostController(mountPath, pluginOptions) {
const totalFailed = const totalFailed =
followingResult.failed + followersResult.failed; followingResult.failed + followersResult.failed;
result = { const allErrors = [
type: "success", ...followingResult.errors,
text: response.locals ...followersResult.errors,
];
let text = response.locals
.__("activitypub.migrate.success") .__("activitypub.migrate.success")
.replace("%d", followingResult.imported) .replace("%d", followingResult.imported)
.replace("%d", followersResult.imported) .replace("%d", followersResult.imported)
.replace("%d", totalFailed), .replace("%d", totalFailed);
if (allErrors.length > 0) {
text += " " + response.locals
.__("activitypub.migrate.failedList")
.replace("%s", allErrors.join(", "));
}
result = {
type: totalFailed > 0 && followingResult.imported + followersResult.imported === 0
? "error"
: "success",
text,
}; };
} }
} }
+51 -11
View File
@@ -54,7 +54,10 @@ export function parseMastodonFollowersList(text) {
*/ */
export async function resolveHandleViaWebFinger(handle) { export async function resolveHandleViaWebFinger(handle) {
const [user, domain] = handle.split("@"); const [user, domain] = handle.split("@");
if (!user || !domain) return null; if (!user || !domain) {
console.warn(`[ActivityPub] Migration: invalid handle "${handle}" — skipping`);
return null;
}
try { try {
// WebFinger lookup // WebFinger lookup
@@ -64,14 +67,20 @@ export async function resolveHandleViaWebFinger(handle) {
signal: AbortSignal.timeout(10_000), signal: AbortSignal.timeout(10_000),
}); });
if (!wfResponse.ok) return null; if (!wfResponse.ok) {
console.warn(`[ActivityPub] Migration: WebFinger failed for ${handle} (HTTP ${wfResponse.status})`);
return null;
}
const jrd = await wfResponse.json(); const jrd = await wfResponse.json();
const selfLink = jrd.links?.find( const selfLink = jrd.links?.find(
(l) => l.rel === "self" && l.type === "application/activity+json", (l) => l.rel === "self" && l.type === "application/activity+json",
); );
if (!selfLink?.href) return null; if (!selfLink?.href) {
console.warn(`[ActivityPub] Migration: no ActivityPub self link for ${handle}`);
return null;
}
// Fetch actor document for inbox and profile // Fetch actor document for inbox and profile
const actorResponse = await fetch(selfLink.href, { const actorResponse = await fetch(selfLink.href, {
@@ -79,7 +88,10 @@ export async function resolveHandleViaWebFinger(handle) {
signal: AbortSignal.timeout(10_000), signal: AbortSignal.timeout(10_000),
}); });
if (!actorResponse.ok) return null; if (!actorResponse.ok) {
console.warn(`[ActivityPub] Migration: actor fetch failed for ${handle} (HTTP ${actorResponse.status})`);
return null;
}
const actor = await actorResponse.json(); const actor = await actorResponse.json();
return { return {
@@ -89,7 +101,8 @@ export async function resolveHandleViaWebFinger(handle) {
name: actor.name || actor.preferredUsername || handle, name: actor.name || actor.preferredUsername || handle,
handle: actor.preferredUsername || user, handle: actor.preferredUsername || user,
}; };
} catch { } catch (error) {
console.warn(`[ActivityPub] Migration: resolve failed for ${handle}: ${error.message}`);
return null; return null;
} }
} }
@@ -99,16 +112,23 @@ export async function resolveHandleViaWebFinger(handle) {
* *
* @param {string[]} handles - Array of handles to import * @param {string[]} handles - Array of handles to import
* @param {Collection} collection - MongoDB ap_following collection * @param {Collection} collection - MongoDB ap_following collection
* @returns {Promise<{imported: number, failed: number}>} * @returns {Promise<{imported: number, failed: number, errors: string[]}>}
*/ */
export async function bulkImportFollowing(handles, collection) { export async function bulkImportFollowing(handles, collection) {
let imported = 0; let imported = 0;
let failed = 0; let failed = 0;
const errors = [];
console.log(`[ActivityPub] Migration: importing ${handles.length} following entries...`);
for (let i = 0; i < handles.length; i++) {
const handle = handles[i];
console.log(`[ActivityPub] Migration: resolving following ${i + 1}/${handles.length}: ${handle}`);
for (const handle of handles) {
const resolved = await resolveHandleViaWebFinger(handle); const resolved = await resolveHandleViaWebFinger(handle);
if (!resolved) { if (!resolved) {
failed++; failed++;
errors.push(handle);
continue; continue;
} }
@@ -130,7 +150,12 @@ export async function bulkImportFollowing(handles, collection) {
imported++; imported++;
} }
return { imported, failed }; console.log(`[ActivityPub] Migration: following import complete — ${imported} imported, ${failed} failed`);
if (errors.length > 0) {
console.log(`[ActivityPub] Migration: failed handles: ${errors.join(", ")}`);
}
return { imported, failed, errors };
} }
/** /**
@@ -140,15 +165,24 @@ export async function bulkImportFollowing(handles, collection) {
* *
* @param {string[]} entries - Array of handles or actor URLs * @param {string[]} entries - Array of handles or actor URLs
* @param {Collection} collection - MongoDB ap_followers collection * @param {Collection} collection - MongoDB ap_followers collection
* @returns {Promise<{imported: number, failed: number}>} * @returns {Promise<{imported: number, failed: number, errors: string[]}>}
*/ */
export async function bulkImportFollowers(entries, collection) { export async function bulkImportFollowers(entries, collection) {
let imported = 0; let imported = 0;
let failed = 0; let failed = 0;
const errors = [];
for (const entry of entries) { console.log(`[ActivityPub] Migration: importing ${entries.length} follower entries...`);
for (let i = 0; i < entries.length; i++) {
const entry = entries[i];
// If it's a URL, store directly; if it's a handle, resolve via WebFinger // If it's a URL, store directly; if it's a handle, resolve via WebFinger
const isUrl = entry.startsWith("http"); const isUrl = entry.startsWith("http");
if (!isUrl) {
console.log(`[ActivityPub] Migration: resolving follower ${i + 1}/${entries.length}: ${entry}`);
}
let actorData; let actorData;
if (isUrl) { if (isUrl) {
@@ -159,6 +193,7 @@ export async function bulkImportFollowers(entries, collection) {
if (!actorData) { if (!actorData) {
failed++; failed++;
errors.push(entry);
continue; continue;
} }
@@ -180,5 +215,10 @@ export async function bulkImportFollowers(entries, collection) {
imported++; imported++;
} }
return { imported, failed }; console.log(`[ActivityPub] Migration: follower import complete — ${imported} imported, ${failed} failed`);
if (errors.length > 0) {
console.log(`[ActivityPub] Migration: failed entries: ${errors.join(", ")}`);
}
return { imported, failed, errors };
} }
+1
View File
@@ -43,6 +43,7 @@
"step3Desc": "Once you have saved your alias and imported your data, go to your Mastodon instance → Preferences → Account → <strong>Move to a different account</strong>. Enter your new fediverse handle and confirm. Mastodon will notify all your followers, and those whose servers support it will automatically re-follow you here. This step is irreversible — your old account will become a redirect.", "step3Desc": "Once you have saved your alias and imported your data, go to your Mastodon instance → Preferences → Account → <strong>Move to a different account</strong>. Enter your new fediverse handle and confirm. Mastodon will notify all your followers, and those whose servers support it will automatically re-follow you here. This step is irreversible — your old account will become a redirect.",
"errorNoFile": "Please select a CSV file before importing.", "errorNoFile": "Please select a CSV file before importing.",
"success": "Imported %d following, %d followers (%d failed).", "success": "Imported %d following, %d followers (%d failed).",
"failedList": "Could not resolve: %s",
"aliasSuccess": "Alias saved — your actor document now includes this account as alsoKnownAs." "aliasSuccess": "Alias saved — your actor document now includes this account as alsoKnownAs."
} }
} }
+1 -1
View File
@@ -1,6 +1,6 @@
{ {
"name": "@rmdes/indiekit-endpoint-activitypub", "name": "@rmdes/indiekit-endpoint-activitypub",
"version": "0.1.7", "version": "0.1.8",
"description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.", "description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.",
"keywords": [ "keywords": [
"indiekit", "indiekit",