fix: payload too large error and add migration logging
- Add express.urlencoded({ limit: '5mb' }) to migration POST route
to handle large CSV files (default 100KB was too small)
- Add per-handle progress logging to console for monitoring imports
- Log failed handles with reasons (WebFinger failure, no AP link, etc.)
- Show failed handles in the UI result notification
- Use error notification type when all imports fail
This commit is contained in:
@@ -179,7 +179,7 @@ export default class ActivityPubEndpoint {
|
|||||||
router.get("/admin/following", followingController(mp));
|
router.get("/admin/following", followingController(mp));
|
||||||
router.get("/admin/activities", activitiesController(mp));
|
router.get("/admin/activities", activitiesController(mp));
|
||||||
router.get("/admin/migrate", migrateGetController(mp, this.options));
|
router.get("/admin/migrate", migrateGetController(mp, this.options));
|
||||||
router.post("/admin/migrate", migratePostController(mp, this.options));
|
router.post("/admin/migrate", express.urlencoded({ extended: true, limit: "5mb" }), migratePostController(mp, this.options));
|
||||||
|
|
||||||
return router;
|
return router;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -63,11 +63,12 @@ export function migratePostController(mountPath, pluginOptions) {
|
|||||||
text: response.locals.__("activitypub.migrate.errorNoFile"),
|
text: response.locals.__("activitypub.migrate.errorNoFile"),
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
let followingResult = { imported: 0, failed: 0 };
|
let followingResult = { imported: 0, failed: 0, errors: [] };
|
||||||
let followersResult = { imported: 0, failed: 0 };
|
let followersResult = { imported: 0, failed: 0, errors: [] };
|
||||||
|
|
||||||
if (importFollowing && followingCollection) {
|
if (importFollowing && followingCollection) {
|
||||||
const handles = parseMastodonFollowingCsv(fileContent);
|
const handles = parseMastodonFollowingCsv(fileContent);
|
||||||
|
console.log(`[ActivityPub] Migration: parsed ${handles.length} following handles from CSV`);
|
||||||
followingResult = await bulkImportFollowing(
|
followingResult = await bulkImportFollowing(
|
||||||
handles,
|
handles,
|
||||||
followingCollection,
|
followingCollection,
|
||||||
@@ -76,6 +77,7 @@ export function migratePostController(mountPath, pluginOptions) {
|
|||||||
|
|
||||||
if (importFollowers && followersCollection) {
|
if (importFollowers && followersCollection) {
|
||||||
const entries = parseMastodonFollowersList(fileContent);
|
const entries = parseMastodonFollowersList(fileContent);
|
||||||
|
console.log(`[ActivityPub] Migration: parsed ${entries.length} follower entries from CSV`);
|
||||||
followersResult = await bulkImportFollowers(
|
followersResult = await bulkImportFollowers(
|
||||||
entries,
|
entries,
|
||||||
followersCollection,
|
followersCollection,
|
||||||
@@ -84,13 +86,28 @@ export function migratePostController(mountPath, pluginOptions) {
|
|||||||
|
|
||||||
const totalFailed =
|
const totalFailed =
|
||||||
followingResult.failed + followersResult.failed;
|
followingResult.failed + followersResult.failed;
|
||||||
result = {
|
const allErrors = [
|
||||||
type: "success",
|
...followingResult.errors,
|
||||||
text: response.locals
|
...followersResult.errors,
|
||||||
|
];
|
||||||
|
|
||||||
|
let text = response.locals
|
||||||
.__("activitypub.migrate.success")
|
.__("activitypub.migrate.success")
|
||||||
.replace("%d", followingResult.imported)
|
.replace("%d", followingResult.imported)
|
||||||
.replace("%d", followersResult.imported)
|
.replace("%d", followersResult.imported)
|
||||||
.replace("%d", totalFailed),
|
.replace("%d", totalFailed);
|
||||||
|
|
||||||
|
if (allErrors.length > 0) {
|
||||||
|
text += " " + response.locals
|
||||||
|
.__("activitypub.migrate.failedList")
|
||||||
|
.replace("%s", allErrors.join(", "));
|
||||||
|
}
|
||||||
|
|
||||||
|
result = {
|
||||||
|
type: totalFailed > 0 && followingResult.imported + followersResult.imported === 0
|
||||||
|
? "error"
|
||||||
|
: "success",
|
||||||
|
text,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+51
-11
@@ -54,7 +54,10 @@ export function parseMastodonFollowersList(text) {
|
|||||||
*/
|
*/
|
||||||
export async function resolveHandleViaWebFinger(handle) {
|
export async function resolveHandleViaWebFinger(handle) {
|
||||||
const [user, domain] = handle.split("@");
|
const [user, domain] = handle.split("@");
|
||||||
if (!user || !domain) return null;
|
if (!user || !domain) {
|
||||||
|
console.warn(`[ActivityPub] Migration: invalid handle "${handle}" — skipping`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// WebFinger lookup
|
// WebFinger lookup
|
||||||
@@ -64,14 +67,20 @@ export async function resolveHandleViaWebFinger(handle) {
|
|||||||
signal: AbortSignal.timeout(10_000),
|
signal: AbortSignal.timeout(10_000),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!wfResponse.ok) return null;
|
if (!wfResponse.ok) {
|
||||||
|
console.warn(`[ActivityPub] Migration: WebFinger failed for ${handle} (HTTP ${wfResponse.status})`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const jrd = await wfResponse.json();
|
const jrd = await wfResponse.json();
|
||||||
const selfLink = jrd.links?.find(
|
const selfLink = jrd.links?.find(
|
||||||
(l) => l.rel === "self" && l.type === "application/activity+json",
|
(l) => l.rel === "self" && l.type === "application/activity+json",
|
||||||
);
|
);
|
||||||
|
|
||||||
if (!selfLink?.href) return null;
|
if (!selfLink?.href) {
|
||||||
|
console.warn(`[ActivityPub] Migration: no ActivityPub self link for ${handle}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
// Fetch actor document for inbox and profile
|
// Fetch actor document for inbox and profile
|
||||||
const actorResponse = await fetch(selfLink.href, {
|
const actorResponse = await fetch(selfLink.href, {
|
||||||
@@ -79,7 +88,10 @@ export async function resolveHandleViaWebFinger(handle) {
|
|||||||
signal: AbortSignal.timeout(10_000),
|
signal: AbortSignal.timeout(10_000),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!actorResponse.ok) return null;
|
if (!actorResponse.ok) {
|
||||||
|
console.warn(`[ActivityPub] Migration: actor fetch failed for ${handle} (HTTP ${actorResponse.status})`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
const actor = await actorResponse.json();
|
const actor = await actorResponse.json();
|
||||||
return {
|
return {
|
||||||
@@ -89,7 +101,8 @@ export async function resolveHandleViaWebFinger(handle) {
|
|||||||
name: actor.name || actor.preferredUsername || handle,
|
name: actor.name || actor.preferredUsername || handle,
|
||||||
handle: actor.preferredUsername || user,
|
handle: actor.preferredUsername || user,
|
||||||
};
|
};
|
||||||
} catch {
|
} catch (error) {
|
||||||
|
console.warn(`[ActivityPub] Migration: resolve failed for ${handle}: ${error.message}`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -99,16 +112,23 @@ export async function resolveHandleViaWebFinger(handle) {
|
|||||||
*
|
*
|
||||||
* @param {string[]} handles - Array of handles to import
|
* @param {string[]} handles - Array of handles to import
|
||||||
* @param {Collection} collection - MongoDB ap_following collection
|
* @param {Collection} collection - MongoDB ap_following collection
|
||||||
* @returns {Promise<{imported: number, failed: number}>}
|
* @returns {Promise<{imported: number, failed: number, errors: string[]}>}
|
||||||
*/
|
*/
|
||||||
export async function bulkImportFollowing(handles, collection) {
|
export async function bulkImportFollowing(handles, collection) {
|
||||||
let imported = 0;
|
let imported = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
|
const errors = [];
|
||||||
|
|
||||||
|
console.log(`[ActivityPub] Migration: importing ${handles.length} following entries...`);
|
||||||
|
|
||||||
|
for (let i = 0; i < handles.length; i++) {
|
||||||
|
const handle = handles[i];
|
||||||
|
console.log(`[ActivityPub] Migration: resolving following ${i + 1}/${handles.length}: ${handle}`);
|
||||||
|
|
||||||
for (const handle of handles) {
|
|
||||||
const resolved = await resolveHandleViaWebFinger(handle);
|
const resolved = await resolveHandleViaWebFinger(handle);
|
||||||
if (!resolved) {
|
if (!resolved) {
|
||||||
failed++;
|
failed++;
|
||||||
|
errors.push(handle);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -130,7 +150,12 @@ export async function bulkImportFollowing(handles, collection) {
|
|||||||
imported++;
|
imported++;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { imported, failed };
|
console.log(`[ActivityPub] Migration: following import complete — ${imported} imported, ${failed} failed`);
|
||||||
|
if (errors.length > 0) {
|
||||||
|
console.log(`[ActivityPub] Migration: failed handles: ${errors.join(", ")}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { imported, failed, errors };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -140,15 +165,24 @@ export async function bulkImportFollowing(handles, collection) {
|
|||||||
*
|
*
|
||||||
* @param {string[]} entries - Array of handles or actor URLs
|
* @param {string[]} entries - Array of handles or actor URLs
|
||||||
* @param {Collection} collection - MongoDB ap_followers collection
|
* @param {Collection} collection - MongoDB ap_followers collection
|
||||||
* @returns {Promise<{imported: number, failed: number}>}
|
* @returns {Promise<{imported: number, failed: number, errors: string[]}>}
|
||||||
*/
|
*/
|
||||||
export async function bulkImportFollowers(entries, collection) {
|
export async function bulkImportFollowers(entries, collection) {
|
||||||
let imported = 0;
|
let imported = 0;
|
||||||
let failed = 0;
|
let failed = 0;
|
||||||
|
const errors = [];
|
||||||
|
|
||||||
for (const entry of entries) {
|
console.log(`[ActivityPub] Migration: importing ${entries.length} follower entries...`);
|
||||||
|
|
||||||
|
for (let i = 0; i < entries.length; i++) {
|
||||||
|
const entry = entries[i];
|
||||||
// If it's a URL, store directly; if it's a handle, resolve via WebFinger
|
// If it's a URL, store directly; if it's a handle, resolve via WebFinger
|
||||||
const isUrl = entry.startsWith("http");
|
const isUrl = entry.startsWith("http");
|
||||||
|
|
||||||
|
if (!isUrl) {
|
||||||
|
console.log(`[ActivityPub] Migration: resolving follower ${i + 1}/${entries.length}: ${entry}`);
|
||||||
|
}
|
||||||
|
|
||||||
let actorData;
|
let actorData;
|
||||||
|
|
||||||
if (isUrl) {
|
if (isUrl) {
|
||||||
@@ -159,6 +193,7 @@ export async function bulkImportFollowers(entries, collection) {
|
|||||||
|
|
||||||
if (!actorData) {
|
if (!actorData) {
|
||||||
failed++;
|
failed++;
|
||||||
|
errors.push(entry);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -180,5 +215,10 @@ export async function bulkImportFollowers(entries, collection) {
|
|||||||
imported++;
|
imported++;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { imported, failed };
|
console.log(`[ActivityPub] Migration: follower import complete — ${imported} imported, ${failed} failed`);
|
||||||
|
if (errors.length > 0) {
|
||||||
|
console.log(`[ActivityPub] Migration: failed entries: ${errors.join(", ")}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { imported, failed, errors };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -43,6 +43,7 @@
|
|||||||
"step3Desc": "Once you have saved your alias and imported your data, go to your Mastodon instance → Preferences → Account → <strong>Move to a different account</strong>. Enter your new fediverse handle and confirm. Mastodon will notify all your followers, and those whose servers support it will automatically re-follow you here. This step is irreversible — your old account will become a redirect.",
|
"step3Desc": "Once you have saved your alias and imported your data, go to your Mastodon instance → Preferences → Account → <strong>Move to a different account</strong>. Enter your new fediverse handle and confirm. Mastodon will notify all your followers, and those whose servers support it will automatically re-follow you here. This step is irreversible — your old account will become a redirect.",
|
||||||
"errorNoFile": "Please select a CSV file before importing.",
|
"errorNoFile": "Please select a CSV file before importing.",
|
||||||
"success": "Imported %d following, %d followers (%d failed).",
|
"success": "Imported %d following, %d followers (%d failed).",
|
||||||
|
"failedList": "Could not resolve: %s",
|
||||||
"aliasSuccess": "Alias saved — your actor document now includes this account as alsoKnownAs."
|
"aliasSuccess": "Alias saved — your actor document now includes this account as alsoKnownAs."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@rmdes/indiekit-endpoint-activitypub",
|
"name": "@rmdes/indiekit-endpoint-activitypub",
|
||||||
"version": "0.1.7",
|
"version": "0.1.8",
|
||||||
"description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.",
|
"description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"indiekit",
|
"indiekit",
|
||||||
|
|||||||
Reference in New Issue
Block a user