svemagie.net/indiekit-endpoint-activitypub
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: switch CSV import to JSON fetch to bypass body size limit

Browse Source 
The app-level Express urlencoded parser (100KB limit) runs before
route-level middleware, so overriding the limit on the route doesn't
help. Solution: POST the CSV as JSON via fetch() to a dedicated
/admin/migrate/import endpoint with its own express.json({ limit: '5mb' }).

- Import button now shows "Importing..." while working
- Results appear inline without page reload
- Failed handles shown in a collapsible details element
- Import button disabled until a file is selected
- Alias form remains a regular POST (small payload, no issue)
This commit is contained in:
Ricardo
2026-02-19 10:22:34 +01:00
parent de00d3a16c
commit d159e79998
5 changed files with 162 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
index.js
+3 -2
View File
@@ -9,7 +9,7 @@ import { dashboardController } from "./lib/controllers/dashboard.js";
import { followersController } from "./lib/controllers/followers.js";
import { followingController } from "./lib/controllers/following.js";
import { activitiesController } from "./lib/controllers/activities.js";
import { migrateGetController, migratePostController } from "./lib/controllers/migrate.js";
import { migrateGetController, migratePostController, migrateImportController } from "./lib/controllers/migrate.js";
const defaults = {
mountPath: "/activitypub",
@@ -179,7 +179,8 @@ export default class ActivityPubEndpoint {
router.get("/admin/following", followingController(mp));
router.get("/admin/activities", activitiesController(mp));
router.get("/admin/migrate", migrateGetController(mp, this.options));
router.post("/admin/migrate", express.urlencoded({ extended: true, limit: "5mb" }), migratePostController(mp, this.options));
router.post("/admin/migrate", migratePostController(mp, this.options));
router.post("/admin/migrate/import", express.json({ limit: "5mb" }), migrateImportController(mp, this.options));
return router;
}
lib/controllers/migrate.js
+70 -79
View File
@@ -2,7 +2,8 @@
* Migration controller — handles Mastodon account migration UI.
*
* GET: shows the 3-step migration page
* POST: processes alias update or CSV file import
* POST /admin/migrate: alias update (small form POST)
* POST /admin/migrate/import: CSV import (JSON via fetch, bypasses body size limit)
*/
import {
@@ -30,86 +31,16 @@ export function migrateGetController(mountPath, pluginOptions) {
export function migratePostController(mountPath, pluginOptions) {
return async (request, response, next) => {
try {
const { application } = request.app.locals;
const action = request.body.action;
let result = null;
if (action === "alias") {
// Update alsoKnownAs on the actor config
const aliasUrl = request.body.aliasUrl?.trim();
if (aliasUrl) {
pluginOptions.alsoKnownAs = aliasUrl;
result = {
type: "success",
text: response.locals.__("activitypub.migrate.aliasSuccess"),
};
}
}
if (action === "import") {
const followingCollection =
application?.collections?.get("ap_following");
const followersCollection =
application?.collections?.get("ap_followers");
const importFollowing = request.body.importTypes?.includes("following");
const importFollowers = request.body.importTypes?.includes("followers");
// Read file content (submitted as text via client-side FileReader)
const fileContent = request.body.csvContent?.trim();
if (!fileContent) {
result = {
type: "error",
text: response.locals.__("activitypub.migrate.errorNoFile"),
};
} else {
let followingResult = { imported: 0, failed: 0, errors: [] };
let followersResult = { imported: 0, failed: 0, errors: [] };
if (importFollowing && followingCollection) {
const handles = parseMastodonFollowingCsv(fileContent);
console.log(`[ActivityPub] Migration: parsed ${handles.length} following handles from CSV`);
followingResult = await bulkImportFollowing(
handles,
followingCollection,
);
}
if (importFollowers && followersCollection) {
const entries = parseMastodonFollowersList(fileContent);
console.log(`[ActivityPub] Migration: parsed ${entries.length} follower entries from CSV`);
followersResult = await bulkImportFollowers(
entries,
followersCollection,
);
}
const totalFailed =
followingResult.failed + followersResult.failed;
const allErrors = [
...followingResult.errors,
...followersResult.errors,
];
let text = response.locals
.__("activitypub.migrate.success")
.replace("%d", followingResult.imported)
.replace("%d", followersResult.imported)
.replace("%d", totalFailed);
if (allErrors.length > 0) {
text += " " + response.locals
.__("activitypub.migrate.failedList")
.replace("%s", allErrors.join(", "));
}
result = {
type: totalFailed > 0 && followingResult.imported + followersResult.imported === 0
? "error"
: "success",
text,
};
}
// Only handles alias updates (small payload, regular form POST)
const aliasUrl = request.body.aliasUrl?.trim();
if (aliasUrl) {
pluginOptions.alsoKnownAs = aliasUrl;
result = {
type: "success",
text: response.locals.__("activitypub.migrate.aliasSuccess"),
};
}
response.render("activitypub-migrate", {
@@ -124,3 +55,63 @@ export function migratePostController(mountPath, pluginOptions) {
};
}
/**
* JSON endpoint for CSV import — receives { csvContent, importTypes }
* via fetch() to bypass Express's app-level urlencoded body size limit.
*/
export function migrateImportController(mountPath, pluginOptions) {
return async (request, response, next) => {
try {
const { application } = request.app.locals;
const { csvContent, importTypes } = request.body;
if (!csvContent?.trim()) {
return response.status(400).json({
type: "error",
text: "No CSV content provided.",
});
}
const followingCollection =
application?.collections?.get("ap_following");
const followersCollection =
application?.collections?.get("ap_followers");
const importFollowing = importTypes?.includes("following");
const importFollowers = importTypes?.includes("followers");
let followingResult = { imported: 0, failed: 0, errors: [] };
let followersResult = { imported: 0, failed: 0, errors: [] };
if (importFollowing && followingCollection) {
const handles = parseMastodonFollowingCsv(csvContent);
console.log(`[ActivityPub] Migration: parsed ${handles.length} following handles from CSV`);
followingResult = await bulkImportFollowing(handles, followingCollection);
}
if (importFollowers && followersCollection) {
const entries = parseMastodonFollowersList(csvContent);
console.log(`[ActivityPub] Migration: parsed ${entries.length} follower entries from CSV`);
followersResult = await bulkImportFollowers(entries, followersCollection);
}
const totalFailed = followingResult.failed + followersResult.failed;
const totalImported = followingResult.imported + followersResult.imported;
const allErrors = [...followingResult.errors, ...followersResult.errors];
return response.json({
type: totalFailed > 0 && totalImported === 0 ? "error" : "success",
followingImported: followingResult.imported,
followersImported: followersResult.imported,
failed: totalFailed,
errors: allErrors,
});
} catch (error) {
console.error("[ActivityPub] Migration import error:", error.message);
return response.status(500).json({
type: "error",
text: error.message,
});
}
};
}
locales/en.json
+1
View File
@@ -44,6 +44,7 @@
"errorNoFile": "Please select a CSV file before importing.",
"success": "Imported %d following, %d followers (%d failed).",
"failedList": "Could not resolve: %s",
"failedListSummary": "Failed handles",
"aliasSuccess": "Alias saved — your actor document now includes this account as alsoKnownAs."
}
}
package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@rmdes/indiekit-endpoint-activitypub",
"version": "0.1.8",
"version": "0.1.9",
"description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.",
"keywords": [
"indiekit",
views/activitypub-migrate.njk
+87 -6
View File
@@ -47,9 +47,7 @@
{{ heading({ text: __("activitypub.migrate.step2Title"), level: 2 }) }}
{{ prose({ text: __("activitypub.migrate.step2Desc") }) }}
<form method="post" novalidate x-data="csvImport()">
<input type="hidden" name="action" value="import">
<input type="hidden" name="csvContent" :value="csvContent">
<div x-data="csvImport('{{ mountPath }}')">
{{ checkboxes({
name: "importTypes",
@@ -86,8 +84,31 @@
</template>
</div>
{{ button({ text: __("activitypub.migrate.importButton") }) }}
</form>
<button class="button" type="button"
:disabled="importing || !csvContent"
@click="startImport()">
<span x-show="!importing">{{ __("activitypub.migrate.importButton") }}</span>
<span x-show="importing" x-text="statusText"></span>
</button>
{# Result notification #}
<template x-if="resultType">
<div :class="'notification-banner notification-banner--' + resultType"
role="alert" style="margin-top: 1em">
<p x-text="resultText"></p>
<template x-if="resultErrors.length > 0">
<details style="margin-top: 0.5em">
<summary>{{ __("activitypub.migrate.failedListSummary") }} (<span x-text="resultErrors.length"></span>)</summary>
<ul style="margin-top: 0.5em; font-size: 0.875em">
<template x-for="err in resultErrors" :key="err">
<li x-text="err"></li>
</template>
</ul>
</details>
</template>
</div>
</template>
</div>
<hr>
@@ -96,12 +117,17 @@
{{ prose({ text: __("activitypub.migrate.step3Desc") }) }}
<script>
function csvImport() {
function csvImport(mountPath) {
return {
csvContent: '',
fileName: '',
lineCount: 0,
fileError: '',
importing: false,
statusText: '',
resultType: '',
resultText: '',
resultErrors: [],
readFile(event) {
var self = this;
@@ -109,6 +135,9 @@
self.fileName = '';
self.lineCount = 0;
self.fileError = '';
self.resultType = '';
self.resultText = '';
self.resultErrors = [];
var file = event.target.files[0];
if (!file) return;
@@ -130,6 +159,58 @@
self.fileError = 'Could not read file';
};
reader.readAsText(file);
},
async startImport() {
var self = this;
self.importing = true;
self.resultType = '';
self.resultText = '';
self.resultErrors = [];
self.statusText = 'Importing\u2026';
// Collect checked import types
var checkboxes = document.querySelectorAll('input[name="importTypes"]:checked');
var importTypes = [];
for (var i = 0; i < checkboxes.length; i++) {
importTypes.push(checkboxes[i].value);
}
if (importTypes.length === 0) {
self.importing = false;
self.resultType = 'error';
self.resultText = 'Please select at least one import type.';
return;
}
try {
var res = await fetch(mountPath + '/admin/migrate/import', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
csvContent: self.csvContent,
importTypes: importTypes
})
});
if (!res.ok) {
var errBody = await res.text();
throw new Error('Server error (' + res.status + '): ' + errBody);
}
var data = await res.json();
self.resultType = data.type;
self.resultText = 'Imported ' + (data.followingImported || 0) + ' following, '
+ (data.followersImported || 0) + ' followers'
+ (data.failed > 0 ? ' (' + data.failed + ' failed)' : '') + '.';
self.resultErrors = data.errors || [];
} catch (err) {
self.resultType = 'error';
self.resultText = err.message;
}
self.importing = false;
self.statusText = '';
}
};
}