fix: switch CSV import to JSON fetch to bypass body size limit
The app-level Express urlencoded parser (100KB limit) runs before
route-level middleware, so overriding the limit on the route doesn't
help. Solution: POST the CSV as JSON via fetch() to a dedicated
/admin/migrate/import endpoint with its own express.json({ limit: '5mb' }).
- Import button now shows "Importing..." while working
- Results appear inline without page reload
- Failed handles shown in a collapsible details element
- Import button disabled until a file is selected
- Alias form remains a regular POST (small payload, no issue)
This commit is contained in:
@@ -9,7 +9,7 @@ import { dashboardController } from "./lib/controllers/dashboard.js";
|
|||||||
import { followersController } from "./lib/controllers/followers.js";
|
import { followersController } from "./lib/controllers/followers.js";
|
||||||
import { followingController } from "./lib/controllers/following.js";
|
import { followingController } from "./lib/controllers/following.js";
|
||||||
import { activitiesController } from "./lib/controllers/activities.js";
|
import { activitiesController } from "./lib/controllers/activities.js";
|
||||||
import { migrateGetController, migratePostController } from "./lib/controllers/migrate.js";
|
import { migrateGetController, migratePostController, migrateImportController } from "./lib/controllers/migrate.js";
|
||||||
|
|
||||||
const defaults = {
|
const defaults = {
|
||||||
mountPath: "/activitypub",
|
mountPath: "/activitypub",
|
||||||
@@ -179,7 +179,8 @@ export default class ActivityPubEndpoint {
|
|||||||
router.get("/admin/following", followingController(mp));
|
router.get("/admin/following", followingController(mp));
|
||||||
router.get("/admin/activities", activitiesController(mp));
|
router.get("/admin/activities", activitiesController(mp));
|
||||||
router.get("/admin/migrate", migrateGetController(mp, this.options));
|
router.get("/admin/migrate", migrateGetController(mp, this.options));
|
||||||
router.post("/admin/migrate", express.urlencoded({ extended: true, limit: "5mb" }), migratePostController(mp, this.options));
|
router.post("/admin/migrate", migratePostController(mp, this.options));
|
||||||
|
router.post("/admin/migrate/import", express.json({ limit: "5mb" }), migrateImportController(mp, this.options));
|
||||||
|
|
||||||
return router;
|
return router;
|
||||||
}
|
}
|
||||||
|
|||||||
+63
-72
@@ -2,7 +2,8 @@
|
|||||||
* Migration controller — handles Mastodon account migration UI.
|
* Migration controller — handles Mastodon account migration UI.
|
||||||
*
|
*
|
||||||
* GET: shows the 3-step migration page
|
* GET: shows the 3-step migration page
|
||||||
* POST: processes alias update or CSV file import
|
* POST /admin/migrate: alias update (small form POST)
|
||||||
|
* POST /admin/migrate/import: CSV import (JSON via fetch, bypasses body size limit)
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import {
|
import {
|
||||||
@@ -30,12 +31,9 @@ export function migrateGetController(mountPath, pluginOptions) {
|
|||||||
export function migratePostController(mountPath, pluginOptions) {
|
export function migratePostController(mountPath, pluginOptions) {
|
||||||
return async (request, response, next) => {
|
return async (request, response, next) => {
|
||||||
try {
|
try {
|
||||||
const { application } = request.app.locals;
|
|
||||||
const action = request.body.action;
|
|
||||||
let result = null;
|
let result = null;
|
||||||
|
|
||||||
if (action === "alias") {
|
// Only handles alias updates (small payload, regular form POST)
|
||||||
// Update alsoKnownAs on the actor config
|
|
||||||
const aliasUrl = request.body.aliasUrl?.trim();
|
const aliasUrl = request.body.aliasUrl?.trim();
|
||||||
if (aliasUrl) {
|
if (aliasUrl) {
|
||||||
pluginOptions.alsoKnownAs = aliasUrl;
|
pluginOptions.alsoKnownAs = aliasUrl;
|
||||||
@@ -44,73 +42,6 @@ export function migratePostController(mountPath, pluginOptions) {
|
|||||||
text: response.locals.__("activitypub.migrate.aliasSuccess"),
|
text: response.locals.__("activitypub.migrate.aliasSuccess"),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (action === "import") {
|
|
||||||
const followingCollection =
|
|
||||||
application?.collections?.get("ap_following");
|
|
||||||
const followersCollection =
|
|
||||||
application?.collections?.get("ap_followers");
|
|
||||||
|
|
||||||
const importFollowing = request.body.importTypes?.includes("following");
|
|
||||||
const importFollowers = request.body.importTypes?.includes("followers");
|
|
||||||
|
|
||||||
// Read file content (submitted as text via client-side FileReader)
|
|
||||||
const fileContent = request.body.csvContent?.trim();
|
|
||||||
if (!fileContent) {
|
|
||||||
result = {
|
|
||||||
type: "error",
|
|
||||||
text: response.locals.__("activitypub.migrate.errorNoFile"),
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
let followingResult = { imported: 0, failed: 0, errors: [] };
|
|
||||||
let followersResult = { imported: 0, failed: 0, errors: [] };
|
|
||||||
|
|
||||||
if (importFollowing && followingCollection) {
|
|
||||||
const handles = parseMastodonFollowingCsv(fileContent);
|
|
||||||
console.log(`[ActivityPub] Migration: parsed ${handles.length} following handles from CSV`);
|
|
||||||
followingResult = await bulkImportFollowing(
|
|
||||||
handles,
|
|
||||||
followingCollection,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (importFollowers && followersCollection) {
|
|
||||||
const entries = parseMastodonFollowersList(fileContent);
|
|
||||||
console.log(`[ActivityPub] Migration: parsed ${entries.length} follower entries from CSV`);
|
|
||||||
followersResult = await bulkImportFollowers(
|
|
||||||
entries,
|
|
||||||
followersCollection,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const totalFailed =
|
|
||||||
followingResult.failed + followersResult.failed;
|
|
||||||
const allErrors = [
|
|
||||||
...followingResult.errors,
|
|
||||||
...followersResult.errors,
|
|
||||||
];
|
|
||||||
|
|
||||||
let text = response.locals
|
|
||||||
.__("activitypub.migrate.success")
|
|
||||||
.replace("%d", followingResult.imported)
|
|
||||||
.replace("%d", followersResult.imported)
|
|
||||||
.replace("%d", totalFailed);
|
|
||||||
|
|
||||||
if (allErrors.length > 0) {
|
|
||||||
text += " " + response.locals
|
|
||||||
.__("activitypub.migrate.failedList")
|
|
||||||
.replace("%s", allErrors.join(", "));
|
|
||||||
}
|
|
||||||
|
|
||||||
result = {
|
|
||||||
type: totalFailed > 0 && followingResult.imported + followersResult.imported === 0
|
|
||||||
? "error"
|
|
||||||
: "success",
|
|
||||||
text,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
response.render("activitypub-migrate", {
|
response.render("activitypub-migrate", {
|
||||||
title: response.locals.__("activitypub.migrate.title"),
|
title: response.locals.__("activitypub.migrate.title"),
|
||||||
@@ -124,3 +55,63 @@ export function migratePostController(mountPath, pluginOptions) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSON endpoint for CSV import — receives { csvContent, importTypes }
|
||||||
|
* via fetch() to bypass Express's app-level urlencoded body size limit.
|
||||||
|
*/
|
||||||
|
export function migrateImportController(mountPath, pluginOptions) {
|
||||||
|
return async (request, response, next) => {
|
||||||
|
try {
|
||||||
|
const { application } = request.app.locals;
|
||||||
|
const { csvContent, importTypes } = request.body;
|
||||||
|
|
||||||
|
if (!csvContent?.trim()) {
|
||||||
|
return response.status(400).json({
|
||||||
|
type: "error",
|
||||||
|
text: "No CSV content provided.",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
const followingCollection =
|
||||||
|
application?.collections?.get("ap_following");
|
||||||
|
const followersCollection =
|
||||||
|
application?.collections?.get("ap_followers");
|
||||||
|
|
||||||
|
const importFollowing = importTypes?.includes("following");
|
||||||
|
const importFollowers = importTypes?.includes("followers");
|
||||||
|
|
||||||
|
let followingResult = { imported: 0, failed: 0, errors: [] };
|
||||||
|
let followersResult = { imported: 0, failed: 0, errors: [] };
|
||||||
|
|
||||||
|
if (importFollowing && followingCollection) {
|
||||||
|
const handles = parseMastodonFollowingCsv(csvContent);
|
||||||
|
console.log(`[ActivityPub] Migration: parsed ${handles.length} following handles from CSV`);
|
||||||
|
followingResult = await bulkImportFollowing(handles, followingCollection);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (importFollowers && followersCollection) {
|
||||||
|
const entries = parseMastodonFollowersList(csvContent);
|
||||||
|
console.log(`[ActivityPub] Migration: parsed ${entries.length} follower entries from CSV`);
|
||||||
|
followersResult = await bulkImportFollowers(entries, followersCollection);
|
||||||
|
}
|
||||||
|
|
||||||
|
const totalFailed = followingResult.failed + followersResult.failed;
|
||||||
|
const totalImported = followingResult.imported + followersResult.imported;
|
||||||
|
const allErrors = [...followingResult.errors, ...followersResult.errors];
|
||||||
|
|
||||||
|
return response.json({
|
||||||
|
type: totalFailed > 0 && totalImported === 0 ? "error" : "success",
|
||||||
|
followingImported: followingResult.imported,
|
||||||
|
followersImported: followersResult.imported,
|
||||||
|
failed: totalFailed,
|
||||||
|
errors: allErrors,
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.error("[ActivityPub] Migration import error:", error.message);
|
||||||
|
return response.status(500).json({
|
||||||
|
type: "error",
|
||||||
|
text: error.message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|||||||
@@ -44,6 +44,7 @@
|
|||||||
"errorNoFile": "Please select a CSV file before importing.",
|
"errorNoFile": "Please select a CSV file before importing.",
|
||||||
"success": "Imported %d following, %d followers (%d failed).",
|
"success": "Imported %d following, %d followers (%d failed).",
|
||||||
"failedList": "Could not resolve: %s",
|
"failedList": "Could not resolve: %s",
|
||||||
|
"failedListSummary": "Failed handles",
|
||||||
"aliasSuccess": "Alias saved — your actor document now includes this account as alsoKnownAs."
|
"aliasSuccess": "Alias saved — your actor document now includes this account as alsoKnownAs."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "@rmdes/indiekit-endpoint-activitypub",
|
"name": "@rmdes/indiekit-endpoint-activitypub",
|
||||||
"version": "0.1.8",
|
"version": "0.1.9",
|
||||||
"description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.",
|
"description": "ActivityPub federation endpoint for Indiekit via Fedify. Adds full fediverse support: actor, inbox, outbox, followers, following, syndication, and Mastodon migration.",
|
||||||
"keywords": [
|
"keywords": [
|
||||||
"indiekit",
|
"indiekit",
|
||||||
|
|||||||
@@ -47,9 +47,7 @@
|
|||||||
{{ heading({ text: __("activitypub.migrate.step2Title"), level: 2 }) }}
|
{{ heading({ text: __("activitypub.migrate.step2Title"), level: 2 }) }}
|
||||||
{{ prose({ text: __("activitypub.migrate.step2Desc") }) }}
|
{{ prose({ text: __("activitypub.migrate.step2Desc") }) }}
|
||||||
|
|
||||||
<form method="post" novalidate x-data="csvImport()">
|
<div x-data="csvImport('{{ mountPath }}')">
|
||||||
<input type="hidden" name="action" value="import">
|
|
||||||
<input type="hidden" name="csvContent" :value="csvContent">
|
|
||||||
|
|
||||||
{{ checkboxes({
|
{{ checkboxes({
|
||||||
name: "importTypes",
|
name: "importTypes",
|
||||||
@@ -86,8 +84,31 @@
|
|||||||
</template>
|
</template>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{{ button({ text: __("activitypub.migrate.importButton") }) }}
|
<button class="button" type="button"
|
||||||
</form>
|
:disabled="importing || !csvContent"
|
||||||
|
@click="startImport()">
|
||||||
|
<span x-show="!importing">{{ __("activitypub.migrate.importButton") }}</span>
|
||||||
|
<span x-show="importing" x-text="statusText"></span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
{# Result notification #}
|
||||||
|
<template x-if="resultType">
|
||||||
|
<div :class="'notification-banner notification-banner--' + resultType"
|
||||||
|
role="alert" style="margin-top: 1em">
|
||||||
|
<p x-text="resultText"></p>
|
||||||
|
<template x-if="resultErrors.length > 0">
|
||||||
|
<details style="margin-top: 0.5em">
|
||||||
|
<summary>{{ __("activitypub.migrate.failedListSummary") }} (<span x-text="resultErrors.length"></span>)</summary>
|
||||||
|
<ul style="margin-top: 0.5em; font-size: 0.875em">
|
||||||
|
<template x-for="err in resultErrors" :key="err">
|
||||||
|
<li x-text="err"></li>
|
||||||
|
</template>
|
||||||
|
</ul>
|
||||||
|
</details>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
</div>
|
||||||
|
|
||||||
<hr>
|
<hr>
|
||||||
|
|
||||||
@@ -96,12 +117,17 @@
|
|||||||
{{ prose({ text: __("activitypub.migrate.step3Desc") }) }}
|
{{ prose({ text: __("activitypub.migrate.step3Desc") }) }}
|
||||||
|
|
||||||
<script>
|
<script>
|
||||||
function csvImport() {
|
function csvImport(mountPath) {
|
||||||
return {
|
return {
|
||||||
csvContent: '',
|
csvContent: '',
|
||||||
fileName: '',
|
fileName: '',
|
||||||
lineCount: 0,
|
lineCount: 0,
|
||||||
fileError: '',
|
fileError: '',
|
||||||
|
importing: false,
|
||||||
|
statusText: '',
|
||||||
|
resultType: '',
|
||||||
|
resultText: '',
|
||||||
|
resultErrors: [],
|
||||||
|
|
||||||
readFile(event) {
|
readFile(event) {
|
||||||
var self = this;
|
var self = this;
|
||||||
@@ -109,6 +135,9 @@
|
|||||||
self.fileName = '';
|
self.fileName = '';
|
||||||
self.lineCount = 0;
|
self.lineCount = 0;
|
||||||
self.fileError = '';
|
self.fileError = '';
|
||||||
|
self.resultType = '';
|
||||||
|
self.resultText = '';
|
||||||
|
self.resultErrors = [];
|
||||||
|
|
||||||
var file = event.target.files[0];
|
var file = event.target.files[0];
|
||||||
if (!file) return;
|
if (!file) return;
|
||||||
@@ -130,6 +159,58 @@
|
|||||||
self.fileError = 'Could not read file';
|
self.fileError = 'Could not read file';
|
||||||
};
|
};
|
||||||
reader.readAsText(file);
|
reader.readAsText(file);
|
||||||
|
},
|
||||||
|
|
||||||
|
async startImport() {
|
||||||
|
var self = this;
|
||||||
|
self.importing = true;
|
||||||
|
self.resultType = '';
|
||||||
|
self.resultText = '';
|
||||||
|
self.resultErrors = [];
|
||||||
|
self.statusText = 'Importing\u2026';
|
||||||
|
|
||||||
|
// Collect checked import types
|
||||||
|
var checkboxes = document.querySelectorAll('input[name="importTypes"]:checked');
|
||||||
|
var importTypes = [];
|
||||||
|
for (var i = 0; i < checkboxes.length; i++) {
|
||||||
|
importTypes.push(checkboxes[i].value);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (importTypes.length === 0) {
|
||||||
|
self.importing = false;
|
||||||
|
self.resultType = 'error';
|
||||||
|
self.resultText = 'Please select at least one import type.';
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
var res = await fetch(mountPath + '/admin/migrate/import', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
csvContent: self.csvContent,
|
||||||
|
importTypes: importTypes
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
var errBody = await res.text();
|
||||||
|
throw new Error('Server error (' + res.status + '): ' + errBody);
|
||||||
|
}
|
||||||
|
|
||||||
|
var data = await res.json();
|
||||||
|
self.resultType = data.type;
|
||||||
|
self.resultText = 'Imported ' + (data.followingImported || 0) + ' following, '
|
||||||
|
+ (data.followersImported || 0) + ' followers'
|
||||||
|
+ (data.failed > 0 ? ' (' + data.failed + ' failed)' : '') + '.';
|
||||||
|
self.resultErrors = data.errors || [];
|
||||||
|
} catch (err) {
|
||||||
|
self.resultType = 'error';
|
||||||
|
self.resultText = err.message;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.importing = false;
|
||||||
|
self.statusText = '';
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user