indiekit-endpoint-activitypub/lib/outbox-backfill.js

/**
 * Outbox backfill — fetch recent posts from a remote actor's outbox
 * and store them in the local ap_timeline after a follow.
 *
 * Runs asynchronously (fire-and-forget) — never blocks followActor().
 * Fetches at most one page (limit 20) from the outbox's `first` page.
 * Uses extractObjectData() from timeline-store for consistent normalization.
 */

import { Note, Article, Create, Announce } from "@fedify/fedify/vocab";
import { extractObjectData } from "./timeline-store.js";
import { addTimelineItem } from "./storage/timeline.js";

/**
 * Fetch up to `limit` recent posts from `actorUrl`'s AP outbox and
 * store them in ap_timeline. Non-blocking — call without await.
 *
 * @param {object} options
 * @param {string} options.actorUrl - Remote actor AP URL
 * @param {object} options.remoteActor - Resolved Fedify actor object
 * @param {object} options.ctx - Fedify context (for documentLoader)
 * @param {string} options.handle - Local actor handle (for signed fetches)
 * @param {object} options.collections - MongoDB collections
 * @param {number} [options.limit=20] - Max posts to backfill
 */
export async function backfillFromOutbox({ actorUrl, remoteActor, ctx, handle, collections, limit = 20 }) {
  if (!collections?.ap_timeline) return;

  const withTimeout = (p, ms = 8000) =>
    Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error("timeout")), ms))]);

  try {
    const documentLoader = await withTimeout(ctx.getDocumentLoader({ identifier: handle }));

    // Fetch outbox collection — cap at 8 s
    const outbox = await withTimeout(remoteActor.getOutbox({ documentLoader })).catch(() => null);
    if (!outbox) return;

    // Guard: if outbox reports more than 500 items without pagination, skip the direct-items path
    // to avoid materialising a huge in-memory array from a hostile server.
    const totalItems = typeof outbox.totalItems === "number" ? outbox.totalItems : null;

    // Collect activity items — try direct items first (only for small collections), then first-page
    const activities = [];
    let itemSource = null;
    if (totalItems === null || totalItems <= 100) {
      try {
        const direct = outbox.getItems?.({ documentLoader });
        if (direct) itemSource = direct;
      } catch { /* not directly iterable */ }
    }

    if (!itemSource) {
      // Paginated — navigate to first page
      const firstPage = await withTimeout(outbox.getFirst({ documentLoader })).catch(() => null);
      if (firstPage?.getItems) itemSource = firstPage.getItems({ documentLoader });
    }

    if (!itemSource) return;

    try {
      for await (const item of itemSource) {
        activities.push(item);
        if (activities.length >= limit) break;
      }
    } catch (iterErr) {
      console.debug(`[ActivityPub] Outbox backfill iterator stopped for ${actorUrl}: ${iterErr?.message}`);
    }

    if (activities.length === 0) return;

    let stored = 0;
    for (const activity of activities) {
      try {
        // Only handle Create(Note/Article) and Announce activities
        let object = null;
        let boostedBy = null;

        if (activity instanceof Create) {
          object = await withTimeout(activity.getObject({ documentLoader }), 5000).catch(() => null);
        } else if (activity instanceof Announce) {
          object = await withTimeout(activity.getObject({ documentLoader }), 5000).catch(() => null);
          if (object) {
            const _username = remoteActor.preferredUsername?.toString() || "";
            const _domain = (() => { try { return new URL(actorUrl).hostname; } catch { return ""; } })();
            let _photo = "";
            try { const _icon = await remoteActor.getIcon(); _photo = _icon?.url?.href || ""; } catch { /* ignore */ }
            boostedBy = {
              name: remoteActor.name?.toString() || _username || "",
              url: actorUrl,
              photo: _photo,
              handle: _username && _domain ? `@${_username}@${_domain}` : "",
            };
          }
        }

        if (!object || !(object instanceof Note || object instanceof Article)) continue;

        const uid = object.id?.href;
        if (!uid) continue;

        // Skip if already in timeline
        const exists = await collections.ap_timeline.findOne({ uid }, { projection: { _id: 1 } });
        if (exists) continue;

        const item = await extractObjectData(object, {
          documentLoader,
          actorFallback: remoteActor,
          boostedBy,
          boostedAt: boostedBy ? new Date().toISOString() : undefined,
        });

        // Ensure visibility field (extractObjectData doesn't set it)
        if (!item.visibility) item.visibility = "public";

        await addTimelineItem({ ap_timeline: collections.ap_timeline }, item);
        stored++;
      } catch (itemErr) {
        console.debug(`[ActivityPub] Outbox backfill skipped item from ${actorUrl}: ${itemErr?.message}`);
      }
    }

    if (stored > 0) {
      console.info(`[ActivityPub] Outbox backfill: stored ${stored} posts from ${actorUrl}`);
    }
  } catch (error) {
    console.warn(`[ActivityPub] Outbox backfill failed for ${actorUrl}:`, error?.message);
  }
}