/** * Feed subscription storage operations * @module storage/feeds */ import { ObjectId } from "mongodb"; import { deleteItemsForFeed } from "./items.js"; /** * Get feeds collection from application * @param {object} application - Indiekit application * @returns {object} MongoDB collection */ function getCollection(application) { return application.collections.get("microsub_feeds"); } /** * Normalize a feed URL for duplicate comparison. * Strips trailing slashes, normalizes protocol to https, lowercases hostname. * @param {string} url - Feed URL * @returns {string} Normalized URL */ export function normalizeUrl(url) { try { const parsed = new URL(url); // Normalize protocol to https parsed.protocol = "https:"; // Lowercase hostname parsed.hostname = parsed.hostname.toLowerCase(); // Remove trailing slash from path (but keep "/" for root) if (parsed.pathname.length > 1 && parsed.pathname.endsWith("/")) { parsed.pathname = parsed.pathname.slice(0, -1); } return parsed.href; } catch { return url; } } /** * Find an existing feed across ALL channels by normalized URL * @param {object} application - Indiekit application * @param {string} url - Feed URL to check * @returns {Promise} Existing feed with channel info, or null */ export async function findFeedAcrossChannels(application, url) { const collection = getCollection(application); const normalized = normalizeUrl(url); // Get all feeds and check normalized URLs // We check a few common URL variants directly for efficiency const variants = new Set(); variants.add(url); variants.add(normalized); // Also try with/without trailing slash if (url.endsWith("/")) { variants.add(url.slice(0, -1)); } else { variants.add(url + "/"); } // Try http/https variants if (url.startsWith("https://")) { variants.add(url.replace("https://", "http://")); } else if (url.startsWith("http://")) { variants.add(url.replace("http://", "https://")); } const existing = await collection.findOne({ url: { $in: [...variants] }, }); if (!existing) return null; // Look up the channel name for a useful error message const channelsCollection = application.collections.get("microsub_channels"); const channel = await channelsCollection.findOne({ _id: existing.channelId }); return { feed: existing, channelName: channel?.name || "unknown channel", }; } /** * Create a new feed subscription * @param {object} application - Indiekit application * @param {object} data - Feed data * @param {ObjectId} data.channelId - Channel ObjectId * @param {string} data.url - Feed URL * @param {string} [data.title] - Feed title * @param {string} [data.photo] - Feed icon URL * @param {string} [data.micropubPostUrl] - Micropub post URL that created this feed (for update tracking) * @returns {Promise} Created feed */ export async function createFeed( application, { channelId, url, title, photo, micropubPostUrl }, ) { const collection = getCollection(application); // Check if feed already exists in this channel (exact match) const existing = await collection.findOne({ channelId, url }); if (existing) { return existing; } // Check for duplicate across ALL channels (normalized URL) const duplicate = await findFeedAcrossChannels(application, url); if (duplicate) { const error = new Error( `Feed already exists in channel "${duplicate.channelName}"`, ); error.code = "DUPLICATE_FEED"; error.existingFeed = duplicate.feed; error.channelName = duplicate.channelName; throw error; } const feed = { channelId, url, title: title || undefined, photo: photo || undefined, micropubPostUrl: micropubPostUrl || undefined, tier: 1, // Start at tier 1 (2 minutes) unmodified: 0, nextFetchAt: new Date(), // Fetch immediately (kept as Date for query compatibility) lastFetchedAt: undefined, websub: undefined, // Will be populated if hub is discovered createdAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }; await collection.insertOne(feed); return feed; } /** * Get all feeds for a channel * @param {object} application - Indiekit application * @param {ObjectId|string} channelId - Channel ObjectId * @returns {Promise} Array of feeds */ export async function getFeedsForChannel(application, channelId) { const collection = getCollection(application); const objectId = typeof channelId === "string" ? new ObjectId(channelId) : channelId; return collection.find({ channelId: objectId }).toArray(); } /** * Get a feed by URL and channel * @param {object} application - Indiekit application * @param {ObjectId|string} channelId - Channel ObjectId * @param {string} url - Feed URL * @returns {Promise} Feed or null */ export async function getFeedByUrl(application, channelId, url) { const collection = getCollection(application); const objectId = typeof channelId === "string" ? new ObjectId(channelId) : channelId; return collection.findOne({ channelId: objectId, url }); } /** * Get a feed by ID * @param {object} application - Indiekit application * @param {ObjectId|string} id - Feed ObjectId * @returns {Promise} Feed or null */ export async function getFeedById(application, id) { const collection = getCollection(application); const objectId = typeof id === "string" ? new ObjectId(id) : id; return collection.findOne({ _id: objectId }); } /** * Get a feed by the micropub post URL that created it. * Used for update/delete tracking when a bookmark post changes. * @param {object} application - Indiekit application * @param {string} postUrl - The micropub post permalink * @returns {Promise<{feed: object, channel: object}|null>} Feed + channel, or null */ export async function getFeedByMicropubPostUrl(application, postUrl) { const collection = getCollection(application); const feed = await collection.findOne({ micropubPostUrl: postUrl }); if (!feed) return null; const channelsCollection = application.collections.get("microsub_channels"); const channel = await channelsCollection.findOne({ _id: feed.channelId }); return { feed, channel }; } /** * Delete a feed by its ObjectId (regardless of channel). * Used when removing a feed without knowing the channel. * @param {object} application - Indiekit application * @param {ObjectId|string} feedId - Feed ObjectId * @returns {Promise} True if deleted */ export async function deleteFeedById(application, feedId) { const collection = getCollection(application); const objectId = typeof feedId === "string" ? new ObjectId(feedId) : feedId; const feed = await collection.findOne({ _id: objectId }); if (!feed) return false; const itemsDeleted = await deleteItemsForFeed(application, feed._id); console.info( `[Microsub] Deleted ${itemsDeleted} items from feed ${feed.url}`, ); const result = await collection.deleteOne({ _id: objectId }); return result.deletedCount > 0; } /** * Update a feed * @param {object} application - Indiekit application * @param {ObjectId|string} id - Feed ObjectId * @param {object} updates - Fields to update * @returns {Promise} Updated feed */ export async function updateFeed(application, id, updates) { const collection = getCollection(application); const objectId = typeof id === "string" ? new ObjectId(id) : id; const result = await collection.findOneAndUpdate( { _id: objectId }, { $set: { ...updates, updatedAt: new Date().toISOString(), }, }, { returnDocument: "after" }, ); return result; } /** * Delete a feed subscription and all its items * @param {object} application - Indiekit application * @param {ObjectId|string} channelId - Channel ObjectId * @param {string} url - Feed URL * @returns {Promise} True if deleted */ export async function deleteFeed(application, channelId, url) { const collection = getCollection(application); const objectId = typeof channelId === "string" ? new ObjectId(channelId) : channelId; // Find the feed first to get its ID for cascade delete const feed = await collection.findOne({ channelId: objectId, url }); if (!feed) { return false; } // Delete all items from this feed const itemsDeleted = await deleteItemsForFeed(application, feed._id); console.info(`[Microsub] Deleted ${itemsDeleted} items from feed ${url}`); // Delete the feed itself const result = await collection.deleteOne({ _id: feed._id }); return result.deletedCount > 0; } /** * Delete all feeds for a channel * @param {object} application - Indiekit application * @param {ObjectId|string} channelId - Channel ObjectId * @returns {Promise} Number of deleted feeds */ export async function deleteFeedsForChannel(application, channelId) { const collection = getCollection(application); const objectId = typeof channelId === "string" ? new ObjectId(channelId) : channelId; const result = await collection.deleteMany({ channelId: objectId }); return result.deletedCount; } /** * Get feeds ready for polling * @param {object} application - Indiekit application * @returns {Promise} Array of feeds to fetch */ export async function getFeedsToFetch(application) { const collection = getCollection(application); const now = new Date(); return collection .find({ $or: [{ nextFetchAt: undefined }, { nextFetchAt: { $lte: now } }], }) .toArray(); } /** * Update feed after fetch * @param {object} application - Indiekit application * @param {ObjectId|string} id - Feed ObjectId * @param {boolean} changed - Whether content changed * @param {object} [extra] - Additional fields to update * @returns {Promise} Updated feed */ export async function updateFeedAfterFetch( application, id, changed, extra = {}, ) { const collection = getCollection(application); const objectId = typeof id === "string" ? new ObjectId(id) : id; // If extra contains tier info, use that (from processor) // Otherwise calculate locally (legacy behavior) let updateData; if (extra.tier === undefined) { // Get current feed state for legacy calculation const feed = await collection.findOne({ _id: objectId }); if (!feed) return; let tier = feed.tier; let unmodified = feed.unmodified; if (changed) { tier = Math.max(0, tier - 1); unmodified = 0; } else { unmodified++; if (unmodified >= 2) { tier = Math.min(10, tier + 1); unmodified = 0; } } const minutes = Math.ceil(Math.pow(2, tier)); const nextFetchAt = new Date(Date.now() + minutes * 60 * 1000); updateData = { tier, unmodified, nextFetchAt, // Kept as Date for query compatibility lastFetchedAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }; } else { updateData = { ...extra, lastFetchedAt: new Date().toISOString(), updatedAt: new Date().toISOString(), }; } return collection.findOneAndUpdate( { _id: objectId }, { $set: updateData }, { returnDocument: "after" }, ); } /** * Update feed WebSub subscription * @param {object} application - Indiekit application * @param {ObjectId|string} id - Feed ObjectId * @param {object} websub - WebSub data * @param {string} websub.hub - Hub URL * @param {string} [websub.topic] - Feed topic URL * @param {string} [websub.secret] - Subscription secret * @param {number} [websub.leaseSeconds] - Lease duration * @returns {Promise} Updated feed */ export async function updateFeedWebsub(application, id, websub) { const collection = getCollection(application); const objectId = typeof id === "string" ? new ObjectId(id) : id; const websubData = { hub: websub.hub, topic: websub.topic, }; // Only set these if provided (subscription confirmed) if (websub.secret) { websubData.secret = websub.secret; } if (websub.leaseSeconds) { websubData.leaseSeconds = websub.leaseSeconds; websubData.expiresAt = new Date(Date.now() + websub.leaseSeconds * 1000); } return collection.findOneAndUpdate( { _id: objectId }, { $set: { websub: websubData, updatedAt: new Date().toISOString(), }, }, { returnDocument: "after" }, ); } /** * Get feed by WebSub subscription ID * Used for WebSub callback handling * @param {object} application - Indiekit application * @param {string} subscriptionId - Subscription ID (feed ObjectId as string) * @returns {Promise} Feed or null */ export async function getFeedBySubscriptionId(application, subscriptionId) { return getFeedById(application, subscriptionId); } /** * Update feed status after processing * Tracks health status, errors, and success metrics * @param {object} application - Indiekit application * @param {ObjectId|string} id - Feed ObjectId * @param {object} status - Status update * @param {boolean} status.success - Whether fetch was successful * @param {string} [status.error] - Error message if failed * @param {number} [status.itemCount] - Number of items in feed * @returns {Promise} Updated feed */ export async function updateFeedStatus(application, id, status) { const collection = getCollection(application); const objectId = typeof id === "string" ? new ObjectId(id) : id; const updateFields = { updatedAt: new Date().toISOString(), }; if (status.success) { updateFields.status = "active"; updateFields.lastSuccessAt = new Date().toISOString(); updateFields.consecutiveErrors = 0; updateFields.lastError = undefined; updateFields.lastErrorAt = undefined; if (status.itemCount !== undefined) { updateFields.itemCount = status.itemCount; } } else { updateFields.status = "error"; updateFields.lastError = status.error; updateFields.lastErrorAt = new Date().toISOString(); } // Use $set for most fields, $inc for consecutiveErrors on failure const updateOp = { $set: updateFields }; if (!status.success) { // Increment consecutive errors updateOp.$inc = { consecutiveErrors: 1 }; } return collection.findOneAndUpdate({ _id: objectId }, updateOp, { returnDocument: "after", }); } /** * Get feeds with errors * @param {object} application - Indiekit application * @param {number} [minErrors=3] - Minimum consecutive errors * @returns {Promise} Array of feeds with errors */ export async function getFeedsWithErrors(application, minErrors = 3) { const collection = getCollection(application); return collection .find({ status: "error", consecutiveErrors: { $gte: minErrors }, }) .toArray(); }