diff --git a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts b/fluxer_api/src/api/worker/tasks/HarvestUserData.ts index 70334e5a6..0ae9c7fd9 100644 --- a/fluxer_api/src/api/worker/tasks/HarvestUserData.ts +++ b/fluxer_api/src/api/worker/tasks/HarvestUserData.ts @@ -171,6 +171,10 @@ interface ArchiveResult { downloadUrl: string; } +// A harvest is every message the account wrote, so the read pages to the end of +// the account rather than stopping at a count. The page size is what bounds one +// query, not what bounds the archive. +const HARVEST_MESSAGE_CHUNK_SIZE = 1000; const CONCURRENT_MESSAGE_LIMIT = 10; const INITIAL_PROGRESS = 5; const MESSAGES_PROGRESS_MAX = 55; @@ -231,6 +235,7 @@ async function harvestMessages( listMessagesByAuthor: ( userId: UserID, limit: number, + lastMessageId?: MessageID, ) => Promise< Array<{ channelId: ChannelID; @@ -252,7 +257,19 @@ async function harvestMessages( const channelMessagesMap = new Map>(); Logger.debug('Fetching all user messages'); const startFetchTime = Date.now(); - const messageRefs = await channelRepository.listMessagesByAuthor(userId, 100000); + const messageRefs: Array<{channelId: ChannelID; messageId: MessageID}> = []; + let lastMessageId: MessageID | undefined; + while (true) { + const page = await channelRepository.listMessagesByAuthor(userId, HARVEST_MESSAGE_CHUNK_SIZE, lastMessageId); + if (page.length === 0) { + break; + } + messageRefs.push(...page); + lastMessageId = page[page.length - 1].messageId; + if (page.length < HARVEST_MESSAGE_CHUNK_SIZE) { + break; + } + } Logger.debug( { totalMessages: messageRefs.length, diff --git a/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx b/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx index a64e2d778..e9cb7796d 100644 --- a/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx +++ b/fluxer_docs/src/content/docs/http-api/users/data-harvest.mdx @@ -191,7 +191,7 @@ Fluxer prepares the archive in the background. It contains: Attachment metadata appears with its message and has the attachment ID, filename, size, content type, CDN URL, and pixel dimensions. The attachment files themselves are never included. -At most 100,000 authored messages are collected. Fluxer skips a message it cannot read, and the harvest still succeeds. When the archive completes, Fluxer sends one email containing a download URL if the account has an email address and the instance has email delivery enabled. That URL expires seven days after it was issued. +Every authored message is collected, with no ceiling on the count. Fluxer skips a message it cannot read, and the harvest still succeeds. When the archive completes, Fluxer sends one email containing a download URL if the account has an email address and the instance has email delivery enabled. That URL expires seven days after it was issued. ### Rate limit