import httpSignature from '@peertube/http-signature'; import { v4 as uuid } from 'uuid'; import config from '@/config/index.js'; import { envOption } from '../env.js'; import processDeliver from './processors/deliver.js'; import processInbox from './processors/inbox.js'; import processDb from './processors/db/index.js'; import processObjectStorage from './processors/object-storage/index.js'; import processSystemQueue from './processors/system/index.js'; import processWebhookDeliver from './processors/webhook-deliver.js'; import { endedPollNotification } from './processors/ended-poll-notification.js'; import { queueLogger } from './logger.js'; import { DriveFile } from '@/models/entities/drive-file.js'; import { getJobInfo } from './get-job-info.js'; import { systemQueue, dbQueue, deliverQueue, inboxQueue, objectStorageQueue, endedPollNotificationQueue, webhookDeliverQueue } from './queues.js'; import { ThinUser } from './types.js'; import { IActivity } from '@/remote/activitypub/type.js'; import { Webhook, webhookEventTypes } from '@/models/entities/webhook.js'; function renderError(e: Error): any { return { stack: e?.stack, message: e?.message, name: e?.name, }; } const systemLogger = queueLogger.createSubLogger('system'); const deliverLogger = queueLogger.createSubLogger('deliver'); const webhookLogger = queueLogger.createSubLogger('webhook'); const inboxLogger = queueLogger.createSubLogger('inbox'); const dbLogger = queueLogger.createSubLogger('db'); const objectStorageLogger = queueLogger.createSubLogger('objectStorage'); systemQueue .on('waiting', (jobId) => systemLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => systemLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => systemLogger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => systemLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) .on('error', (job: any, err: Error) => systemLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => systemLogger.warn(`stalled id=${job.id}`)); deliverQueue .on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => deliverLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => deliverLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => deliverLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) .on('error', (job: any, err: Error) => deliverLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => deliverLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); inboxQueue .on('waiting', (jobId) => inboxLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => inboxLogger.debug(`active ${getJobInfo(job, true)}`)) .on('completed', (job, result) => inboxLogger.debug(`completed(${result}) ${getJobInfo(job, true)}`)) .on('failed', (job, err) => inboxLogger.warn(`failed(${err}) ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`, { job, e: renderError(err) })) .on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => inboxLogger.warn(`stalled ${getJobInfo(job)} activity=${job.data.activity ? job.data.activity.id : 'none'}`)); dbQueue .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); objectStorageQueue .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`)); webhookDeliverQueue .on('waiting', (jobId) => webhookLogger.debug(`waiting id=${jobId}`)) .on('active', (job) => webhookLogger.debug(`active ${getJobInfo(job, true)} to=${job.data.to}`)) .on('completed', (job, result) => webhookLogger.debug(`completed(${result}) ${getJobInfo(job, true)} to=${job.data.to}`)) .on('failed', (job, err) => webhookLogger.warn(`failed(${err}) ${getJobInfo(job)} to=${job.data.to}`)) .on('error', (job: any, err: Error) => webhookLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => webhookLogger.warn(`stalled ${getJobInfo(job)} to=${job.data.to}`)); export function deliver(user: ThinUser, content: unknown, to: string | null) { if (content == null) return null; if (to == null) return null; const data = { user: { id: user.id, }, content, to, }; return deliverQueue.add(data, { attempts: config.deliverJobMaxAttempts || 12, timeout: 1 * 60 * 1000, // 1min backoff: { type: 'apBackoff', }, removeOnComplete: true, removeOnFail: true, }); } export function inbox(activity: IActivity, signature: httpSignature.IParsedSignature) { const data = { activity: activity, signature, }; return inboxQueue.add(data, { attempts: config.inboxJobMaxAttempts || 8, timeout: 5 * 60 * 1000, // 5min backoff: { type: 'apBackoff', }, removeOnComplete: true, removeOnFail: true, }); } export function createDeleteDriveFilesJob(user: ThinUser) { return dbQueue.add('deleteDriveFiles', { user: user, }, { removeOnComplete: true, removeOnFail: true, }); } export function createExportCustomEmojisJob(user: ThinUser) { return dbQueue.add('exportCustomEmojis', { user: user, }, { removeOnComplete: true, removeOnFail: true, }); } export function createExportNotesJob(user: ThinUser) { return dbQueue.add('exportNotes', { user: user, }, { removeOnComplete: true, removeOnFail: true, }); } export function createExportFollowingJob(user: ThinUser, excludeMuting = false, excludeInactive = false) { return dbQueue.add('exportFollowing', { user: user, excludeMuting, excludeInactive, }, { removeOnComplete: true, removeOnFail: true, }); } export function createExportMuteJob(user: ThinUser) { return dbQueue.add('exportMute', { user: user, }, { removeOnComplete: true, removeOnFail: true, }); } export function createExportBlockingJob(user: ThinUser) { return dbQueue.add('exportBlocking', { user: user, }, { removeOnComplete: true, removeOnFail: true, }); } export function createExportUserListsJob(user: ThinUser) { return dbQueue.add('exportUserLists', { user: user, }, { removeOnComplete: true, removeOnFail: true, }); } export function createImportFollowingJob(user: ThinUser, fileId: DriveFile['id']) { return dbQueue.add('importFollowing', { user: user, fileId: fileId, }, { removeOnComplete: true, removeOnFail: true, }); } export function createImportMutingJob(user: ThinUser, fileId: DriveFile['id']) { return dbQueue.add('importMuting', { user: user, fileId: fileId, }, { removeOnComplete: true, removeOnFail: true, }); } export function createImportBlockingJob(user: ThinUser, fileId: DriveFile['id']) { return dbQueue.add('importBlocking', { user: user, fileId: fileId, }, { removeOnComplete: true, removeOnFail: true, }); } export function createImportUserListsJob(user: ThinUser, fileId: DriveFile['id']) { return dbQueue.add('importUserLists', { user: user, fileId: fileId, }, { removeOnComplete: true, removeOnFail: true, }); } export function createImportCustomEmojisJob(user: ThinUser, fileId: DriveFile['id']) { return dbQueue.add('importCustomEmojis', { user: user, fileId: fileId, }, { removeOnComplete: true, removeOnFail: true, }); } export function createDeleteAccountJob(user: ThinUser, opts: { soft?: boolean; } = {}) { return dbQueue.add('deleteAccount', { user: user, soft: opts.soft, }, { removeOnComplete: true, removeOnFail: true, }); } export function createDeleteObjectStorageFileJob(key: string) { return objectStorageQueue.add('deleteFile', { key: key, }, { removeOnComplete: true, removeOnFail: true, }); } export function createCleanRemoteFilesJob() { return objectStorageQueue.add('cleanRemoteFiles', {}, { removeOnComplete: true, removeOnFail: true, }); } export function webhookDeliver(webhook: Webhook, type: typeof webhookEventTypes[number], content: unknown) { const data = { type, content, webhookId: webhook.id, userId: webhook.userId, to: webhook.url, secret: webhook.secret, createdAt: Date.now(), eventId: uuid(), }; return webhookDeliverQueue.add(data, { attempts: 4, timeout: 1 * 60 * 1000, // 1min backoff: { type: 'apBackoff', }, removeOnComplete: true, removeOnFail: true, }); } export default function() { if (envOption.onlyServer) return; deliverQueue.process(config.deliverJobConcurrency || 128, processDeliver); inboxQueue.process(config.inboxJobConcurrency || 16, processInbox); endedPollNotificationQueue.process(endedPollNotification); webhookDeliverQueue.process(64, processWebhookDeliver); processDb(dbQueue); processObjectStorage(objectStorageQueue); systemQueue.add('tickCharts', { }, { repeat: { cron: '55 * * * *' }, removeOnComplete: true, }); systemQueue.add('resyncCharts', { }, { repeat: { cron: '0 0 * * *' }, removeOnComplete: true, }); systemQueue.add('cleanCharts', { }, { repeat: { cron: '0 0 * * *' }, removeOnComplete: true, }); systemQueue.add('checkExpiredMutings', { }, { repeat: { cron: '*/5 * * * *' }, removeOnComplete: true, }); processSystemQueue(systemQueue); } export function destroy() { deliverQueue.once('cleaned', (jobs, status) => { deliverLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); deliverQueue.clean(0, 'delayed'); inboxQueue.once('cleaned', (jobs, status) => { inboxLogger.succ(`Cleaned ${jobs.length} ${status} jobs`); }); inboxQueue.clean(0, 'delayed'); }