From 7c7f32d9a6597fdc7bea02da0cfd4a843fd32d22 Mon Sep 17 00:00:00 2001 From: syuilo Date: Tue, 23 Oct 2018 05:36:35 +0900 Subject: [PATCH] Refactoring --- src/chart/drive.ts | 122 ++ src/chart/hashtag.ts | 37 + src/chart/index.ts | 285 +++++ src/chart/network.ts | 64 + src/chart/notes.ts | 114 ++ src/chart/per-user-drive.ts | 101 ++ src/chart/per-user-following.ts | 128 ++ src/chart/per-user-notes.ts | 94 ++ src/chart/per-user-reactions.ts | 45 + src/chart/users.ts | 75 ++ src/remote/activitypub/models/person.ts | 4 +- src/server/api/endpoints/charts/drive.ts | 6 +- src/server/api/endpoints/charts/hashtag.ts | 6 +- src/server/api/endpoints/charts/network.ts | 6 +- src/server/api/endpoints/charts/notes.ts | 6 +- src/server/api/endpoints/charts/user/drive.ts | 6 +- .../api/endpoints/charts/user/following.ts | 6 +- src/server/api/endpoints/charts/user/notes.ts | 6 +- .../api/endpoints/charts/user/reactions.ts | 6 +- src/server/api/endpoints/charts/users.ts | 6 +- src/server/api/private/signup.ts | 4 +- src/server/index.ts | 4 +- src/services/drive/add-file.ts | 7 +- src/services/drive/delete-file.ts | 7 +- src/services/following/create.ts | 4 +- src/services/following/delete.ts | 4 +- src/services/following/requests/accept.ts | 4 +- src/services/note/create.ts | 8 +- src/services/note/delete.ts | 7 +- src/services/note/reaction/create.ts | 4 +- src/services/register-hashtag.ts | 4 +- src/services/stats.ts | 1056 ----------------- 32 files changed, 1125 insertions(+), 1111 deletions(-) create mode 100644 src/chart/drive.ts create mode 100644 src/chart/hashtag.ts create mode 100644 src/chart/index.ts create mode 100644 src/chart/network.ts create mode 100644 src/chart/notes.ts create mode 100644 src/chart/per-user-drive.ts create mode 100644 src/chart/per-user-following.ts create mode 100644 src/chart/per-user-notes.ts create mode 100644 src/chart/per-user-reactions.ts create mode 100644 src/chart/users.ts delete mode 100644 src/services/stats.ts diff --git a/src/chart/drive.ts b/src/chart/drive.ts new file mode 100644 index 000000000..ff454c750 --- /dev/null +++ b/src/chart/drive.ts @@ -0,0 +1,122 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj } from './'; +import DriveFile, { IDriveFile } from '../models/drive-file'; +import { isLocalUser } from '../models/user'; + +/** + * ドライブに関するチャート + */ +type DriveLog = { + local: { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: number; + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: number; + + /** + * 増加したドライブファイル数 + */ + incCount: number; + + /** + * 増加したドライブ使用量 + */ + incSize: number; + + /** + * 減少したドライブファイル数 + */ + decCount: number; + + /** + * 減少したドライブ使用量 + */ + decSize: number; + }; + + remote: DriveLog['local']; +}; + +class DriveChart extends Chart { + constructor() { + super('drive'); + } + + @autobind + protected async getTemplate(init: boolean, latest?: DriveLog): Promise { + const calcSize = (local: boolean) => DriveFile + .aggregate([{ + $match: { + 'metadata._user.host': local ? null : { $ne: null }, + 'metadata.deletedAt': { $exists: false } + } + }, { + $project: { + length: true + } + }, { + $group: { + _id: null, + usage: { $sum: '$length' } + } + }]) + .then(res => res.length > 0 ? res[0].usage : 0); + + const [localCount, remoteCount, localSize, remoteSize] = init ? await Promise.all([ + DriveFile.count({ 'metadata._user.host': null }), + DriveFile.count({ 'metadata._user.host': { $ne: null } }), + calcSize(true), + calcSize(false) + ]) : [ + latest ? latest.local.totalCount : 0, + latest ? latest.remote.totalCount : 0, + latest ? latest.local.totalSize : 0, + latest ? latest.remote.totalSize : 0 + ]; + + return { + local: { + totalCount: localCount, + totalSize: localSize, + incCount: 0, + incSize: 0, + decCount: 0, + decSize: 0 + }, + remote: { + totalCount: remoteCount, + totalSize: remoteSize, + incCount: 0, + incSize: 0, + decCount: 0, + decSize: 0 + } + }; + } + + @autobind + public async update(file: IDriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.length : -file.length; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.length; + } else { + update.decCount = 1; + update.decSize = file.length; + } + + await this.inc({ + [isLocalUser(file.metadata._user) ? 'local' : 'remote']: update + }); + } +} + +export default new DriveChart(); diff --git a/src/chart/hashtag.ts b/src/chart/hashtag.ts new file mode 100644 index 000000000..976fd0c84 --- /dev/null +++ b/src/chart/hashtag.ts @@ -0,0 +1,37 @@ +import autobind from 'autobind-decorator'; +import * as mongo from 'mongodb'; +import Chart, { Partial } from './'; + +/** + * ハッシュタグに関するチャート + */ +type HashtagLog = { + /** + * 投稿された数 + */ + count: number; +}; + +class HashtagChart extends Chart { + constructor() { + super('hashtag', true); + } + + @autobind + protected async getTemplate(init: boolean, latest?: HashtagLog): Promise { + return { + count: 0 + }; + } + + @autobind + public async update(hashtag: string, userId: mongo.ObjectId) { + const inc: Partial = { + count: 1 + }; + + await this.incIfUnique(inc, 'users', userId.toHexString(), hashtag); + } +} + +export default new HashtagChart(); diff --git a/src/chart/index.ts b/src/chart/index.ts new file mode 100644 index 000000000..48934dba3 --- /dev/null +++ b/src/chart/index.ts @@ -0,0 +1,285 @@ +/** + * チャートエンジン + */ + +const nestedProperty = require('nested-property'); +import autobind from 'autobind-decorator'; +import * as mongo from 'mongodb'; +import db from '../db/mongodb'; +import { ICollection } from 'monk'; + +export type Obj = { [key: string]: any }; + +export type Partial = { + [P in keyof T]?: Partial; +}; + +type ArrayValue = { + [P in keyof T]: T[P] extends number ? Array : ArrayValue; +}; + +type Span = 'day' | 'hour'; + +//#region Chart Core +type Log = { + _id: mongo.ObjectID; + + /** + * 集計のグループ + */ + group?: any; + + /** + * 集計日時 + */ + date: Date; + + /** + * 集計期間 + */ + span: Span; + + /** + * データ + */ + data: T; + + /** + * ユニークインクリメント用 + */ + unique?: Obj; +}; + +/** + * 様々なチャートの管理を司るクラス + */ +export default abstract class Chart { + protected collection: ICollection>; + protected abstract async getTemplate(init: boolean, latest?: T, group?: any): Promise; + + constructor(name: string, grouped = false) { + this.collection = db.get>(`chart.${name}`); + if (grouped) { + this.collection.createIndex({ span: -1, date: -1, group: -1 }, { unique: true }); + } else { + this.collection.createIndex({ span: -1, date: -1 }, { unique: true }); + } + } + + @autobind + private convertQuery(x: Obj, path: string): Obj { + const query: Obj = {}; + + const dive = (x: Obj, path: string) => { + Object.entries(x).forEach(([k, v]) => { + const p = path ? `${path}.${k}` : k; + if (typeof v === 'number') { + query[p] = v; + } else { + dive(v, p); + } + }); + }; + + dive(x, path); + + return query; + } + + @autobind + private async getCurrentLog(span: Span, group?: any): Promise> { + const now = new Date(); + const y = now.getFullYear(); + const m = now.getMonth(); + const d = now.getDate(); + const h = now.getHours(); + + const current = + span == 'day' ? new Date(y, m, d) : + span == 'hour' ? new Date(y, m, d, h) : + null; + + // 現在(今日または今のHour)のログ + const currentLog = await this.collection.findOne({ + group: group, + span: span, + date: current + }); + + if (currentLog) { + return currentLog; + } + + // 集計期間が変わってから、初めてのチャート更新なら + // 最も最近のログを持ってくる + // * 例えば集計期間が「日」である場合で考えると、 + // * 昨日何もチャートを更新するような出来事がなかった場合は、 + // * ログがそもそも作られずドキュメントが存在しないということがあり得るため、 + // * 「昨日の」と決め打ちせずに「もっとも最近の」とします + const latest = await this.collection.findOne({ + group: group, + span: span + }, { + sort: { + date: -1 + } + }); + + if (latest) { + // 現在のログを初期挿入 + const data = await this.getTemplate(false, latest.data); + + const log = await this.collection.insert({ + group: group, + span: span, + date: current, + data: data + }); + + return log; + } else { + // ログが存在しなかったら + // * Misskeyインスタンスを建てて初めてのチャート更新時など + + // 空のログを作成 + const data = await this.getTemplate(true, null, group); + + const log = await this.collection.insert({ + group: group, + span: span, + date: current, + data: data + }); + + return log; + } + } + + @autobind + protected commit(query: Obj, group?: any, uniqueKey?: string, uniqueValue?: string): void { + const update = (log: Log) => { + // ユニークインクリメントの場合、指定のキーに指定の値が既に存在していたら弾く + if ( + uniqueKey && + log.unique && + log.unique[uniqueKey] && + log.unique[uniqueKey].includes(uniqueValue) + ) return; + + // ユニークインクリメントの指定のキーに値を追加 + if (uniqueKey) { + query['$push'] = { + [`unique.${uniqueKey}`]: uniqueValue + }; + } + + this.collection.update({ + _id: log._id + }, query); + }; + + this.getCurrentLog('day', group).then(log => update(log)); + this.getCurrentLog('hour', group).then(log => update(log)); + } + + @autobind + protected inc(inc: Partial, group?: any): void { + this.commit({ + $inc: this.convertQuery(inc, 'data') + }, group); + } + + @autobind + protected incIfUnique(inc: Partial, key: string, value: string, group?: any): void { + this.commit({ + $inc: this.convertQuery(inc, 'data') + }, group, key, value); + } + + @autobind + public async getChart(span: Span, range: number, group?: any): Promise> { + const promisedChart: Promise[] = []; + + const now = new Date(); + const y = now.getFullYear(); + const m = now.getMonth(); + const d = now.getDate(); + const h = now.getHours(); + + const gt = + span == 'day' ? new Date(y, m, d - range) : + span == 'hour' ? new Date(y, m, d, h - range) : null; + + const logs = await this.collection.find({ + group: group, + span: span, + date: { + $gt: gt + } + }, { + sort: { + date: -1 + }, + fields: { + _id: 0 + } + }); + + for (let i = (range - 1); i >= 0; i--) { + const current = + span == 'day' ? new Date(y, m, d - i) : + span == 'hour' ? new Date(y, m, d, h - i) : + null; + + const log = logs.find(l => l.date.getTime() == current.getTime()); + + if (log) { + promisedChart.unshift(Promise.resolve(log.data)); + } else { // 隙間埋め + const latest = logs.find(l => l.date.getTime() < current.getTime()); + promisedChart.unshift(this.getTemplate(false, latest ? latest.data : null)); + } + } + + const chart = await Promise.all(promisedChart); + + const res: ArrayValue = {} as any; + + /** + * [{ + * xxxxx: 1, + * yyyyy: 5 + * }, { + * xxxxx: 2, + * yyyyy: 6 + * }, { + * xxxxx: 3, + * yyyyy: 7 + * }] + * + * を + * + * { + * xxxxx: [1, 2, 3], + * yyyyy: [5, 6, 7] + * } + * + * にする + */ + const dive = (x: Obj, path?: string) => { + Object.entries(x).forEach(([k, v]) => { + const p = path ? `${path}.${k}` : k; + if (typeof v == 'object') { + dive(v, p); + } else { + nestedProperty.set(res, p, chart.map(s => nestedProperty.get(s, p))); + } + }); + }; + + dive(chart[0]); + + return res; + } +} +//#endregion diff --git a/src/chart/network.ts b/src/chart/network.ts new file mode 100644 index 000000000..fce47099d --- /dev/null +++ b/src/chart/network.ts @@ -0,0 +1,64 @@ +import autobind from 'autobind-decorator'; +import Chart, { Partial } from './'; + +/** + * ネットワークに関するチャート + */ +type NetworkLog = { + /** + * 受信したリクエスト数 + */ + incomingRequests: number; + + /** + * 送信したリクエスト数 + */ + outgoingRequests: number; + + /** + * 応答時間の合計 + * TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる + */ + totalTime: number; + + /** + * 合計受信データ量 + */ + incomingBytes: number; + + /** + * 合計送信データ量 + */ + outgoingBytes: number; +}; + +class NetworkChart extends Chart { + constructor() { + super('network'); + } + + @autobind + protected async getTemplate(init: boolean, latest?: NetworkLog): Promise { + return { + incomingRequests: 0, + outgoingRequests: 0, + totalTime: 0, + incomingBytes: 0, + outgoingBytes: 0 + }; + } + + @autobind + public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) { + const inc: Partial = { + incomingRequests: incomingRequests, + totalTime: time, + incomingBytes: incomingBytes, + outgoingBytes: outgoingBytes + }; + + await this.inc(inc); + } +} + +export default new NetworkChart(); diff --git a/src/chart/notes.ts b/src/chart/notes.ts new file mode 100644 index 000000000..738778e72 --- /dev/null +++ b/src/chart/notes.ts @@ -0,0 +1,114 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj } from '.'; +import Note, { INote } from '../models/note'; +import { isLocalUser } from '../models/user'; + +/** + * 投稿に関するチャート + */ +type NotesLog = { + local: { + /** + * 集計期間時点での、全投稿数 + */ + total: number; + + /** + * 増加した投稿数 + */ + inc: number; + + /** + * 減少した投稿数 + */ + dec: number; + + diffs: { + /** + * 通常の投稿数の差分 + */ + normal: number; + + /** + * リプライの投稿数の差分 + */ + reply: number; + + /** + * Renoteの投稿数の差分 + */ + renote: number; + }; + }; + + remote: NotesLog['local']; +}; + +class NotesChart extends Chart { + constructor() { + super('notes'); + } + + @autobind + protected async getTemplate(init: boolean, latest?: NotesLog): Promise { + const [localCount, remoteCount] = init ? await Promise.all([ + Note.count({ '_user.host': null }), + Note.count({ '_user.host': { $ne: null } }) + ]) : [ + latest ? latest.local.total : 0, + latest ? latest.remote.total : 0 + ]; + + return { + local: { + total: localCount, + inc: 0, + dec: 0, + diffs: { + normal: 0, + reply: 0, + renote: 0 + } + }, + remote: { + total: remoteCount, + inc: 0, + dec: 0, + diffs: { + normal: 0, + reply: 0, + renote: 0 + } + } + }; + } + + @autobind + public async update(note: INote, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc({ + [isLocalUser(note._user) ? 'local' : 'remote']: update + }); + } +} + +export default new NotesChart(); diff --git a/src/chart/per-user-drive.ts b/src/chart/per-user-drive.ts new file mode 100644 index 000000000..3decedeb3 --- /dev/null +++ b/src/chart/per-user-drive.ts @@ -0,0 +1,101 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj } from './'; +import DriveFile, { IDriveFile } from '../models/drive-file'; + +/** + * ユーザーごとのドライブに関するチャート + */ +type PerUserDriveLog = { + /** + * 集計期間時点での、全ドライブファイル数 + */ + totalCount: number; + + /** + * 集計期間時点での、全ドライブファイルの合計サイズ + */ + totalSize: number; + + /** + * 増加したドライブファイル数 + */ + incCount: number; + + /** + * 増加したドライブ使用量 + */ + incSize: number; + + /** + * 減少したドライブファイル数 + */ + decCount: number; + + /** + * 減少したドライブ使用量 + */ + decSize: number; +}; + +class PerUserDriveChart extends Chart { + constructor() { + super('perUserDrive', true); + } + + @autobind + protected async getTemplate(init: boolean, latest?: PerUserDriveLog, group?: any): Promise { + const calcSize = () => DriveFile + .aggregate([{ + $match: { + 'metadata.userId': group, + 'metadata.deletedAt': { $exists: false } + } + }, { + $project: { + length: true + } + }, { + $group: { + _id: null, + usage: { $sum: '$length' } + } + }]) + .then(res => res.length > 0 ? res[0].usage : 0); + + const [count, size] = init ? await Promise.all([ + DriveFile.count({ 'metadata.userId': group }), + calcSize() + ]) : [ + latest ? latest.totalCount : 0, + latest ? latest.totalSize : 0 + ]; + + return { + totalCount: count, + totalSize: size, + incCount: 0, + incSize: 0, + decCount: 0, + decSize: 0 + }; + } + + @autobind + public async update(file: IDriveFile, isAdditional: boolean) { + const update: Obj = {}; + + update.totalCount = isAdditional ? 1 : -1; + update.totalSize = isAdditional ? file.length : -file.length; + if (isAdditional) { + update.incCount = 1; + update.incSize = file.length; + } else { + update.decCount = 1; + update.decSize = file.length; + } + + await this.inc(update, file.metadata.userId); + } +} + +export default new PerUserDriveChart(); diff --git a/src/chart/per-user-following.ts b/src/chart/per-user-following.ts new file mode 100644 index 000000000..fac4a1619 --- /dev/null +++ b/src/chart/per-user-following.ts @@ -0,0 +1,128 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj } from './'; +import Following from '../models/following'; +import { IUser, isLocalUser } from '../models/user'; + +/** + * ユーザーごとのフォローに関するチャート + */ +type PerUserFollowingLog = { + local: { + /** + * フォローしている + */ + followings: { + /** + * 合計 + */ + total: number; + + /** + * フォローした数 + */ + inc: number; + + /** + * フォロー解除した数 + */ + dec: number; + }; + + /** + * フォローされている + */ + followers: { + /** + * 合計 + */ + total: number; + + /** + * フォローされた数 + */ + inc: number; + + /** + * フォロー解除された数 + */ + dec: number; + }; + }; + + remote: PerUserFollowingLog['local']; +}; + +class PerUserFollowingChart extends Chart { + constructor() { + super('perUserFollowing', true); + } + + @autobind + protected async getTemplate(init: boolean, latest?: PerUserFollowingLog, group?: any): Promise { + const [ + localFollowingsCount, + localFollowersCount, + remoteFollowingsCount, + remoteFollowersCount + ] = init ? await Promise.all([ + Following.count({ followerId: group, '_followee.host': null }), + Following.count({ followeeId: group, '_follower.host': null }), + Following.count({ followerId: group, '_followee.host': { $ne: null } }), + Following.count({ followeeId: group, '_follower.host': { $ne: null } }) + ]) : [ + latest ? latest.local.followings.total : 0, + latest ? latest.local.followers.total : 0, + latest ? latest.remote.followings.total : 0, + latest ? latest.remote.followers.total : 0 + ]; + + return { + local: { + followings: { + total: localFollowingsCount, + inc: 0, + dec: 0 + }, + followers: { + total: localFollowersCount, + inc: 0, + dec: 0 + } + }, + remote: { + followings: { + total: remoteFollowingsCount, + inc: 0, + dec: 0 + }, + followers: { + total: remoteFollowersCount, + inc: 0, + dec: 0 + } + } + }; + } + + @autobind + public async update(follower: IUser, followee: IUser, isFollow: boolean) { + const update: Obj = {}; + + update.total = isFollow ? 1 : -1; + + if (isFollow) { + update.inc = 1; + } else { + update.dec = 1; + } + + this.inc({ + [isLocalUser(follower) ? 'local' : 'remote']: { followings: update } + }, follower._id); + this.inc({ + [isLocalUser(followee) ? 'local' : 'remote']: { followers: update } + }, followee._id); + } +} + +export default new PerUserFollowingChart(); diff --git a/src/chart/per-user-notes.ts b/src/chart/per-user-notes.ts new file mode 100644 index 000000000..9558f5c83 --- /dev/null +++ b/src/chart/per-user-notes.ts @@ -0,0 +1,94 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj } from './'; +import Note, { INote } from '../models/note'; +import { IUser } from '../models/user'; + +/** + * ユーザーごとの投稿に関するチャート + */ +type PerUserNotesLog = { + /** + * 集計期間時点での、全投稿数 + */ + total: number; + + /** + * 増加した投稿数 + */ + inc: number; + + /** + * 減少した投稿数 + */ + dec: number; + + diffs: { + /** + * 通常の投稿数の差分 + */ + normal: number; + + /** + * リプライの投稿数の差分 + */ + reply: number; + + /** + * Renoteの投稿数の差分 + */ + renote: number; + }; +}; + +class PerUserNotesChart extends Chart { + constructor() { + super('perUserNotes', true); + } + + @autobind + protected async getTemplate(init: boolean, latest?: PerUserNotesLog, group?: any): Promise { + const [count] = init ? await Promise.all([ + Note.count({ userId: group, deletedAt: null }), + ]) : [ + latest ? latest.total : 0 + ]; + + return { + total: count, + inc: 0, + dec: 0, + diffs: { + normal: 0, + reply: 0, + renote: 0 + } + }; + } + + @autobind + public async update(user: IUser, note: INote, isAdditional: boolean) { + const update: Obj = { + diffs: {} + }; + + update.total = isAdditional ? 1 : -1; + + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + if (note.replyId != null) { + update.diffs.reply = isAdditional ? 1 : -1; + } else if (note.renoteId != null) { + update.diffs.renote = isAdditional ? 1 : -1; + } else { + update.diffs.normal = isAdditional ? 1 : -1; + } + + await this.inc(update, user._id); + } +} + +export default new PerUserNotesChart(); diff --git a/src/chart/per-user-reactions.ts b/src/chart/per-user-reactions.ts new file mode 100644 index 000000000..a31952ea2 --- /dev/null +++ b/src/chart/per-user-reactions.ts @@ -0,0 +1,45 @@ +import autobind from 'autobind-decorator'; +import Chart from './'; +import { IUser, isLocalUser } from '../models/user'; +import { INote } from '../models/note'; + +/** + * ユーザーごとのリアクションに関するチャート + */ +type PerUserReactionsLog = { + local: { + /** + * リアクションされた数 + */ + count: number; + }; + + remote: PerUserReactionsLog['local']; +}; + +class PerUserReactionsChart extends Chart { + constructor() { + super('perUserReaction', true); + } + + @autobind + protected async getTemplate(init: boolean, latest?: PerUserReactionsLog, group?: any): Promise { + return { + local: { + count: 0 + }, + remote: { + count: 0 + } + }; + } + + @autobind + public async update(user: IUser, note: INote) { + this.inc({ + [isLocalUser(user) ? 'local' : 'remote']: { count: 1 } + }, note.userId); + } +} + +export default new PerUserReactionsChart(); diff --git a/src/chart/users.ts b/src/chart/users.ts new file mode 100644 index 000000000..547e595b0 --- /dev/null +++ b/src/chart/users.ts @@ -0,0 +1,75 @@ +import autobind from 'autobind-decorator'; +import Chart, { Obj } from './'; +import User, { IUser, isLocalUser } from '../models/user'; + +/** + * ユーザーに関するチャート + */ +type UsersLog = { + local: { + /** + * 集計期間時点での、全ユーザー数 + */ + total: number; + + /** + * 増加したユーザー数 + */ + inc: number; + + /** + * 減少したユーザー数 + */ + dec: number; + }; + + remote: UsersLog['local']; +}; + +class UsersChart extends Chart { + constructor() { + super('users'); + } + + @autobind + protected async getTemplate(init: boolean, latest?: UsersLog): Promise { + const [localCount, remoteCount] = init ? await Promise.all([ + User.count({ host: null }), + User.count({ host: { $ne: null } }) + ]) : [ + latest ? latest.local.total : 0, + latest ? latest.remote.total : 0 + ]; + + return { + local: { + total: localCount, + inc: 0, + dec: 0 + }, + remote: { + total: remoteCount, + inc: 0, + dec: 0 + } + }; + } + + @autobind + public async update(user: IUser, isAdditional: boolean) { + const update: Obj = {}; + + update.total = isAdditional ? 1 : -1; + if (isAdditional) { + update.inc = 1; + } else { + update.dec = 1; + } + + await this.inc({ + [isLocalUser(user) ? 'local' : 'remote']: update + }); + } +} + +export default new UsersChart(); diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts index bf1df8791..a39c8d33c 100644 --- a/src/remote/activitypub/models/person.ts +++ b/src/remote/activitypub/models/person.ts @@ -10,7 +10,7 @@ import { isCollectionOrOrderedCollection, isCollection, IPerson } from '../type' import { IDriveFile } from '../../../models/drive-file'; import Meta from '../../../models/meta'; import htmlToMFM from '../../../mfm/html-to-mfm'; -import { usersStats } from '../../../services/stats'; +import usersChart from '../../../chart/users'; import { URL } from 'url'; import { resolveNote } from './note'; @@ -180,7 +180,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await driveStats.getChart(ps.span as any, ps.limit); + const stats = await driveChart.getChart(ps.span as any, ps.limit); res(stats); }); diff --git a/src/server/api/endpoints/charts/hashtag.ts b/src/server/api/endpoints/charts/hashtag.ts index b42bc97ef..c97a8c11d 100644 --- a/src/server/api/endpoints/charts/hashtag.ts +++ b/src/server/api/endpoints/charts/hashtag.ts @@ -1,10 +1,10 @@ import $ from 'cafy'; import getParams from '../../get-params'; -import { hashtagStats } from '../../../../services/stats'; +import hashtagChart from '../../../../chart/hashtag'; export const meta = { desc: { - 'ja-JP': 'ハッシュタグごとの統計を取得します。' + 'ja-JP': 'ハッシュタグごとのチャートを取得します。' }, params: { @@ -33,7 +33,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await hashtagStats.getChart(ps.span as any, ps.limit, ps.tag); + const stats = await hashtagChart.getChart(ps.span as any, ps.limit, ps.tag); res(stats); }); diff --git a/src/server/api/endpoints/charts/network.ts b/src/server/api/endpoints/charts/network.ts index 49d87bfc2..ed3a9c232 100644 --- a/src/server/api/endpoints/charts/network.ts +++ b/src/server/api/endpoints/charts/network.ts @@ -1,10 +1,10 @@ import $ from 'cafy'; import getParams from '../../get-params'; -import { networkStats } from '../../../../services/stats'; +import networkChart from '../../../../chart/network'; export const meta = { desc: { - 'ja-JP': 'ネットワークの統計を取得します。' + 'ja-JP': 'ネットワークのチャートを取得します。' }, params: { @@ -27,7 +27,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await networkStats.getChart(ps.span as any, ps.limit); + const stats = await networkChart.getChart(ps.span as any, ps.limit); res(stats); }); diff --git a/src/server/api/endpoints/charts/notes.ts b/src/server/api/endpoints/charts/notes.ts index a9dc06826..b24bfc638 100644 --- a/src/server/api/endpoints/charts/notes.ts +++ b/src/server/api/endpoints/charts/notes.ts @@ -1,10 +1,10 @@ import $ from 'cafy'; import getParams from '../../get-params'; -import { notesStats } from '../../../../services/stats'; +import notesChart from '../../../../chart/notes'; export const meta = { desc: { - 'ja-JP': '投稿の統計を取得します。' + 'ja-JP': '投稿のチャートを取得します。' }, params: { @@ -27,7 +27,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await notesStats.getChart(ps.span as any, ps.limit); + const stats = await notesChart.getChart(ps.span as any, ps.limit); res(stats); }); diff --git a/src/server/api/endpoints/charts/user/drive.ts b/src/server/api/endpoints/charts/user/drive.ts index d32088795..092f697f5 100644 --- a/src/server/api/endpoints/charts/user/drive.ts +++ b/src/server/api/endpoints/charts/user/drive.ts @@ -1,11 +1,11 @@ import $ from 'cafy'; import getParams from '../../../get-params'; -import { perUserDriveStats } from '../../../../../services/stats'; +import perUserDriveChart from '../../../../../chart/per-user-drive'; import ID from '../../../../../misc/cafy-id'; export const meta = { desc: { - 'ja-JP': 'ユーザーごとのドライブの統計を取得します。' + 'ja-JP': 'ユーザーごとのドライブのチャートを取得します。' }, params: { @@ -35,7 +35,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await perUserDriveStats.getChart(ps.span as any, ps.limit, ps.userId); + const stats = await perUserDriveChart.getChart(ps.span as any, ps.limit, ps.userId); res(stats); }); diff --git a/src/server/api/endpoints/charts/user/following.ts b/src/server/api/endpoints/charts/user/following.ts index dbb2b46df..7918b9a9d 100644 --- a/src/server/api/endpoints/charts/user/following.ts +++ b/src/server/api/endpoints/charts/user/following.ts @@ -1,11 +1,11 @@ import $ from 'cafy'; import getParams from '../../../get-params'; -import { perUserFollowingStats } from '../../../../../services/stats'; +import perUserFollowingChart from '../../../../../chart/per-user-following'; import ID from '../../../../../misc/cafy-id'; export const meta = { desc: { - 'ja-JP': 'ユーザーごとのフォロー/フォロワーの統計を取得します。' + 'ja-JP': 'ユーザーごとのフォロー/フォロワーのチャートを取得します。' }, params: { @@ -35,7 +35,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await perUserFollowingStats.getChart(ps.span as any, ps.limit, ps.userId); + const stats = await perUserFollowingChart.getChart(ps.span as any, ps.limit, ps.userId); res(stats); }); diff --git a/src/server/api/endpoints/charts/user/notes.ts b/src/server/api/endpoints/charts/user/notes.ts index a256ed96f..cd028d88a 100644 --- a/src/server/api/endpoints/charts/user/notes.ts +++ b/src/server/api/endpoints/charts/user/notes.ts @@ -1,11 +1,11 @@ import $ from 'cafy'; import getParams from '../../../get-params'; -import { perUserNotesStats } from '../../../../../services/stats'; +import perUserNotesChart from '../../../../../chart/per-user-notes'; import ID from '../../../../../misc/cafy-id'; export const meta = { desc: { - 'ja-JP': 'ユーザーごとの投稿の統計を取得します。' + 'ja-JP': 'ユーザーごとの投稿のチャートを取得します。' }, params: { @@ -35,7 +35,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await perUserNotesStats.getChart(ps.span as any, ps.limit, ps.userId); + const stats = await perUserNotesChart.getChart(ps.span as any, ps.limit, ps.userId); res(stats); }); diff --git a/src/server/api/endpoints/charts/user/reactions.ts b/src/server/api/endpoints/charts/user/reactions.ts index 1d1e7d5a4..8632044ff 100644 --- a/src/server/api/endpoints/charts/user/reactions.ts +++ b/src/server/api/endpoints/charts/user/reactions.ts @@ -1,11 +1,11 @@ import $ from 'cafy'; import getParams from '../../../get-params'; -import { perUserReactionsStats } from '../../../../../services/stats'; +import perUserReactionsChart from '../../../../../chart/per-user-reactions'; import ID from '../../../../../misc/cafy-id'; export const meta = { desc: { - 'ja-JP': 'ユーザーごとの被リアクション数の統計を取得します。' + 'ja-JP': 'ユーザーごとの被リアクション数のチャートを取得します。' }, params: { @@ -35,7 +35,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await perUserReactionsStats.getChart(ps.span as any, ps.limit, ps.userId); + const stats = await perUserReactionsChart.getChart(ps.span as any, ps.limit, ps.userId); res(stats); }); diff --git a/src/server/api/endpoints/charts/users.ts b/src/server/api/endpoints/charts/users.ts index 78d35fb64..48dc31c88 100644 --- a/src/server/api/endpoints/charts/users.ts +++ b/src/server/api/endpoints/charts/users.ts @@ -1,10 +1,10 @@ import $ from 'cafy'; import getParams from '../../get-params'; -import { usersStats } from '../../../../services/stats'; +import usersChart from '../../../../chart/users'; export const meta = { desc: { - 'ja-JP': 'ユーザーの統計を取得します。' + 'ja-JP': 'ユーザーのチャートを取得します。' }, params: { @@ -27,7 +27,7 @@ export default (params: any) => new Promise(async (res, rej) => { const [ps, psErr] = getParams(meta, params); if (psErr) throw psErr; - const stats = await usersStats.getChart(ps.span as any, ps.limit); + const stats = await usersChart.getChart(ps.span as any, ps.limit); res(stats); }); diff --git a/src/server/api/private/signup.ts b/src/server/api/private/signup.ts index caab0267c..d6eba6981 100644 --- a/src/server/api/private/signup.ts +++ b/src/server/api/private/signup.ts @@ -7,7 +7,7 @@ import generateUserToken from '../common/generate-native-user-token'; import config from '../../../config'; import Meta from '../../../models/meta'; import RegistrationTicket from '../../../models/registration-tickets'; -import { usersStats } from '../../../services/stats'; +import usersChart from '../../../chart/users'; if (config.recaptcha) { recaptcha.init({ @@ -130,7 +130,7 @@ export default async (ctx: Koa.Context) => { }, { upsert: true }); //#endregion - usersStats.update(account, true); + usersChart.update(account, true); const res = await pack(account, account, { detail: true, diff --git a/src/server/index.ts b/src/server/index.ts index 848727fb1..f1933dc40 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -17,7 +17,7 @@ const requestStats = require('request-stats'); import activityPub from './activitypub'; import webFinger from './webfinger'; import config from '../config'; -import { networkStats } from '../services/stats'; +import networkChart from '../chart/network'; import apiServer from './api'; // Init app @@ -104,7 +104,7 @@ export default () => new Promise(resolve => { const outgoingBytes = queue.reduce((a, b) => a + b.res.bytes, 0); queue = []; - networkStats.update(requests, time, incomingBytes, outgoingBytes); + networkChart.update(requests, time, incomingBytes, outgoingBytes); }, 5000); //#endregion }); diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index 273b05637..ea5a295e0 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -17,7 +17,8 @@ import { isLocalUser, IUser, IRemoteUser } from '../../models/user'; import delFile from './delete-file'; import config from '../../config'; import { getDriveFileThumbnailBucket } from '../../models/drive-file-thumbnail'; -import { driveStats, perUserDriveStats } from '../stats'; +import driveChart from '../../chart/drive'; +import perUserDriveChart from '../../chart/per-user-drive'; const log = debug('misskey:drive:add-file'); @@ -399,8 +400,8 @@ export default async function( }); // 統計を更新 - driveStats.update(driveFile, true); - perUserDriveStats.update(driveFile, true); + driveChart.update(driveFile, true); + perUserDriveChart.update(driveFile, true); return driveFile; } diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index 761a5d6d4..3e2f42003 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -2,7 +2,8 @@ import * as Minio from 'minio'; import DriveFile, { DriveFileChunk, IDriveFile } from '../../models/drive-file'; import DriveFileThumbnail, { DriveFileThumbnailChunk } from '../../models/drive-file-thumbnail'; import config from '../../config'; -import { driveStats, perUserDriveStats } from '../stats'; +import driveChart from '../../chart/drive'; +import perUserDriveChart from '../../chart/per-user-drive'; export default async function(file: IDriveFile, isExpired = false) { if (file.metadata.storage == 'minio') { @@ -48,6 +49,6 @@ export default async function(file: IDriveFile, isExpired = false) { //#endregion // 統計を更新 - driveStats.update(file, false); - perUserDriveStats.update(file, false); + driveChart.update(file, false); + perUserDriveChart.update(file, false); } diff --git a/src/services/following/create.ts b/src/services/following/create.ts index 44c543885..87d13c444 100644 --- a/src/services/following/create.ts +++ b/src/services/following/create.ts @@ -7,7 +7,7 @@ import renderFollow from '../../remote/activitypub/renderer/follow'; import renderAccept from '../../remote/activitypub/renderer/accept'; import { deliver } from '../../queue'; import createFollowRequest from './requests/create'; -import { perUserFollowingStats } from '../stats'; +import perUserFollowingChart from '../../chart/per-user-following'; export default async function(follower: IUser, followee: IUser, requestId?: string) { // フォロー対象が鍵アカウントである or @@ -53,7 +53,7 @@ export default async function(follower: IUser, followee: IUser, requestId?: stri }); //#endregion - perUserFollowingStats.update(follower, followee, true); + perUserFollowingChart.update(follower, followee, true); // Publish follow event if (isLocalUser(follower)) { diff --git a/src/services/following/delete.ts b/src/services/following/delete.ts index da286ee98..9f82af2bf 100644 --- a/src/services/following/delete.ts +++ b/src/services/following/delete.ts @@ -5,7 +5,7 @@ import pack from '../../remote/activitypub/renderer'; import renderFollow from '../../remote/activitypub/renderer/follow'; import renderUndo from '../../remote/activitypub/renderer/undo'; import { deliver } from '../../queue'; -import { perUserFollowingStats } from '../stats'; +import perUserFollowingChart from '../../chart/per-user-following'; export default async function(follower: IUser, followee: IUser) { const following = await Following.findOne({ @@ -38,7 +38,7 @@ export default async function(follower: IUser, followee: IUser) { }); //#endregion - perUserFollowingStats.update(follower, followee, false); + perUserFollowingChart.update(follower, followee, false); // Publish unfollow event if (isLocalUser(follower)) { diff --git a/src/services/following/requests/accept.ts b/src/services/following/requests/accept.ts index bccf632bf..32453c74d 100644 --- a/src/services/following/requests/accept.ts +++ b/src/services/following/requests/accept.ts @@ -6,7 +6,7 @@ import renderAccept from '../../../remote/activitypub/renderer/accept'; import { deliver } from '../../../queue'; import Following from '../../../models/following'; import { publishMainStream } from '../../../stream'; -import { perUserFollowingStats } from '../../stats'; +import perUserFollowingChart from '../../../chart/per-user-following'; export default async function(followee: IUser, follower: IUser) { await Following.insert({ @@ -58,7 +58,7 @@ export default async function(followee: IUser, follower: IUser) { }); //#endregion - perUserFollowingStats.update(follower, followee, true); + perUserFollowingChart.update(follower, followee, true); await User.update({ _id: followee._id }, { $inc: { diff --git a/src/services/note/create.ts b/src/services/note/create.ts index ef0c783d1..ac6cc8651 100644 --- a/src/services/note/create.ts +++ b/src/services/note/create.ts @@ -23,7 +23,9 @@ import registerHashtag from '../register-hashtag'; import isQuote from '../../misc/is-quote'; import { TextElementMention } from '../../mfm/parse/elements/mention'; import { TextElementHashtag } from '../../mfm/parse/elements/hashtag'; -import { notesStats, perUserNotesStats } from '../stats'; +import notesChart from '../../chart/notes'; +import perUserNotesChart from '../../chart/per-user-notes'; + import { erase, unique } from '../../prelude/array'; import insertNoteUnread from './unread'; @@ -165,8 +167,8 @@ export default async (user: IUser, data: Option, silent = false) => new Promise< } // 統計を更新 - notesStats.update(note, true); - perUserNotesStats.update(user, note, true); + notesChart.update(note, true); + perUserNotesChart.update(user, note, true); // ハッシュタグ登録 tags.map(tag => registerHashtag(user, tag)); diff --git a/src/services/note/delete.ts b/src/services/note/delete.ts index fd23c10b8..d86ca6e50 100644 --- a/src/services/note/delete.ts +++ b/src/services/note/delete.ts @@ -6,7 +6,8 @@ import pack from '../../remote/activitypub/renderer'; import { deliver } from '../../queue'; import Following from '../../models/following'; import renderTombstone from '../../remote/activitypub/renderer/tombstone'; -import { notesStats, perUserNotesStats } from '../stats'; +import notesChart from '../../chart/notes'; +import perUserNotesChart from '../../chart/per-user-notes'; import config from '../../config'; import NoteUnread from '../../models/note-unread'; import read from './read'; @@ -63,6 +64,6 @@ export default async function(user: IUser, note: INote) { //#endregion // 統計を更新 - notesStats.update(note, false); - perUserNotesStats.update(user, note, false); + notesChart.update(note, false); + perUserNotesChart.update(user, note, false); } diff --git a/src/services/note/reaction/create.ts b/src/services/note/reaction/create.ts index 13bb44ff3..edf648109 100644 --- a/src/services/note/reaction/create.ts +++ b/src/services/note/reaction/create.ts @@ -8,7 +8,7 @@ import watch from '../watch'; import renderLike from '../../../remote/activitypub/renderer/like'; import { deliver } from '../../../queue'; import pack from '../../../remote/activitypub/renderer'; -import { perUserReactionsStats } from '../../stats'; +import perUserReactionsChart from '../../../chart/per-user-reactions'; export default async (user: IUser, note: INote, reaction: string) => new Promise(async (res, rej) => { // Myself @@ -44,7 +44,7 @@ export default async (user: IUser, note: INote, reaction: string) => new Promise $inc: inc }); - perUserReactionsStats.update(user, note); + perUserReactionsChart.update(user, note); publishNoteStream(note._id, 'reacted', { reaction: reaction, diff --git a/src/services/register-hashtag.ts b/src/services/register-hashtag.ts index 58222c1f4..106df377b 100644 --- a/src/services/register-hashtag.ts +++ b/src/services/register-hashtag.ts @@ -1,6 +1,6 @@ import { IUser } from '../models/user'; import Hashtag from '../models/hashtag'; -import { hashtagStats } from './stats'; +import hashtagChart from '../chart/hashtag'; export default async function(user: IUser, tag: string) { tag = tag.toLowerCase(); @@ -27,5 +27,5 @@ export default async function(user: IUser, tag: string) { }); } - hashtagStats.update(tag, user._id); + hashtagChart.update(tag, user._id); } diff --git a/src/services/stats.ts b/src/services/stats.ts deleted file mode 100644 index a7b584f4d..000000000 --- a/src/services/stats.ts +++ /dev/null @@ -1,1056 +0,0 @@ -/** - * このファイルでは、チャートに関する処理を行います。 - */ - -const nestedProperty = require('nested-property'); -import autobind from 'autobind-decorator'; -import * as mongo from 'mongodb'; -import db from '../db/mongodb'; -import Note, { INote } from '../models/note'; -import User, { isLocalUser, IUser } from '../models/user'; -import DriveFile, { IDriveFile } from '../models/drive-file'; -import { ICollection } from 'monk'; -import Following from '../models/following'; - -type Obj = { [key: string]: any }; - -type Partial = { - [P in keyof T]?: Partial; -}; - -type ArrayValue = { - [P in keyof T]: T[P] extends number ? Array : ArrayValue; -}; - -type Span = 'day' | 'hour'; - -//#region Chart Core -type Log = { - _id: mongo.ObjectID; - - /** - * 集計のグループ - */ - group?: any; - - /** - * 集計日時 - */ - date: Date; - - /** - * 集計期間 - */ - span: Span; - - /** - * データ - */ - data: T; - - /** - * ユニークインクリメント用 - */ - unique?: Obj; -}; - -/** - * 様々なチャートの管理を司るクラス - */ -abstract class Stats { - protected collection: ICollection>; - protected abstract async getTemplate(init: boolean, latest?: T, group?: any): Promise; - - constructor(name: string, grouped = false) { - this.collection = db.get>(`stats.${name}`); - if (grouped) { - this.collection.createIndex({ span: -1, date: -1, group: -1 }, { unique: true }); - } else { - this.collection.createIndex({ span: -1, date: -1 }, { unique: true }); - } - } - - @autobind - private convertQuery(x: Obj, path: string): Obj { - const query: Obj = {}; - - const dive = (x: Obj, path: string) => { - Object.entries(x).forEach(([k, v]) => { - const p = path ? `${path}.${k}` : k; - if (typeof v === 'number') { - query[p] = v; - } else { - dive(v, p); - } - }); - }; - - dive(x, path); - - return query; - } - - @autobind - private async getCurrentLog(span: Span, group?: any): Promise> { - const now = new Date(); - const y = now.getFullYear(); - const m = now.getMonth(); - const d = now.getDate(); - const h = now.getHours(); - - const current = - span == 'day' ? new Date(y, m, d) : - span == 'hour' ? new Date(y, m, d, h) : - null; - - // 現在(今日または今のHour)の統計 - const currentLog = await this.collection.findOne({ - group: group, - span: span, - date: current - }); - - if (currentLog) { - return currentLog; - } - - // 集計期間が変わってから、初めてのチャート更新なら - // 最も最近の統計を持ってくる - // * 例えば集計期間が「日」である場合で考えると、 - // * 昨日何もチャートを更新するような出来事がなかった場合は、 - // * 統計がそもそも作られずドキュメントが存在しないということがあり得るため、 - // * 「昨日の」と決め打ちせずに「もっとも最近の」とします - const latest = await this.collection.findOne({ - group: group, - span: span - }, { - sort: { - date: -1 - } - }); - - if (latest) { - // 現在の統計を初期挿入 - const data = await this.getTemplate(false, latest.data); - - const log = await this.collection.insert({ - group: group, - span: span, - date: current, - data: data - }); - - return log; - } else { - // 統計が存在しなかったら - // * Misskeyインスタンスを建てて初めてのチャート更新時など - - // 空の統計を作成 - const data = await this.getTemplate(true, null, group); - - const log = await this.collection.insert({ - group: group, - span: span, - date: current, - data: data - }); - - return log; - } - } - - @autobind - protected commit(query: Obj, group?: any, uniqueKey?: string, uniqueValue?: string): void { - const update = (log: Log) => { - // ユニークインクリメントの場合、指定のキーに指定の値が既に存在していたら弾く - if ( - uniqueKey && - log.unique && - log.unique[uniqueKey] && - log.unique[uniqueKey].includes(uniqueValue) - ) return; - - // ユニークインクリメントの指定のキーに値を追加 - if (uniqueKey) { - query['$push'] = { - [`unique.${uniqueKey}`]: uniqueValue - }; - } - - this.collection.update({ - _id: log._id - }, query); - }; - - this.getCurrentLog('day', group).then(log => update(log)); - this.getCurrentLog('hour', group).then(log => update(log)); - } - - @autobind - protected inc(inc: Partial, group?: any): void { - this.commit({ - $inc: this.convertQuery(inc, 'data') - }, group); - } - - @autobind - protected incIfUnique(inc: Partial, key: string, value: string, group?: any): void { - this.commit({ - $inc: this.convertQuery(inc, 'data') - }, group, key, value); - } - - @autobind - public async getChart(span: Span, range: number, group?: any): Promise> { - const promisedChart: Promise[] = []; - - const now = new Date(); - const y = now.getFullYear(); - const m = now.getMonth(); - const d = now.getDate(); - const h = now.getHours(); - - const gt = - span == 'day' ? new Date(y, m, d - range) : - span == 'hour' ? new Date(y, m, d, h - range) : null; - - const logs = await this.collection.find({ - group: group, - span: span, - date: { - $gt: gt - } - }, { - sort: { - date: -1 - }, - fields: { - _id: 0 - } - }); - - for (let i = (range - 1); i >= 0; i--) { - const current = - span == 'day' ? new Date(y, m, d - i) : - span == 'hour' ? new Date(y, m, d, h - i) : - null; - - const log = logs.find(l => l.date.getTime() == current.getTime()); - - if (log) { - promisedChart.unshift(Promise.resolve(log.data)); - } else { // 隙間埋め - const latest = logs.find(l => l.date.getTime() < current.getTime()); - promisedChart.unshift(this.getTemplate(false, latest ? latest.data : null)); - } - } - - const chart = await Promise.all(promisedChart); - - const res: ArrayValue = {} as any; - - /** - * [{ - * xxxxx: 1, - * yyyyy: 5 - * }, { - * xxxxx: 2, - * yyyyy: 6 - * }, { - * xxxxx: 3, - * yyyyy: 7 - * }] - * - * を - * - * { - * xxxxx: [1, 2, 3], - * yyyyy: [5, 6, 7] - * } - * - * にする - */ - const dive = (x: Obj, path?: string) => { - Object.entries(x).forEach(([k, v]) => { - const p = path ? `${path}.${k}` : k; - if (typeof v == 'object') { - dive(v, p); - } else { - nestedProperty.set(res, p, chart.map(s => nestedProperty.get(s, p))); - } - }); - }; - - dive(chart[0]); - - return res; - } -} -//#endregion - -//#region Users stats -/** - * ユーザーに関する統計 - */ -type UsersLog = { - local: { - /** - * 集計期間時点での、全ユーザー数 - */ - total: number; - - /** - * 増加したユーザー数 - */ - inc: number; - - /** - * 減少したユーザー数 - */ - dec: number; - }; - - remote: UsersLog['local']; -}; - -class UsersStats extends Stats { - constructor() { - super('users'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: UsersLog): Promise { - const [localCount, remoteCount] = init ? await Promise.all([ - User.count({ host: null }), - User.count({ host: { $ne: null } }) - ]) : [ - latest ? latest.local.total : 0, - latest ? latest.remote.total : 0 - ]; - - return { - local: { - total: localCount, - inc: 0, - dec: 0 - }, - remote: { - total: remoteCount, - inc: 0, - dec: 0 - } - }; - } - - @autobind - public async update(user: IUser, isAdditional: boolean) { - const update: Obj = {}; - - update.total = isAdditional ? 1 : -1; - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - await this.inc({ - [isLocalUser(user) ? 'local' : 'remote']: update - }); - } -} - -export const usersStats = new UsersStats(); -//#endregion - -//#region Notes stats -/** - * 投稿に関する統計 - */ -type NotesLog = { - local: { - /** - * 集計期間時点での、全投稿数 - */ - total: number; - - /** - * 増加した投稿数 - */ - inc: number; - - /** - * 減少した投稿数 - */ - dec: number; - - diffs: { - /** - * 通常の投稿数の差分 - */ - normal: number; - - /** - * リプライの投稿数の差分 - */ - reply: number; - - /** - * Renoteの投稿数の差分 - */ - renote: number; - }; - }; - - remote: NotesLog['local']; -}; - -class NotesStats extends Stats { - constructor() { - super('notes'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: NotesLog): Promise { - const [localCount, remoteCount] = init ? await Promise.all([ - Note.count({ '_user.host': null }), - Note.count({ '_user.host': { $ne: null } }) - ]) : [ - latest ? latest.local.total : 0, - latest ? latest.remote.total : 0 - ]; - - return { - local: { - total: localCount, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }, - remote: { - total: remoteCount, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - } - }; - } - - @autobind - public async update(note: INote, isAdditional: boolean) { - const update: Obj = { - diffs: {} - }; - - update.total = isAdditional ? 1 : -1; - - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - if (note.replyId != null) { - update.diffs.reply = isAdditional ? 1 : -1; - } else if (note.renoteId != null) { - update.diffs.renote = isAdditional ? 1 : -1; - } else { - update.diffs.normal = isAdditional ? 1 : -1; - } - - await this.inc({ - [isLocalUser(note._user) ? 'local' : 'remote']: update - }); - } -} - -export const notesStats = new NotesStats(); -//#endregion - -//#region Drive stats -/** - * ドライブに関する統計 - */ -type DriveLog = { - local: { - /** - * 集計期間時点での、全ドライブファイル数 - */ - totalCount: number; - - /** - * 集計期間時点での、全ドライブファイルの合計サイズ - */ - totalSize: number; - - /** - * 増加したドライブファイル数 - */ - incCount: number; - - /** - * 増加したドライブ使用量 - */ - incSize: number; - - /** - * 減少したドライブファイル数 - */ - decCount: number; - - /** - * 減少したドライブ使用量 - */ - decSize: number; - }; - - remote: DriveLog['local']; -}; - -class DriveStats extends Stats { - constructor() { - super('drive'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: DriveLog): Promise { - const calcSize = (local: boolean) => DriveFile - .aggregate([{ - $match: { - 'metadata._user.host': local ? null : { $ne: null }, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(res => res.length > 0 ? res[0].usage : 0); - - const [localCount, remoteCount, localSize, remoteSize] = init ? await Promise.all([ - DriveFile.count({ 'metadata._user.host': null }), - DriveFile.count({ 'metadata._user.host': { $ne: null } }), - calcSize(true), - calcSize(false) - ]) : [ - latest ? latest.local.totalCount : 0, - latest ? latest.remote.totalCount : 0, - latest ? latest.local.totalSize : 0, - latest ? latest.remote.totalSize : 0 - ]; - - return { - local: { - totalCount: localCount, - totalSize: localSize, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - }, - remote: { - totalCount: remoteCount, - totalSize: remoteSize, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - } - }; - } - - @autobind - public async update(file: IDriveFile, isAdditional: boolean) { - const update: Obj = {}; - - update.totalCount = isAdditional ? 1 : -1; - update.totalSize = isAdditional ? file.length : -file.length; - if (isAdditional) { - update.incCount = 1; - update.incSize = file.length; - } else { - update.decCount = 1; - update.decSize = file.length; - } - - await this.inc({ - [isLocalUser(file.metadata._user) ? 'local' : 'remote']: update - }); - } -} - -export const driveStats = new DriveStats(); -//#endregion - -//#region Network stats -/** - * ネットワークに関する統計 - */ -type NetworkLog = { - /** - * 受信したリクエスト数 - */ - incomingRequests: number; - - /** - * 送信したリクエスト数 - */ - outgoingRequests: number; - - /** - * 応答時間の合計 - * TIP: (totalTime / incomingRequests) でひとつのリクエストに平均でどれくらいの時間がかかったか知れる - */ - totalTime: number; - - /** - * 合計受信データ量 - */ - incomingBytes: number; - - /** - * 合計送信データ量 - */ - outgoingBytes: number; -}; - -class NetworkStats extends Stats { - constructor() { - super('network'); - } - - @autobind - protected async getTemplate(init: boolean, latest?: NetworkLog): Promise { - return { - incomingRequests: 0, - outgoingRequests: 0, - totalTime: 0, - incomingBytes: 0, - outgoingBytes: 0 - }; - } - - @autobind - public async update(incomingRequests: number, time: number, incomingBytes: number, outgoingBytes: number) { - const inc: Partial = { - incomingRequests: incomingRequests, - totalTime: time, - incomingBytes: incomingBytes, - outgoingBytes: outgoingBytes - }; - - await this.inc(inc); - } -} - -export const networkStats = new NetworkStats(); -//#endregion - -//#region Hashtag stats -/** - * ハッシュタグに関する統計 - */ -type HashtagLog = { - /** - * 投稿された数 - */ - count: number; -}; - -class HashtagStats extends Stats { - constructor() { - super('hashtag', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: HashtagLog): Promise { - return { - count: 0 - }; - } - - @autobind - public async update(hashtag: string, userId: mongo.ObjectId) { - const inc: Partial = { - count: 1 - }; - - await this.incIfUnique(inc, 'users', userId.toHexString(), hashtag); - } -} - -export const hashtagStats = new HashtagStats(); -//#endregion - -//#region Per user following stats -/** - * ユーザーごとのフォローに関する統計 - */ -type PerUserFollowingLog = { - local: { - /** - * フォローしている - */ - followings: { - /** - * 合計 - */ - total: number; - - /** - * フォローした数 - */ - inc: number; - - /** - * フォロー解除した数 - */ - dec: number; - }; - - /** - * フォローされている - */ - followers: { - /** - * 合計 - */ - total: number; - - /** - * フォローされた数 - */ - inc: number; - - /** - * フォロー解除された数 - */ - dec: number; - }; - }; - - remote: PerUserFollowingLog['local']; -}; - -class PerUserFollowingStats extends Stats { - constructor() { - super('perUserFollowing', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserFollowingLog, group?: any): Promise { - const [ - localFollowingsCount, - localFollowersCount, - remoteFollowingsCount, - remoteFollowersCount - ] = init ? await Promise.all([ - Following.count({ followerId: group, '_followee.host': null }), - Following.count({ followeeId: group, '_follower.host': null }), - Following.count({ followerId: group, '_followee.host': { $ne: null } }), - Following.count({ followeeId: group, '_follower.host': { $ne: null } }) - ]) : [ - latest ? latest.local.followings.total : 0, - latest ? latest.local.followers.total : 0, - latest ? latest.remote.followings.total : 0, - latest ? latest.remote.followers.total : 0 - ]; - - return { - local: { - followings: { - total: localFollowingsCount, - inc: 0, - dec: 0 - }, - followers: { - total: localFollowersCount, - inc: 0, - dec: 0 - } - }, - remote: { - followings: { - total: remoteFollowingsCount, - inc: 0, - dec: 0 - }, - followers: { - total: remoteFollowersCount, - inc: 0, - dec: 0 - } - } - }; - } - - @autobind - public async update(follower: IUser, followee: IUser, isFollow: boolean) { - const update: Obj = {}; - - update.total = isFollow ? 1 : -1; - - if (isFollow) { - update.inc = 1; - } else { - update.dec = 1; - } - - this.inc({ - [isLocalUser(follower) ? 'local' : 'remote']: { followings: update } - }, follower._id); - this.inc({ - [isLocalUser(followee) ? 'local' : 'remote']: { followers: update } - }, followee._id); - } -} - -export const perUserFollowingStats = new PerUserFollowingStats(); -//#endregion - -//#region Per user notes stats -/** - * ユーザーごとの投稿に関する統計 - */ -type PerUserNotesLog = { - /** - * 集計期間時点での、全投稿数 - */ - total: number; - - /** - * 増加した投稿数 - */ - inc: number; - - /** - * 減少した投稿数 - */ - dec: number; - - diffs: { - /** - * 通常の投稿数の差分 - */ - normal: number; - - /** - * リプライの投稿数の差分 - */ - reply: number; - - /** - * Renoteの投稿数の差分 - */ - renote: number; - }; -}; - -class PerUserNotesStats extends Stats { - constructor() { - super('perUserNotes', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserNotesLog, group?: any): Promise { - const [count] = init ? await Promise.all([ - Note.count({ userId: group, deletedAt: null }), - ]) : [ - latest ? latest.total : 0 - ]; - - return { - total: count, - inc: 0, - dec: 0, - diffs: { - normal: 0, - reply: 0, - renote: 0 - } - }; - } - - @autobind - public async update(user: IUser, note: INote, isAdditional: boolean) { - const update: Obj = { - diffs: {} - }; - - update.total = isAdditional ? 1 : -1; - - if (isAdditional) { - update.inc = 1; - } else { - update.dec = 1; - } - - if (note.replyId != null) { - update.diffs.reply = isAdditional ? 1 : -1; - } else if (note.renoteId != null) { - update.diffs.renote = isAdditional ? 1 : -1; - } else { - update.diffs.normal = isAdditional ? 1 : -1; - } - - await this.inc(update, user._id); - } -} - -export const perUserNotesStats = new PerUserNotesStats(); -//#endregion - -//#region Per user reactions stats -/** - * ユーザーごとのリアクションに関する統計 - */ -type PerUserReactionsLog = { - local: { - /** - * リアクションされた数 - */ - count: number; - }; - - remote: PerUserReactionsLog['local']; -}; - -class PerUserReactionsStats extends Stats { - constructor() { - super('perUserReaction', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserReactionsLog, group?: any): Promise { - return { - local: { - count: 0 - }, - remote: { - count: 0 - } - }; - } - - @autobind - public async update(user: IUser, note: INote) { - this.inc({ - [isLocalUser(user) ? 'local' : 'remote']: { count: 1 } - }, note.userId); - } -} - -export const perUserReactionsStats = new PerUserReactionsStats(); -//#endregion - -//#region Per user drive stats -/** - * ユーザーごとのドライブに関する統計 - */ -type PerUserDriveLog = { - /** - * 集計期間時点での、全ドライブファイル数 - */ - totalCount: number; - - /** - * 集計期間時点での、全ドライブファイルの合計サイズ - */ - totalSize: number; - - /** - * 増加したドライブファイル数 - */ - incCount: number; - - /** - * 増加したドライブ使用量 - */ - incSize: number; - - /** - * 減少したドライブファイル数 - */ - decCount: number; - - /** - * 減少したドライブ使用量 - */ - decSize: number; -}; - -class PerUserDriveStats extends Stats { - constructor() { - super('perUserDrive', true); - } - - @autobind - protected async getTemplate(init: boolean, latest?: PerUserDriveLog, group?: any): Promise { - const calcSize = () => DriveFile - .aggregate([{ - $match: { - 'metadata.userId': group, - 'metadata.deletedAt': { $exists: false } - } - }, { - $project: { - length: true - } - }, { - $group: { - _id: null, - usage: { $sum: '$length' } - } - }]) - .then(res => res.length > 0 ? res[0].usage : 0); - - const [count, size] = init ? await Promise.all([ - DriveFile.count({ 'metadata.userId': group }), - calcSize() - ]) : [ - latest ? latest.totalCount : 0, - latest ? latest.totalSize : 0 - ]; - - return { - totalCount: count, - totalSize: size, - incCount: 0, - incSize: 0, - decCount: 0, - decSize: 0 - }; - } - - @autobind - public async update(file: IDriveFile, isAdditional: boolean) { - const update: Obj = {}; - - update.totalCount = isAdditional ? 1 : -1; - update.totalSize = isAdditional ? file.length : -file.length; - if (isAdditional) { - update.incCount = 1; - update.incSize = file.length; - } else { - update.decCount = 1; - update.decSize = file.length; - } - - await this.inc(update, file.metadata.userId); - } -} - -export const perUserDriveStats = new PerUserDriveStats(); -//#endregion