yume/yumechi-no-kuni
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: リモートサーバーのサーバー情報を収集しないオプション (#14634)

Browse source 
* wip

* wip

* Update FetchInstanceMetadataService.ts

* Update FetchInstanceMetadataService.ts

* Update types.ts
This commit is contained in:
syuilo 2024-10-13 20:22:16 +09:00 committed by GitHub
parent 45d42b8641
commit ff47fef572
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 185 additions and 86 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
locales/index.d.ts vendored
View file

@ -4366,6 +4366,10 @@ export interface Locale extends ILocale {
* リモートサーバーのチャートを生成
*/
"enableChartsForFederatedInstances": string;
/**
* リモートサーバーの情報を取得
*/
"enableStatsForFederatedInstances": string;
/**
* ノートのアクションにクリップを追加
*/

1
locales/ja-JP.yml
View file

@ -1087,6 +1087,7 @@ retryAllQueuesConfirmTitle: "今すぐ再試行しますか？"
retryAllQueuesConfirmText: "一時的にサーバーの負荷が増大することがあります。"
enableChartsForRemoteUser: "リモートユーザーのチャートを生成"
enableChartsForFederatedInstances: "リモートサーバーのチャートを生成"
enableStatsForFederatedInstances: "リモートサーバーの情報を取得"
showClipButtonInNoteFooter: "ノートのアクションにクリップを追加"
reactionsDisplaySize: "リアクションの表示サイズ"
limitWidthOfReaction: "リアクションの最大横幅を制限し、縮小して表示する"

16
packages/backend/migration/1727318020265-enableStatsForFederatedInstances.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class EnableStatsForFederatedInstances1727318020265 {
name = 'EnableStatsForFederatedInstances1727318020265'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "enableStatsForFederatedInstances" boolean NOT NULL DEFAULT true`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "enableStatsForFederatedInstances"`);
}
}

4
packages/backend/src/core/AccountMoveService.ts
View file

@ -274,14 +274,16 @@ export class AccountMoveService {
}
// Update instance stats by decreasing remote followers count by the number of local followers who were following the old account.
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(oldAccount)) {
this.federatedInstanceService.fetch(oldAccount.host).then(async i => {
this.federatedInstanceService.fetchOrRegister(oldAccount.host).then(async i => {
this.instancesRepository.decrement({ id: i.id }, 'followersCount', localFollowerIds.length);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.updateFollowers(i.host, false);
}
});
}
}
// FIXME: expensive?
for (const followerId of localFollowerIds) {

20
packages/backend/src/core/FederatedInstanceService.ts
View file

@ -47,7 +47,7 @@ export class FederatedInstanceService implements OnApplicationShutdown {
}
@bindThis
public async fetch(host: string): Promise<MiInstance> {
public async fetchOrRegister(host: string): Promise<MiInstance> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
@ -70,6 +70,24 @@ export class FederatedInstanceService implements OnApplicationShutdown {
}
}
@bindThis
public async fetch(host: string): Promise<MiInstance | null> {
host = this.utilityService.toPuny(host);
const cached = await this.federatedInstanceCache.get(host);
if (cached !== undefined) return cached;
const index = await this.instancesRepository.findOneBy({ host });
if (index == null) {
this.federatedInstanceCache.set(host, null);
return null;
} else {
this.federatedInstanceCache.set(host, index);
return index;
}
}
@bindThis
public async update(id: MiInstance['id'], data: Partial<MiInstance>): Promise<void> {
const result = await this.instancesRepository.createQueryBuilder().update()

2
packages/backend/src/core/FetchInstanceMetadataService.ts
View file

@ -82,7 +82,7 @@ export class FetchInstanceMetadataService {
try {
if (!force) {
const _instance = await this.federatedInstanceService.fetch(host);
const _instance = await this.federatedInstanceService.fetchOrRegister(host);
const now = Date.now();
if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
// unlock at the finally caluse

4
packages/backend/src/core/NoteCreateService.ts
View file

@ -511,14 +511,16 @@ export class NoteCreateService implements OnApplicationShutdown {
}
// Register host
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(user)) {
this.federatedInstanceService.fetch(user.host).then(async i => {
this.federatedInstanceService.fetchOrRegister(user.host).then(async i => {
this.updateNotesCountQueue.enqueue(i.id, 1);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.updateNote(i.host, note, true);
}
});
}
}
// ハッシュタグ更新
if (data.visibility === 'public' || data.visibility === 'home') {

4
packages/backend/src/core/NoteDeleteService.ts
View file

@ -106,8 +106,9 @@ export class NoteDeleteService {
this.perUserNotesChart.update(user, note, false);
}
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(user)) {
this.federatedInstanceService.fetch(user.host).then(async i => {
this.federatedInstanceService.fetchOrRegister(user.host).then(async i => {
this.instancesRepository.decrement({ id: i.id }, 'notesCount', 1);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.updateNote(i.host, note, false);
@ -115,6 +116,7 @@ export class NoteDeleteService {
});
}
}
}
for (const cascadingNote of cascadingNotes) {
this.searchService.unindexNote(cascadingNote);

12
packages/backend/src/core/UserFollowingService.ts
View file

@ -305,21 +305,23 @@ export class UserFollowingService implements OnModuleInit {
//#endregion
//#region Update instance stats
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) {
this.federatedInstanceService.fetch(follower.host).then(async i => {
this.federatedInstanceService.fetchOrRegister(follower.host).then(async i => {
this.instancesRepository.increment({ id: i.id }, 'followingCount', 1);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.updateFollowing(i.host, true);
}
});
} else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) {
this.federatedInstanceService.fetch(followee.host).then(async i => {
this.federatedInstanceService.fetchOrRegister(followee.host).then(async i => {
this.instancesRepository.increment({ id: i.id }, 'followersCount', 1);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.updateFollowers(i.host, true);
}
});
}
}
//#endregion
this.perUserFollowingChart.update(follower, followee, true);
@ -437,21 +439,23 @@ export class UserFollowingService implements OnModuleInit {
//#endregion
//#region Update instance stats
if (this.meta.enableStatsForFederatedInstances) {
if (this.userEntityService.isRemoteUser(follower) && this.userEntityService.isLocalUser(followee)) {
this.federatedInstanceService.fetch(follower.host).then(async i => {
this.federatedInstanceService.fetchOrRegister(follower.host).then(async i => {
this.instancesRepository.decrement({ id: i.id }, 'followingCount', 1);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.updateFollowing(i.host, false);
}
});
} else if (this.userEntityService.isLocalUser(follower) && this.userEntityService.isRemoteUser(followee)) {
this.federatedInstanceService.fetch(followee.host).then(async i => {
this.federatedInstanceService.fetchOrRegister(followee.host).then(async i => {
this.instancesRepository.decrement({ id: i.id }, 'followersCount', 1);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.updateFollowers(i.host, false);
}
});
}
}
//#endregion
this.perUserFollowingChart.update(follower, followee, false);

6
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -408,13 +408,15 @@ export class ApPersonService implements OnModuleInit {
this.cacheService.uriPersonCache.set(user.uri, user);
// Register host
this.federatedInstanceService.fetch(host).then(i => {
if (this.meta.enableStatsForFederatedInstances) {
this.federatedInstanceService.fetchOrRegister(host).then(i => {
this.instancesRepository.increment({ id: i.id }, 'usersCount', 1);
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.newUser(i.host);
}
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
});
}
this.usersChart.update(user, true);

5
packages/backend/src/models/Meta.ts
View file

@ -529,6 +529,11 @@ export class MiMeta {
})
public enableChartsForFederatedInstances: boolean;
@Column('boolean', {
default: true,
})
public enableStatsForFederatedInstances: boolean;
@Column('boolean', {
default: false,
})

29
packages/backend/src/queue/processors/DeliverProcessorService.ts
View file

@ -74,8 +74,17 @@ export class DeliverProcessorService {
try {
await this.apRequestService.signedPost(job.data.user, job.data.to, job.data.content, job.data.digest);
// Update stats
this.federatedInstanceService.fetch(host).then(i => {
this.apRequestChart.deliverSucc();
this.federationChart.deliverd(host, true);
// Update instance stats
process.nextTick(async () => {
const i = await (this.meta.enableStatsForFederatedInstances
? this.federatedInstanceService.fetchOrRegister(host)
: this.federatedInstanceService.fetch(host));
if (i == null) return;
if (i.isNotResponding) {
this.federatedInstanceService.update(i.id, {
isNotResponding: false,
@ -83,9 +92,9 @@ export class DeliverProcessorService {
});
}
if (this.meta.enableStatsForFederatedInstances) {
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
this.apRequestChart.deliverSucc();
this.federationChart.deliverd(i.host, true);
}
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.requestSent(i.host, true);
@ -94,8 +103,11 @@ export class DeliverProcessorService {
return 'Success';
} catch (res) {
// Update stats
this.federatedInstanceService.fetch(host).then(i => {
this.apRequestChart.deliverFail();
this.federationChart.deliverd(host, false);
// Update instance stats
this.federatedInstanceService.fetchOrRegister(host).then(i => {
if (!i.isNotResponding) {
this.federatedInstanceService.update(i.id, {
isNotResponding: true,
@ -116,9 +128,6 @@ export class DeliverProcessorService {
});
}
this.apRequestChart.deliverFail();
this.federationChart.deliverd(i.host, false);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.requestSent(i.host, false);
}
@ -129,7 +138,7 @@ export class DeliverProcessorService {
if (!res.isRetryable) {
// 相手が閉鎖していることを明示しているため、配送停止する
if (job.data.isSharedInbox && res.statusCode === 410) {
this.federatedInstanceService.fetch(host).then(i => {
this.federatedInstanceService.fetchOrRegister(host).then(i => {
this.federatedInstanceService.update(i.id, {
suspensionState: 'goneSuspended',
});

20
packages/backend/src/queue/processors/InboxProcessorService.ts
View file

@ -192,21 +192,27 @@ export class InboxProcessorService implements OnApplicationShutdown {
}
}
// Update stats
this.federatedInstanceService.fetch(authUser.user.host).then(i => {
this.apRequestChart.inbox();
this.federationChart.inbox(authUser.user.host);
// Update instance stats
process.nextTick(async () => {
const i = await (this.meta.enableStatsForFederatedInstances
? this.federatedInstanceService.fetchOrRegister(authUser.user.host)
: this.federatedInstanceService.fetch(authUser.user.host));
if (i == null) return;
this.updateInstanceQueue.enqueue(i.id, {
latestRequestReceivedAt: new Date(),
shouldUnsuspend: i.suspensionState === 'autoSuspendedForNotResponding',
});
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
this.apRequestChart.inbox();
this.federationChart.inbox(i.host);
if (this.meta.enableChartsForFederatedInstances) {
this.instanceChart.requestReceived(i.host);
}
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
});
// アクティビティを処理

5
packages/backend/src/server/api/endpoints/admin/meta.ts
View file

@ -348,6 +348,10 @@ export const meta = {
type: 'boolean',
optional: false, nullable: false,
},
enableStatsForFederatedInstances: {
type: 'boolean',
optional: false, nullable: false,
},
enableServerMachineStats: {
type: 'boolean',
optional: false, nullable: false,
@ -635,6 +639,7 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
truemailAuthKey: instance.truemailAuthKey,
enableChartsForRemoteUser: instance.enableChartsForRemoteUser,
enableChartsForFederatedInstances: instance.enableChartsForFederatedInstances,
enableStatsForFederatedInstances: instance.enableStatsForFederatedInstances,
enableServerMachineStats: instance.enableServerMachineStats,
enableIdenticonGeneration: instance.enableIdenticonGeneration,
bannedEmailDomains: instance.bannedEmailDomains,

5
packages/backend/src/server/api/endpoints/admin/update-meta.ts
View file

@ -136,6 +136,7 @@ export const paramDef = {
truemailAuthKey: { type: 'string', nullable: true },
enableChartsForRemoteUser: { type: 'boolean' },
enableChartsForFederatedInstances: { type: 'boolean' },
enableStatsForFederatedInstances: { type: 'boolean' },
enableServerMachineStats: { type: 'boolean' },
enableIdenticonGeneration: { type: 'boolean' },
serverRules: { type: 'array', items: { type: 'string' } },
@ -578,6 +579,10 @@ export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-
set.enableChartsForFederatedInstances = ps.enableChartsForFederatedInstances;
}
if (ps.enableStatsForFederatedInstances !== undefined) {
set.enableStatsForFederatedInstances = ps.enableStatsForFederatedInstances;
}
if (ps.enableServerMachineStats !== undefined) {
set.enableServerMachineStats = ps.enableServerMachineStats;
}

20
packages/backend/test/unit/FetchInstanceMetadataService.ts
View file

@ -8,6 +8,7 @@ process.env.NODE_ENV = 'test';
import { jest } from '@jest/globals';
import { Test } from '@nestjs/testing';
import { Redis } from 'ioredis';
import type { TestingModule } from '@nestjs/testing';
import { GlobalModule } from '@/GlobalModule.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
@ -16,7 +17,6 @@ import { LoggerService } from '@/core/LoggerService.js';
import { UtilityService } from '@/core/UtilityService.js';
import { IdService } from '@/core/IdService.js';
import { DI } from '@/di-symbols.js';
import type { TestingModule } from '@nestjs/testing';
function mockRedis() {
const hash = {} as any;
@ -52,7 +52,7 @@ describe('FetchInstanceMetadataService', () => {
if (token === HttpRequestService) {
return { getJson: jest.fn(), getHtml: jest.fn(), send: jest.fn() };
} else if (token === FederatedInstanceService) {
return { fetch: jest.fn() };
return { fetchOrRegister: jest.fn() };
} else if (token === DI.redis) {
return mockRedis;
}
@ -75,7 +75,7 @@ describe('FetchInstanceMetadataService', () => {
test('Lock and update', async () => {
redisClient.set = mockRedis();
const now = Date.now();
federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => { return now - 10 * 1000 * 60 * 60 * 24; } } } as any);
federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => { return now - 10 * 1000 * 60 * 60 * 24; } } } as any);
httpRequestService.getJson.mockImplementation(() => { throw Error(); });
const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock');
const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock');
@ -83,14 +83,14 @@ describe('FetchInstanceMetadataService', () => {
await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any);
expect(tryLockSpy).toHaveBeenCalledTimes(1);
expect(unlockSpy).toHaveBeenCalledTimes(1);
expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(1);
expect(federatedInstanceService.fetchOrRegister).toHaveBeenCalledTimes(1);
expect(httpRequestService.getJson).toHaveBeenCalled();
});
test('Lock and don\'t update', async () => {
redisClient.set = mockRedis();
const now = Date.now();
federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now } } as any);
federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => now } } as any);
httpRequestService.getJson.mockImplementation(() => { throw Error(); });
const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock');
const unlockSpy = jest.spyOn(fetchInstanceMetadataService, 'unlock');
@ -98,14 +98,14 @@ describe('FetchInstanceMetadataService', () => {
await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any);
expect(tryLockSpy).toHaveBeenCalledTimes(1);
expect(unlockSpy).toHaveBeenCalledTimes(1);
expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(1);
expect(federatedInstanceService.fetchOrRegister).toHaveBeenCalledTimes(1);
expect(httpRequestService.getJson).toHaveBeenCalledTimes(0);
});
test('Do nothing when lock not acquired', async () => {
redisClient.set = mockRedis();
const now = Date.now();
federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any);
federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any);
httpRequestService.getJson.mockImplementation(() => { throw Error(); });
await fetchInstanceMetadataService.tryLock('example.com');
const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock');
@ -114,14 +114,14 @@ describe('FetchInstanceMetadataService', () => {
await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any);
expect(tryLockSpy).toHaveBeenCalledTimes(1);
expect(unlockSpy).toHaveBeenCalledTimes(0);
expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(0);
expect(federatedInstanceService.fetchOrRegister).toHaveBeenCalledTimes(0);
expect(httpRequestService.getJson).toHaveBeenCalledTimes(0);
});
test('Do when lock not acquired but forced', async () => {
redisClient.set = mockRedis();
const now = Date.now();
federatedInstanceService.fetch.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any);
federatedInstanceService.fetchOrRegister.mockResolvedValue({ infoUpdatedAt: { getTime: () => now - 10 * 1000 * 60 * 60 * 24 } } as any);
httpRequestService.getJson.mockImplementation(() => { throw Error(); });
await fetchInstanceMetadataService.tryLock('example.com');
const tryLockSpy = jest.spyOn(fetchInstanceMetadataService, 'tryLock');
@ -130,7 +130,7 @@ describe('FetchInstanceMetadataService', () => {
await fetchInstanceMetadataService.fetchInstanceMetadata({ host: 'example.com' } as any, true);
expect(tryLockSpy).toHaveBeenCalledTimes(0);
expect(unlockSpy).toHaveBeenCalledTimes(1);
expect(federatedInstanceService.fetch).toHaveBeenCalledTimes(0);
expect(federatedInstanceService.fetchOrRegister).toHaveBeenCalledTimes(0);
expect(httpRequestService.getJson).toHaveBeenCalled();
});
});

16
packages/frontend/src/pages/admin/performance.vue
View file

@ -29,6 +29,13 @@ SPDX-License-Identifier: AGPL-3.0-only
</MkSwitch>
</div>
<div class="_panel" style="padding: 16px;">
<MkSwitch v-model="enableStatsForFederatedInstances" @change="onChange_enableStatsForFederatedInstances">
<template #label>{{ i18n.ts.enableStatsForFederatedInstances }}</template>
<template #caption>{{ i18n.ts.turnOffToImprovePerformance }}</template>
</MkSwitch>
</div>
<div class="_panel" style="padding: 16px;">
<MkSwitch v-model="enableChartsForFederatedInstances" @change="onChange_enableChartsForFederatedInstances">
<template #label>{{ i18n.ts.enableChartsForFederatedInstances }}</template>
@ -120,6 +127,7 @@ const meta = await misskeyApi('admin/meta');
const enableServerMachineStats = ref(meta.enableServerMachineStats);
const enableIdenticonGeneration = ref(meta.enableIdenticonGeneration);
const enableChartsForRemoteUser = ref(meta.enableChartsForRemoteUser);
const enableStatsForFederatedInstances = ref(meta.enableStatsForFederatedInstances);
const enableChartsForFederatedInstances = ref(meta.enableChartsForFederatedInstances);
function onChange_enableServerMachineStats(value: boolean) {
@ -146,6 +154,14 @@ function onChange_enableChartsForRemoteUser(value: boolean) {
});
}
function onChange_enableStatsForFederatedInstances(value: boolean) {
os.apiWithDialog('admin/update-meta', {
enableStatsForFederatedInstances: value,
}).then(() => {
fetchInstance(true);
});
}
function onChange_enableChartsForFederatedInstances(value: boolean) {
os.apiWithDialog('admin/update-meta', {
enableChartsForFederatedInstances: value,

2
packages/misskey-js/src/autogen/types.ts
View file

@ -5165,6 +5165,7 @@ export type operations = {
truemailAuthKey: string | null;
enableChartsForRemoteUser: boolean;
enableChartsForFederatedInstances: boolean;
enableStatsForFederatedInstances: boolean;
enableServerMachineStats: boolean;
enableIdenticonGeneration: boolean;
manifestJsonOverride: string;
@ -9547,6 +9548,7 @@ export type operations = {
truemailAuthKey?: string | null;
enableChartsForRemoteUser?: boolean;
enableChartsForFederatedInstances?: boolean;
enableStatsForFederatedInstances?: boolean;
enableServerMachineStats?: boolean;
enableIdenticonGeneration?: boolean;
serverRules?: string[];