From 77c9b90e6d72e8d1ca5038611fa967d789bacf69 Mon Sep 17 00:00:00 2001
From: syuilo <Syuilotan@yahoo.co.jp>
Date: Tue, 5 Nov 2019 22:14:42 +0900
Subject: [PATCH] Store nodeinfo per federated instances (#5578)
* Store nodeinfo per federated instances
* Update fetch-nodeinfo.ts
* Update fetch-nodeinfo.ts
* update
---
migration/1572760203493-nodeinfo.ts | 29 ++++++
src/misc/app-lock.ts | 4 +
src/models/entities/instance.ts | 50 ++++++++--
src/queue/processors/deliver.ts | 3 +
src/queue/processors/inbox.ts | 3 +
src/remote/activitypub/models/person.ts | 2 +
src/services/fetch-nodeinfo.ts | 91 +++++++++++++++++++
.../register-or-fetch-instance-doc.ts | 1 -
8 files changed, 173 insertions(+), 10 deletions(-)
create mode 100644 migration/1572760203493-nodeinfo.ts
create mode 100644 src/services/fetch-nodeinfo.ts
diff --git a/migration/1572760203493-nodeinfo.ts b/migration/1572760203493-nodeinfo.ts
new file mode 100644
index 0000000000..88d8df7239
--- /dev/null
+++ b/migration/1572760203493-nodeinfo.ts
@@ -0,0 +1,29 @@
+import {MigrationInterface, QueryRunner} from "typeorm";
+
+export class nodeinfo1572760203493 implements MigrationInterface {
+
+ public async up(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "system"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "softwareName" character varying(64) DEFAULT null`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "softwareVersion" character varying(64) DEFAULT null`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "openRegistrations" boolean DEFAULT null`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "name" character varying(256) DEFAULT null`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "description" character varying(4096) DEFAULT null`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerName" character varying(128) DEFAULT null`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "maintainerEmail" character varying(256) DEFAULT null`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "infoUpdatedAt" TIMESTAMP WITH TIME ZONE`, undefined);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<any> {
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "infoUpdatedAt"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerEmail"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "maintainerName"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "description"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "name"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "openRegistrations"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareVersion"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" DROP COLUMN "softwareName"`, undefined);
+ await queryRunner.query(`ALTER TABLE "instance" ADD "system" character varying(64)`, undefined);
+ }
+
+}
diff --git a/src/misc/app-lock.ts b/src/misc/app-lock.ts
index 30579ed934..3d5ff91882 100644
--- a/src/misc/app-lock.ts
+++ b/src/misc/app-lock.ts
@@ -20,3 +20,7 @@ const lock: (key: string, timeout?: number) => Promise<() => void>
export function getApLock(uri: string, timeout = 30 * 1000) {
return lock(`ap-object:${uri}`, timeout);
}
+
+export function getNodeinfoLock(host: string, timeout = 30 * 1000) {
+ return lock(`nodeinfo:${host}`, timeout);
+}
diff --git a/src/models/entities/instance.ts b/src/models/entities/instance.ts
index 52c5215f14..dd0de100d6 100644
--- a/src/models/entities/instance.ts
+++ b/src/models/entities/instance.ts
@@ -25,15 +25,6 @@ export class Instance {
})
public host: string;
- /**
- * インスタンスのシステム (MastodonとかMisskeyとかPleromaとか)
- */
- @Column('varchar', {
- length: 64, nullable: true,
- comment: 'The system of the Instance.'
- })
- public system: string | null;
-
/**
* インスタンスのユーザー数
*/
@@ -129,4 +120,45 @@ export class Instance {
default: false
})
public isMarkedAsClosed: boolean;
+
+ @Column('varchar', {
+ length: 64, nullable: true, default: null,
+ comment: 'The software of the Instance.'
+ })
+ public softwareName: string | null;
+
+ @Column('varchar', {
+ length: 64, nullable: true, default: null,
+ })
+ public softwareVersion: string | null;
+
+ @Column('boolean', {
+ nullable: true, default: null,
+ })
+ public openRegistrations: boolean | null;
+
+ @Column('varchar', {
+ length: 256, nullable: true, default: null,
+ })
+ public name: string | null;
+
+ @Column('varchar', {
+ length: 4096, nullable: true, default: null,
+ })
+ public description: string | null;
+
+ @Column('varchar', {
+ length: 128, nullable: true, default: null,
+ })
+ public maintainerName: string | null;
+
+ @Column('varchar', {
+ length: 256, nullable: true, default: null,
+ })
+ public maintainerEmail: string | null;
+
+ @Column('timestamp with time zone', {
+ nullable: true,
+ })
+ public infoUpdatedAt: Date | null;
}
diff --git a/src/queue/processors/deliver.ts b/src/queue/processors/deliver.ts
index 8837c80d87..b252c20163 100644
--- a/src/queue/processors/deliver.ts
+++ b/src/queue/processors/deliver.ts
@@ -4,6 +4,7 @@ import { registerOrFetchInstanceDoc } from '../../services/register-or-fetch-ins
import Logger from '../../services/logger';
import { Instances } from '../../models';
import { instanceChart } from '../../services/chart';
+import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
const logger = new Logger('deliver');
@@ -28,6 +29,8 @@ export default async (job: Bull.Job) => {
isNotResponding: false
});
+ fetchNodeinfo(i);
+
instanceChart.requestSent(i.host, true);
});
diff --git a/src/queue/processors/inbox.ts b/src/queue/processors/inbox.ts
index e71181ee73..1a583ec865 100644
--- a/src/queue/processors/inbox.ts
+++ b/src/queue/processors/inbox.ts
@@ -13,6 +13,7 @@ import { fetchMeta } from '../../misc/fetch-meta';
import { toPuny } from '../../misc/convert-host';
import { validActor } from '../../remote/activitypub/type';
import { ensure } from '../../prelude/ensure';
+import { fetchNodeinfo } from '../../services/fetch-nodeinfo';
const logger = new Logger('inbox');
@@ -105,6 +106,8 @@ export default async (job: Bull.Job): Promise<void> => {
isNotResponding: false
});
+ fetchNodeinfo(i);
+
instanceChart.requestReceived(i.host);
});
diff --git a/src/remote/activitypub/models/person.ts b/src/remote/activitypub/models/person.ts
index 198fd78bd5..c7a6d56638 100644
--- a/src/remote/activitypub/models/person.ts
+++ b/src/remote/activitypub/models/person.ts
@@ -27,6 +27,7 @@ import { validActor } from '../../../remote/activitypub/type';
import { getConnection } from 'typeorm';
import { ensure } from '../../../prelude/ensure';
import { toArray } from '../../../prelude/array';
+import { fetchNodeinfo } from '../../../services/fetch-nodeinfo';
const logger = apLogger;
@@ -191,6 +192,7 @@ export async function createPerson(uri: string, resolver?: Resolver): Promise<Us
registerOrFetchInstanceDoc(host).then(i => {
Instances.increment({ id: i.id }, 'usersCount', 1);
instanceChart.newUser(i.host);
+ fetchNodeinfo(i);
});
usersChart.update(user!, true);
diff --git a/src/services/fetch-nodeinfo.ts b/src/services/fetch-nodeinfo.ts
new file mode 100644
index 0000000000..e5d652a6b3
--- /dev/null
+++ b/src/services/fetch-nodeinfo.ts
@@ -0,0 +1,91 @@
+import * as request from 'request-promise-native';
+import { Instance } from '../models/entities/instance';
+import { Instances } from '../models';
+import config from '../config';
+import { getNodeinfoLock } from '../misc/app-lock';
+import Logger from '../services/logger';
+
+export const logger = new Logger('nodeinfo', 'cyan');
+
+export async function fetchNodeinfo(instance: Instance) {
+ const unlock = await getNodeinfoLock(instance.host);
+
+ const _instance = await Instances.findOne({ host: instance.host });
+ const now = Date.now();
+ if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) {
+ unlock();
+ return;
+ }
+
+ logger.info(`Fetching nodeinfo of ${instance.host} ...`);
+
+ try {
+ const wellknown = await request({
+ url: 'https://' + instance.host + '/.well-known/nodeinfo',
+ proxy: config.proxy,
+ timeout: 1000 * 10,
+ forever: true,
+ headers: {
+ 'User-Agent': config.userAgent,
+ Accept: 'application/json, */*'
+ },
+ json: true
+ }).catch(e => {
+ if (e.statusCode === 404) {
+ throw 'No nodeinfo provided';
+ } else {
+ throw e.statusCode || e.message;
+ }
+ });
+
+ if (wellknown.links == null || !Array.isArray(wellknown.links)) {
+ throw 'No wellknown links';
+ }
+
+ const links = wellknown.links as any[];
+
+ const lnik1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0');
+ const lnik2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0');
+ const lnik2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1');
+ const link = lnik2_1 || lnik2_0 || lnik1_0;
+
+ if (link == null) {
+ throw 'No nodeinfo link provided';
+ }
+
+ const info = await request({
+ url: link.href,
+ proxy: config.proxy,
+ timeout: 1000 * 10,
+ forever: true,
+ headers: {
+ 'User-Agent': config.userAgent,
+ Accept: 'application/json, */*'
+ },
+ json: true
+ }).catch(e => {
+ throw e.statusCode || e.message;
+ });
+
+ await Instances.update(instance.id, {
+ infoUpdatedAt: new Date(),
+ softwareName: info.software.name.toLowerCase(),
+ softwareVersion: info.software.version,
+ openRegistrations: info.openRegistrations,
+ name: info.metadata ? (info.metadata.nodeName || info.metadata.name || null) : null,
+ description: info.metadata ? (info.metadata.nodeDescription || info.metadata.description || null) : null,
+ maintainerName: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name || null) : null : null,
+ maintainerEmail: info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email || null) : null : null,
+ });
+
+ logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`);
+ } catch (e) {
+ logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${e}`);
+
+ await Instances.update(instance.id, {
+ infoUpdatedAt: new Date(),
+ });
+ } finally {
+ unlock();
+ }
+}
diff --git a/src/services/register-or-fetch-instance-doc.ts b/src/services/register-or-fetch-instance-doc.ts
index 9957edd3db..3501e20de1 100644
--- a/src/services/register-or-fetch-instance-doc.ts
+++ b/src/services/register-or-fetch-instance-doc.ts
@@ -15,7 +15,6 @@ export async function registerOrFetchInstanceDoc(host: string): Promise<Instance
host,
caughtAt: new Date(),
lastCommunicatedAt: new Date(),
- system: null // TODO
});
federationChart.update(true);