From 27d16c6a120d05e69eb069035a851c48c44548c5 Mon Sep 17 00:00:00 2001
From: syuilo <syuilotan@yahoo.co.jp>
Date: Wed, 6 Feb 2019 20:56:48 +0900
Subject: [PATCH] Resolve #4151
---
locales/ja-JP.yml | 6 +-
.../views/components/profile-editor.vue | 19 +++-
src/queue/processors/export-blocking.ts | 90 +++++++++++++++++++
src/queue/processors/export-following.ts | 90 +++++++++++++++++++
src/queue/processors/export-mute.ts | 90 +++++++++++++++++++
src/queue/processors/index.ts | 6 ++
src/server/api/endpoints/i/export-blocking.ts | 18 ++++
.../api/endpoints/i/export-following.ts | 18 ++++
src/server/api/endpoints/i/export-mute.ts | 18 ++++
9 files changed, 351 insertions(+), 4 deletions(-)
create mode 100644 src/queue/processors/export-blocking.ts
create mode 100644 src/queue/processors/export-following.ts
create mode 100644 src/queue/processors/export-mute.ts
create mode 100644 src/server/api/endpoints/i/export-blocking.ts
create mode 100644 src/server/api/endpoints/i/export-following.ts
create mode 100644 src/server/api/endpoints/i/export-mute.ts
diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml
index 77bcad3ff5..41b891cf71 100644
--- a/locales/ja-JP.yml
+++ b/locales/ja-JP.yml
@@ -558,7 +558,11 @@ common/views/components/profile-editor.vue:
email-verified: "メールアドレスが確認されました"
email-not-verified: "メールアドレスが確認されていません。メールボックスをご確認ください。"
export: "エクスポート"
- export-notes: "すべての投稿のエクスポート"
+ export-targets:
+ all-notes: "すべての投稿データ"
+ following-list: "フォロー"
+ mute-list: "ミュート"
+ blocking-list: "ブロック"
export-requested: "エクスポートをリクエストしました。これには時間がかかる場合があります。エクスポートが終わると、ドライブにファイルが追加されます。"
common/views/components/user-list-editor.vue:
diff --git a/src/client/app/common/views/components/profile-editor.vue b/src/client/app/common/views/components/profile-editor.vue
index a3b1c6d870..472124fa84 100644
--- a/src/client/app/common/views/components/profile-editor.vue
+++ b/src/client/app/common/views/components/profile-editor.vue
@@ -92,7 +92,14 @@
<header>{{ $t('export') }}</header>
<div>
- <ui-button @click="exportNotes()"><fa :icon="faDownload"/> {{ $t('export-notes') }}</ui-button>
+ <ui-select v-model="exportTarget">
+ <span slot="label">{{ $t('export-target') }}</span>
+ <option value="notes">{{ $t('export-targets.all-notes') }}</option>
+ <option value="following">{{ $t('export-targets.following-list') }}</option>
+ <option value="mute">{{ $t('export-targets.mute-list') }}</option>
+ <option value="blocking">{{ $t('export-targets.blocking-list') }}</option>
+ </ui-select>
+ <ui-button @click="doExport()"><fa :icon="faDownload"/> {{ $t('export') }}</ui-button>
</div>
</section>
</ui-card>
@@ -133,6 +140,7 @@ export default Vue.extend({
saving: false,
avatarUploading: false,
bannerUploading: false,
+ exportTarget: 'notes',
faDownload
};
},
@@ -264,8 +272,13 @@ export default Vue.extend({
});
},
- exportNotes() {
- this.$root.api('i/export-notes', {});
+ doExport() {
+ this.$root.api(
+ this.exportTarget == 'notes' ? 'i/export-notes' :
+ this.exportTarget == 'following' ? 'i/export-following' :
+ this.exportTarget == 'mute' ? 'i/export-mute' :
+ this.exportTarget == 'blocking' ? 'i/export-blocking' :
+ null, {});
this.$root.dialog({
type: 'info',
diff --git a/src/queue/processors/export-blocking.ts b/src/queue/processors/export-blocking.ts
new file mode 100644
index 0000000000..b7a2d3122f
--- /dev/null
+++ b/src/queue/processors/export-blocking.ts
@@ -0,0 +1,90 @@
+
+import * as bq from 'bee-queue';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+import * as mongo from 'mongodb';
+
+import { queueLogger } from '../logger';
+import addFile from '../../services/drive/add-file';
+import User from '../../models/user';
+import dateFormat = require('dateformat');
+import Blocking from '../../models/blocking';
+import config from '../../config';
+
+const logger = queueLogger.createSubLogger('export-blocking');
+
+export async function exportBlocking(job: bq.Job, done: any): Promise<void> {
+ logger.info(`Exporting blocking of ${job.data.user._id} ...`);
+
+ const user = await User.findOne({
+ _id: new mongo.ObjectID(job.data.user._id.toString())
+ });
+
+ // Create temp file
+ const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
+ tmp.file((e, path, fd, cleanup) => {
+ if (e) return rej(e);
+ res([path, cleanup]);
+ });
+ });
+
+ logger.info(`Temp file is ${path}`);
+
+ const stream = fs.createWriteStream(path, { flags: 'a' });
+
+ let exportedCount = 0;
+ let ended = false;
+ let cursor: any = null;
+
+ while (!ended) {
+ const blockings = await Blocking.find({
+ blockerId: user._id,
+ ...(cursor ? { _id: { $gt: cursor } } : {})
+ }, {
+ limit: 100,
+ sort: {
+ _id: 1
+ }
+ });
+
+ if (blockings.length === 0) {
+ ended = true;
+ job.reportProgress(100);
+ break;
+ }
+
+ cursor = blockings[blockings.length - 1]._id;
+
+ for (const block of blockings) {
+ const u = await User.findOne({ _id: block.blockeeId }, { fields: { username: true, host: true } });
+ const content = u.host ? `${u.username}@${u.host}` : `${u.username}@${config.host}`;
+ await new Promise((res, rej) => {
+ stream.write(content + '\n', err => {
+ if (err) {
+ logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ exportedCount++;
+ }
+
+ const total = await Blocking.count({
+ blockerId: user._id,
+ });
+
+ job.reportProgress(exportedCount / total);
+ }
+
+ stream.end();
+ logger.succ(`Exported to: ${path}`);
+
+ const fileName = dateFormat(new Date(), 'blocking-yyyy-mm-dd-HH-MM-ss') + '.json';
+ const driveFile = await addFile(user, path, fileName);
+
+ logger.succ(`Exported to: ${driveFile._id}`);
+ cleanup();
+ done();
+}
diff --git a/src/queue/processors/export-following.ts b/src/queue/processors/export-following.ts
new file mode 100644
index 0000000000..7515926361
--- /dev/null
+++ b/src/queue/processors/export-following.ts
@@ -0,0 +1,90 @@
+
+import * as bq from 'bee-queue';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+import * as mongo from 'mongodb';
+
+import { queueLogger } from '../logger';
+import addFile from '../../services/drive/add-file';
+import User from '../../models/user';
+import dateFormat = require('dateformat');
+import Following from '../../models/following';
+import config from '../../config';
+
+const logger = queueLogger.createSubLogger('export-following');
+
+export async function exportFollowing(job: bq.Job, done: any): Promise<void> {
+ logger.info(`Exporting following of ${job.data.user._id} ...`);
+
+ const user = await User.findOne({
+ _id: new mongo.ObjectID(job.data.user._id.toString())
+ });
+
+ // Create temp file
+ const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
+ tmp.file((e, path, fd, cleanup) => {
+ if (e) return rej(e);
+ res([path, cleanup]);
+ });
+ });
+
+ logger.info(`Temp file is ${path}`);
+
+ const stream = fs.createWriteStream(path, { flags: 'a' });
+
+ let exportedCount = 0;
+ let ended = false;
+ let cursor: any = null;
+
+ while (!ended) {
+ const followings = await Following.find({
+ followerId: user._id,
+ ...(cursor ? { _id: { $gt: cursor } } : {})
+ }, {
+ limit: 100,
+ sort: {
+ _id: 1
+ }
+ });
+
+ if (followings.length === 0) {
+ ended = true;
+ job.reportProgress(100);
+ break;
+ }
+
+ cursor = followings[followings.length - 1]._id;
+
+ for (const following of followings) {
+ const u = await User.findOne({ _id: following.followeeId }, { fields: { username: true, host: true } });
+ const content = u.host ? `${u.username}@${u.host}` : `${u.username}@${config.host}`;
+ await new Promise((res, rej) => {
+ stream.write(content + '\n', err => {
+ if (err) {
+ logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ exportedCount++;
+ }
+
+ const total = await Following.count({
+ followerId: user._id,
+ });
+
+ job.reportProgress(exportedCount / total);
+ }
+
+ stream.end();
+ logger.succ(`Exported to: ${path}`);
+
+ const fileName = dateFormat(new Date(), 'following-yyyy-mm-dd-HH-MM-ss') + '.json';
+ const driveFile = await addFile(user, path, fileName);
+
+ logger.succ(`Exported to: ${driveFile._id}`);
+ cleanup();
+ done();
+}
diff --git a/src/queue/processors/export-mute.ts b/src/queue/processors/export-mute.ts
new file mode 100644
index 0000000000..e526aa06f6
--- /dev/null
+++ b/src/queue/processors/export-mute.ts
@@ -0,0 +1,90 @@
+
+import * as bq from 'bee-queue';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+import * as mongo from 'mongodb';
+
+import { queueLogger } from '../logger';
+import addFile from '../../services/drive/add-file';
+import User from '../../models/user';
+import dateFormat = require('dateformat');
+import Mute from '../../models/mute';
+import config from '../../config';
+
+const logger = queueLogger.createSubLogger('export-mute');
+
+export async function exportMute(job: bq.Job, done: any): Promise<void> {
+ logger.info(`Exporting mute of ${job.data.user._id} ...`);
+
+ const user = await User.findOne({
+ _id: new mongo.ObjectID(job.data.user._id.toString())
+ });
+
+ // Create temp file
+ const [path, cleanup] = await new Promise<[string, any]>((res, rej) => {
+ tmp.file((e, path, fd, cleanup) => {
+ if (e) return rej(e);
+ res([path, cleanup]);
+ });
+ });
+
+ logger.info(`Temp file is ${path}`);
+
+ const stream = fs.createWriteStream(path, { flags: 'a' });
+
+ let exportedCount = 0;
+ let ended = false;
+ let cursor: any = null;
+
+ while (!ended) {
+ const mutes = await Mute.find({
+ muterId: user._id,
+ ...(cursor ? { _id: { $gt: cursor } } : {})
+ }, {
+ limit: 100,
+ sort: {
+ _id: 1
+ }
+ });
+
+ if (mutes.length === 0) {
+ ended = true;
+ job.reportProgress(100);
+ break;
+ }
+
+ cursor = mutes[mutes.length - 1]._id;
+
+ for (const mute of mutes) {
+ const u = await User.findOne({ _id: mute.muteeId }, { fields: { username: true, host: true } });
+ const content = u.host ? `${u.username}@${u.host}` : `${u.username}@${config.host}`;
+ await new Promise((res, rej) => {
+ stream.write(content + '\n', err => {
+ if (err) {
+ logger.error(err);
+ rej(err);
+ } else {
+ res();
+ }
+ });
+ });
+ exportedCount++;
+ }
+
+ const total = await Mute.count({
+ muterId: user._id,
+ });
+
+ job.reportProgress(exportedCount / total);
+ }
+
+ stream.end();
+ logger.succ(`Exported to: ${path}`);
+
+ const fileName = dateFormat(new Date(), 'mute-yyyy-mm-dd-HH-MM-ss') + '.json';
+ const driveFile = await addFile(user, path, fileName);
+
+ logger.succ(`Exported to: ${driveFile._id}`);
+ cleanup();
+ done();
+}
diff --git a/src/queue/processors/index.ts b/src/queue/processors/index.ts
index 3f08fe29fb..6869983205 100644
--- a/src/queue/processors/index.ts
+++ b/src/queue/processors/index.ts
@@ -1,12 +1,18 @@
import deliver from './http/deliver';
import processInbox from './http/process-inbox';
import { exportNotes } from './export-notes';
+import { exportFollowing } from './export-following';
+import { exportMute } from './export-mute';
+import { exportBlocking } from './export-blocking';
import { queueLogger } from '../logger';
const handlers: any = {
deliver,
processInbox,
exportNotes,
+ exportFollowing,
+ exportMute,
+ exportBlocking,
};
export default (job: any, done: any) => {
diff --git a/src/server/api/endpoints/i/export-blocking.ts b/src/server/api/endpoints/i/export-blocking.ts
new file mode 100644
index 0000000000..12a08c6d7d
--- /dev/null
+++ b/src/server/api/endpoints/i/export-blocking.ts
@@ -0,0 +1,18 @@
+import define from '../../define';
+import { createExportBlockingJob } from '../../../../queue';
+import ms = require('ms');
+
+export const meta = {
+ secure: true,
+ requireCredential: true,
+ limit: {
+ duration: ms('1hour'),
+ max: 1,
+ },
+};
+
+export default define(meta, (ps, user) => new Promise(async (res, rej) => {
+ createExportBlockingJob(user);
+
+ res();
+}));
diff --git a/src/server/api/endpoints/i/export-following.ts b/src/server/api/endpoints/i/export-following.ts
new file mode 100644
index 0000000000..e819898d9b
--- /dev/null
+++ b/src/server/api/endpoints/i/export-following.ts
@@ -0,0 +1,18 @@
+import define from '../../define';
+import { createExportFollowingJob } from '../../../../queue';
+import ms = require('ms');
+
+export const meta = {
+ secure: true,
+ requireCredential: true,
+ limit: {
+ duration: ms('1hour'),
+ max: 1,
+ },
+};
+
+export default define(meta, (ps, user) => new Promise(async (res, rej) => {
+ createExportFollowingJob(user);
+
+ res();
+}));
diff --git a/src/server/api/endpoints/i/export-mute.ts b/src/server/api/endpoints/i/export-mute.ts
new file mode 100644
index 0000000000..c719a1f6a5
--- /dev/null
+++ b/src/server/api/endpoints/i/export-mute.ts
@@ -0,0 +1,18 @@
+import define from '../../define';
+import { createExportMuteJob } from '../../../../queue';
+import ms = require('ms');
+
+export const meta = {
+ secure: true,
+ requireCredential: true,
+ limit: {
+ duration: ms('1hour'),
+ max: 1,
+ },
+};
+
+export default define(meta, (ps, user) => new Promise(async (res, rej) => {
+ createExportMuteJob(user);
+
+ res();
+}));