From 3b0daeac3c73cd2b22fbdb90ef26a5987e9f0ae4 Mon Sep 17 00:00:00 2001 From: Aya Morisawa Date: Sun, 26 May 2019 22:05:09 +0900 Subject: [PATCH 01/14] Fix #4986 (#4991) --- src/boot/master.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/boot/master.ts b/src/boot/master.ts index 503f09c1f..6c23a528f 100644 --- a/src/boot/master.ts +++ b/src/boot/master.ts @@ -50,7 +50,7 @@ export async function masterMain() { // initialize app config = await init(); - if (config.port == null) { + if (config.port == null || Number.isNaN(config.port)) { bootLogger.error('The port is not configured. Please configure port.', null, true); process.exit(1); } From 41cdae07c384fbd12383f1792a1bad5f1a429ac2 Mon Sep 17 00:00:00 2001 From: Aya Morisawa Date: Sun, 26 May 2019 22:35:25 +0900 Subject: [PATCH 02/14] Fix #4989 (#4992) * Fix #4989 * Update --- locales/ja-JP.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 9153655f3..a8ec210a9 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1235,6 +1235,19 @@ admin/views/queue.vue: title: "キュー" remove-all-jobs: "すべてのジョブをクリア" +admin/views/logs.vue: + logs: "ログ" + domain: "ドメイン" + level: "レベル" + levels: + all: "全て" + info: "情報" + success: "成功" + warning: "警告" + error: "エラー" + debug: "デバッグ" + delete-all: "全て削除" + admin/views/abuse.vue: title: "スパム報告" target: "対象" From fad1352d51b08a4baa67b24ca366754fbd10fee3 Mon Sep 17 00:00:00 2001 From: Aya Morisawa Date: Sun, 26 May 2019 23:14:09 +0900 Subject: [PATCH 03/14] Add locale for admin/views/federation.vue (#4994) --- locales/ja-JP.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index a8ec210a9..fea404b64 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1554,6 +1554,7 @@ admin/views/federation.vue: day: "1日ごと" blocked-hosts: "ブロック" blocked-hosts-info: "ブロックしたいホストを改行で区切って記述します。" + save: "保存" desktop/views/pages/welcome.vue: about: "詳しく..." From a6a88af513f916246673576c3fbf4c90b7f1a8b6 Mon Sep 17 00:00:00 2001 From: Aya Morisawa Date: Sun, 26 May 2019 23:14:31 +0900 Subject: [PATCH 04/14] Add locales for admin/views/queue.vue (#4993) --- locales/ja-JP.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index fea404b64..99bcb1b50 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1234,6 +1234,15 @@ admin/views/dashboard.vue: admin/views/queue.vue: title: "キュー" remove-all-jobs: "すべてのジョブをクリア" + jobs: "ジョブ" + queue: "キュー" + domains: + deliver: "配送" + inbox: "受信" + state: "状態" + states: + delayed: "遅延" + result-is-truncated: "結果は省略されています" admin/views/logs.vue: logs: "ログ" From 911f5c14553574b22a5f059e3e1e56e85ac5ce7c Mon Sep 17 00:00:00 2001 From: Satsuki Yanagi <17376330+u1-liquid@users.noreply.github.com> Date: Mon, 27 May 2019 00:03:39 +0900 Subject: [PATCH 05/14] =?UTF-8?q?i18n:=20=E3=83=9A=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E7=B7=A8=E9=9B=86=E3=81=AE=E3=82=AB=E3=82=A6=E3=83=B3=E3=82=BF?= =?UTF-8?q?=E3=83=BC=E3=81=AE=E5=A2=97=E5=8A=A0=E5=80=A4=20(#4995)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../views/pages/page-editor/els/page-editor.el.counter.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/app/common/views/pages/page-editor/els/page-editor.el.counter.vue b/src/client/app/common/views/pages/page-editor/els/page-editor.el.counter.vue index 95c15b01b..4fc2aac8f 100644 --- a/src/client/app/common/views/pages/page-editor/els/page-editor.el.counter.vue +++ b/src/client/app/common/views/pages/page-editor/els/page-editor.el.counter.vue @@ -5,7 +5,7 @@
{{ $t('blocks._counter.name') }} {{ $t('blocks._counter.text') }} - {{ $t('blocks._counter.increment') }} + {{ $t('blocks._counter.inc') }}
From 94ce4fa780f3c2db5a2b2b33152a9938cfe4065c Mon Sep 17 00:00:00 2001 From: Satsuki Yanagi <17376330+u1-liquid@users.noreply.github.com> Date: Mon, 27 May 2019 08:41:24 +0900 Subject: [PATCH 06/14] Fix #4996 (#4997) --- src/models/repositories/user.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/models/repositories/user.ts b/src/models/repositories/user.ts index f81fa6bc7..d3cd75b17 100644 --- a/src/models/repositories/user.ts +++ b/src/models/repositories/user.ts @@ -176,6 +176,7 @@ export class UserRepository extends Repository { autoWatch: profile!.autoWatch, alwaysMarkNsfw: profile!.alwaysMarkNsfw, carefulBot: profile!.carefulBot, + autoAcceptFollowed: profile!.autoAcceptFollowed, hasUnreadMessagingMessage: this.getHasUnreadMessagingMessage(user.id), hasUnreadNotification: Notifications.count({ where: { From 9403ee649569d923df03ca1870982d325d5432a1 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 27 May 2019 15:38:14 +0900 Subject: [PATCH 07/14] Improve post form Resolve #4998 Resolve #1151 --- locales/ja-JP.yml | 62 +-- package.json | 2 +- src/client/app/common/scripts/post-form.ts | 478 +++++++++++++++++ .../desktop/views/components/post-form.vue | 499 ++---------------- .../app/mobile/views/components/post-form.vue | 344 +----------- 5 files changed, 556 insertions(+), 829 deletions(-) create mode 100644 src/client/app/common/scripts/post-form.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 99bcb1b50..25c052bc4 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -101,6 +101,34 @@ common: follow-users-to-make-your-timeline: "ユーザーをフォローすると投稿がタイムラインに表示されます。" explore: "ユーザーを探索する" + post-form: + attach-location-information: "位置情報を添付する" + hide-contents: "内容を隠す" + reply-placeholder: "この投稿への返信..." + quote-placeholder: "この投稿を引用..." + option-quote-placeholder: "この投稿を引用... (オプション)" + quote-attached: "引用付き" + quote-question: "引用として添付しますか？" + submit: "投稿" + reply: "返信" + renote: "Renote" + posting: "投稿中" + attach-media-from-local: "PCからメディアを添付" + attach-media-from-drive: "ドライブからメディアを添付" + insert-a-kao: "v('ω')v" + create-poll: "アンケートを作成" + text-remain: "残り{}文字" + recent-tags: "最近" + local-only-message: "この投稿はローカルにのみ公開されます" + click-to-tagging: "クリックでタグ付け" + visibility: "公開範囲" + geolocation-alert: "お使いの端末は位置情報に対応していません" + error: "エラー" + enter-username: "ユーザー名を入力してください" + add-visible-user: "ユーザーを追加" + cw-placeholder: "内容への注釈 (オプション)" + username-prompt: "ユーザー名を入力してください" + weekday-short: sunday: "日" monday: "月" @@ -1017,34 +1045,12 @@ desktop/views/components/notifications.vue: empty: "ありません！" desktop/views/components/post-form.vue: - add-visible-user: "+ユーザーを追加" - attach-location-information: "位置情報を添付する" - hide-contents: "内容を隠す" - reply-placeholder: "この投稿への返信..." - quote-placeholder: "この投稿を引用..." - submit: "投稿" - reply: "返信" - renote: "Renote" posted: "投稿しました！" replied: "返信しました！" reposted: "Renoteしました！" note-failed: "投稿に失敗しました" reply-failed: "返信に失敗しました" renote-failed: "Renoteに失敗しました" - posting: "投稿中" - attach-media-from-local: "PCからメディアを添付" - attach-media-from-drive: "ドライブからメディアを添付" - insert-a-kao: "v('ω')v" - create-poll: "アンケートを作成" - text-remain: "残り{}文字" - recent-tags: "最近" - local-only-message: "この投稿はローカルにのみ公開されます" - click-to-tagging: "クリックでタグ付け" - visibility: "公開範囲" - geolocation-alert: "お使いの端末は位置情報に対応していません" - error: "エラー" - enter-username: "ユーザー名を入力してください" - annotations: "内容への注釈 (オプション)" desktop/views/components/post-form-window.vue: note: "新規投稿" @@ -1726,18 +1732,6 @@ mobile/views/components/note-sub.vue: mobile/views/components/notifications.vue: empty: "ありません！" -mobile/views/components/post-form.vue: - add-visible-user: "ユーザーを追加" - submit: "投稿" - reply: "返信" - renote: "Renote" - quote-placeholder: "この投稿を引用... (オプション)" - reply-placeholder: "この投稿への返信..." - cw-placeholder: "内容への注釈 (オプション)" - geolocation-alert: "お使いの端末は位置情報に対応していません" - error: "エラー" - username-prompt: "ユーザー名を入力してください" - mobile/views/components/sub-note-content.vue: private: "この投稿は非公開です" deleted: "この投稿は削除されました" diff --git a/package.json b/package.json index 6fc8a0648..b6615ed5b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "misskey", "author": "syuilo ", - "version": "11.19.1", + "version": "11.20.0", "codename": "daybreak", "repository": { "type": "git", diff --git a/src/client/app/common/scripts/post-form.ts b/src/client/app/common/scripts/post-form.ts new file mode 100644 index 000000000..85a578484 --- /dev/null +++ b/src/client/app/common/scripts/post-form.ts @@ -0,0 +1,478 @@ +import insertTextAtCursor from 'insert-text-at-cursor'; +import { length } from 'stringz'; +import { toASCII } from 'punycode'; +import MkVisibilityChooser from '../views/components/visibility-chooser.vue'; +import getFace from './get-face'; +import { parse } from '../../../../mfm/parse'; +import { host, url } from '../../config'; +import i18n from '../../i18n'; +import { erase, unique } from '../../../../prelude/array'; +import extractMentions from '../../../../misc/extract-mentions'; + +export default (opts) => ({ + i18n: i18n(), + + components: { + XPostFormAttaches: () => import('../views/components/post-form-attaches.vue').then(m => m.default), + XPollEditor: () => import('../views/components/poll-editor.vue').then(m => m.default) + }, + + props: { + reply: { + type: Object, + required: false + }, + renote: { + type: Object, + required: false + }, + mention: { + type: Object, + required: false + }, + initialText: { + type: String, + required: false + }, + instant: { + type: Boolean, + required: false, + default: false + } + }, + + data() { + return { + posting: false, + text: '', + files: [], + uploadings: [], + poll: false, + pollChoices: [], + pollMultiple: false, + pollExpiration: [], + useCw: false, + cw: null, + geo: null, + visibility: 'public', + visibleUsers: [], + localOnly: false, + autocomplete: null, + draghover: false, + quoteId: null, + recentHashtags: JSON.parse(localStorage.getItem('hashtags') || '[]'), + maxNoteTextLength: 1000 + }; + }, + + computed: { + draftId(): string { + return this.renote + ? `renote:${this.renote.id}` + : this.reply + ? `reply:${this.reply.id}` + : 'note'; + }, + + placeholder(): string { + const xs = [ + this.$t('@.note-placeholders.a'), + this.$t('@.note-placeholders.b'), + this.$t('@.note-placeholders.c'), + this.$t('@.note-placeholders.d'), + this.$t('@.note-placeholders.e'), + this.$t('@.note-placeholders.f') + ]; + const x = xs[Math.floor(Math.random() * xs.length)]; + + return this.renote + ? opts.mobile ? this.$t('@.post-form.option-quote-placeholder') : this.$t('@.post-form.quote-placeholder') + : this.reply + ? this.$t('@.post-form.reply-placeholder') + : x; + }, + + submitText(): string { + return this.renote + ? this.$t('@.post-form.renote') + : this.reply + ? this.$t('@.post-form.reply') + : this.$t('@.post-form.submit'); + }, + + canPost(): boolean { + return !this.posting && + (1 <= this.text.length || 1 <= this.files.length || this.poll || this.renote) && + (length(this.text.trim()) <= this.maxNoteTextLength) && + (!this.poll || this.pollChoices.length >= 2); + } + }, + + created() { + this.$root.getMeta().then(meta => { + this.maxNoteTextLength = meta.maxNoteTextLength; + }); + }, + + mounted() { + if (this.initialText) { + this.text = this.initialText; + } + + if (this.mention) { + this.text = this.mention.host ? `@${this.mention.username}@${toASCII(this.mention.host)}` : `@${this.mention.username}`; + this.text += ' '; + } + + if (this.reply && this.reply.user.host != null) { + this.text = `@${this.reply.user.username}@${toASCII(this.reply.user.host)} `; + } + + if (this.reply && this.reply.text != null) { + const ast = parse(this.reply.text); + + for (const x of extractMentions(ast)) { + const mention = x.host ? `@${x.username}@${toASCII(x.host)}` : `@${x.username}`; + + // 自分は除外 + if (this.$store.state.i.username == x.username && x.host == null) continue; + if (this.$store.state.i.username == x.username && x.host == host) continue; + + // 重複は除外 + if (this.text.indexOf(`${mention} `) != -1) continue; + + this.text += `${mention} `; + } + } + + // デフォルト公開範囲 + this.applyVisibility(this.$store.state.settings.rememberNoteVisibility ? (this.$store.state.device.visibility || this.$store.state.settings.defaultNoteVisibility) : this.$store.state.settings.defaultNoteVisibility); + + // 公開以外へのリプライ時は元の公開範囲を引き継ぐ + if (this.reply && ['home', 'followers', 'specified'].includes(this.reply.visibility)) { + this.visibility = this.reply.visibility; + } + + if (this.reply) { + this.$root.api('users/show', { userId: this.reply.userId }).then(user => { + this.visibleUsers.push(user); + }); + } + + // keep cw when reply + if (this.$store.state.settings.keepCw && this.reply && this.reply.cw) { + this.useCw = true; + this.cw = this.reply.cw; + } + + this.focus(); + + this.$nextTick(() => { + this.focus(); + }); + + this.$nextTick(() => { + // 書きかけの投稿を復元 + if (!this.instant && !this.mention) { + const draft = JSON.parse(localStorage.getItem('drafts') || '{}')[this.draftId]; + if (draft) { + this.text = draft.data.text; + this.files = (draft.data.files || []).filter(e => e); + if (draft.data.poll) { + this.poll = true; + this.$nextTick(() => { + (this.$refs.poll as any).set(draft.data.poll); + }); + } + this.$emit('change-attached-files', this.files); + } + } + + this.$nextTick(() => this.watch()); + }); + }, + + methods: { + watch() { + this.$watch('text', () => this.saveDraft()); + this.$watch('poll', () => this.saveDraft()); + this.$watch('files', () => this.saveDraft()); + }, + + trimmedLength(text: string) { + return length(text.trim()); + }, + + addTag(tag: string) { + insertTextAtCursor(this.$refs.text, ` #${tag} `); + }, + + focus() { + (this.$refs.text as any).focus(); + }, + + chooseFile() { + (this.$refs.file as any).click(); + }, + + chooseFileFromDrive() { + this.$chooseDriveFile({ + multiple: true + }).then(files => { + for (const x of files) this.attachMedia(x); + }); + }, + + attachMedia(driveFile) { + this.files.push(driveFile); + this.$emit('change-attached-files', this.files); + }, + + detachMedia(id) { + this.files = this.files.filter(x => x.id != id); + this.$emit('change-attached-files', this.files); + }, + + onChangeFile() { + for (const x of Array.from((this.$refs.file as any).files)) this.upload(x); + }, + + upload(file) { + (this.$refs.uploader as any).upload(file); + }, + + onChangeUploadings(uploads) { + this.$emit('change-uploadings', uploads); + }, + + onPollUpdate() { + const got = this.$refs.poll.get(); + this.pollChoices = got.choices; + this.pollMultiple = got.multiple; + this.pollExpiration = [got.expiration, got.expiresAt || got.expiredAfter]; + this.saveDraft(); + }, + + setGeo() { + if (navigator.geolocation == null) { + this.$root.dialog({ + type: 'warning', + text: this.$t('@.post-form.geolocation-alert') + }); + return; + } + + navigator.geolocation.getCurrentPosition(pos => { + this.geo = pos.coords; + this.$emit('geo-attached', this.geo); + }, err => { + this.$root.dialog({ + type: 'error', + title: this.$t('@.post-form.error'), + text: err.message + }); + }, { + enableHighAccuracy: true + }); + }, + + removeGeo() { + this.geo = null; + this.$emit('geo-dettached'); + }, + + setVisibility() { + const w = this.$root.new(MkVisibilityChooser, { + source: this.$refs.visibilityButton, + currentVisibility: this.visibility + }); + w.$once('chosen', v => { + this.applyVisibility(v); + }); + }, + + applyVisibility(v: string) { + const m = v.match(/^local-(.+)/); + if (m) { + this.localOnly = true; + this.visibility = m[1]; + } else { + this.localOnly = false; + this.visibility = v; + } + }, + + addVisibleUser() { + this.$root.dialog({ + title: this.$t('@.post-form.enter-username'), + user: true + }).then(({ canceled, result: user }) => { + if (canceled) return; + this.visibleUsers.push(user); + }); + }, + + removeVisibleUser(user) { + this.visibleUsers = erase(user, this.visibleUsers); + }, + + clear() { + this.text = ''; + this.files = []; + this.poll = false; + this.$emit('change-attached-files', this.files); + }, + + onKeydown(e) { + if ((e.which == 10 || e.which == 13) && (e.ctrlKey || e.metaKey) && this.canPost) this.post(); + }, + + async onPaste(e) { + for (const item of Array.from(e.clipboardData.items)) { + if (item.kind == 'file') { + this.upload(item.getAsFile()); + } + } + + const paste = e.clipboardData.getData('text'); + + if (paste.startsWith(url + '/notes/')) { + e.preventDefault(); + + this.$root.dialog({ + type: 'info', + text: this.$t('@.post-form.quote-question'), + showCancelButton: true + }).then(({ canceled }) => { + if (canceled) { + insertTextAtCursor(this.$refs.text, paste); + return; + } + + this.quoteId = paste.substr(url.length).match(/^\/notes\/(.+?)\/?$/)[1]; + }); + } + }, + + onDragover(e) { + const isFile = e.dataTransfer.items[0].kind == 'file'; + const isDriveFile = e.dataTransfer.types[0] == 'mk_drive_file'; + if (isFile || isDriveFile) { + e.preventDefault(); + this.draghover = true; + e.dataTransfer.dropEffect = e.dataTransfer.effectAllowed == 'all' ? 'copy' : 'move'; + } + }, + + onDragenter(e) { + this.draghover = true; + }, + + onDragleave(e) { + this.draghover = false; + }, + + onDrop(e): void { + this.draghover = false; + + // ファイルだったら + if (e.dataTransfer.files.length > 0) { + e.preventDefault(); + for (const x of Array.from(e.dataTransfer.files)) this.upload(x); + return; + } + + //#region ドライブのファイル + const driveFile = e.dataTransfer.getData('mk_drive_file'); + if (driveFile != null && driveFile != '') { + const file = JSON.parse(driveFile); + this.files.push(file); + this.$emit('change-attached-files', this.files); + e.preventDefault(); + } + //#endregion + }, + + async emoji() { + const Picker = await import('../../desktop/views/components/emoji-picker-dialog.vue').then(m => m.default); + const button = this.$refs.emoji; + const rect = button.getBoundingClientRect(); + const vm = this.$root.new(Picker, { + x: button.offsetWidth + rect.left + window.pageXOffset, + y: rect.top + window.pageYOffset + }); + vm.$once('chosen', emoji => { + insertTextAtCursor(this.$refs.text, emoji); + }); + }, + + saveDraft() { + if (this.instant) return; + + const data = JSON.parse(localStorage.getItem('drafts') || '{}'); + + data[this.draftId] = { + updatedAt: new Date(), + data: { + text: this.text, + files: this.files, + poll: this.poll && this.$refs.poll ? (this.$refs.poll as any).get() : undefined + } + }; + + localStorage.setItem('drafts', JSON.stringify(data)); + }, + + deleteDraft() { + const data = JSON.parse(localStorage.getItem('drafts') || '{}'); + + delete data[this.draftId]; + + localStorage.setItem('drafts', JSON.stringify(data)); + }, + + kao() { + this.text += getFace(); + }, + + post() { + this.posting = true; + const viaMobile = opts.mobile && !this.$store.state.settings.disableViaMobile; + this.$root.api('notes/create', { + text: this.text == '' ? undefined : this.text, + fileIds: this.files.length > 0 ? this.files.map(f => f.id) : undefined, + replyId: this.reply ? this.reply.id : undefined, + renoteId: this.renote ? this.renote.id : this.quoteId ? this.quoteId : undefined, + poll: this.poll ? (this.$refs.poll as any).get() : undefined, + cw: this.useCw ? this.cw || '' : undefined, + visibility: this.visibility, + visibleUserIds: this.visibility == 'specified' ? this.visibleUsers.map(u => u.id) : undefined, + localOnly: this.localOnly, + geo: this.geo ? { + coordinates: [this.geo.longitude, this.geo.latitude], + altitude: this.geo.altitude, + accuracy: this.geo.accuracy, + altitudeAccuracy: this.geo.altitudeAccuracy, + heading: isNaN(this.geo.heading) ? null : this.geo.heading, + speed: this.geo.speed, + } : null, + viaMobile: viaMobile + }).then(data => { + this.clear(); + this.deleteDraft(); + this.$emit('posted'); + if (opts.onSuccess) opts.onSuccess(this); + }).catch(err => { + if (opts.onSuccess) opts.onFailure(this); + }).then(() => { + this.posting = false; + }); + + if (this.text && this.text != '') { + const hashtags = parse(this.text).filter(x => x.node.type === 'hashtag').map(x => x.node.props.hashtag); + const history = JSON.parse(localStorage.getItem('hashtags') || '[]') as string[]; + localStorage.setItem('hashtags', JSON.stringify(unique(hashtags.concat(history)))); + } + }, + } +}); diff --git a/src/client/app/desktop/views/components/post-form.vue b/src/client/app/desktop/views/components/post-form.vue index 64652e0eb..99f508a92 100644 --- a/src/client/app/desktop/views/components/post-form.vue +++ b/src/client/app/desktop/views/components/post-form.vue @@ -10,14 +10,15 @@ [x] - {{ $t('add-visible-user') }} + {{ $t('@.post-form.add-visible-user') }}
- {{ $t('recent-tags') }}: - #{{ tag }} + {{ $t('@.post-form.recent-tags') }}: + #{{ tag }}
-
{{ $t('local-only-message') }}
- +
{{ $t('@.post-form.quote-attached') }}
+
{{ $t('@.post-form.local-only-message') }}
+
+ + +
{{ $t('@.post-form.quote-attached') }}
@@ -50,337 +56,21 @@ From 72fb23f4d51db1b999a10b7d8bb50293a81af163 Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 27 May 2019 16:54:47 +0900 Subject: [PATCH 08/14] Improve drive management --- locales/ja-JP.yml | 3 ++ src/client/app/admin/views/drive.vue | 27 +++++++++++++++++ src/queue/index.ts | 30 +++++++++++++++++++ src/queue/processors/db/delete-drive-files.ts | 2 +- .../processors/object-storage/delete-file.ts | 22 ++++++++++++++ src/queue/processors/object-storage/index.ts | 12 ++++++++ .../admin/delete-all-files-of-a-user.ts | 4 +-- .../admin/drive/clean-remote-files.ts | 21 +++++++++++++ .../api/endpoints/admin/drive/cleanup.ts | 21 +++++++++++++ .../admin/federation/delete-all-files.ts | 4 +-- .../api/endpoints/drive/files/delete.ts | 4 +-- src/services/drive/add-file.ts | 4 +-- src/services/drive/delete-file.ts | 26 +++++----------- src/tools/clean-remote-files.ts | 26 ---------------- 14 files changed, 152 insertions(+), 54 deletions(-) create mode 100644 src/queue/processors/object-storage/delete-file.ts create mode 100644 src/queue/processors/object-storage/index.ts create mode 100644 src/server/api/endpoints/admin/drive/clean-remote-files.ts create mode 100644 src/server/api/endpoints/admin/drive/cleanup.ts delete mode 100644 src/tools/clean-remote-files.ts diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 25c052bc4..493875af6 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1417,6 +1417,9 @@ admin/views/drive.vue: unmark-as-sensitive: "閲覧注意を解除" marked-as-sensitive: "閲覧注意に設定しました" unmarked-as-sensitive: "閲覧注意を解除しました" + clean-remote-files: "リモートファイルのキャッシュを削除" + clean-remote-files-are-you-sure: "すべてのリモートファイルのキャッシュを削除してもよろしいですか？" + clean-up: "クリーンアップ" admin/views/users.vue: operation: "操作" diff --git a/src/client/app/admin/views/drive.vue b/src/client/app/admin/views/drive.vue index 22b3b7eb9..1152db2b9 100644 --- a/src/client/app/admin/views/drive.vue +++ b/src/client/app/admin/views/drive.vue @@ -14,6 +14,10 @@ {{ $t('lookup') }} +
+ {{ $t('clean-up') }} + {{ $t('clean-remote-files') }} +
@@ -227,6 +231,29 @@ export default Vue.extend({ }); }); }, + + cleanRemoteFiles() { + this.$root.dialog({ + type: 'warning', + text: this.$t('clean-remote-files-are-you-sure'), + showCancelButton: true + }).then(({ canceled }) => { + if (canceled) return; + this.$root.api('admin/drive/clean-remote-files'); + this.$root.dialog({ + type: 'success', + splash: true + }); + }); + }, + + cleanUp() { + this.$root.api('admin/drive/cleanup'); + this.$root.dialog({ + type: 'success', + splash: true + }); + } } }); diff --git a/src/queue/index.ts b/src/queue/index.ts index a010004f1..70233d831 100644 --- a/src/queue/index.ts +++ b/src/queue/index.ts @@ -8,6 +8,7 @@ import { program } from '../argv'; import processDeliver from './processors/deliver'; import processInbox from './processors/inbox'; import processDb from './processors/db'; +import procesObjectStorage from './processors/object-storage'; import { queueLogger } from './logger'; import { DriveFile } from '../models/entities/drive-file'; @@ -34,9 +35,12 @@ function renderError(e: Error): any { export const deliverQueue = initializeQueue('deliver'); export const inboxQueue = initializeQueue('inbox'); export const dbQueue = initializeQueue('db'); +export const objectStorageQueue = initializeQueue('objectStorage'); const deliverLogger = queueLogger.createSubLogger('deliver'); const inboxLogger = queueLogger.createSubLogger('inbox'); +const dbLogger = queueLogger.createSubLogger('db'); +const objectStorageLogger = queueLogger.createSubLogger('objectStorage'); deliverQueue .on('waiting', (jobId) => deliverLogger.debug(`waiting id=${jobId}`)) @@ -54,6 +58,22 @@ inboxQueue .on('error', (job: any, err: Error) => inboxLogger.error(`error ${err}`, { job, e: renderError(err) })) .on('stalled', (job) => inboxLogger.warn(`stalled id=${job.id} activity=${job.data.activity ? job.data.activity.id : 'none'}`)); +dbQueue + .on('waiting', (jobId) => dbLogger.debug(`waiting id=${jobId}`)) + .on('active', (job) => dbLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => dbLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => dbLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => dbLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => dbLogger.warn(`stalled id=${job.id}`)); + +objectStorageQueue + .on('waiting', (jobId) => objectStorageLogger.debug(`waiting id=${jobId}`)) + .on('active', (job) => objectStorageLogger.debug(`active id=${job.id}`)) + .on('completed', (job, result) => objectStorageLogger.debug(`completed(${result}) id=${job.id}`)) + .on('failed', (job, err) => objectStorageLogger.warn(`failed(${err}) id=${job.id}`, { job, e: renderError(err) })) + .on('error', (job: any, err: Error) => objectStorageLogger.error(`error ${err}`, { job, e: renderError(err) })) + .on('stalled', (job) => objectStorageLogger.warn(`stalled id=${job.id}`)); + export function deliver(user: ILocalUser, content: any, to: any) { if (content == null) return null; @@ -165,11 +185,21 @@ export function createImportUserListsJob(user: ILocalUser, fileId: DriveFile['id }); } +export function createDeleteObjectStorageFileJob(key: string) { + return objectStorageQueue.add('deleteFile', { + key: key + }, { + removeOnComplete: true, + removeOnFail: true + }); +} + export default function() { if (!program.onlyServer) { deliverQueue.process(128, processDeliver); inboxQueue.process(128, processInbox); processDb(dbQueue); + procesObjectStorage(objectStorageQueue); } } diff --git a/src/queue/processors/db/delete-drive-files.ts b/src/queue/processors/db/delete-drive-files.ts index 4e4eab86b..5ecee9139 100644 --- a/src/queue/processors/db/delete-drive-files.ts +++ b/src/queue/processors/db/delete-drive-files.ts @@ -1,7 +1,7 @@ import * as Bull from 'bull'; import { queueLogger } from '../../logger'; -import deleteFile from '../../../services/drive/delete-file'; +import { deleteFile } from '../../../services/drive/delete-file'; import { Users, DriveFiles } from '../../../models'; import { MoreThan } from 'typeorm'; diff --git a/src/queue/processors/object-storage/delete-file.ts b/src/queue/processors/object-storage/delete-file.ts new file mode 100644 index 000000000..8e6b5f959 --- /dev/null +++ b/src/queue/processors/object-storage/delete-file.ts @@ -0,0 +1,22 @@ +import * as Bull from 'bull'; +import * as Minio from 'minio'; +import { fetchMeta } from '../../../misc/fetch-meta'; + +export default async (job: Bull.Job) => { + const meta = await fetchMeta(); + + const minio = new Minio.Client({ + endPoint: meta.objectStorageEndpoint!, + region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, + port: meta.objectStoragePort ? meta.objectStoragePort : undefined, + useSSL: meta.objectStorageUseSSL, + accessKey: meta.objectStorageAccessKey!, + secretKey: meta.objectStorageSecretKey!, + }); + + const key: string = job.data.key; + + await minio.removeObject(meta.objectStorageBucket!, key); + + return 'Success'; +}; diff --git a/src/queue/processors/object-storage/index.ts b/src/queue/processors/object-storage/index.ts new file mode 100644 index 000000000..f2fc08f1a --- /dev/null +++ b/src/queue/processors/object-storage/index.ts @@ -0,0 +1,12 @@ +import * as Bull from 'bull'; +import deleteFile from './delete-file'; + +const jobs = { + deleteFile, +} as any; + +export default function(q: Bull.Queue) { + for (const [k, v] of Object.entries(jobs)) { + q.process(k, v as any); + } +} diff --git a/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts index 84e9c363e..276d1f37b 100644 --- a/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts +++ b/src/server/api/endpoints/admin/delete-all-files-of-a-user.ts @@ -1,6 +1,6 @@ import $ from 'cafy'; import define from '../../define'; -import del from '../../../../services/drive/delete-file'; +import { deleteFile } from '../../../../services/drive/delete-file'; import { DriveFiles } from '../../../../models'; import { ID } from '../../../../misc/cafy-id'; @@ -27,6 +27,6 @@ export default define(meta, async (ps, me) => { }); for (const file of files) { - del(file); + deleteFile(file); } }); diff --git a/src/server/api/endpoints/admin/drive/clean-remote-files.ts b/src/server/api/endpoints/admin/drive/clean-remote-files.ts new file mode 100644 index 000000000..a861052bb --- /dev/null +++ b/src/server/api/endpoints/admin/drive/clean-remote-files.ts @@ -0,0 +1,21 @@ +import { Not, IsNull } from 'typeorm'; +import define from '../../../define'; +import { deleteFile } from '../../../../../services/drive/delete-file'; +import { DriveFiles } from '../../../../../models'; + +export const meta = { + tags: ['admin'], + + requireCredential: true, + requireModerator: true, +}; + +export default define(meta, async (ps, me) => { + const files = await DriveFiles.find({ + userHost: Not(IsNull()) + }); + + for (const file of files) { + deleteFile(file, true); + } +}); diff --git a/src/server/api/endpoints/admin/drive/cleanup.ts b/src/server/api/endpoints/admin/drive/cleanup.ts new file mode 100644 index 000000000..272416dee --- /dev/null +++ b/src/server/api/endpoints/admin/drive/cleanup.ts @@ -0,0 +1,21 @@ +import { IsNull } from 'typeorm'; +import define from '../../../define'; +import { deleteFile } from '../../../../../services/drive/delete-file'; +import { DriveFiles } from '../../../../../models'; + +export const meta = { + tags: ['admin'], + + requireCredential: true, + requireModerator: true, +}; + +export default define(meta, async (ps, me) => { + const files = await DriveFiles.find({ + userId: IsNull() + }); + + for (const file of files) { + deleteFile(file); + } +}); diff --git a/src/server/api/endpoints/admin/federation/delete-all-files.ts b/src/server/api/endpoints/admin/federation/delete-all-files.ts index befb36226..24ad9ce80 100644 --- a/src/server/api/endpoints/admin/federation/delete-all-files.ts +++ b/src/server/api/endpoints/admin/federation/delete-all-files.ts @@ -1,6 +1,6 @@ import $ from 'cafy'; import define from '../../../define'; -import del from '../../../../../services/drive/delete-file'; +import { deleteFile } from '../../../../../services/drive/delete-file'; import { DriveFiles } from '../../../../../models'; export const meta = { @@ -22,6 +22,6 @@ export default define(meta, async (ps, me) => { }); for (const file of files) { - del(file); + deleteFile(file); } }); diff --git a/src/server/api/endpoints/drive/files/delete.ts b/src/server/api/endpoints/drive/files/delete.ts index d8cc5ec0a..d6a20f352 100644 --- a/src/server/api/endpoints/drive/files/delete.ts +++ b/src/server/api/endpoints/drive/files/delete.ts @@ -1,6 +1,6 @@ import $ from 'cafy'; import { ID } from '../../../../../misc/cafy-id'; -import del from '../../../../../services/drive/delete-file'; +import { deleteFile } from '../../../../../services/drive/delete-file'; import { publishDriveStream } from '../../../../../services/stream'; import define from '../../../define'; import { ApiError } from '../../../error'; @@ -57,7 +57,7 @@ export default define(meta, async (ps, user) => { } // Delete - await del(file); + await deleteFile(file); // Publish fileDeleted event publishDriveStream(user.id, 'fileDeleted', file.id); diff --git a/src/services/drive/add-file.ts b/src/services/drive/add-file.ts index eb0b0abaa..d21c67d18 100644 --- a/src/services/drive/add-file.ts +++ b/src/services/drive/add-file.ts @@ -7,7 +7,7 @@ import * as uuid from 'uuid'; import * as sharp from 'sharp'; import { publishMainStream, publishDriveStream } from '../stream'; -import delFile from './delete-file'; +import { deleteFile } from './delete-file'; import { fetchMeta } from '../../misc/fetch-meta'; import { GenerateVideoThumbnail } from './generate-video-thumbnail'; import { driveLogger } from './logger'; @@ -233,7 +233,7 @@ async function deleteOldFile(user: IRemoteUser) { const oldFile = await q.getOne(); if (oldFile) { - delFile(oldFile, true); + deleteFile(oldFile, true); } } diff --git a/src/services/drive/delete-file.ts b/src/services/drive/delete-file.ts index b429ca0db..258c0853f 100644 --- a/src/services/drive/delete-file.ts +++ b/src/services/drive/delete-file.ts @@ -1,11 +1,10 @@ -import * as Minio from 'minio'; import { DriveFile } from '../../models/entities/drive-file'; import { InternalStorage } from './internal-storage'; import { DriveFiles, Instances, Notes } from '../../models'; import { driveChart, perUserDriveChart, instanceChart } from '../chart'; -import { fetchMeta } from '../../misc/fetch-meta'; +import { createDeleteObjectStorageFileJob } from '../../queue'; -export default async function(file: DriveFile, isExpired = false) { +export async function deleteFile(file: DriveFile, isExpired = false) { if (file.storedInternal) { InternalStorage.del(file.accessKey!); @@ -17,25 +16,14 @@ export default async function(file: DriveFile, isExpired = false) { InternalStorage.del(file.webpublicAccessKey!); } } else if (!file.isLink) { - const meta = await fetchMeta(); - - const minio = new Minio.Client({ - endPoint: meta.objectStorageEndpoint!, - region: meta.objectStorageRegion ? meta.objectStorageRegion : undefined, - port: meta.objectStoragePort ? meta.objectStoragePort : undefined, - useSSL: meta.objectStorageUseSSL, - accessKey: meta.objectStorageAccessKey!, - secretKey: meta.objectStorageSecretKey!, - }); - - await minio.removeObject(meta.objectStorageBucket!, file.accessKey!); + createDeleteObjectStorageFileJob(file.accessKey!); if (file.thumbnailUrl) { - await minio.removeObject(meta.objectStorageBucket!, file.thumbnailAccessKey!); + createDeleteObjectStorageFileJob(file.thumbnailAccessKey!); } if (file.webpublicUrl) { - await minio.removeObject(meta.objectStorageBucket!, file.webpublicAccessKey!); + createDeleteObjectStorageFileJob(file.webpublicAccessKey!); } } @@ -44,8 +32,8 @@ export default async function(file: DriveFile, isExpired = false) { DriveFiles.update(file.id, { isLink: true, url: file.uri, - thumbnailUrl: null, - webpublicUrl: null + thumbnailUrl: file.uri, + webpublicUrl: file.uri }); } else { DriveFiles.delete(file.id); diff --git a/src/tools/clean-remote-files.ts b/src/tools/clean-remote-files.ts deleted file mode 100644 index 633a6fc04..000000000 --- a/src/tools/clean-remote-files.ts +++ /dev/null @@ -1,26 +0,0 @@ -import * as promiseLimit from 'promise-limit'; -import del from '../services/drive/delete-file'; -import { DriveFiles } from '../models'; -import { Not, IsNull } from 'typeorm'; -import { DriveFile } from '../models/entities/drive-file'; -import { ensure } from '../prelude/ensure'; - -const limit = promiseLimit(16); - -DriveFiles.find({ - userHost: Not(IsNull()) -}).then(async files => { - console.log(`there is ${files.length} files`); - - await Promise.all(files.map(file => limit(() => job(file)))); - - console.log('ALL DONE'); -}); - -async function job(file: DriveFile): Promise { - file = await DriveFiles.findOne(file.id).then(ensure); - - await del(file, true); - - console.log('done', file.id); -} From 85d8e6f220eacf2f1cd94b72a7b5ebc8ab3991ca Mon Sep 17 00:00:00 2001 From: syuilo Date: Mon, 27 May 2019 17:23:05 +0900 Subject: [PATCH 09/14] Refactoring --- src/client/app/admin/views/queue.chart.vue | 181 ++++++++++++++++++ src/client/app/admin/views/queue.vue | 206 ++------------------- src/daemons/queue-stats.ts | 30 ++- 3 files changed, 225 insertions(+), 192 deletions(-) create mode 100644 src/client/app/admin/views/queue.chart.vue diff --git a/src/client/app/admin/views/queue.chart.vue b/src/client/app/admin/views/queue.chart.vue new file mode 100644 index 000000000..c3d68d75a --- /dev/null +++ b/src/client/app/admin/views/queue.chart.vue @@ -0,0 +1,181 @@ + + + + + diff --git a/src/client/app/admin/views/queue.vue b/src/client/app/admin/views/queue.vue index b379ba185..358ed6cf7 100644 --- a/src/client/app/admin/views/queue.vue +++ b/src/client/app/admin/views/queue.vue @@ -4,57 +4,11 @@
Deliver
- The queue is jammed. - - - Process - - - - - Active - - - - - Waiting - - - - - Delayed - - - - -
+
Inbox
- The queue is jammed. - - - Process - - - - - Active - - - - - Waiting - - - - - Delayed - - - - -
+
{{ $t('remove-all-jobs') }} @@ -94,74 +48,31 @@ diff --git a/src/client/app/admin/views/queue.vue b/src/client/app/admin/views/queue.vue index 358ed6cf7..43a41ffbe 100644 --- a/src/client/app/admin/views/queue.vue +++ b/src/client/app/admin/views/queue.vue @@ -2,14 +2,28 @@
-
-
Deliver
+
+
{{ $t('domains.deliver') }}
-
-
Inbox
+
+
{{ $t('domains.inbox') }}
+
+
+ {{ $t('other-queues') }} +
+
{{ $t('domains.db') }}
+ +
+ +
+
{{ $t('domains.objectStorage') }}
+ +
+
+
{{ $t('remove-all-jobs') }}
@@ -23,9 +37,13 @@ + + + + @@ -48,7 +66,7 @@