enhance(frontend): 通知音にドライブのファイルを使用できるように (#12447)

* (enhance) サウンドにドライブのファイルを使用できるように

* Update Changelog

* fix

* fix design

* fix design

* Update store.ts

* (fix) ファイル名表示

* refactor

* (refactor) better types

* operationTypeとsoundTypeの混同を防止

* (refactor)

* (fix)

* enhance jsdoc

* driveFile -> _driveFile_

CHANGELOG.md

@@ -24,6 +24,7 @@
 - Enhance: 絵文字のオートコンプリート機能強化 #12364
 - Enhance: ユーザーのRawデータを表示するページが復活
 - Enhance: リアクション選択時に音を鳴らせるように
+- Enhance: サウンドにドライブのファイルを使用できるように
 - fix: 「設定のバックアップ」で一部の項目がバックアップに含まれていなかった問題を修正
 - Fix: ウィジェットのジョブキューにて音声の発音方法変更に追従できていなかったのを修正 #12367
 - Fix: コードエディタが正しく表示されない問題を修正

locales/index.d.ts

@@ -1947,6 +1947,14 @@ export interface Locale {
         "channel": string;
         "reaction": string;
     };
+    "_soundSettings": {
+        "driveFile": string;
+        "driveFileWarn": string;
+        "driveFileTypeWarn": string;
+        "driveFileTypeWarnDescription": string;
+        "driveFileDurationWarn": string;
+        "driveFileDurationWarnDescription": string;
+    };
     "_ago": {
         "future": string;
         "justNow": string;

locales/ja-JP.yml

@@ -1852,6 +1852,14 @@ _sfx:
   channel: "チャンネル通知"
   reaction: "リアクション選択時"
 
+_soundSettings:
+  driveFile: "ドライブの音声を使用"
+  driveFileWarn: "ドライブのファイルを選択してください"
+  driveFileTypeWarn: "このファイルは対応していません"
+  driveFileTypeWarnDescription: "音声ファイルを選択してください"
+  driveFileDurationWarn: "音声が長すぎます"
+  driveFileDurationWarnDescription: "長い音声を使用するとMisskeyの使用に支障をきたす可能性があります。それでも続行しますか？"
+
 _ago:
   future: "未来"
   justNow: "たった今"

packages/frontend/src/pages/settings/sounds.sound.vue

@@ -7,8 +7,12 @@ SPDX-License-Identifier: AGPL-3.0-only
 <div class="_gaps_m">
 	<MkSelect v-model="type">
 		<template #label>{{ i18n.ts.sound }}</template>
-		<option v-for="x in soundsTypes" :key="x" :value="x">{{ x == null ? i18n.ts.none : x }}</option>
+		<option v-for="x in soundsTypes" :key="x ?? 'null'" :value="x">{{ getSoundTypeName(x) }}</option>
 	</MkSelect>
+	<div v-if="type === '_driveFile_'" :class="$style.fileSelectorRoot">
+		<MkButton :class="$style.fileSelectorButton" inline rounded primary @click="selectSound">{{ i18n.ts.selectFile }}</MkButton>
+		<div :class="['_nowrap', !fileUrl && $style.fileNotSelected]">{{ friendlyFileName }}</div>
+	</div>
 	<MkRange v-model="volume" :min="0" :max="1" :step="0.05" :textConverter="(v) => `${Math.floor(v * 100)}%`">
 		<template #label>{{ i18n.ts.volume }}</template>
 	</MkRange>
@@ -21,30 +25,149 @@ SPDX-License-Identifier: AGPL-3.0-only
 </template>
 
 <script lang="ts" setup>
-import { } from 'vue';
+import { ref, computed } from 'vue';
+import type { SoundType } from '@/scripts/sound.js';
 import MkSelect from '@/components/MkSelect.vue';
 import MkButton from '@/components/MkButton.vue';
 import MkRange from '@/components/MkRange.vue';
 import { i18n } from '@/i18n.js';
-import { playFile, soundsTypes } from '@/scripts/sound.js';
+import * as os from '@/os.js';
+import { playFile, soundsTypes, getSoundDuration } from '@/scripts/sound.js';
+import { selectFile } from '@/scripts/select-file.js';
 
 const props = defineProps<{
-	type: string;
+	type: SoundType;
+	fileId?: string;
+	fileUrl?: string;
 	volume: number;
 }>();
 
 const emit = defineEmits<{
-	(ev: 'update', result: { type: string; volume: number; }): void;
+	(ev: 'update', result: { type: SoundType; fileId?: string; fileUrl?: string; volume: number; }): void;
 }>();
 
-let type = $ref(props.type);
-let volume = $ref(props.volume);
+const type = ref<SoundType>(props.type);
+const fileId = ref(props.fileId);
+const fileUrl = ref(props.fileUrl);
+const fileName = ref<string>('');
+const volume = ref(props.volume);
+
+if (type.value === '_driveFile_' && fileId.value) {
+	const apiRes = await os.api('drive/files/show', {
+		fileId: fileId.value,
+	});
+	fileName.value = apiRes.name;
+}
+
+function getSoundTypeName(f: SoundType): string {
+	switch (f) {
+		case null:
+			return i18n.ts.none;
+		case '_driveFile_':
+			return i18n.ts._soundSettings.driveFile;
+		default:
+			return f;
+	}
+}
+
+const friendlyFileName = computed<string>(() => {
+	if (fileName.value) {
+		return fileName.value;
+	}
+	if (fileUrl.value) {
+		return fileUrl.value;
+	}
+
+	return i18n.ts._soundSettings.driveFileWarn;
+});
+
+function selectSound(ev) {
+	selectFile(ev.currentTarget ?? ev.target, i18n.ts._soundSettings.driveFile).then(async (file) => {
+		if (!file.type.startsWith('audio')) {
+			os.alert({
+				type: 'warning',
+				title: i18n.ts._soundSettings.driveFileTypeWarn,
+				text: i18n.ts._soundSettings.driveFileTypeWarnDescription,
+			});
+			return;
+		}
+		const duration = await getSoundDuration(file.url);
+		if (duration >= 2000) {
+			const { canceled } = await os.confirm({
+				type: 'warning',
+				title: i18n.ts._soundSettings.driveFileDurationWarn,
+				text: i18n.ts._soundSettings.driveFileDurationWarnDescription,
+				okText: i18n.ts.continue,
+				cancelText: i18n.ts.cancel,
+			});
+			if (canceled) return;
+		}
+
+		fileUrl.value = file.url;
+		fileName.value = file.name;
+		fileId.value = file.id;
+	});
+}
 
 function listen() {
-	playFile(type, volume);
+	if (type.value === '_driveFile_' && (!fileUrl.value || !fileId.value)) {
+		os.alert({
+			type: 'warning',
+			text: i18n.ts._soundSettings.driveFileWarn,
+		});
+		return;
+	}
+
+	playFile(type.value === '_driveFile_' ? {
+		type: '_driveFile_',
+		fileId: fileId.value as string,
+		fileUrl: fileUrl.value as string,
+		volume: volume.value,
+	} : {
+		type: type.value,
+		volume: volume.value,
+	});
 }
 
 function save() {
-	emit('update', { type, volume });
+	if (type.value === '_driveFile_' && !fileUrl.value) {
+		os.alert({
+			type: 'warning',
+			text: i18n.ts._soundSettings.driveFileWarn,
+		});
+		return;
+	}
+
+	if (type.value !== '_driveFile_') {
+		fileUrl.value = undefined;
+		fileName.value = '';
+		fileId.value = undefined;
+	}
+
+	emit('update', {
+		type: type.value,
+		fileId: fileId.value,
+		fileUrl: fileUrl.value,
+		volume: volume.value,
+	});
+
+	os.success();
 }
 </script>
+
+<style module>
+.fileSelectorRoot {
+	display: flex;
+	align-items: center;
+	gap: 8px;
+}
+
+.fileSelectorButton {
+	flex-shrink: 0;
+}
+
+.fileNotSelected {
+	font-weight: 700;
+	color: var(--infoWarnFg);
+}
+</style>

packages/frontend/src/pages/settings/sounds.vue

@@ -18,11 +18,11 @@ SPDX-License-Identifier: AGPL-3.0-only
 	<FormSection>
 		<template #label>{{ i18n.ts.sounds }}</template>
 		<div class="_gaps_s">
-			<MkFolder v-for="type in soundsKeys" :key="type">
+			<MkFolder v-for="type in operationTypes" :key="type">
 				<template #label>{{ i18n.t('_sfx.' + type) }}</template>
-				<template #suffix>{{ sounds[type].type ?? i18n.ts.none }}</template>
+				<template #suffix>{{ getSoundTypeName(sounds[type].type) }}</template>
 
-				<XSound :type="sounds[type].type" :volume="sounds[type].volume" @update="(res) => updated(type, res)"/>
+				<XSound :type="sounds[type].type" :volume="sounds[type].volume" :fileId="sounds[type].fileId" :fileUrl="sounds[type].fileUrl" @update="(res) => updated(type, res)"/>
 			</MkFolder>
 		</div>
 	</FormSection>
@@ -33,6 +33,8 @@ SPDX-License-Identifier: AGPL-3.0-only
 
 <script lang="ts" setup>
 import { Ref, computed, ref } from 'vue';
+import type { SoundType, OperationType } from '@/scripts/sound.js';
+import type { SoundStore } from '@/store.js';
 import XSound from './sounds.sound.vue';
 import MkRange from '@/components/MkRange.vue';
 import MkButton from '@/components/MkButton.vue';
@@ -40,6 +42,7 @@ import FormSection from '@/components/form/section.vue';
 import MkFolder from '@/components/MkFolder.vue';
 import { i18n } from '@/i18n.js';
 import { definePageMetadata } from '@/scripts/page-metadata.js';
+import { operationTypes } from '@/scripts/sound.js';
 import { defaultStore } from '@/store.js';
 import MkSwitch from '@/components/MkSwitch.vue';
 
@@ -47,9 +50,7 @@ const notUseSound = computed(defaultStore.makeGetterSetter('sound_notUseSound'))
 const useSoundOnlyWhenActive = computed(defaultStore.makeGetterSetter('sound_useSoundOnlyWhenActive'));
 const masterVolume = computed(defaultStore.makeGetterSetter('sound_masterVolume'));
 
-const soundsKeys = ['note', 'noteMy', 'notification', 'antenna', 'channel', 'reaction'] as const;
-
-const sounds = ref<Record<typeof soundsKeys[number], Ref<any>>>({
+const sounds = ref<Record<OperationType, Ref<SoundStore>>>({
 	note: defaultStore.reactiveState.sound_note,
 	noteMy: defaultStore.reactiveState.sound_noteMy,
 	notification: defaultStore.reactiveState.sound_notification,
@@ -58,9 +59,22 @@ const sounds = ref<Record<typeof soundsKeys[number], Ref<any>>>({
 	reaction: defaultStore.reactiveState.sound_reaction,
 });
 
+function getSoundTypeName(f: SoundType): string {
+	switch (f) {
+		case null:
+			return i18n.ts.none;
+		case '_driveFile_':
+			return i18n.ts._soundSettings.driveFile;
+		default:
+			return f;
+	}
+}
+
 async function updated(type: keyof typeof sounds.value, sound) {
-	const v = {
+	const v: SoundStore = {
 		type: sound.type,
+		fileId: sound.fileId,
+		fileUrl: sound.fileUrl,
 		volume: sound.volume,
 	};
 

packages/frontend/src/scripts/sound.ts

@@ -3,14 +3,22 @@
  * SPDX-License-Identifier: AGPL-3.0-only
  */
 
+import type { SoundStore } from '@/store.js';
 import { defaultStore } from '@/store.js';
+import * as os from '@/os.js';
 
 let ctx: AudioContext;
 const cache = new Map<string, AudioBuffer>();
 let canPlay = true;
 
 export const soundsTypes = [
+	// 音声なし
 	null,
+
+	// ドライブの音声
+	'_driveFile_',
+
+	// プリインストール
 	'syuilo/n-aec',
 	'syuilo/n-aec-4va',
 	'syuilo/n-aec-4vb',
@@ -64,32 +72,96 @@ export const soundsTypes = [
 	'noizenecio/kick_gaba7',
 ] as const;
 
-export async function loadAudio(file: string, useCache = true) {
+export const operationTypes = [
+	'noteMy',
+	'note',
+	'antenna',
+	'channel',
+	'notification',
+	'reaction',
+] as const;
+
+/** サウンドの種類 */
+export type SoundType = typeof soundsTypes[number];
+
+/** スプライトの種類 */
+export type OperationType = typeof operationTypes[number];
+
+/**
+ * 音声を読み込む
+ * @param soundStore サウンド設定
+ * @param options `useCache`: デフォルトは`true` 一度再生した音声はキャッシュする
+ */
+export async function loadAudio(soundStore: SoundStore, options?: { useCache?: boolean; }) {
+	if (_DEV_) console.log('loading audio. opts:', options);
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+	if (soundStore.type === null || (soundStore.type === '_driveFile_' && !soundStore.fileUrl)) {
+		return;
+	}
+	// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
 	if (ctx == null) {
 		ctx = new AudioContext();
 	}
-	if (useCache && cache.has(file)) {
-		return cache.get(file)!;
+	if (options?.useCache ?? true) {
+		if (soundStore.type === '_driveFile_' && cache.has(soundStore.fileId)) {
+			if (_DEV_) console.log('use cache');
+			return cache.get(soundStore.fileId) as AudioBuffer;
+		} else if (cache.has(soundStore.type)) {
+			if (_DEV_) console.log('use cache');
+			return cache.get(soundStore.type) as AudioBuffer;
+		}
+	}
+
+	let response: Response;
+
+	if (soundStore.type === '_driveFile_') {
+		try {
+			response = await fetch(soundStore.fileUrl);
+		} catch (err) {
+			try {
+				// URLが変わっている可能性があるのでドライブ側からURLを取得するフォールバック
+				const apiRes = await os.api('drive/files/show', {
+					fileId: soundStore.fileId,
+				});
+				response = await fetch(apiRes.url);
+			} catch (fbErr) {
+				// それでも無理なら諦める
+				return;
+			}
+		}
+	} else {
+		try {
+			response = await fetch(`/client-assets/sounds/${soundStore.type}.mp3`);
+		} catch (err) {
+			return;
+		}
 	}
 
-	const response = await fetch(`/client-assets/sounds/${file}.mp3`);
 	const arrayBuffer = await response.arrayBuffer();
 	const audioBuffer = await ctx.decodeAudioData(arrayBuffer);
 
-	if (useCache) {
-		cache.set(file, audioBuffer);
+	if (options?.useCache ?? true) {
+		if (soundStore.type === '_driveFile_') {
+			cache.set(soundStore.fileId, audioBuffer);
+		} else {
+			cache.set(soundStore.type, audioBuffer);
+		}
 	}
 
 	return audioBuffer;
 }
 
-export function play(type: 'noteMy' | 'note' | 'antenna' | 'channel' | 'notification' | 'reaction') {
-	const sound = defaultStore.state[`sound_${type}`];
-	if (_DEV_) console.log('play', type, sound);
+/**
+ * 既定のスプライトを再生する
+ * @param type スプライトの種類を指定
+ */
+export function play(operationType: OperationType) {
+	const sound = defaultStore.state[`sound_${operationType}`];
+	if (_DEV_) console.log('play', operationType, sound);
 	if (sound.type == null || !canPlay) return;
 
 	canPlay = false;
-	playFile(sound.type, sound.volume).then(() => {
+	playFile(sound).then(() => {
 		// ごく短時間に音が重複しないように
 		setTimeout(() => {
 			canPlay = true;
@@ -97,9 +169,14 @@ export function play(type: 'noteMy' | 'note' | 'antenna' | 'channel' | 'notifica
 	});
 }
 
-export async function playFile(file: string, volume: number) {
-	const buffer = await loadAudio(file);
-	createSourceNode(buffer, volume)?.start();
+/**
+ * サウンド設定形式で指定された音声を再生する
+ * @param soundStore サウンド設定
+ */
+export async function playFile(soundStore: SoundStore) {
+	const buffer = await loadAudio(soundStore);
+	if (!buffer) return;
+	createSourceNode(buffer, soundStore.volume)?.start();
 }
 
 export function createSourceNode(buffer: AudioBuffer, volume: number) : AudioBufferSourceNode | null {
@@ -118,6 +195,27 @@ export function createSourceNode(buffer: AudioBuffer, volume: number) : AudioBuf
 	return soundSource;
 }
 
+/**
+ * 音声の長さをミリ秒で取得する
+ * @param file ファイルのURL（ドライブIDではない）
+ */
+export async function getSoundDuration(file: string): Promise<number> {
+	const audioEl = document.createElement('audio');
+	audioEl.src = file;
+	return new Promise((resolve) => {
+		const si = setInterval(() => {
+			if (audioEl.readyState > 0) {
+				resolve(audioEl.duration * 1000);
+				clearInterval(si);
+				audioEl.remove();
+			}
+		}, 100);
+	});
+}
+
+/**
+ * ミュートすべきかどうかを判断する
+ */
 export function isMute(): boolean {
 	if (defaultStore.state.sound_notUseSound) {
 		// サウンドを出力しない

packages/frontend/src/store.ts

@@ -6,6 +6,7 @@
 import { markRaw, ref } from 'vue';
 import * as Misskey from 'misskey-js';
 import { miLocalStorage } from './local-storage.js';
+import type { SoundType } from '@/scripts/sound.js';
 import { Storage } from '@/pizzax.js';
 
 interface PostFormAction {
@@ -35,6 +36,22 @@ interface PageViewInterruptor {
 	handler: (page: Misskey.entities.Page) => unknown;
 }
 
+/** サウンド設定 */
+export type SoundStore = {
+	type: Exclude<SoundType, '_driveFile_'>;
+	volume: number;
+} | {
+	type: '_driveFile_';
+
+	/** ドライブのファイルID */
+	fileId: string;
+
+	/** ファイルURL（こちらが優先される） */
+	fileUrl: string;
+
+	volume: number;
+}
+
 export const postFormActions: PostFormAction[] = [];
 export const userActions: UserAction[] = [];
 export const noteActions: NoteAction[] = [];
@@ -401,27 +418,27 @@ export const defaultStore = markRaw(new Storage('base', {
 	},
 	sound_note: {
 		where: 'device',
-		default: { type: 'syuilo/n-aec', volume: 1 },
+		default: { type: 'syuilo/n-aec', volume: 1 } as SoundStore,
 	},
 	sound_noteMy: {
 		where: 'device',
-		default: { type: 'syuilo/n-cea-4va', volume: 1 },
+		default: { type: 'syuilo/n-cea-4va', volume: 1 } as SoundStore,
 	},
 	sound_notification: {
 		where: 'device',
-		default: { type: 'syuilo/n-ea', volume: 1 },
+		default: { type: 'syuilo/n-ea', volume: 1 } as SoundStore,
 	},
 	sound_antenna: {
 		where: 'device',
-		default: { type: 'syuilo/triple', volume: 1 },
+		default: { type: 'syuilo/triple', volume: 1 } as SoundStore,
 	},
 	sound_channel: {
 		where: 'device',
-		default: { type: 'syuilo/square-pico', volume: 1 },
+		default: { type: 'syuilo/square-pico', volume: 1 } as SoundStore,
 	},
 	sound_reaction: {
 		where: 'device',
-		default: { type: 'syuilo/bubble2', volume: 1 },
+		default: { type: 'syuilo/bubble2', volume: 1 } as SoundStore,
 	},
 }));
 

packages/frontend/src/widgets/WidgetJobQueue.vue

@@ -104,7 +104,13 @@ let jammedAudioBuffer: AudioBuffer | null = $ref(null);
 let jammedSoundNodePlaying: boolean = $ref(false);
 
 if (defaultStore.state.sound_masterVolume) {
-	sound.loadAudio('syuilo/queue-jammed').then(buf => jammedAudioBuffer = buf);
+	sound.loadAudio({
+		type: 'syuilo/queue-jammed',
+		volume: 1,
+	}).then(buf => {
+		if (!buf) throw new Error('[WidgetJobQueue] Failed to initialize AudioBuffer');
+		jammedAudioBuffer = buf;
+	});
 }
 
 for (const domain of ['inbox', 'deliver']) {