yume/yumechi-no-kuni
1
0
Fork 0
Code Issues Pull requests1 Projects Releases Packages Wiki Activity Actions

Compare commits

..

1 commit
master ... system-acc

Author SHA1 Message Date
eternal-flame-AD
4190358362
fix(backend): fixup migration for incorrect extraction on system accounts table 
Signed-off-by: eternal-flame-AD <yume@yumechi.jp>
 2025-03-06 10:58:01 -06:00
216 changed files with 2533 additions and 16113 deletions
.config
cypress-devcontainer.ymldocker_example.ymlexample.ymlpodman_apply_example.sh
.devcontainer
devcontainer.yml
.dockerignore
.forgejo
misskey
test.yml
workflows
docker.ymllint.ymltest-backend.ymltest-production.yml
.github
ISSUE_TEMPLATE
01_bug-report.yml02_feature-request.ymlconfig.yml
PULL_REQUEST_TEMPLATE
01_bug.md02_enhance.md03_release.md
pull_request_template.md
.gitignore
.okteto
okteto-pipeline.yml
.vscode
settings.json
CHANGELOG.mdContainerfileREADME.mdSECURITY.md
chart/files
default.yml
compose.local-db.ymlcompose_example.yml
locales
en-US.ymlindex.d.tsja-JP.yml
package.json
packages/backend
migration
1730937958242-Pgroonga.js1731757142918-increase_character_limits.js1732071810971-IndexUserDeleted.js1739006797620-GoogleAnalytics.js1741279404074-system-accounts-fixup.js
package.json
scripts
check_connect.js
src
GlobalModule.ts
boot
common.tsentry.tsmaster.tsworker.ts
config.ts
core
ActorKeySignerService.tsAiService.tsCoreModule.tsDeleteAccountService.tsDownloadService.tsDriveService.tsEmailService.tsFeaturedService.tsFileInfoService.tsHttpRequestService.tsImageProcessingService.tsNoteCreateService.tsNotificationService.tsPushNotificationService.tsQueueModule.tsReactionService.tsRemoteUserResolveService.tsRoleService.tsSearchService.tsUserWebhookService.tsVideoProcessingService.tsWebAuthnService.tsWebhookTestService.ts
activitypub
ApDbResolverService.tsApInboxService.tsApRendererService.tsApRequestService.tsApResolverService.tsJsonLdService.ts
misc
check-against-url.tsvalidator.ts
models
ApNoteService.tsApPersonService.tsApQuestionService.ts
type.ts
entities
DriveFileEntityService.tsDriveFolderEntityService.tsEmojiEntityService.tsMetaEntityService.ts
misc
log-sanitization.ts
models
DriveFile.tsFollowing.tsMeta.tsNote.tsUser.tsUserProfile.ts
json-schema
meta.ts
postgres.ts
queue
QueueProcessorService.tsmetrics.ts
processors
DeleteAccountProcessorService.tsInboxProcessorService.ts
server
FileServerService.tsNodeinfoServerService.tsServerModule.ts
Show more

35
.config/cypress-devcontainer.yml
View file

@ -153,13 +153,6 @@ redis:
id: 'aidx'
# ┌──────────┐
#───┘ Metrics └──────────────────────────────────────────
#prometheusMetrics:
# enable: false
# scrapeToken: '' # Set non-empty to require a bearer token for scraping
# ┌────────────────┐
#───┘ Error tracking └──────────────────────────────────────────
@ -175,36 +168,12 @@ id: 'aidx'
# options:
# dsn: 'https://examplePublicKey@o0.ingest.sentry.io/0'
# ┌──────────────┐
#──┘ Web Security └──────────────────────────────────────
# ┌─────────────────────┐
#───┘ Other configuration └─────────────────────────────────────
# Whether disable HSTS
#disableHsts: true
# Whether to enable HSTS preload
# Read these before enabling:
# - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security#preloading_strict_transport_security
# - https://hstspreload.org/
#hstsPreload: false
# Enable additional security headers that reduce the risk of XSS attacks or privacy leaks.
# browserSandboxing:
# # Do not send the Referrer header to other domains. The default when browserSandboxing is missing is true.
# strictOriginReferrer: true
# csp:
# # Do not send a CSP header. The default is a strict CSP header that prevents any form of external fetching or execution.
# disable: false
# # Merge additional directives into the CSP header. The default is an empty object.
# # You may want to list your CDN or other trusted domains here.
# # Media proxies are automatically added to the CSP header. This is an exception, things like Sentry will not be automatically added.
# appendDirectives:
# 'script-src':
# - "'unsafe-eval'" # do not use this ... just an example
# - 'https://example.com'
# ┌─────────────────────┐
#───┘ Other configuration └─────────────────────────────────────
# Number of worker processes
#clusterLimit: 1

35
.config/docker_example.yml
View file

@ -165,13 +165,6 @@ fulltextSearch:
id: 'aidx'
# ┌──────────┐
#───┘ Metrics └──────────────────────────────────────────
#prometheusMetrics:
# enable: false
# scrapeToken: '' # Set non-empty to require a bearer token for scraping
# ┌────────────────┐
#───┘ Error tracking └──────────────────────────────────────────
@ -187,36 +180,12 @@ id: 'aidx'
# options:
# dsn: 'https://examplePublicKey@o0.ingest.sentry.io/0'
# ┌──────────────┐
#──┘ Web Security └──────────────────────────────────────
# ┌─────────────────────┐
#───┘ Other configuration └─────────────────────────────────────
# Whether disable HSTS
#disableHsts: true
# Whether to enable HSTS preload
# Read these before enabling:
# - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security#preloading_strict_transport_security
# - https://hstspreload.org/
#hstsPreload: false
# Enable additional security headers that reduce the risk of XSS attacks or privacy leaks.
# browserSandboxing:
# # Do not send the Referrer header to other domains. The default when browserSandboxing is missing is true.
# strictOriginReferrer: true
# csp:
# # Do not send a CSP header. The default is a strict CSP header that prevents any form of external fetching or execution.
# disable: false
# # Merge additional directives into the CSP header. The default is an empty object.
# # You may want to list your CDN or other trusted domains here.
# # Media proxies are automatically added to the CSP header. This is an exception, things like Sentry will not be automatically added.
# appendDirectives:
# 'script-src':
# - "'unsafe-eval'" # do not use this ... just an example
# - 'https://example.com'
# ┌─────────────────────┐
#───┘ Other configuration └─────────────────────────────────────
# Number of worker processes
#clusterLimit: 1

35
.config/example.yml
View file

@ -247,13 +247,6 @@ fulltextSearch:
id: 'aidx'
# ┌──────────┐
#───┘ Metrics └──────────────────────────────────────────
#prometheusMetrics:
# enable: false
# scrapeToken: '' # Set non-empty to require a bearer token for scraping
# ┌────────────────┐
#───┘ Error tracking └──────────────────────────────────────────
@ -269,36 +262,12 @@ id: 'aidx'
# options:
# dsn: 'https://examplePublicKey@o0.ingest.sentry.io/0'
# ┌──────────────┐
#──┘ Web Security └──────────────────────────────────────
# ┌─────────────────────┐
#───┘ Other configuration └─────────────────────────────────────
# Whether disable HSTS
#disableHsts: true
# Whether to enable HSTS preload
# Read these before enabling:
# - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security#preloading_strict_transport_security
# - https://hstspreload.org/
#hstsPreload: false
# Enable additional security headers that reduce the risk of XSS attacks or privacy leaks.
# browserSandboxing:
# # Do not send the Referrer header to other domains. The default when browserSandboxing is missing is true.
# strictOriginReferrer: true
# csp:
# # Do not send a CSP header. The default is a strict CSP header that prevents any form of external fetching or execution.
# disable: false
# # Merge additional directives into the CSP header. The default is an empty object.
# # You may want to list your CDN or other trusted domains here.
# # Media proxies are automatically added to the CSP header. This is an exception, things like Sentry will not be automatically added.
# appendDirectives:
# 'script-src':
# - "'unsafe-eval'" # do not use this ... just an example
# - 'https://example.com'
# ┌─────────────────────┐
#───┘ Other configuration └─────────────────────────────────────
# Number of worker processes
#clusterLimit: 1

33
.config/podman_apply_example.sh
View file

@ -1,33 +0,0 @@
#!/bin/bash
set -e
vault lease revoke -sync -prefix misskey-db/creds/misskey-test0-runtime
CREDS_JSON=$(vault read -format json misskey-db/creds/misskey-test0-runtime)
if [ "$?" -ne 0 ]; then
echo "Failed to get credentials"
exit 1
fi
export POSTGRES_USER=$(echo "$CREDS_JSON" | jq -r '.data.username')
export POSTGRES_PASSWORD=$(echo "$CREDS_JSON" | jq -r '.data.password')
export POSTGRES_HOST=misskey-db
export POSTGRES_PORT=5432
export POSTGRES_DB=misskey
export POSTGRES_URL="postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}"
podman run --pod misskey-web -d \
--replace \
--network misskey \
--env "POSTGRES_*" \
--volume ../var/files:/misskey/files:U \
--volume .config/:/misskey/.config:ro \
--volume ../run/misskey-podman:/run/misskey:U \
--health-cmd "misskey-auto-deploy-entrypoint :healthcheck" \
--name misskey-web \
--restart always \
misskey-podman

6
.devcontainer/devcontainer.yml
View file

@ -161,12 +161,6 @@ id: 'aidx'
# Whether disable HSTS
#disableHsts: true
# Whether to enable HSTS preload
# Read these before enabling:
# - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security#preloading_strict_transport_security
# - https://hstspreload.org/
#hstsPreload: false
# Number of worker processes
#clusterLimit: 1

4
.dockerignore
View file

@ -1,10 +1,8 @@
.autogen
.github
.forgejo
.travis
.vscode
.config
Dockerfile
build/
built/
@ -30,5 +28,3 @@ fluent-emojis/
.idea/
packages/*/.vscode/
packages/backend/test/compose.yml
/yume-mods/target

17
.forgejo/misskey/test.yml
View file

@ -1,17 +0,0 @@
url: 'http://misskey.local'
setupPassword: example_password_please_change_this_or_you_will_get_hacked
# ローカルでテストするときにポートを被らないようにするためデフォルトのものとは変える(以下同じ)
port: 61812
db:
host: postgres
port: 5432
db: test-misskey
user: postgres
pass: ''
redis:
host: redis
port: 6379
id: aidx

60
.forgejo/workflows/docker.yml
View file

@ -1,60 +0,0 @@
name: Publish Docker image
on:
push:
branches:
- master
- develop
pull_request:
workflow_dispatch:
env:
REGISTRY_IMAGE: l1drm/yumechi-no-kuni
TAGS: |
type=edge
type=ref,event=pr
type=ref,event=branch
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
jobs:
# see https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
build:
name: Build
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
platform:
- linux/amd64
steps:
- name: Prepare
run: |
platform=${{ matrix.platform }}
echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
- name: Check out the repo
uses: actions/checkout@v4.1.1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY_IMAGE }}
tags: ${{ env.TAGS }}
- name: Log in to Docker Hub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and Push to Docker Hub
id: build
uses: docker/build-push-action@v6
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
platforms: ${{ matrix.platform }}
provenance: true
labels: ${{ steps.meta.outputs.labels }}

111
.forgejo/workflows/lint.yml
View file

@ -1,111 +0,0 @@
name: Lint
on:
push:
branches:
- master
- develop
paths:
- packages/backend/**
- packages/frontend/**
- packages/frontend-shared/**
- packages/frontend-embed/**
- packages/sw/**
- packages/misskey-js/**
- packages/misskey-bubble-game/**
- packages/misskey-reversi/**
- packages/shared/eslint.config.js
- .forgejo/workflows/lint.yml
pull_request:
paths:
- packages/backend/**
- packages/frontend/**
- packages/frontend-shared/**
- packages/frontend-embed/**
- packages/sw/**
- packages/misskey-js/**
- packages/misskey-bubble-game/**
- packages/misskey-reversi/**
- packages/shared/eslint.config.js
- .forgejo/workflows/lint.yml
jobs:
pnpm_install:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
submodules: true
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4.0.4
with:
node-version-file: '.node-version'
cache: 'pnpm'
- run: corepack enable
- run: pnpm i --frozen-lockfile
lint:
needs: [pnpm_install]
runs-on: ubuntu-latest
continue-on-error: true
strategy:
matrix:
workspace:
- backend
- frontend
- frontend-shared
- frontend-embed
- sw
- misskey-js
- misskey-bubble-game
- misskey-reversi
env:
eslint-cache-version: v1
eslint-cache-path: ${{ github.workspace }}/node_modules/.cache/eslint-${{ matrix.workspace }}
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
submodules: true
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4.0.4
with:
node-version-file: '.node-version'
cache: 'pnpm'
- run: corepack enable
- run: pnpm i --frozen-lockfile
- name: Restore eslint cache
uses: actions/cache@v4.1.0
with:
path: ${{ env.eslint-cache-path }}
key: eslint-${{ env.eslint-cache-version }}-${{ matrix.workspace }}-${{ hashFiles('**/pnpm-lock.yaml') }}-${{ github.ref_name }}-${{ github.sha }}
restore-keys: eslint-${{ env.eslint-cache-version }}-${{ matrix.workspace }}-${{ hashFiles('**/pnpm-lock.yaml') }}-
- run: pnpm --filter ${{ matrix.workspace }} run eslint --cache --cache-location ${{ env.eslint-cache-path }} --cache-strategy content
typecheck:
needs: [pnpm_install]
runs-on: ubuntu-latest
continue-on-error: true
strategy:
matrix:
workspace:
- backend
- sw
- misskey-js
steps:
- uses: actions/checkout@v4.1.1
with:
fetch-depth: 0
submodules: true
- uses: pnpm/action-setup@v4
- uses: actions/setup-node@v4.0.4
with:
node-version-file: '.node-version'
cache: 'pnpm'
- run: corepack enable
- run: pnpm i --frozen-lockfile
- run: pnpm --filter misskey-js run build
if: ${{ matrix.workspace == 'backend' || matrix.workspace == 'sw' }}
- run: pnpm --filter misskey-reversi run build
if: ${{ matrix.workspace == 'backend' }}
- run: pnpm --filter ${{ matrix.workspace }} run typecheck

98
.forgejo/workflows/test-backend.yml
View file

@ -1,98 +0,0 @@
name: Test (backend)
on:
push:
branches:
- master
- develop
paths:
- packages/backend/**
# for permissions
- packages/misskey-js/**
- .forgejo/workflows/test-backend.yml
- .forgejo/misskey/**
pull_request:
paths:
- packages/backend/**
# for permissions
- packages/misskey-js/**
- .forgejo/workflows/test-backend.yml
- .forgejo/misskey/**
jobs:
unit:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [22.11.0]
services:
postgres:
image: l1drm/postgres-pgroonga:alpine-15-znver4
env:
POSTGRES_DB: test-misskey
POSTGRES_HOST_AUTH_METHOD: trust
redis:
image: redis:7
steps:
- uses: actions/checkout@v4.1.1
with:
submodules: true
- name: Install pnpm
uses: pnpm/action-setup@v4
- name: Install FFmpeg
uses: https://github.com/FedericoCarboni/setup-ffmpeg@v3
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4.0.4
with:
node-version: ${{ matrix.node-version }}
cache: 'pnpm'
- run: corepack enable
- run: pnpm i --frozen-lockfile
- name: Check pnpm-lock.yaml
run: git diff --exit-code pnpm-lock.yaml
- name: Copy Configure
run: cp .forgejo/misskey/test.yml .config
- name: Build
run: pnpm build
- name: Test
run: pnpm --filter backend test-and-coverage
# e2e:
# runs-on: ubuntu-latest
#
# strategy:
# matrix:
# node-version: [22.11.0]
#
# services:
# postgres:
# image: l1drm/postgres-pgroonga:alpine-15-znver4
# env:
# POSTGRES_DB: test-misskey
# POSTGRES_HOST_AUTH_METHOD: trust
# redis:
# image: redis:7
#
# steps:
# - uses: actions/checkout@v4.1.1
# with:
# submodules: true
# - name: Install pnpm
# uses: pnpm/action-setup@v4
# - name: Use Node.js ${{ matrix.node-version }}
# uses: actions/setup-node@v4.0.4
# with:
# node-version: ${{ matrix.node-version }}
# cache: 'pnpm'
# - run: corepack enable
# - run: pnpm i --frozen-lockfile
# - name: Check pnpm-lock.yaml
# run: git diff --exit-code pnpm-lock.yaml
# - name: Copy Configure
# run: cp .forgejo/misskey/test.yml .config
# - name: Build
# run: pnpm build
# - name: Test
# run: pnpm --filter backend test-and-coverage:e2e

39
.forgejo/workflows/test-production.yml
View file

@ -1,39 +0,0 @@
name: Test (production install and build)
on:
push:
branches:
- master
- develop
pull_request:
env:
NODE_ENV: production
jobs:
production:
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [22.11.0]
steps:
- uses: actions/checkout@v4.1.1
with:
submodules: true
- name: Install pnpm
uses: pnpm/action-setup@v4
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v4.0.4
with:
node-version: ${{ matrix.node-version }}
cache: 'pnpm'
- run: corepack enable
- run: pnpm i --frozen-lockfile
- name: Check pnpm-lock.yaml
run: git diff --exit-code pnpm-lock.yaml
- name: Copy Configure
run: cp .github/misskey/test.yml .config/default.yml
- name: Build
run: pnpm build

97
.github/ISSUE_TEMPLATE/01_bug-report.yml vendored Normal file
View file

@ -0,0 +1,97 @@
name: 🐛 Bug Report
description: Create a report to help us improve
labels: ["⚠️bug?"]
body:
- type: markdown
attributes:
value: |
Thanks for reporting!
First, in order to avoid duplicate Issues, please search to see if the problem you found has already been reported.
Also, If you are NOT owner/admin of server, PLEASE DONT REPORT SERVER SPECIFIC ISSUES TO HERE! (e.g. feature XXX is not working in misskey.example) Please try with another misskey servers, and if your issue is only reproducible with specific server, contact your server's owner/admin first.
- type: textarea
attributes:
label: 💡 Summary
description: Tell us what the bug is
validations:
required: true
- type: textarea
attributes:
label: 🥰 Expected Behavior
description: Tell us what should happen
validations:
required: true
- type: textarea
attributes:
label: 🤬 Actual Behavior
description: |
Tell us what happens instead of the expected behavior.
Please include errors from the developer console and/or server log files if you have access to them.
validations:
required: true
- type: textarea
attributes:
label: 📝 Steps to Reproduce
placeholder: |
1.
2.
3.
validations:
required: false
- type: textarea
attributes:
label: 💻 Frontend Environment
description: |
Tell us where on the platform it happens
DO NOT WRITE "latest". Please provide the specific version.
Examples:
* Model and OS of the device(s): MacBook Pro (14inch, 2021), macOS Ventura 13.4
* Browser: Chrome 113.0.5672.126
* Server URL: misskey.example.com
* Misskey: 2025.x.x
value: |
* Model and OS of the device(s):
* Browser:
* Server URL:
* Misskey:
render: markdown
validations:
required: false
- type: textarea
attributes:
label: 🛰 Backend Environment (for server admin)
description: |
Tell us where on the platform it happens
DO NOT WRITE "latest". Please provide the specific version.
If you are using a managed service, put that after the version.
Examples:
* Installation Method or Hosting Service: docker compose, k8s/docker, systemd, "Misskey install shell script", development environment
* Misskey: 2025.x.x
* Node: 20.x.x
* PostgreSQL: 15.x.x
* Redis: 7.x.x
* OS and Architecture: Ubuntu 24.04.2 LTS aarch64
value: |
* Installation Method or Hosting Service:
* Misskey:
* Node:
* PostgreSQL:
* Redis:
* OS and Architecture:
render: markdown
validations:
required: false
- type: checkboxes
attributes:
label: Do you want to address this bug yourself?
options:
- label: Yes, I will patch the bug myself and send a pull request

22
.github/ISSUE_TEMPLATE/02_feature-request.yml vendored Normal file
View file

@ -0,0 +1,22 @@
name: ✨ Feature Request
description: Suggest an idea for this project
labels: ["✨Feature"]
body:
- type: textarea
attributes:
label: Summary
description: Tell us what the suggestion is
validations:
required: true
- type: textarea
attributes:
label: Purpose
description: Describe the specific problem or need you think this feature will solve, and who it will help.
validations:
required: true
- type: checkboxes
attributes:
label: Do you want to implement this feature yourself?
options:
- label: Yes, I will implement this by myself and send a pull request

8
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View file

@ -0,0 +1,8 @@
contact_links:
- name: 💬 Misskey official Discord
url: https://discord.gg/Wp8gVStHW3
about: Chat freely about Misskey
# 仮
- name: 💬 Start discussion
url: https://github.com/misskey-dev/misskey/discussions
about: The official forum to join conversation and ask question

23
.github/PULL_REQUEST_TEMPLATE/01_bug.md vendored Normal file
View file

@ -0,0 +1,23 @@
<!-- ℹ お読みください / README
PRありがとうございます！ PRを作成する前に、コントリビューションガイドをご確認ください:
Thank you for your PR! Before creating a PR, please check the contribution guide:
https://github.com/misskey-dev/misskey/blob/develop/CONTRIBUTING.md
-->
## What
<!-- このPRで何をしたのか？ どう変わるのか？ -->
<!-- What did you do with this PR? How will it change things? -->
## Why
<!-- なぜそうするのか？ どういう意図なのか？ 何が困っているのか？ -->
<!-- Why do you do it? What are your intentions? What is the problem? -->
## Additional info (optional)
<!-- テスト観点など -->
<!-- Test perspective, etc -->
## Checklist
- [ ] Read the [contribution guide](https://github.com/misskey-dev/misskey/blob/develop/CONTRIBUTING.md)
- [ ] Test working in a local environment
- [ ] (If needed) Update CHANGELOG.md
- [ ] (If possible) Add tests

23
.github/PULL_REQUEST_TEMPLATE/02_enhance.md vendored Normal file
View file

@ -0,0 +1,23 @@
<!-- ℹ お読みください / README
PRありがとうございます！ PRを作成する前に、コントリビューションガイドをご確認ください:
Thank you for your PR! Before creating a PR, please check the contribution guide:
https://github.com/misskey-dev/misskey/blob/develop/CONTRIBUTING.md
-->
## What
<!-- このPRで何をしたのか？ どう変わるのか？ -->
<!-- What did you do with this PR? How will it change things? -->
## Why
<!-- なぜそうするのか？ どういう意図なのか？ 何が困っているのか？ -->
<!-- Why do you do it? What are your intentions? What is the problem? -->
## Additional info (optional)
<!-- テスト観点など -->
<!-- Test perspective, etc -->
## Checklist
- [ ] Read the [contribution guide](https://github.com/misskey-dev/misskey/blob/develop/CONTRIBUTING.md)
- [ ] Test working in a local environment
- [ ] (If needed) Update CHANGELOG.md
- [ ] (If possible) Add tests

20
.github/PULL_REQUEST_TEMPLATE/03_release.md vendored Normal file
View file

@ -0,0 +1,20 @@
## Summary
This is a release PR.
For more information on the release instructions, please see:
https://github.com/misskey-dev/misskey/blob/develop/CONTRIBUTING.md#release
## For reviewers
- CHANGELOGに抜け漏れは無いか
- バージョンの上げ方は適切か
- 他にこのリリースに含めなければならない変更は無いか
- 全体的な変更内容を俯瞰し問題は無いか
- レビューされていないコミットがある場合は、それが問題ないか
- 最終的な動作確認を行い問題は無いか
などを確認し、リリースする準備が整っていると思われる場合は approve してください。
## Checklist
- [ ] package.jsonのバージョンが正しく更新されている
- [ ] CHANGELOGが過不足無く更新されている
- [ ] CIが全て通っている

4
.github/pull_request_template.md vendored
View file

@ -1,7 +1,7 @@
<!-- ℹ お読みください / README
PRありがとうございます！ PRを作成する前に、コントリビューションガイドをご確認ください:
Thank you for your PR! Before creating a PR, please check the contribution guide:
https://forge.yumechi.jp/yume/yumechi-no-kuni/src/branch/master/CONTRIBUTING.md
https://github.com/misskey-dev/misskey/blob/develop/CONTRIBUTING.md
-->
## What
@ -17,7 +17,7 @@ https://forge.yumechi.jp/yume/yumechi-no-kuni/src/branch/master/CONTRIBUTING.md
<!-- Test perspective, etc -->
## Checklist
- [ ] Read the [contribution guide](https://forge.yumechi.jp/yume/yumechi-no-kuni/src/branch/master/CONTRIBUTING.md)
- [ ] Read the [contribution guide](https://github.com/misskey-dev/misskey/blob/develop/CONTRIBUTING.md)
- [ ] Test working in a local environment
- [ ] (If needed) Add story of storybook
- [ ] (If needed) Update CHANGELOG.md

1
.gitignore vendored
View file

@ -36,7 +36,6 @@ coverage
!/.config/docker_example.yml
!/.config/docker_example.env
!/.config/cypress-devcontainer.yml
!/.config/podman_apply_example.sh
docker-compose.yml
./compose.yml
.devcontainer/compose.yml

6
.okteto/okteto-pipeline.yml Normal file
View file

@ -0,0 +1,6 @@
build:
misskey:
args:
- NODE_ENV=development
deploy:
- helm upgrade --install misskey chart --set image=${OKTETO_BUILD_MISSKEY_IMAGE} --set url="https://misskey-$(kubectl config view --minify -o jsonpath='{..namespace}').cloud.okteto.net" --set environment=development

5
.vscode/settings.json vendored
View file

@ -11,8 +11,5 @@
"editor.codeActionsOnSave": {
"source.fixAll": "explicit"
},
"editor.formatOnSave": false,
"rust-analyzer.linkedProjects": [
"yume-mods/Cargo.toml",
]
"editor.formatOnSave": false
}

71
CHANGELOG.md
View file

@ -1,9 +1,15 @@
## Unreleased
## 2025.3.0-yumechinokuni.0
### General
-
### Client
-
### Server
- Fix: DBマイグレーション際にシステムアカウントのユーザーID判定が正しくない問題を修正
## 2025.3.0
### General
@ -25,14 +31,12 @@
- Fix: S3互換オブジェクトストレージでファイルのアップロードに失敗することがある問題を修正
(Cherry-picked from https://github.com/MisskeyIO/misskey/pull/895)
## 2025.2.1-yumechinokuni.0
- Reflect upstream changes
## 2025.2.1
### General
- Feat: アクセストークン発行時に通知するように
- Feat: 実験的なGoogleAnalyticsサポートを追加
- 依存関係の更新
### Client
@ -69,17 +73,6 @@
- Fix: クリップの説明欄を更新する際に空にできない問題を修正
- Fix: フォロワーではないユーザーにリノートもしくは返信された場合にノートのDeleteアクティビティが送られていない問題を修正
## 2025.2.0-yumechinokuni.2
- Enhance: 成り済まし対策として、ActivityPub照会された時にリモートのリダイレクトを拒否できるように (config.disallowExternalApRedirect)
- Fix: リモートのリダイレクトのOriginが正しくチェックされるように
- Revert: 2025.2.0-yumechinokuni.1で追加した一時措置を元に戻す
## 2025.2.0-yumechinokuni.1
- Security: Revert miskey-dev/misskey#14897
- Security: AP請求を外部ドメーンにリダイレクトしないように
## 2025.2.0
### General
@ -177,52 +170,8 @@
- Fix: 連合OFFで投稿されたノートに対する冗長な処理を抑止 ( #15018 )
- Fix: `/api.json`のレスポンスが2回目のリクエスト以降おかしくなる問題を修正
## 2024.11.0-yumechinokuni.8
- Frontend: SSRでユーザープロフィールが表示されない問題を修正
- Security: SSRプライバシー方面の改善
- Security: AP Payloadの検証を強化
## 2024.11.0-yumechinokuni.7
- Misskey Trademark内容をWebUIから削除
- Service Worker キャッシュが正しく動作しない問題を修正
## 2024.11.0-yumechinokuni.6
- Upstream: 2024.11.0-alpha.4 タッグをマージする
- Performance: EmojiのリクエストをProxyでキャッシュするように
- Performance: Service Workerのキャッシュを最適化
- Security: AP Payloadの検証を強化
- Security: Image/Video Processorはドライブ機能だけを使うように
## 2024.11.0-yumechinokuni.5
- Upstream: 2024.11.0-alpha.2 タッグをマージする
- Reliability: Activitypub event deduplication
- DevOps: Prometheus サーバーメトリクス
- Enhance: ハッシュタグランギングを改善
- Enhance: PgroongaのCWサーチ + パフォーマンス改善
## 2024.11.0-yumechinokuni.4
- Upstream: 2024.11.0-alpha.1 タッグをマージする
- DevOps: 管理者アクセストークンがユーザー登録できるようになる (write:admin:create-account)
- Frontend: Stream再接続ロジックdata raceを修正
- Security: CSPにCDNなどの外部ホストはホワイトリストできるように
### 2024.11.0-yumechinokuni.4p1
PgroongaのCWサーチ (github.com/paricafe/misskey#d30db97b59d264450901c1dd86808dcb43875ea9)
### 2024.11.0-yumechinokuni.4p2
- fix(backend): アナウンスメントを作成ときにWebUIフォームの画像URLを後悔できないのを修正 (/admin/announcement/create)
## 2024.11.0-yumechinokuni.3
- Security: CSPの設定を強化
- Fix: flaky testの修正
### Misskey.js
- Feat: allow setting `binaryType` of WebSocket connection
## 2024.11.0

81
Containerfile
View file

@ -1,81 +0,0 @@
# IMPORTANT:
#
# This container has no user isolation and is designed specifically for use with Podman and ideally protected by a MAC (SELinux, AppArmor, etc).
#
# For general Docker usage, please make sure use the official Dockerfile instead.
ARG NODE_VERSION=22.11.0-bookworm
FROM rust:1-bookworm AS rust-builder
RUN apt-get update \
&& apt-get install -yqq --no-install-recommends \
build-essential libcap-ng-dev libapparmor-dev
COPY ./yume-mods /yume-mods
WORKDIR /yume-mods
ARG ENTRYPOINT_FEATURES=""
RUN cargo build -p misskey-auto-deploy-entrypoint --release --features "${ENTRYPOINT_FEATURES}"
FROM node:${NODE_VERSION} AS builder
ENV COREPACK_DEFAULT_TO_LATEST=0
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
--mount=type=cache,target=/var/lib/apt,sharing=locked \
rm -f /etc/apt/apt.conf.d/docker-clean \
; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache \
&& apt-get update \
&& apt-get install -yqq --no-install-recommends \
build-essential
RUN corepack enable
WORKDIR /misskey
ENV COREPACK_DEFAULT_TO_LATEST=0
COPY . ./
RUN --mount=type=cache,target=/root/.local/share/pnpm/store,sharing=locked \
pnpm i --frozen-lockfile --aggregate-output
RUN git submodule update --init
RUN pnpm build
RUN rm -rf .git/
RUN chmod -R -w /misskey
FROM --platform=$TARGETPLATFORM node:${NODE_VERSION}-slim AS runner
ENV COREPACK_DEFAULT_TO_LATEST=0
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ffmpeg curl libjemalloc-dev libjemalloc2 libcap-ng0 libapparmor1 \
&& ln -s /usr/lib/$(uname -m)-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so \
&& corepack enable \
&& mkdir -p /misskey \
&& find / -type d -path /sys -prune -o -type d -path /proc -prune -o -type f -perm /u+s -ignore_readdir_race -exec chmod u-s {} \; \
&& find / -type d -path /sys -prune -o -type d -path /proc -prune -o -type f -perm /g+s -ignore_readdir_race -exec chmod g-s {} \; \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists
WORKDIR /misskey
# add package.json to add pnpm
COPY ./package.json ./package.json
RUN corepack install
COPY --from=rust-builder --chown=0:0 --chmod=755 /yume-mods/target/release/misskey-auto-deploy-entrypoint /usr/local/bin/misskey-auto-deploy-entrypoint
COPY --from=builder --chown=0:0 /misskey/ ./
ENV LD_PRELOAD=/usr/local/lib/libjemalloc.so
ARG NODE_ENV=production
ENV NODE_ENV=${NODE_ENV}
HEALTHCHECK --interval=10s --timeout=10s --retries=3 \
CMD [":healthcheck"]
ENTRYPOINT ["/usr/local/bin/misskey-auto-deploy-entrypoint"]
CMD ["/usr/local/bin/pnpm", "run", "migrateandstart"]

62
README.md
View file

@ -1,31 +1,49 @@
# ゆめちのくに
<div align="center">
<a href="https://misskey-hub.net">
<img src="./assets/title_float.svg" alt="Misskey logo" style="border-radius:50%" width="300"/>
</a>
YumechiNoKuni is a fork of Misskey, with a focus on security, observability and reliability.
**🌎 **Misskey** is an open source, federated social media platform that's free forever! 🚀**
[mi.yumechi.jp](https://mi.yumechi.jp) is running this version.
[Learn more](https://misskey-hub.net/)
[Learn more about Misskey](https://misskey-hub.net/)
---
## Main differences
<a href="https://misskey-hub.net/servers/">
<img src="https://custom-icon-badges.herokuapp.com/badge/find_an-instance-acea31?logoColor=acea31&style=for-the-badge&logo=misskey&labelColor=363B40" alt="find an instance"/></a>
### Unique features
<a href="https://misskey-hub.net/docs/for-admin/install/guides/">
<img src="https://custom-icon-badges.herokuapp.com/badge/create_an-instance-FBD53C?logoColor=FBD53C&style=for-the-badge&logo=server&labelColor=363B40" alt="create an instance"/></a>
- Strict ActivityPub sanitization by whitelisting properties and normalizing all referential properties.
- Strict Content Security Policy.
- Require TLSv1.2+ over port 443 for all ActivityPub requests.
- Strongly-typed inbox filtering in Rust.
- Reduce needless retries by marking more errors as permanent.
- Detailed prometheus metrics for slow requests, DB queries, AP processing, failed auths, etc.
- Disable unauthenticated media processing and use custom AppArmored media proxy.
- Enable active users in nodeinfo back.
- Advertise Git information over nodeinfo for better observability and easy tracking of the actual code running.
- Logical replication for the database over mTLS.
- More atomic operations in API handlers.
<a href="./CONTRIBUTING.md">
<img src="https://custom-icon-badges.herokuapp.com/badge/become_a-contributor-A371F7?logoColor=A371F7&style=for-the-badge&logo=git-merge&labelColor=363B40" alt="become a contributor"/></a>
### Picked from github.com/paricafe/misskey
<a href="https://discord.gg/Wp8gVStHW3">
<img src="https://custom-icon-badges.herokuapp.com/badge/join_the-community-5865F2?logoColor=5865F2&style=for-the-badge&logo=discord&labelColor=363B40" alt="join the community"/></a>
- pgroonga full-text search (with modifications).
- Better Service Worker caching.
- Better hashtag statistics.
- Better handling of deep recursive AP objects.
<a href="https://www.patreon.com/syuilo">
<img src="https://custom-icon-badges.herokuapp.com/badge/become_a-patron-F96854?logoColor=F96854&style=for-the-badge&logo=patreon&labelColor=363B40" alt="become a patron"/></a>
</div>
## Thanks
<a href="https://sentry.io/"><img src="https://github.com/misskey-dev/misskey/assets/4439005/98576556-222f-467a-94be-e98dbda1d852" height="30" alt="Sentry" /></a>
Thanks to [Sentry](https://sentry.io/) for providing the error tracking platform that helps us catch unexpected errors.
<a href="https://www.chromatic.com/"><img src="https://user-images.githubusercontent.com/321738/84662277-e3db4f80-af1b-11ea-88f5-91d67a5e59f6.png" height="30" alt="Chromatic" /></a>
Thanks to [Chromatic](https://www.chromatic.com/) for providing the visual testing platform that helps us review UI changes and catch visual regressions.
<a href="https://about.codecov.io/for/open-source/"><img src="https://about.codecov.io/wp-content/themes/codecov/assets/brand/sentry-cobranding/logos/codecov-by-sentry-logo.svg" height="30" alt="Codecov" /></a>
Thanks to [Codecov](https://about.codecov.io/for/open-source/) for providing the code coverage platform that helps us improve our test coverage.
<a href="https://crowdin.com/"><img src="https://user-images.githubusercontent.com/20679825/230709597-1299a011-171a-4294-a91e-355a9b37c672.svg" height="30" alt="Crowdin" /></a>
Thanks to [Crowdin](https://crowdin.com/) for providing the localization platform that helps us translate Misskey into many languages.
<a href="https://hub.docker.com/"><img src="https://user-images.githubusercontent.com/20679825/230148221-f8e73a32-a49b-47c3-9029-9a15c3824f92.png" height="30" alt="Docker" /></a>
Thanks to [Docker](https://hub.docker.com/) for providing the container platform that helps us run Misskey in production.

13
SECURITY.md
View file

@ -1,11 +1,11 @@
# Reporting Security Issues
If you discover a security issue in this project, please use the `git blame` command to identify the source of the issue,
if it was introduced by this fork please contact me at secity<at>yumechi.jp.
If you discover a security issue in Misskey, please report it by **[this form](https://github.com/misskey-dev/misskey/security/advisories/new)**.
For upstream issues please report by **[this form](https://github.com/misskey-dev/misskey/security/advisories/new)**.
This will allow us to assess the risk, and make a fix available before we add a
bug report to the GitHub repository.
Thanks for helping make YumechiNoKuni safe for everyone.
Thanks for helping make Misskey safe for everyone.
> [!note]
> CNA [requires](https://www.cve.org/ResourcesSupport/AllResources/CNARules#section_5-2_Description) that CVEs include a description in English for inclusion in the CVE Catalog.
@ -14,4 +14,7 @@ Thanks for helping make YumechiNoKuni safe for everyone.
## When create a patch
If you can also create a patch to fix the vulnerability, please send a diff file with the report.
If you can also create a patch to fix the vulnerability, please create a PR on the private fork.
> [!note]
> There is a GitHub bug that prevents merging if a PR not following the develop branch of upstream, so please keep follow the develop branch.

6
chart/files/default.yml
View file

@ -182,12 +182,6 @@ id: "aidx"
# Whether disable HSTS
#disableHsts: true
# Whether to enable HSTS preload
# Read these before enabling:
# - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Strict-Transport-Security#preloading_strict_transport_security
# - https://hstspreload.org/
#hstsPreload: false
# Number of worker processes
#clusterLimit: 1

14
compose.local-db.yml
View file

@ -15,7 +15,7 @@ services:
db:
restart: always
image: l1drm/postgres-pgroonga:alpine-15-znver4
image: postgres:15-alpine
ports:
- "5432:5432"
env_file:
@ -27,18 +27,6 @@ services:
interval: 5s
retries: 20
# db-publish:
# restart: always
# image: alpine/socat
# ports:
# - "5433:5433"
# volumes:
# - ./replication:/data
# command: [
# "OPENSSL-LISTEN:5433,openssl-min-proto-version=TLS1.2,cert=/data/certs/server.pub.pem,key=/data/certs/server.key.pem,cafile=/data/certs/ca.pem,fork,reuseaddr",
# "TCP4:db:5432"
# ]
# meilisearch:
# restart: always
# image: getmeili/meilisearch:v1.3.4

131
compose_example.yml
View file

@ -1,31 +1,7 @@
services:
nyuukyou:
build: yume-mods/nyuukyou
restart: always
user: "${MISSKEY_UID}:${MISSKEY_GID}"
profiles: [web]
links:
- web
depends_on:
web:
condition: service_healthy
networks:
- internal_network
- external_network
ports:
- "3001:3001"
volumes:
- ./nyuukyou:/store
web:
build:
context: .
args:
- UID: "${MISSKEY_UID}"
- GID: "${MISSKEY_GID}"
profiles: [web]
build: .
restart: always
user: "${MISSKEY_UID}:${MISSKEY_GID}"
links:
- db
- redis
@ -50,7 +26,6 @@ services:
redis:
restart: always
image: redis:7-alpine
user: "${MISSKEY_UID}:${MISSKEY_GID}"
networks:
- internal_network
volumes:
@ -62,9 +37,7 @@ services:
db:
restart: always
image: l1drm/postgres-pgroonga:alpine-15-znver4
user: "${MISSKEY_UID}:${MISSKEY_GID}"
shm_size: 2gb
image: postgres:15-alpine
networks:
- internal_network
env_file:
@ -76,106 +49,6 @@ services:
interval: 5s
retries: 20
replikey:
restart: always
image: l1drm/replikey:latest
profiles: [replikey-master]
user: "${MISSKEY_UID}:${MISSKEY_GID}"
links:
- db
- redis
networks:
- internal_network
- external_network
ports:
- "5443:5443"
volumes:
- ./replikey:/etc/replikey:ro
command: [
"network",
"reverse-proxy",
"--listen",
"0.0.0.0:5443",
"--cert",
"/etc/replikey/cert.pem",
"--key",
"/etc/replikey/key.pem",
"--ca",
"/etc/replikey/ca.pem",
"--redis-sni",
"${MTLS_REDIS_SNI}",
"--redis-target",
"redis:6379",
"--postgres-sni",
"${MTLS_POSTGRES_SNI}",
"--postgres-target",
"db:5432",
]
replikey-postgres-slave:
restart: always
image: l1drm/replikey:latest
profiles: [replikey-slave]
user: "${MISSKEY_UID}:${MISSKEY_GID}"
links:
- db
- redis
networks:
- internal_network
- external_network
ports:
# - "4001:4001"
volumes:
- ./replikey:/etc/replikey:ro
command: [
"network",
"forward-proxy",
"--listen",
"0.0.0.0:4001",
"--sni",
"${MTLS_POSTGRES_SNI}",
"--target",
"db:5432",
"--cert",
"/etc/replikey/cert.pem",
"--key",
"/etc/replikey/key.pem",
"--ca",
"/etc/replikey/ca.pem",
]
replikey-redis-slave:
restart: always
image: l1drm/replikey:latest
profiles: [replikey-slave]
user: "${MISSKEY_UID}:${MISSKEY_GID}"
links:
- db
- redis
networks:
- internal_network
- external_network
ports:
# - "4002:4002"
volumes:
- ./replikey:/etc/replikey:ro
command: [
"network",
"forward-proxy",
"--listen",
"0.0.0.0:4002",
"--sni",
"${MTLS_REDIS_SNI}",
"--target",
"redis:6379",
"--cert",
"/etc/replikey/cert.pem",
"--key",
"/etc/replikey/key.pem",
"--ca",
"/etc/replikey/ca.pem",
]
# mcaptcha:
# restart: always
# image: mcaptcha/mcaptcha:latest

1
locales/en-US.yml
View file

@ -2134,7 +2134,6 @@ _permissions:
"read:flash-likes": "View list of liked Plays"
"write:flash-likes": "Edit list of liked Plays"
"read:admin:abuse-user-reports": "View user reports"
"write:admin:create-account": "Create user account"
"write:admin:delete-account": "Delete user account"
"write:admin:delete-all-files-of-a-user": "Delete all files of a user"
"read:admin:index-stats": "View database index stats"

4
locales/index.d.ts vendored
View file

@ -8306,10 +8306,6 @@ export interface Locale extends ILocale {
* ユーザーからの通報を見る
*/
"read:admin:abuse-user-reports": string;
/**
* ユーザーアカウントを作成する
*/
"write:admin:create-account": string;
/**
* ユーザーアカウントを削除する
*/

1
locales/ja-JP.yml
View file

@ -2180,7 +2180,6 @@ _permissions:
"read:flash-likes": "Playのいいねを見る"
"write:flash-likes": "Playのいいねを操作する"
"read:admin:abuse-user-reports": "ユーザーからの通報を見る"
"write:admin:create-account": "ユーザーアカウントを作成する"
"write:admin:delete-account": "ユーザーアカウントを削除する"
"write:admin:delete-all-files-of-a-user": "ユーザーのすべてのファイルを削除する"
"read:admin:index-stats": "データベースインデックスに関する情報を見る"

13
package.json
View file

@ -1,10 +1,10 @@
{
"name": "misskey",
"version": "2025.3.0-yumechinokuni.0",
"version": "2025.3.0",
"codename": "nasubi",
"repository": {
"type": "git",
"url": "https://forge.yumechi.jp/yume.yumechi-no-kuni.git"
"url": "https://github.com/misskey-dev/misskey.git"
},
"packageManager": "pnpm@9.15.4",
"workspaces": [
@ -24,15 +24,15 @@
"build": "pnpm build-pre && pnpm -r build && pnpm build-assets",
"build-storybook": "pnpm --filter frontend build-storybook",
"build-misskey-js-with-types": "pnpm build-pre && pnpm --filter backend... --filter=!misskey-js build && pnpm --filter backend generate-api-json --no-build && ncp packages/backend/built/api.json packages/misskey-js/generator/api.json && pnpm --filter misskey-js update-autogen-code && pnpm --filter misskey-js build && pnpm --filter misskey-js api",
"start": "pnpm check:connect && cd packages/backend && cross-env RUN_MODE=web node ./built/boot/entry.js",
"start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cd packages/backend && cross-env NODE_ENV=test RUN_MODE=test node ./built/boot/entry.js",
"start": "pnpm check:connect && cd packages/backend && node ./built/boot/entry.js",
"start:test": "ncp ./.github/misskey/test.yml ./.config/test.yml && cd packages/backend && cross-env NODE_ENV=test node ./built/boot/entry.js",
"init": "pnpm migrate",
"migrate": "cd packages/backend && pnpm migrate",
"revert": "cd packages/backend && pnpm revert",
"check:connect": "cd packages/backend && pnpm check:connect",
"migrateandstart": "pnpm migrate && pnpm start",
"watch": "pnpm dev",
"dev": "cross-env RUN_MODE=dev node scripts/dev.mjs",
"dev": "node scripts/dev.mjs",
"lint": "pnpm -r lint",
"cy:open": "pnpm cypress open --browser --e2e --config-file=cypress.config.ts",
"cy:run": "pnpm cypress run",
@ -74,5 +74,8 @@
"globals": "15.15.0",
"ncp": "2.0.0",
"start-server-and-test": "2.0.10"
},
"optionalDependencies": {
"@tensorflow/tfjs-core": "4.22.0"
}
}

18
packages/backend/migration/1730937958242-Pgroonga.js
View file

@ -1,18 +0,0 @@
export class Pgroonga1730937958242 {
name = 'Pgroonga1730937958242'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_f27f5d88941e57442be75ba9c8" ON "note" USING "pgroonga" ("text")`);
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_7cc8d9b0ee7861b4e5dc86ad85" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2)`);
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_065d4d8f3b5adb4a08841eae3c" ON "user" USING "pgroonga" ("name" pgroonga_varchar_full_text_search_ops_v2)`);
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_fcb770976ff8240af5799e3ffc" ON "user_profile" USING "pgroonga" ("description" pgroonga_varchar_full_text_search_ops_v2) `);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX "public"."IDX_f27f5d88941e57442be75ba9c8"`);
await queryRunner.query(`DROP INDEX "public"."IDX_7cc8d9b0ee7861b4e5dc86ad85"`);
await queryRunner.query(`DROP INDEX "public"."IDX_065d4d8f3b5adb4a08841eae3c"`);
await queryRunner.query(`DROP INDEX "public"."IDX_fcb770976ff8240af5799e3ffc"`);
}
}

22
packages/backend/migration/1731757142918-increase_character_limits.js
View file

@ -1,22 +0,0 @@
/*
* SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class IncreaseCharacterLimits1731757142918 {
name = 'IncreaseCharacterLimits1731757142918'
async up(queryRunner) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_7cc8d9b0ee7861b4e5dc86ad85"`);
await queryRunner.query(`ALTER TABLE "drive_file" ALTER COLUMN "comment" TYPE varchar(32768)`);
await queryRunner.query(`ALTER TABLE "note" ALTER COLUMN "cw" TYPE text`);
await queryRunner.query(`CREATE INDEX "IDX_7cc8d9b0ee7861b4e5dc86ad85" ON "note" USING "pgroonga" ("cw")`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "note" ALTER COLUMN "cw" TYPE varchar(512)`);
await queryRunner.query(`ALTER TABLE "drive_file" ALTER COLUMN "comment" TYPE varchar(8192)`);
await queryRunner.query(`DROP INDEX "IDX_7cc8d9b0ee7861b4e5dc86ad85"`);
await queryRunner.query(`CREATE INDEX "IDX_7cc8d9b0ee7861b4e5dc86ad85" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops)`);
}
}

16
packages/backend/migration/1732071810971-IndexUserDeleted.js
View file

@ -1,16 +0,0 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project and yumechi
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class IndexUserDeleted1732071810971 {
name = 'IndexUserDeleted1732071810971'
async up(queryRunner) {
await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_199b79e682bdc5ba946f491686" ON "user" ("isDeleted")`);
}
async down(queryRunner) {
await queryRunner.query(`DROP INDEX IF EXISTS "IDX_199b79e682bdc5ba946f491686"`);
}
}

16
packages/backend/migration/1739006797620-GoogleAnalytics.js Normal file
View file

@ -0,0 +1,16 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
export class GoogleAnalytics1739006797620 {
name = 'GoogleAnalytics1739006797620'
async up(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" ADD "googleAnalyticsMeasurementId" character varying(64)`);
}
async down(queryRunner) {
await queryRunner.query(`ALTER TABLE "meta" DROP COLUMN "googleAnalyticsMeasurementId"`);
}
}

4
packages/backend/migration/1741279404074-system-accounts-fixup.js
View file

@ -7,13 +7,13 @@ export class SystemAccounts1741279404074 {
name = 'SystemAccounts1741279404074'
async up(queryRunner) {
const instanceActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'instance.actor' AND "host" IS NULL AND "id" NOT IN (SELECT "userId" FROM "system_account" WHERE "type" = 'actor')`);
const instanceActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'instance.actor' AND "host" IS NULL AND "userId" NOT IN (SELECT "userId" FROM "system_account" WHERE "type" = 'actor')`);
if (instanceActor.length > 0) {
console.warn('instance.actor was incorrect, updating...');
await queryRunner.query(`UPDATE "system_account" SET "id" = '${instanceActor[0].id}', "userId" = '${instanceActor[0].id}' WHERE "type" = 'actor'`);
}
const relayActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'relay.actor' AND "host" IS NULL AND "id" NOT IN (SELECT "userId" FROM "system_account" WHERE "type" = 'relay')`);
const relayActor = await queryRunner.query(`SELECT "id" FROM "user" WHERE "username" = 'relay.actor' AND "host" IS NULL AND "userId" NOT IN (SELECT "userId" FROM "system_account" WHERE "type" = 'relay')`);
if (relayActor.length > 0) {
console.warn('relay.actor was incorrect, updating...');
await queryRunner.query(`UPDATE "system_account" SET "id" = '${relayActor[0].id}', "userId" = '${relayActor[0].id}' WHERE "type" = 'relay'`);

4
packages/backend/package.json
View file

@ -48,6 +48,8 @@
"@swc/core-win32-arm64-msvc": "1.10.16",
"@swc/core-win32-ia32-msvc": "1.10.16",
"@swc/core-win32-x64-msvc": "1.10.16",
"@tensorflow/tfjs": "4.22.0",
"@tensorflow/tfjs-node": "4.22.0",
"bufferutil": "4.0.9",
"slacc-android-arm-eabi": "0.0.10",
"slacc-android-arm64": "0.0.10",
@ -144,6 +146,7 @@
"nested-property": "4.0.0",
"node-fetch": "3.3.2",
"nodemailer": "6.10.0",
"nsfwjs": "4.2.0",
"oauth": "0.10.0",
"oauth2orize": "1.12.0",
"oauth2orize-pkce": "0.1.2",
@ -153,7 +156,6 @@
"pg": "8.13.3",
"pkce-challenge": "4.1.0",
"probe-image-size": "7.2.3",
"prom-client": "^15.1.3",
"promise-limit": "2.7.0",
"pug": "3.0.3",
"qrcode": "1.5.4",

7
packages/backend/scripts/check_connect.js
View file

@ -22,7 +22,6 @@ async function connectToRedis(redisOptions) {
lazyConnect: true,
reconnectOnError: false,
showFriendlyErrorStack: true,
connectTimeout: 10000,
});
redis.on('error', e => reject(e));
@ -51,9 +50,7 @@ const promises = Array
]))
.map(connectToRedis)
.concat([
connectToPostgres().then(() => { console.log('Connected to PostgreSQL.'); }),
connectToPostgres()
]);
await Promise.allSettled(promises);
process.exit(0);
await Promise.all(promises);

2
packages/backend/src/GlobalModule.ts
View file

@ -24,7 +24,7 @@ const $config: Provider = {
const $db: Provider = {
provide: DI.db,
useFactory: async (config) => {
const db = createPostgresDataSource(config, true);
const db = createPostgresDataSource(config);
return await db.initialize();
},
inject: [DI.config],

7
packages/backend/src/boot/common.ts
View file

@ -4,7 +4,6 @@
*/
import { NestFactory } from '@nestjs/core';
import * as prom from 'prom-client';
import { ChartManagementService } from '@/core/chart/ChartManagementService.js';
import { QueueProcessorService } from '@/queue/QueueProcessorService.js';
import { NestLogger } from '@/NestLogger.js';
@ -13,9 +12,8 @@ import { QueueStatsService } from '@/daemons/QueueStatsService.js';
import { ServerStatsService } from '@/daemons/ServerStatsService.js';
import { ServerService } from '@/server/ServerService.js';
import { MainModule } from '@/MainModule.js';
import { MetricsService } from '@/server/api/MetricsService.js';
export async function server(workerRegistry?: prom.AggregatorRegistry<prom.PrometheusContentType>) {
export async function server() {
const app = await NestFactory.createApplicationContext(MainModule, {
logger: new NestLogger(),
});
@ -24,9 +22,6 @@ export async function server(workerRegistry?: prom.AggregatorRegistry<prom.Prome
await serverService.launch();
if (process.env.NODE_ENV !== 'test') {
if (workerRegistry) {
app.get(MetricsService).setWorkerRegistry(workerRegistry);
}
app.get(ChartManagementService).start();
app.get(QueueStatsService).start();
app.get(ServerStatsService).start();

12
packages/backend/src/boot/entry.ts
View file

@ -8,7 +8,6 @@
*/
import cluster from 'node:cluster';
import * as prom from 'prom-client';
import { EventEmitter } from 'node:events';
import chalk from 'chalk';
import Xev from 'xev';
@ -18,15 +17,6 @@ import { masterMain } from './master.js';
import { workerMain } from './worker.js';
import { readyRef } from './ready.js';
const workerRegistry = new prom.AggregatorRegistry<prom.PrometheusContentType>();
prom.collectDefaultMetrics({
labels: {
cluster_type: `${cluster.isPrimary ? 'master' : 'worker'}`,
worker_id: cluster.worker?.id.toString() || 'none'
}
});
import 'reflect-metadata';
process.title = `Misskey (${cluster.isPrimary ? 'master' : 'worker'})`;
@ -81,7 +71,7 @@ process.on('exit', code => {
if (!envOption.disableClustering) {
if (cluster.isPrimary) {
logger.info(`Start main process... pid: ${process.pid}`);
await masterMain(workerRegistry);
await masterMain();
ev.mount();
} else if (cluster.isWorker) {
logger.info(`Start worker process... pid: ${process.pid}`);

59
packages/backend/src/boot/master.ts
View file

@ -7,7 +7,6 @@ import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import * as os from 'node:os';
import * as prom from 'prom-client';
import cluster from 'node:cluster';
import chalk from 'chalk';
import chalkTemplate from 'chalk-template';
@ -19,7 +18,6 @@ import type { Config } from '@/config.js';
import { showMachineInfo } from '@/misc/show-machine-info.js';
import { envOption } from '@/env.js';
import { jobQueue, server } from './common.js';
import { metricGauge } from '@/server/api/MetricsService.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@ -31,24 +29,6 @@ const bootLogger = logger.createSubLogger('boot', 'magenta');
const themeColor = chalk.hex('#86b300');
const mBuildInfo = metricGauge({
name: 'misskey_build_info',
help: 'Misskey build information',
labelNames: ['gitCommit', 'gitDescribe', 'node_version']
});
mBuildInfo?.set({
gitCommit: meta.gitCommit || 'unknown',
gitDescribe: meta.gitDescribe || 'unknown',
node_version: process.version
}, 1);
const mStartupTime = metricGauge({
name: 'misskey_startup_time',
help: 'Misskey startup time',
labelNames: ['pid']
});
function greet() {
if (!envOption.quiet) {
//#region Misskey logo
@ -74,7 +54,7 @@ function greet() {
/**
* Init master process
*/
export async function masterMain(workerRegistry?: prom.AggregatorRegistry<prom.PrometheusContentType>) {
export async function masterMain() {
let config!: Config;
// initialize app
@ -84,7 +64,6 @@ export async function masterMain(workerRegistry?: prom.AggregatorRegistry<prom.P
await showMachineInfo(bootLogger);
showNodejsVersion();
config = loadConfigBoot();
//await connectDb();
if (config.pidFile) fs.writeFileSync(config.pidFile, process.pid.toString());
} catch (e) {
@ -112,28 +91,36 @@ export async function masterMain(workerRegistry?: prom.AggregatorRegistry<prom.P
});
}
mStartupTime?.set({ pid: process.pid }, Date.now());
bootLogger.info(
`mode: [disableClustering: ${envOption.disableClustering}, onlyServer: ${envOption.onlyServer}, onlyQueue: ${envOption.onlyQueue}]`,
);
if (!envOption.disableClustering) {
// clusterモジュール有効時
if (envOption.disableClustering) {
if (envOption.onlyServer) {
await server(workerRegistry);
// onlyServer かつ enableCluster な場合、メインプロセスはforkのみに制限する(listenしない)。
// ワーカープロセス側でlistenすると、メインプロセスでポートへの着信を受け入れてワーカープロセスへの分配を行う動作をする。
// そのため、メインプロセスでも直接listenするとポートの競合が発生して起動に失敗してしまう。
// see: https://nodejs.org/api/cluster.html#cluster
} else if (envOption.onlyQueue) {
await jobQueue();
} else {
await server(workerRegistry);
await jobQueue();
}
} else {
if (envOption.onlyServer) {
// nop
} else if (envOption.onlyQueue) {
// nop
await jobQueue();
} else {
await server(workerRegistry);
await server();
}
await spawnWorkers(config.clusterLimit);
} else {
// clusterモジュール無効時
if (envOption.onlyServer) {
await server();
} else if (envOption.onlyQueue) {
await jobQueue();
} else {
await server();
await jobQueue();
}
}
if (envOption.onlyQueue) {

1
packages/backend/src/boot/worker.ts
View file

@ -4,7 +4,6 @@
*/
import cluster from 'node:cluster';
import { collectDefaultMetrics, AggregatorRegistry, RegistryContentType } from 'prom-client';
import * as Sentry from '@sentry/node';
import { nodeProfilingIntegration } from '@sentry/profiling-node';
import { envOption } from '@/env.js';

47
packages/backend/src/config.ts
View file

@ -9,7 +9,6 @@ import { dirname, resolve } from 'node:path';
import * as yaml from 'js-yaml';
import * as Sentry from '@sentry/node';
import type { RedisOptions } from 'ioredis';
import { type CSPHashed, hashResource, hashSourceFile } from './server/csp.js';
type RedisOptionsSource = Partial<RedisOptions> & {
host: string;
@ -20,18 +19,6 @@ type RedisOptionsSource = Partial<RedisOptions> & {
prefix?: string;
};
type BrowserSandboxing = {
// send Referrer-Policy: strict-origin
strictOriginReferrer?: boolean;
csp?: {
disable?: boolean;
appendDirectives?: {
[directive: string]: string | string[];
}
};
};
/**
* 設定ファイルの型
*/
@ -41,7 +28,6 @@ type Source = {
socket?: string;
chmodSocket?: string;
disableHsts?: boolean;
hstsPreload?: boolean;
db: {
host: string;
port: number;
@ -75,16 +61,11 @@ type Source = {
index: string;
scope?: 'local' | 'global' | string[];
};
prometheusMetrics?: { enable: boolean, scrapeToken?: string };
sentryForBackend?: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; };
sentryForFrontend?: { options: Partial<Sentry.NodeOptions> };
publishTarballInsteadOfProvideRepositoryUrl?: boolean;
browserSandboxing?: BrowserSandboxing;
setupPassword?: string;
proxy?: string;
@ -137,7 +118,6 @@ export type Config = {
socket: string | undefined;
chmodSocket: string | undefined;
disableHsts: boolean | undefined;
hstsPreload: boolean | undefined;
db: {
host: string;
port: number;
@ -193,13 +173,7 @@ export type Config = {
}
}
browserSandboxing: BrowserSandboxing;
cspPrerenderedContent: Map<string, CSPHashed>;
version: string;
gitDescribe: string;
gitCommit: string;
publishTarballInsteadOfProvideRepositoryUrl: boolean;
setupPassword: string | undefined;
host: string;
@ -223,12 +197,8 @@ export type Config = {
redisForJobQueue: RedisOptions & RedisOptionsSource;
redisForTimelines: RedisOptions & RedisOptionsSource;
redisForReactions: RedisOptions & RedisOptionsSource;
prometheusMetrics: { enable: boolean, scrapeToken?: string } | undefined;
sentryForBackend: { options: Partial<Sentry.NodeOptions>; enableNodeProfiling: boolean; } | undefined;
sentryForFrontend: { options: Partial<Sentry.NodeOptions> } | undefined;
perChannelMaxNoteCacheCount: number;
perUserNotificationsMaxCount: number;
deactivateAntennaThreshold: number;
@ -269,7 +239,7 @@ export function loadConfig(): Config {
const config = yaml.load(fs.readFileSync(path, 'utf-8')) as Source;
const url = tryCreateUrl(config.url ?? process.env.MISSKEY_URL ?? '');
const { version, gitDescribe, gitCommit } = meta;
const version = meta.version;
const host = url.host;
const hostname = url.hostname;
const scheme = url.protocol.replace(/:$/, '');
@ -284,21 +254,9 @@ export function loadConfig(): Config {
: null;
const internalMediaProxy = `${scheme}://${host}/proxy`;
const redis = convertRedisOptions(config.redis, host);
const htmlScriptPrelude = `var VERSION = ${JSON.stringify(version)}; var CLIENT_ENTRY = ${JSON.stringify(frontendManifest['src/_boot_.ts'].file)};`;
const htmlEmbedScriptPrelude = `var VERSION = ${JSON.stringify(version)}; var CLIENT_ENTRY = ${JSON.stringify(frontendEmbedManifest['src/boot.ts'].file)};`;
const cspPrerenderedContent = new Map([
['.prelude.js', hashResource(htmlScriptPrelude)],
['.prelude.embed.js', hashResource(htmlEmbedScriptPrelude)],
...['boot.js', 'style.css', 'style.embed.css', 'boot.embed.js',
'bios.css', 'bios.js', 'cli.css', 'cli.js', 'error.css'
].map((file) => [file, hashSourceFile(`${_dirname}/server/web/${file}`)] as [string, CSPHashed]),
]);
return {
version,
gitCommit,
gitDescribe,
browserSandboxing: config.browserSandboxing ?? { strictOriginReferrer: true },
publishTarballInsteadOfProvideRepositoryUrl: !!config.publishTarballInsteadOfProvideRepositoryUrl,
setupPassword: config.setupPassword,
url: url.origin,
@ -306,8 +264,6 @@ export function loadConfig(): Config {
socket: config.socket,
chmodSocket: config.chmodSocket,
disableHsts: config.disableHsts,
hstsPreload: config.hstsPreload ?? false,
cspPrerenderedContent,
host,
hostname,
scheme,
@ -326,7 +282,6 @@ export function loadConfig(): Config {
redisForJobQueue: config.redisForJobQueue ? convertRedisOptions(config.redisForJobQueue, host) : redis,
redisForTimelines: config.redisForTimelines ? convertRedisOptions(config.redisForTimelines, host) : redis,
redisForReactions: config.redisForReactions ? convertRedisOptions(config.redisForReactions, host) : redis,
prometheusMetrics: config.prometheusMetrics,
sentryForBackend: config.sentryForBackend,
sentryForFrontend: config.sentryForFrontend,
id: config.id,

32
packages/backend/src/core/ActorKeySignerService.ts
View file

@ -1,32 +0,0 @@
import crypto from 'node:crypto';
import { Inject, Injectable } from "@nestjs/common";
@Injectable()
export class ActorKeySignerService {
private proxyUri?: string;
constructor() {
this.proxyUri = process.env.MISSKEY_ACTOR_KEY_PROXY_URL;
}
public async sign(id: string, privateKey: string, data: string): Promise<string> {
if (this.proxyUri && privateKey === 'proxy') {
const response = await fetch(`${this.proxyUri}signature/${id}`, {
method: 'POST',
body: data,
});
if (!response.ok) {
throw new Error(`Failed to sign data: ${response.statusText}`);
}
return response.text();
}
const signer = crypto.createSign('sha256');
signer.update(data);
signer.end();
const signature = signer.sign(privateKey);
return signature.toString('base64');
}
}

89
packages/backend/src/core/AiService.ts Normal file
View file

@ -0,0 +1,89 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import { Injectable } from '@nestjs/common';
import * as nsfw from 'nsfwjs';
import si from 'systeminformation';
import { Mutex } from 'async-mutex';
import fetch from 'node-fetch';
import { bindThis } from '@/decorators.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const REQUIRED_CPU_FLAGS_X64 = ['avx2', 'fma'];
let isSupportedCpu: undefined | boolean = undefined;
@Injectable()
export class AiService {
private model: nsfw.NSFWJS;
private modelLoadMutex: Mutex = new Mutex();
constructor(
) {
}
@bindThis
public async detectSensitive(path: string): Promise<nsfw.PredictionType[] | null> {
try {
if (isSupportedCpu === undefined) {
isSupportedCpu = await this.computeIsSupportedCpu();
}
if (!isSupportedCpu) {
console.error('This CPU cannot use TensorFlow.');
return null;
}
const tf = await import('@tensorflow/tfjs-node');
tf.env().global.fetch = fetch;
if (this.model == null) {
await this.modelLoadMutex.runExclusive(async () => {
if (this.model == null) {
this.model = await nsfw.load(`file://${_dirname}/../../nsfw-model/`, { size: 299 });
}
});
}
const buffer = await fs.promises.readFile(path);
const image = await tf.node.decodeImage(buffer, 3) as any;
try {
const predictions = await this.model.classify(image);
return predictions;
} finally {
image.dispose();
}
} catch (err) {
console.error(err);
return null;
}
}
private async computeIsSupportedCpu(): Promise<boolean> {
switch (process.arch) {
case 'x64': {
const cpuFlags = await this.getCpuFlags();
return REQUIRED_CPU_FLAGS_X64.every(required => cpuFlags.includes(required));
}
case 'arm64': {
// As far as I know, no required CPU flags for ARM64.
return true;
}
default: {
return false;
}
}
}
@bindThis
private async getCpuFlags(): Promise<string[]> {
const str = await si.cpuFlags();
return str.split(/\s+/);
}
}

24
packages/backend/src/core/CoreModule.ts
View file

@ -17,6 +17,7 @@ import { WebhookTestService } from '@/core/WebhookTestService.js';
import { FlashService } from '@/core/FlashService.js';
import { AccountMoveService } from './AccountMoveService.js';
import { AccountUpdateService } from './AccountUpdateService.js';
import { AiService } from './AiService.js';
import { AnnouncementService } from './AnnouncementService.js';
import { AntennaService } from './AntennaService.js';
import { AppLockService } from './AppLockService.js';
@ -34,7 +35,7 @@ import { GlobalEventService } from './GlobalEventService.js';
import { HashtagService } from './HashtagService.js';
import { HttpRequestService } from './HttpRequestService.js';
import { IdService } from './IdService.js';
import { __YUME_PRIVATE_ImageProcessingService } from './ImageProcessingService.js';
import { ImageProcessingService } from './ImageProcessingService.js';
import { SystemAccountService } from './SystemAccountService.js';
import { InternalStorageService } from './InternalStorageService.js';
import { MetaService } from './MetaService.js';
@ -65,7 +66,7 @@ import { UserMutingService } from './UserMutingService.js';
import { UserRenoteMutingService } from './UserRenoteMutingService.js';
import { UserSuspendService } from './UserSuspendService.js';
import { UserAuthService } from './UserAuthService.js';
import { __YUME_PRIVATE_VideoProcessingService } from './VideoProcessingService.js';
import { VideoProcessingService } from './VideoProcessingService.js';
import { UserWebhookService } from './UserWebhookService.js';
import { UtilityService } from './UtilityService.js';
import { FileInfoService } from './FileInfoService.js';
@ -150,7 +151,6 @@ import { QueueModule } from './QueueModule.js';
import { QueueService } from './QueueService.js';
import { LoggerService } from './LoggerService.js';
import type { Provider } from '@nestjs/common';
import { ActorKeySignerService } from './ActorKeySignerService.js';
//#region 文字列ベースでのinjection用(循環参照対応のため)
const $LoggerService: Provider = { provide: 'LoggerService', useExisting: LoggerService };
@ -158,6 +158,7 @@ const $AbuseReportService: Provider = { provide: 'AbuseReportService', useExisti
const $AbuseReportNotificationService: Provider = { provide: 'AbuseReportNotificationService', useExisting: AbuseReportNotificationService };
const $AccountMoveService: Provider = { provide: 'AccountMoveService', useExisting: AccountMoveService };
const $AccountUpdateService: Provider = { provide: 'AccountUpdateService', useExisting: AccountUpdateService };
const $AiService: Provider = { provide: 'AiService', useExisting: AiService };
const $AnnouncementService: Provider = { provide: 'AnnouncementService', useExisting: AnnouncementService };
const $AntennaService: Provider = { provide: 'AntennaService', useExisting: AntennaService };
const $AppLockService: Provider = { provide: 'AppLockService', useExisting: AppLockService };
@ -175,7 +176,7 @@ const $GlobalEventService: Provider = { provide: 'GlobalEventService', useExisti
const $HashtagService: Provider = { provide: 'HashtagService', useExisting: HashtagService };
const $HttpRequestService: Provider = { provide: 'HttpRequestService', useExisting: HttpRequestService };
const $IdService: Provider = { provide: 'IdService', useExisting: IdService };
const $ImageProcessingService: Provider = { provide: '__YUME_PRIVATE_ImageProcessingService', useExisting: __YUME_PRIVATE_ImageProcessingService };
const $ImageProcessingService: Provider = { provide: 'ImageProcessingService', useExisting: ImageProcessingService };
const $InternalStorageService: Provider = { provide: 'InternalStorageService', useExisting: InternalStorageService };
const $MetaService: Provider = { provide: 'MetaService', useExisting: MetaService };
const $MfmService: Provider = { provide: 'MfmService', useExisting: MfmService };
@ -207,7 +208,7 @@ const $UserRenoteMutingService: Provider = { provide: 'UserRenoteMutingService',
const $UserSearchService: Provider = { provide: 'UserSearchService', useExisting: UserSearchService };
const $UserSuspendService: Provider = { provide: 'UserSuspendService', useExisting: UserSuspendService };
const $UserAuthService: Provider = { provide: 'UserAuthService', useExisting: UserAuthService };
const $VideoProcessingService: Provider = { provide: '__YUME_PRIVATE_VideoProcessingService', useExisting: __YUME_PRIVATE_VideoProcessingService };
const $VideoProcessingService: Provider = { provide: 'VideoProcessingService', useExisting: VideoProcessingService };
const $UserWebhookService: Provider = { provide: 'UserWebhookService', useExisting: UserWebhookService };
const $SystemWebhookService: Provider = { provide: 'SystemWebhookService', useExisting: SystemWebhookService };
const $WebhookTestService: Provider = { provide: 'WebhookTestService', useExisting: WebhookTestService };
@ -306,6 +307,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
AbuseReportNotificationService,
AccountMoveService,
AccountUpdateService,
AiService,
AnnouncementService,
AntennaService,
AppLockService,
@ -323,7 +325,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
HashtagService,
HttpRequestService,
IdService,
__YUME_PRIVATE_ImageProcessingService,
ImageProcessingService,
InternalStorageService,
MetaService,
MfmService,
@ -355,7 +357,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
UserSearchService,
UserSuspendService,
UserAuthService,
__YUME_PRIVATE_VideoProcessingService,
VideoProcessingService,
UserWebhookService,
SystemWebhookService,
WebhookTestService,
@ -432,7 +434,6 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
ApMfmService,
ApRendererService,
ApRequestService,
ActorKeySignerService,
ApResolverService,
JsonLdService,
RemoteLoggerService,
@ -451,6 +452,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$AbuseReportNotificationService,
$AccountMoveService,
$AccountUpdateService,
$AiService,
$AnnouncementService,
$AntennaService,
$AppLockService,
@ -596,6 +598,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
AbuseReportNotificationService,
AccountMoveService,
AccountUpdateService,
AiService,
AnnouncementService,
AntennaService,
AppLockService,
@ -613,7 +616,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
HashtagService,
HttpRequestService,
IdService,
__YUME_PRIVATE_ImageProcessingService,
ImageProcessingService,
InternalStorageService,
MetaService,
MfmService,
@ -645,7 +648,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
UserSearchService,
UserSuspendService,
UserAuthService,
__YUME_PRIVATE_VideoProcessingService,
VideoProcessingService,
UserWebhookService,
SystemWebhookService,
WebhookTestService,
@ -739,6 +742,7 @@ const $ApQuestionService: Provider = { provide: 'ApQuestionService', useExisting
$AbuseReportNotificationService,
$AccountMoveService,
$AccountUpdateService,
$AiService,
$AnnouncementService,
$AntennaService,
$AppLockService,

4
packages/backend/src/core/DeleteAccountService.ts
View file

@ -57,10 +57,6 @@ export class DeleteAccountService {
});
}
if (!(await this.usersRepository.update({ id: user.id, isDeleted: false }, { isDeleted: true })).affected) {
return;
}
// 物理削除する前にDelete activityを送信する
if (this.userEntityService.isLocalUser(user)) {
// 知り得る全SharedInboxにDelete配信

11
packages/backend/src/core/DownloadService.ts
View file

@ -44,14 +44,6 @@ export class DownloadService {
const maxSize = this.config.maxFileSize;
const urlObj = new URL(url);
if (urlObj.protocol && urlObj.protocol !== 'https:') {
throw new Error(`Unsupported protocol: ${urlObj.protocol}, only HTTPS is supported`);
}
urlObj.protocol = 'https:';
if (urlObj.port && urlObj.port !== '443') {
throw new Error(`Unsupported port: ${urlObj.port}, only 443 is supported`);
}
let filename = urlObj.pathname.split('/').pop() ?? 'untitled';
const req = got.stream(url, {
@ -68,7 +60,8 @@ export class DownloadService {
request: operationTimeout, // whole operation timeout
},
agent: {
https: this.httpRequestService.httpsAgent,
http: this.httpRequestService.getAgentForHttp(urlObj, true),
https: this.httpRequestService.getAgentForHttps(urlObj, true),
},
http2: false, // default
retry: {

52
packages/backend/src/core/DriveService.ts
View file

@ -22,8 +22,8 @@ import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { contentDisposition } from '@/misc/content-disposition.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { __YUME_PRIVATE_VideoProcessingService } from '@/core/VideoProcessingService.js';
import { __YUME_PRIVATE_ImageProcessingService } from '@/core/ImageProcessingService.js';
import { VideoProcessingService } from '@/core/VideoProcessingService.js';
import { ImageProcessingService } from '@/core/ImageProcessingService.js';
import type { IImage } from '@/core/ImageProcessingService.js';
import { QueueService } from '@/core/QueueService.js';
import type { MiDriveFolder } from '@/models/DriveFolder.js';
@ -87,9 +87,9 @@ type UploadFromUrlArgs = {
@Injectable()
export class DriveService {
public static NoSuchFolderError = class extends Error { };
public static InvalidFileNameError = class extends Error { };
public static CannotUnmarkSensitiveError = class extends Error { };
public static NoSuchFolderError = class extends Error {};
public static InvalidFileNameError = class extends Error {};
public static CannotUnmarkSensitiveError = class extends Error {};
private registerLogger: Logger;
private downloaderLogger: Logger;
private deleteLogger: Logger;
@ -120,8 +120,8 @@ export class DriveService {
private downloadService: DownloadService,
private internalStorageService: InternalStorageService,
private s3Service: S3Service,
private privateImageProcessingService: __YUME_PRIVATE_ImageProcessingService,
private privateVideoProcessingService: __YUME_PRIVATE_VideoProcessingService,
private imageProcessingService: ImageProcessingService,
private videoProcessingService: VideoProcessingService,
private globalEventService: GlobalEventService,
private queueService: QueueService,
private roleService: RoleService,
@ -147,11 +147,11 @@ export class DriveService {
*/
@bindThis
private async save(file: MiDriveFile, path: string, name: string, type: string, hash: string, size: number): Promise<MiDriveFile> {
// thunbnail, webpublic を必要なら生成
// thunbnail, webpublic を必要なら生成
const alts = await this.generateAlts(path, type, !file.uri);
if (this.meta.useObjectStorage) {
//#region ObjectStorage params
//#region ObjectStorage params
let [ext] = (name.match(/\.([a-zA-Z0-9_-]+)$/) ?? ['']);
if (ext === '') {
@ -170,7 +170,7 @@ export class DriveService {
}
const baseUrl = this.meta.objectStorageBaseUrl
?? `${this.meta.objectStorageUseSSL ? 'https' : 'http'}://${this.meta.objectStorageEndpoint}${this.meta.objectStoragePort ? `:${this.meta.objectStoragePort}` : ''}/${this.meta.objectStorageBucket}`;
?? `${ this.meta.objectStorageUseSSL ? 'https' : 'http' }://${ this.meta.objectStorageEndpoint }${ this.meta.objectStoragePort ? `:${this.meta.objectStoragePort}` : '' }/${ this.meta.objectStorageBucket }`;
// for original
const prefix = this.meta.objectStoragePrefix ? `${this.meta.objectStoragePrefix}/` : '';
@ -278,7 +278,7 @@ export class DriveService {
}
try {
const thumbnail = await this.privateVideoProcessingService.generateVideoThumbnail(path);
const thumbnail = await this.videoProcessingService.generateVideoThumbnail(path);
return {
webpublic: null,
thumbnail,
@ -312,9 +312,9 @@ export class DriveService {
satisfyWebpublic = !!(
type !== 'image/svg+xml' && // security reason
type !== 'image/avif' && // not supported by Mastodon and MS Edge
!(metadata.exif ?? metadata.iptc ?? metadata.xmp ?? metadata.tifftagPhotoshop) &&
metadata.width && metadata.width <= 2048 &&
metadata.height && metadata.height <= 2048
!(metadata.exif ?? metadata.iptc ?? metadata.xmp ?? metadata.tifftagPhotoshop) &&
metadata.width && metadata.width <= 2048 &&
metadata.height && metadata.height <= 2048
);
} catch (err) {
this.registerLogger.warn(`sharp failed: ${err}`);
@ -332,9 +332,9 @@ export class DriveService {
try {
if (['image/jpeg', 'image/webp', 'image/avif'].includes(type)) {
webpublic = await this.privateImageProcessingService.convertSharpToWebp(img, 2048, 2048);
webpublic = await this.imageProcessingService.convertSharpToWebp(img, 2048, 2048);
} else if (['image/png', 'image/bmp', 'image/svg+xml'].includes(type)) {
webpublic = await this.privateImageProcessingService.convertSharpToPng(img, 2048, 2048);
webpublic = await this.imageProcessingService.convertSharpToPng(img, 2048, 2048);
} else {
this.registerLogger.debug('web image not created (not an required image)');
}
@ -353,9 +353,9 @@ export class DriveService {
try {
if (isAnimated) {
thumbnail = await this.privateImageProcessingService.convertSharpToWebp(sharp(path, { animated: true }), 374, 317, { alphaQuality: 70 });
thumbnail = await this.imageProcessingService.convertSharpToWebp(sharp(path, { animated: true }), 374, 317, { alphaQuality: 70 });
} else {
thumbnail = await this.privateImageProcessingService.convertSharpToWebp(img, 498, 422);
thumbnail = await this.imageProcessingService.convertSharpToWebp(img, 498, 422);
}
} catch (err) {
this.registerLogger.warn('thumbnail not created (an error occurred)', err as Error);
@ -471,11 +471,11 @@ export class DriveService {
const info = await this.fileInfoService.getFileInfo(path, {
skipSensitiveDetection: skipNsfwCheck,
sensitiveThreshold: // 感度が高いほどしきい値は低くすることになる
this.meta.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 0.1 :
this.meta.sensitiveMediaDetectionSensitivity === 'high' ? 0.3 :
this.meta.sensitiveMediaDetectionSensitivity === 'low' ? 0.7 :
this.meta.sensitiveMediaDetectionSensitivity === 'veryLow' ? 0.9 :
0.5,
this.meta.sensitiveMediaDetectionSensitivity === 'veryHigh' ? 0.1 :
this.meta.sensitiveMediaDetectionSensitivity === 'high' ? 0.3 :
this.meta.sensitiveMediaDetectionSensitivity === 'low' ? 0.7 :
this.meta.sensitiveMediaDetectionSensitivity === 'veryLow' ? 0.9 :
0.5,
sensitiveThresholdForPorn: 0.75,
enableSensitiveMediaDetectionForVideos: this.meta.enableSensitiveMediaDetectionForVideos,
});
@ -495,7 +495,7 @@ export class DriveService {
);
if (user && !force) {
// Check if there is a file with the same hash
// Check if there is a file with the same hash
const matched = await this.driveFilesRepository.findOneBy({
md5: info.md5,
userId: user.id,
@ -583,7 +583,7 @@ export class DriveService {
file.maybePorn = info.porn;
file.isSensitive = user
? this.userEntityService.isLocalUser(user) && profile!.alwaysMarkNsfw ? true :
sensitive ?? false
sensitive ?? false
: false;
if (user && this.utilityService.isMediaSilencedHost(this.meta.mediaSilencedHosts, user.host)) file.isSensitive = true;
@ -617,7 +617,7 @@ export class DriveService {
file = await this.driveFilesRepository.insertOne(file);
} catch (err) {
// duplicate key error (when already registered)
// duplicate key error (when already registered)
if (isDuplicateKeyValueError(err)) {
this.registerLogger.info(`already registered ${file.uri}`);

1
packages/backend/src/core/EmailService.ts
View file

@ -145,7 +145,6 @@ export class EmailService {
try {
// TODO: htmlサニタイズ
const info = await transporter.sendMail({
replyTo: this.meta.maintainerEmail ? { name: this.meta.maintainerName || 'Instance Maintainer', address: this.meta.maintainerEmail } : undefined,
from: this.meta.email!,
to: to,
subject: subject,

2
packages/backend/src/core/FeaturedService.ts
View file

@ -12,7 +12,7 @@ import { bindThis } from '@/decorators.js';
const GLOBAL_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
export const GALLERY_POSTS_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 3; // 3日ごと
const PER_USER_NOTES_RANKING_WINDOW = 1000 * 60 * 60 * 24 * 7; // 1週間ごと
const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60 * 48; // 48時間ごと
const HASHTAG_RANKING_WINDOW = 1000 * 60 * 60; // 1時間ごと
const featuredEpoc = new Date('2023-01-01T00:00:00Z').getTime();

101
packages/backend/src/core/FileInfoService.ts
View file

@ -15,9 +15,12 @@ import isSvg from 'is-svg';
import probeImageSize from 'probe-image-size';
import { sharpBmp } from '@misskey-dev/sharp-read-bmp';
import * as blurhash from 'blurhash';
import { createTempDir } from '@/misc/create-temp.js';
import { AiService } from '@/core/AiService.js';
import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import type { PredictionType } from 'nsfwjs';
export type FileInfo = {
size: number;
@ -50,6 +53,7 @@ export class FileInfoService {
private logger: Logger;
constructor(
private aiService: AiService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('file-info');
@ -163,7 +167,102 @@ export class FileInfoService {
@bindThis
private async detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> {
return [false, false];
let sensitive = false;
let porn = false;
function judgePrediction(result: readonly PredictionType[]): [sensitive: boolean, porn: boolean] {
let sensitive = false;
let porn = false;
if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true;
return [sensitive, porn];
}
if ([
'image/jpeg',
'image/png',
'image/webp',
].includes(mime)) {
const result = await this.aiService.detectSensitive(source);
if (result) {
[sensitive, porn] = judgePrediction(result);
}
} else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) {
const [outDir, disposeOutDir] = await createTempDir();
try {
const command = FFmpeg()
.input(source)
.inputOptions([
'-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする（そうなるとは限らない）
'-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする（そうなるとは限らない）
])
.noAudio()
.videoFilters([
{
filter: 'select', // フレームのフィルタリング
options: {
e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタする（VP9 とかはデコードしてみないとわからないっぽい）
},
},
{
filter: 'blackframe', // 暗いフレームの検出
options: {
amount: '0', // 暗さに関わらず全てのフレームで測定値を取る
},
},
{
filter: 'metadata',
options: {
mode: 'select', // フレーム選択モード
key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率（前のフィルタからのメタデータを参照する）
value: '50',
function: 'less', // 50% 未満のフレームを選択する（50% 以上暗部があるフレームだと誤検知を招くかもしれないので）
},
},
{
filter: 'scale',
options: {
w: 299,
h: 299,
},
},
])
.format('image2')
.output(join(outDir, '%d.png'))
.outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない
const results: ReturnType<typeof judgePrediction>[] = [];
let frameIndex = 0;
let targetIndex = 0;
let nextIndex = 1;
for await (const path of this.asyncIterateFrames(outDir, command)) {
try {
const index = frameIndex++;
if (index !== targetIndex) {
continue;
}
targetIndex = nextIndex;
nextIndex += index; // fibonacci sequence によってフレーム数制限を掛ける
const result = await this.aiService.detectSensitive(path);
if (result) {
results.push(judgePrediction(result));
}
} finally {
fs.promises.unlink(path);
}
}
sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold);
porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn);
} finally {
disposeOutDir();
}
}
return [sensitive, porn];
}
private async *asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator<string, void> {

37
packages/backend/src/core/HttpRequestService.ts
View file

@ -9,7 +9,7 @@ import * as net from 'node:net';
import ipaddr from 'ipaddr.js';
import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
import { HttpsProxyAgent } from 'hpagent';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import { Inject, Injectable } from '@nestjs/common';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
@ -19,7 +19,7 @@ import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/val
import { assertActivityMatchesUrls, FetchAllowSoftFailMask } from '@/core/activitypub/misc/check-against-url.js';
import type { IObject } from '@/core/activitypub/type.js';
import type { Response } from 'node-fetch';
import { URL } from 'node:url';
import type { URL } from 'node:url';
export type HttpRequestSendOptions = {
throwErrorWhenResponseNotOk: boolean;
@ -59,12 +59,14 @@ class HttpRequestServiceAgent extends http.Agent {
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
@ -96,12 +98,14 @@ class HttpsRequestServiceAgent extends https.Agent {
@bindThis
private isPrivateIp(ip: string): boolean {
const parsedIp = ipaddr.parse(ip);
for (const net of this.config.allowedPrivateNetworks ?? []) {
const cidr = ipaddr.parseCIDR(net);
if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) {
return false;
}
}
return parsedIp.range() !== 'unicast';
}
}
@ -153,15 +157,30 @@ export class HttpRequestService {
keepAliveMsecs: 30 * 1000,
lookup: cache.lookup as unknown as net.LookupFunction,
localAddress: config.outgoingAddress,
minVersion: 'TLSv1.2' as const,
};
this.httpNative = new http.Agent(agentOption);
this.httpsNative = new https.Agent(agentOption);
this.http = new HttpRequestServiceAgent(config, agentOption);
this.https = new HttpsRequestServiceAgent(config, agentOption);
const maxSockets = Math.max(256, config.deliverJobConcurrency ?? 128);
this.httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
localAddress: config.outgoingAddress,
})
: this.http;
this.httpsAgent = config.proxy
? new HttpsProxyAgent({
keepAlive: true,
@ -303,15 +322,7 @@ export class HttpRequestService {
controller.abort();
}, timeout);
const urlParsed = new URL(url);
if (urlParsed.protocol !== 'https:') {
throw new Error('Invalid protocol');
}
if (urlParsed.port && urlParsed.port !== '443') {
throw new Error('Invalid port');
}
const isLocalAddressAllowed = args.isLocalAddressAllowed ?? false;
const res = await fetch(url, {
method: args.method ?? 'GET',
@ -321,7 +332,7 @@ export class HttpRequestService {
},
body: args.body,
size: args.size ?? 10 * 1024 * 1024,
agent: (url) => this.getAgentByUrl(url, false),
agent: (url) => this.getAgentByUrl(url, false, isLocalAddressAllowed),
signal: controller.signal,
});

4
packages/backend/src/core/ImageProcessingService.ts
View file

@ -46,9 +46,7 @@ import { bindThis } from '@/decorators.js';
import { Readable } from 'node:stream';
@Injectable()
// Prevent accidental import by upstream merge
// eslint-disable-next-line
export class __YUME_PRIVATE_ImageProcessingService {
export class ImageProcessingService {
constructor(
) {
}

5
packages/backend/src/core/NoteCreateService.ts
View file

@ -7,7 +7,6 @@ import { setImmediate } from 'node:timers/promises';
import * as mfm from 'mfm-js';
import { In, DataSource, IsNull, LessThan } from 'typeorm';
import * as Redis from 'ioredis';
import * as Bull from 'bullmq';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import { extractMentions } from '@/misc/extract-mentions.js';
import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mfm.js';
@ -294,7 +293,7 @@ export class NoteCreateService implements OnApplicationShutdown {
case 'followers':
// 他人のfollowers noteはreject
if (data.renote.userId !== user.id) {
throw new Bull.UnrecoverableError('Renote target is not public or home');
throw new Error('Renote target is not public or home');
}
// Renote対象がfollowersならfollowersにする
@ -302,7 +301,7 @@ export class NoteCreateService implements OnApplicationShutdown {
break;
case 'specified':
// specified / direct noteはreject
throw new Bull.UnrecoverableError('Renote target is not public or home');
throw new Error('Renote target is not public or home');
}
}

9
packages/backend/src/core/NotificationService.ts
View file

@ -21,13 +21,6 @@ import type { Config } from '@/config.js';
import { UserListService } from '@/core/UserListService.js';
import type { FilterUnionByProperty } from '@/types.js';
import { trackPromise } from '@/misc/promise-tracker.js';
import { metricCounter } from '@/server/api/MetricsService.js';
const mNotificationsCreated = metricCounter({
name: 'misskey_notifications_created',
help: 'Notifications created',
labelNames: ['event_type'],
});
@Injectable()
export class NotificationService implements OnApplicationShutdown {
@ -172,8 +165,6 @@ export class NotificationService implements OnApplicationShutdown {
if (packed == null) return null;
mNotificationsCreated?.inc({ event_type: notification.type });
// Publish notification event
this.globalEventService.publishMainStream(notifieeId, 'notification', packed);

17
packages/backend/src/core/PushNotificationService.ts
View file

@ -13,19 +13,6 @@ import { getNoteSummary } from '@/misc/get-note-summary.js';
import type { MiMeta, MiSwSubscription, SwSubscriptionsRepository } from '@/models/_.js';
import { bindThis } from '@/decorators.js';
import { RedisKVCache } from '@/misc/cache.js';
import { metricCounter } from '@/server/api/MetricsService.js';
const mWebPushCreated = metricCounter({
name: 'misskey_webpush_created',
help: 'WebPush event',
labelNames: ['event_type'],
});
const mWebPushError = metricCounter({
name: 'misskey_webpush_error',
help: 'WebPush error',
labelNames: ['event_type', 'status'],
});
// Defined also packages/sw/types.ts#L13
type PushNotificationsTypes = {
@ -108,8 +95,6 @@ export class PushNotificationService implements OnApplicationShutdown {
},
};
mWebPushCreated?.inc({ event_type: type });
push.sendNotification(pushSubscription, JSON.stringify({
type,
body: (type === 'notification' || type === 'unreadAntennaNote') ? truncateBody(type, body) : body,
@ -131,8 +116,6 @@ export class PushNotificationService implements OnApplicationShutdown {
}).then(() => {
this.refreshCache(userId);
});
} else {
mWebPushError?.inc({ event_type: type, status: err.statusCode || 'unknown' });
}
});
}

36
packages/backend/src/core/QueueModule.ts
View file

@ -18,7 +18,6 @@ import {
SystemWebhookDeliverJobData,
} from '../queue/types.js';
import type { Provider } from '@nestjs/common';
import { mActiveJobs, mDelayedJobs, mFailedJobs, mJobBlockedCounter, mWaitingJobs } from '@/queue/metrics.js';
export type SystemQueue = Bull.Queue<Record<string, unknown>>;
export type EndedPollNotificationQueue = Bull.Queue<EndedPollNotificationJobData>;
@ -30,74 +29,57 @@ export type ObjectStorageQueue = Bull.Queue;
export type UserWebhookDeliverQueue = Bull.Queue<UserWebhookDeliverJobData>;
export type SystemWebhookDeliverQueue = Bull.Queue<SystemWebhookDeliverJobData>;
function withMetrics<T>(queue: Bull.Queue<T>): Bull.Queue<T> {
if (process.env.NODE_ENV !== 'test') {
setInterval(async () => {
mActiveJobs?.set({ queue: queue.name }, await queue.getActiveCount());
mDelayedJobs?.set({ queue: queue.name }, await queue.getDelayedCount());
mWaitingJobs?.set({ queue: queue.name }, await queue.getWaitingCount());
mFailedJobs?.set({ queue: queue.name }, await queue.getFailedCount());
}, 2000);
queue.on('waiting', () => {
mJobBlockedCounter?.inc({ queue: queue.name });
});
}
return queue;
}
const $system: Provider = {
provide: 'queue:system',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM, baseQueueOptions(config, QUEUE.SYSTEM)),
inject: [DI.config],
};
const $endedPollNotification: Provider = {
provide: 'queue:endedPollNotification',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.ENDED_POLL_NOTIFICATION, baseQueueOptions(config, QUEUE.ENDED_POLL_NOTIFICATION)),
inject: [DI.config],
};
const $deliver: Provider = {
provide: 'queue:deliver',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.DELIVER, baseQueueOptions(config, QUEUE.DELIVER)),
inject: [DI.config],
};
const $inbox: Provider = {
provide: 'queue:inbox',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.INBOX, baseQueueOptions(config, QUEUE.INBOX)),
inject: [DI.config],
};
const $db: Provider = {
provide: 'queue:db',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.DB, baseQueueOptions(config, QUEUE.DB)),
inject: [DI.config],
};
const $relationship: Provider = {
provide: 'queue:relationship',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.RELATIONSHIP, baseQueueOptions(config, QUEUE.RELATIONSHIP)),
inject: [DI.config],
};
const $objectStorage: Provider = {
provide: 'queue:objectStorage',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.OBJECT_STORAGE, baseQueueOptions(config, QUEUE.OBJECT_STORAGE)),
inject: [DI.config],
};
const $userWebhookDeliver: Provider = {
provide: 'queue:userWebhookDeliver',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.USER_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.USER_WEBHOOK_DELIVER))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.USER_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.USER_WEBHOOK_DELIVER)),
inject: [DI.config],
};
const $systemWebhookDeliver: Provider = {
provide: 'queue:systemWebhookDeliver',
useFactory: (config: Config) => withMetrics(new Bull.Queue(QUEUE.SYSTEM_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.SYSTEM_WEBHOOK_DELIVER))),
useFactory: (config: Config) => new Bull.Queue(QUEUE.SYSTEM_WEBHOOK_DELIVER, baseQueueOptions(config, QUEUE.SYSTEM_WEBHOOK_DELIVER)),
inject: [DI.config],
};

32
packages/backend/src/core/ReactionService.ts
View file

@ -30,9 +30,6 @@ import { trackPromise } from '@/misc/promise-tracker.js';
import { isQuote, isRenote } from '@/misc/is-renote.js';
import { ReactionsBufferingService } from '@/core/ReactionsBufferingService.js';
import { PER_NOTE_REACTION_USER_PAIR_CACHE_MAX } from '@/const.js';
import { UserWebhookPayload, UserWebhookService } from './UserWebhookService.js';
import { QueueService } from './QueueService.js';
import { Packed } from '@/misc/json-schema.js';
const FALLBACK = '\u2764';
@ -97,8 +94,6 @@ export class ReactionService {
private reactionsBufferingService: ReactionsBufferingService,
private idService: IdService,
private featuredService: FeaturedService,
private queueService: QueueService,
private webhookService: UserWebhookService,
private globalEventService: GlobalEventService,
private apRendererService: ApRendererService,
private apDeliverManagerService: ApDeliverManagerService,
@ -169,10 +164,6 @@ export class ReactionService {
}
}
if (/['\\]/.test(reaction)) {
throw new IdentifiableError('cf61d38c-598a-49e2-b75a-6c38671fcc43', 'Invalid reaction.');
}
const record: MiNoteReaction = {
id: this.idService.gen(),
noteId: note.id,
@ -263,33 +254,12 @@ export class ReactionService {
userId: user.id,
});
// リアクションされたユーザーがローカルユーザーなら通知を作成してWebhookを送信
// リアクションされたユーザーがローカルユーザーなら通知を作成
if (note.userHost === null) {
this.notificationService.createNotification(note.userId, 'reaction', {
noteId: note.id,
reaction: reaction,
}, user.id);
this.webhookService.getActiveWebhooks().then(async webhooks => {
webhooks = webhooks.filter(x => x.userId === note.userId && x.on.includes('reaction'));
if (webhooks.length === 0) return;
const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true });
const userObj = await this.userEntityService.pack(user.id, null, { schema: 'UserLite' });
const payload: UserWebhookPayload<'reaction'> = {
note: noteObj,
reaction: {
id: record.id,
user: userObj,
reaction: reaction,
},
};
for (const webhook of webhooks) {
this.queueService.userWebhookDeliver(webhook, 'reaction', payload);
}
});
}
//#region 配信

4
packages/backend/src/core/RemoteUserResolveService.ts
View file

@ -18,7 +18,6 @@ import { RemoteLoggerService } from '@/core/RemoteLoggerService.js';
import { ApDbResolverService } from '@/core/activitypub/ApDbResolverService.js';
import { ApPersonService } from '@/core/activitypub/models/ApPersonService.js';
import { bindThis } from '@/decorators.js';
import { ApResolverService } from './activitypub/ApResolverService.js';
@Injectable()
export class RemoteUserResolveService {
@ -36,7 +35,6 @@ export class RemoteUserResolveService {
private remoteLoggerService: RemoteLoggerService,
private apDbResolverService: ApDbResolverService,
private apPersonService: ApPersonService,
private apResolverService: ApResolverService,
) {
this.logger = this.remoteLoggerService.logger.createSubLogger('resolve-user');
}
@ -93,7 +91,7 @@ export class RemoteUserResolveService {
}
this.logger.succ(`return new remote user: ${chalk.magenta(acctLower)}`);
return await this.apPersonService.createPerson(self.href, this.apResolverService.createResolver());
return await this.apPersonService.createPerson(self.href);
}
// ユーザー情報が古い場合は、WebFingerからやりなおして返す

1
packages/backend/src/core/RoleService.ts
View file

@ -478,7 +478,6 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit {
return ids.length > 0
? await this.usersRepository.findBy({
id: In(ids),
isDeleted: false,
})
: [];
}

2
packages/backend/src/core/SearchService.ts
View file

@ -4,7 +4,7 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { Brackets, In } from 'typeorm';
import { In } from 'typeorm';
import { DI } from '@/di-symbols.js';
import { type Config, FulltextSearchProvider } from '@/config.js';
import { bindThis } from '@/decorators.js';

8
packages/backend/src/core/UserWebhookService.ts
View file

@ -15,14 +15,6 @@ import { QueueService } from '@/core/QueueService.js';
import type { OnApplicationShutdown } from '@nestjs/common';
export type UserWebhookPayload<T extends WebhookEventTypes> =
T extends 'reaction' ? {
reaction: {
id: string,
user: Packed<'UserLite'>,
reaction: string,
}
note: Packed<'Note'>,
} :
T extends 'note' | 'reply' | 'renote' | 'mention' ? {
note: Packed<'Note'>,
} :

8
packages/backend/src/core/VideoProcessingService.ts
View file

@ -7,21 +7,19 @@ import { Inject, Injectable } from '@nestjs/common';
import FFmpeg from 'fluent-ffmpeg';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { __YUME_PRIVATE_ImageProcessingService } from '@/core/ImageProcessingService.js';
import { ImageProcessingService } from '@/core/ImageProcessingService.js';
import type { IImage } from '@/core/ImageProcessingService.js';
import { createTempDir } from '@/misc/create-temp.js';
import { bindThis } from '@/decorators.js';
import { appendQuery, query } from '@/misc/prelude/url.js';
@Injectable()
// Prevent accidental import by upstream merge
// eslint-disable-next-line
export class __YUME_PRIVATE_VideoProcessingService {
export class VideoProcessingService {
constructor(
@Inject(DI.config)
private config: Config,
private imageProcessingService: __YUME_PRIVATE_ImageProcessingService,
private imageProcessingService: ImageProcessingService,
) {
}

18
packages/backend/src/core/WebAuthnService.ts
View file

@ -83,11 +83,7 @@ export class WebAuthnService {
}
@bindThis
public async verifyRegistration(
userId: MiUser['id'],
response: RegistrationResponseJSON,
twoFactorOnly: boolean = false,
): Promise<{
public async verifyRegistration(userId: MiUser['id'], response: RegistrationResponseJSON): Promise<{
credentialID: string;
credentialPublicKey: Uint8Array;
attestationObject: Uint8Array;
@ -115,7 +111,7 @@ export class WebAuthnService {
expectedChallenge: challenge,
expectedOrigin: relyingParty.origin,
expectedRPID: relyingParty.rpId,
requireUserVerification: !twoFactorOnly,
requireUserVerification: true,
});
} catch (error) {
console.error(error);
@ -247,12 +243,8 @@ export class WebAuthnService {
}
@bindThis
public async verifyAuthentication(
userId: MiUser['id'],
response: AuthenticationResponseJSON,
twoFactorOnly: boolean = false,
): Promise<boolean> {
const challenge = await this.redisClient.get(`webauthn:challenge:${userId}`);
public async verifyAuthentication(userId: MiUser['id'], response: AuthenticationResponseJSON): Promise<boolean> {
const challenge = await this.redisClient.getdel(`webauthn:challenge:${userId}`);
if (!challenge) {
throw new IdentifiableError('2d16e51c-007b-4edd-afd2-f7dd02c947f6', 'challenge not found');
@ -306,7 +298,7 @@ export class WebAuthnService {
counter: key.counter,
transports: key.transports ? key.transports as AuthenticatorTransportFuture[] : undefined,
},
requireUserVerification: !twoFactorOnly,
requireUserVerification: true,
});
} catch (error) {
console.error(error);

18
packages/backend/src/core/WebhookTestService.ts
View file

@ -252,20 +252,6 @@ function toPackedUserDetailedNotMe(user: MiUser, override?: Packed<'UserDetailed
};
}
function generateDummyReactionPayload(note_override?: Partial<MiNote>): UserWebhookPayload<'reaction'> {
const dummyNote = generateDummyNote(note_override);
const dummyReaction = {
id: 'dummy-reaction-1',
user: toPackedUserLite(generateDummyUser()),
reaction: 'test_reaction',
};
return {
note: toPackedNote(dummyNote),
reaction: dummyReaction,
};
}
const dummyUser1 = generateDummyUser();
const dummyUser2 = generateDummyUser({
id: 'dummy-user-2',
@ -292,10 +278,6 @@ const dummyUser3 = generateDummyUser({
notesCount: 15900,
});
function wrapBodyEnum<T extends string, U>(tag: T, body: U): { [K in T]: U } {
return { [tag]: body } as { [K in T]: U };
}
@Injectable()
export class WebhookTestService {
public static NoSuchWebhookError = class extends Error {

1
packages/backend/src/core/activitypub/ApDbResolverService.ts
View file

@ -17,7 +17,6 @@ import { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import { getApId } from './type.js';
import { ApPersonService } from './models/ApPersonService.js';
import type { IObject } from './type.js';
import { toASCII } from 'node:punycode';
export type UriParseResult = {
/** wether the URI was generated by us */

159
packages/backend/src/core/activitypub/ApInboxService.ts
View file

@ -5,7 +5,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import * as prom from 'prom-client';
import { DI } from '@/di-symbols.js';
import type { Config } from '@/config.js';
import { UserFollowingService } from '@/core/UserFollowingService.js';
@ -29,7 +28,8 @@ import { bindThis } from '@/decorators.js';
import type { MiRemoteUser } from '@/models/User.js';
import { GlobalEventService } from '@/core/GlobalEventService.js';
import { AbuseReportService } from '@/core/AbuseReportService.js';
import { getApHrefNullable, getApId, getApIds, getApType, isActor, isCollection, isCollectionOrOrderedCollection, isPost, isTombstone, validActor, validPost, yumeDowncastAccept, yumeDowncastAdd, yumeDowncastAnnounce, yumeDowncastBlock, yumeDowncastCreate, yumeDowncastDelete, yumeDowncastFlag, yumeDowncastFollow, yumeDowncastLike, yumeDowncastMove, yumeDowncastReject, yumeDowncastRemove, yumeDowncastUndo, yumeDowncastUpdate } from './type.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { getApHrefNullable, getApId, getApIds, getApType, isAccept, isActor, isAdd, isAnnounce, isBlock, isCollection, isCollectionOrOrderedCollection, isCreate, isDelete, isFlag, isFollow, isLike, isMove, isPost, isReject, isRemove, isTombstone, isUndo, isUpdate, validActor, validPost } from './type.js';
import { ApNoteService } from './models/ApNoteService.js';
import { ApLoggerService } from './ApLoggerService.js';
import { ApDbResolverService } from './ApDbResolverService.js';
@ -39,14 +39,6 @@ import { ApPersonService } from './models/ApPersonService.js';
import { ApQuestionService } from './models/ApQuestionService.js';
import type { Resolver } from './ApResolverService.js';
import type { IAccept, IAdd, IAnnounce, IBlock, ICreate, IDelete, IFlag, IFollow, ILike, IObject, IReject, IRemove, IUndo, IUpdate, IMove, IPost } from './type.js';
import { metricCounter } from '@/server/api/MetricsService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
const mInboxReceived = metricCounter({
name: 'misskey_ap_inbox_received_total',
help: 'Total number of activities received by AP inbox',
labelNames: ['host', 'type'],
});
@Injectable()
export class ApInboxService {
@ -151,92 +143,37 @@ export class ApInboxService {
public async performOneActivity(actor: MiRemoteUser, activity: IObject, resolver?: Resolver): Promise<string | void> {
if (actor.isSuspended) return;
const create = yumeDowncastCreate(activity);
if (create) {
mInboxReceived?.inc({ host: actor.host, type: 'create' });
return await this.create(actor, create);
if (isCreate(activity)) {
return await this.create(actor, activity, resolver);
} else if (isDelete(activity)) {
return await this.delete(actor, activity);
} else if (isUpdate(activity)) {
return await this.update(actor, activity, resolver);
} else if (isFollow(activity)) {
return await this.follow(actor, activity);
} else if (isAccept(activity)) {
return await this.accept(actor, activity, resolver);
} else if (isReject(activity)) {
return await this.reject(actor, activity, resolver);
} else if (isAdd(activity)) {
return await this.add(actor, activity, resolver);
} else if (isRemove(activity)) {
return await this.remove(actor, activity, resolver);
} else if (isAnnounce(activity)) {
return await this.announce(actor, activity, resolver);
} else if (isLike(activity)) {
return await this.like(actor, activity);
} else if (isUndo(activity)) {
return await this.undo(actor, activity, resolver);
} else if (isBlock(activity)) {
return await this.block(actor, activity);
} else if (isFlag(activity)) {
return await this.flag(actor, activity);
} else if (isMove(activity)) {
return await this.move(actor, activity, resolver);
} else {
return `unrecognized activity type: ${activity.type}`;
}
const update = yumeDowncastUpdate(activity);
if (update) {
mInboxReceived?.inc({ host: actor.host, type: 'update' });
return await this.update(actor, update);
}
const del = yumeDowncastDelete(activity);
if (del) {
mInboxReceived?.inc({ host: actor.host, type: 'delete' });
return await this.delete(actor, del);
}
const follow = yumeDowncastFollow(activity);
if (follow) {
mInboxReceived?.inc({ host: actor.host, type: 'follow' });
return await this.follow(actor, follow);
}
const accept = yumeDowncastAccept(activity);
if (accept) {
mInboxReceived?.inc({ host: actor.host, type: 'accept' });
return await this.accept(actor, accept);
}
const reject = yumeDowncastReject(activity);
if (reject) {
mInboxReceived?.inc({ host: actor.host, type: 'reject' });
return await this.reject(actor, reject);
}
const add = yumeDowncastAdd(activity);
if (add) {
mInboxReceived?.inc({ host: actor.host, type: 'add' });
return await this.add(actor, add);
}
const remove = yumeDowncastRemove(activity);
if (remove) {
mInboxReceived?.inc({ host: actor.host, type: 'remove' });
return await this.remove(actor, remove);
}
const announce = yumeDowncastAnnounce(activity);
if (announce) {
mInboxReceived?.inc({ host: actor.host, type: 'announce' });
return await this.announce(actor, announce);
}
const like = yumeDowncastLike(activity);
if (like) {
mInboxReceived?.inc({ host: actor.host, type: 'like' });
return await this.like(actor, like);
}
const move = yumeDowncastMove(activity);
if (move) {
mInboxReceived?.inc({ host: actor.host, type: 'move' });
return await this.move(actor, move);
}
const undo = yumeDowncastUndo(activity);
if (undo) {
mInboxReceived?.inc({ host: actor.host, type: 'undo' });
return await this.undo(actor, undo);
}
const block = yumeDowncastBlock(activity);
if (block) {
mInboxReceived?.inc({ host: actor.host, type: 'block' });
return await this.block(actor, block);
}
const flag = yumeDowncastFlag(activity);
if (flag) {
mInboxReceived?.inc({ host: actor.host, type: 'flag' });
return await this.flag(actor, flag);
}
mInboxReceived?.inc({ host: actor.host, type: 'unknown' });
return `unrecognized activity type: ${activity.type}`;
}
@bindThis
@ -291,8 +228,7 @@ export class ApInboxService {
throw err;
});
const follow = yumeDowncastFollow(object);
if (follow) return await this.acceptFollow(actor, follow);
if (isFollow(object)) return await this.acceptFollow(actor, object);
return `skip: Unknown Accept type: ${getApType(object)}`;
}
@ -554,9 +490,9 @@ export class ApInboxService {
formerType = 'Note';
}
if (validPost?.includes(formerType)) {
if (validPost.includes(formerType)) {
return await this.deleteNote(actor, uri);
} else if (validActor?.includes(formerType)) {
} else if (validActor.includes(formerType)) {
return await this.deleteActor(actor, uri);
} else {
return `Unknown type ${formerType}`;
@ -575,10 +511,11 @@ export class ApInboxService {
return 'skip: already deleted or actor not found';
}
const job = await this.queueService.createDeleteAccountJob(actor);
this.globalEventService.publishInternalEvent('remoteUserUpdated', { id: actor.id });
return 'ok';
return `ok: queued ${job.name} ${job.id}`;
}
@bindThis
@ -645,8 +582,7 @@ export class ApInboxService {
throw e;
});
const follow = yumeDowncastFollow(object);
if (follow) return await this.rejectFollow(actor, follow);
if (isFollow(object)) return await this.rejectFollow(actor, object);
return `skip: Unknown Reject type: ${getApType(object)}`;
}
@ -714,20 +650,11 @@ export class ApInboxService {
});
// don't queue because the sender may attempt again when timeout
const follow = yumeDowncastFollow(object);
if (follow) return await this.undoFollow(actor, follow);
const block = yumeDowncastBlock(object);
if (block) return await this.undoBlock(actor, block);
const like = yumeDowncastLike(object);
if (like) return await this.undoLike(actor, like);
const announce = yumeDowncastAnnounce(object);
if (announce) return await this.undoAnnounce(actor, announce);
const accept = yumeDowncastAccept(object);
if (accept) return await this.undoAccept(actor, accept);
if (isFollow(object)) return await this.undoFollow(actor, object);
if (isBlock(object)) return await this.undoBlock(actor, object);
if (isLike(object)) return await this.undoLike(actor, object);
if (isAnnounce(object)) return await this.undoAnnounce(actor, object);
if (isAccept(object)) return await this.undoAccept(actor, object);
return `skip: unknown object type ${getApType(object)}`;
}

106
packages/backend/src/core/activitypub/ApRendererService.ts
View file

@ -31,7 +31,7 @@ import { UtilityService } from '@/core/UtilityService.js';
import { JsonLdService } from './JsonLdService.js';
import { ApMfmService } from './ApMfmService.js';
import { CONTEXT } from './misc/contexts.js';
import { markOutgoing, type IAccept, type IActivity, type IAdd, type IAnnounce, type IApDocument, type IApEmoji, type IApHashtag, type IApImage, type IApMention, type IBlock, type ICreate, type IDelete, type IFlag, type IFollow, type IKey, type ILike, type IMove, type IObject, type IPost, type IQuestion, type IReject, type IRemove, type ITombstone, type IUndo, type IUpdate } from './type.js';
import type { IAccept, IActivity, IAdd, IAnnounce, IApDocument, IApEmoji, IApHashtag, IApImage, IApMention, IBlock, ICreate, IDelete, IFlag, IFollow, IKey, ILike, IMove, IObject, IPost, IQuestion, IReject, IRemove, ITombstone, IUndo, IUpdate } from './type.js';
@Injectable()
export class ApRendererService {
@ -71,21 +71,21 @@ export class ApRendererService {
@bindThis
public renderAccept(object: string | IObject, user: { id: MiUser['id']; host: null }): IAccept {
return markOutgoing({
return {
type: 'Accept',
actor: this.userEntityService.genLocalUserUri(user.id),
object,
}, undefined);
};
}
@bindThis
public renderAdd(user: MiLocalUser, target: string | IObject | undefined, object: string | IObject): IAdd {
return markOutgoing({
return {
type: 'Add',
actor: this.userEntityService.genLocalUserUri(user.id),
target,
object,
}, undefined);
};
}
@bindThis
@ -108,7 +108,7 @@ export class ApRendererService {
throw new Error('renderAnnounce: cannot render non-public note');
}
return markOutgoing({
return {
id: `${this.config.url}/notes/${note.id}/activity`,
actor: this.userEntityService.genLocalUserUri(note.userId),
type: 'Announce',
@ -116,7 +116,7 @@ export class ApRendererService {
to,
cc,
object,
}, undefined);
};
}
/**
@ -130,23 +130,23 @@ export class ApRendererService {
throw new Error('renderBlock: missing blockee uri');
}
return markOutgoing({
return {
type: 'Block',
id: `${this.config.url}/blocks/${block.id}`,
actor: this.userEntityService.genLocalUserUri(block.blockerId),
object: block.blockee.uri,
}, undefined);
};
}
@bindThis
public renderCreate(object: IObject, note: MiNote): ICreate {
const activity: ICreate = markOutgoing({
const activity: ICreate = {
id: `${this.config.url}/notes/${note.id}/activity`,
actor: this.userEntityService.genLocalUserUri(note.userId),
type: 'Create',
published: this.idService.parse(note.id).date.toISOString(),
object,
}, undefined);
};
if (object.to) activity.to = object.to;
if (object.cc) activity.cc = object.cc;
@ -156,28 +156,28 @@ export class ApRendererService {
@bindThis
public renderDelete(object: IObject | string, user: { id: MiUser['id']; host: null }): IDelete {
return markOutgoing({
return {
type: 'Delete',
actor: this.userEntityService.genLocalUserUri(user.id),
object,
published: new Date().toISOString(),
}, undefined);
};
}
@bindThis
public renderDocument(file: MiDriveFile): IApDocument {
return markOutgoing({
return {
type: 'Document',
mediaType: file.webpublicType ?? file.type,
url: this.driveFileEntityService.getPublicUrl(file),
name: file.comment,
sensitive: file.isSensitive,
}, undefined);
};
}
@bindThis
public renderEmoji(emoji: MiEmoji): IApEmoji {
return markOutgoing( {
return {
id: `${this.config.url}/emojis/${emoji.name}`,
type: 'Emoji',
name: `:${emoji.name}:`,
@ -191,28 +191,28 @@ export class ApRendererService {
_misskey_license: {
freeText: emoji.license,
},
}, undefined);
};
}
// to anonymise reporters, the reporting actor must be a system user
@bindThis
public renderFlag(user: MiLocalUser, object: IObject | string, content: string): IFlag {
return markOutgoing({
return {
type: 'Flag',
actor: this.userEntityService.genLocalUserUri(user.id),
content,
object,
}, undefined);
};
}
@bindThis
public renderFollowRelay(relay: MiRelay, relayActor: MiLocalUser): IFollow {
return markOutgoing({
return {
id: `${this.config.url}/activities/follow-relay/${relay.id}`,
type: 'Follow',
actor: this.userEntityService.genLocalUserUri(relayActor.id),
object: 'https://www.w3.org/ns/activitystreams#Public',
}, undefined);
};
}
/**
@ -231,31 +231,31 @@ export class ApRendererService {
followee: MiPartialLocalUser | MiPartialRemoteUser,
requestId?: string,
): IFollow {
return markOutgoing({
return {
id: requestId ?? `${this.config.url}/follows/${follower.id}/${followee.id}`,
type: 'Follow',
actor: this.userEntityService.getUserUri(follower),
object: this.userEntityService.getUserUri(followee),
}, undefined);
};
}
@bindThis
public renderHashtag(tag: string): IApHashtag {
return markOutgoing({
return {
type: 'Hashtag',
href: `${this.config.url}/tags/${encodeURIComponent(tag)}`,
name: `#${tag}`,
}, undefined);
};
}
@bindThis
public renderImage(file: MiDriveFile): IApImage {
return markOutgoing({
return {
type: 'Image',
url: this.driveFileEntityService.getPublicUrl(file),
sensitive: file.isSensitive,
name: file.comment,
}, undefined);
};
}
@bindThis
@ -292,7 +292,7 @@ export class ApRendererService {
@bindThis
public renderKey(user: MiLocalUser, key: MiUserKeypair, postfix?: string): IKey {
return markOutgoing({
return {
id: `${this.config.url}/users/${user.id}${postfix ?? '/publickey'}`,
type: 'Key',
owner: this.userEntityService.genLocalUserUri(user.id),
@ -300,21 +300,21 @@ export class ApRendererService {
type: 'spki',
format: 'pem',
}),
}, undefined);
};
}
@bindThis
public async renderLike(noteReaction: MiNoteReaction, note: { uri: string | null }): Promise<ILike> {
const reaction = noteReaction.reaction;
const object: ILike = markOutgoing({
const object: ILike = {
type: 'Like',
id: `${this.config.url}/likes/${noteReaction.id}`,
actor: `${this.config.url}/users/${noteReaction.userId}`,
object: note.uri ? note.uri : `${this.config.url}/notes/${noteReaction.noteId}`,
content: reaction,
_misskey_reaction: reaction,
}, undefined);
};
if (reaction.startsWith(':')) {
const name = reaction.replaceAll(':', '');
@ -328,11 +328,11 @@ export class ApRendererService {
@bindThis
public renderMention(mention: MiPartialLocalUser | MiPartialRemoteUser): IApMention {
return markOutgoing({
return {
type: 'Mention',
href: this.userEntityService.getUserUri(mention),
name: this.userEntityService.isRemoteUser(mention) ? `@${mention.username}@${mention.host}` : `@${(mention as MiLocalUser).username}`,
}, undefined);
};
}
@bindThis
@ -342,13 +342,13 @@ export class ApRendererService {
): IMove {
const actor = this.userEntityService.getUserUri(src);
const target = this.userEntityService.getUserUri(dst);
return markOutgoing({
return {
id: `${this.config.url}/moves/${src.id}/${dst.id}`,
actor,
type: 'Move',
object: actor,
target,
}, undefined);
};
}
@bindThis
@ -462,7 +462,7 @@ export class ApRendererService {
})),
} as const : {};
return markOutgoing({
return {
id: `${this.config.url}/notes/${note.id}`,
type: 'Note',
attributedTo,
@ -485,7 +485,7 @@ export class ApRendererService {
sensitive: note.cw != null || files.some(file => file.isSensitive),
tag,
...asPoll,
}, undefined);
};
}
@bindThis
@ -569,7 +569,7 @@ export class ApRendererService {
@bindThis
public renderQuestion(user: { id: MiUser['id'] }, note: MiNote, poll: MiPoll): IQuestion {
return markOutgoing({
return {
type: 'Question',
id: `${this.config.url}/questions/${note.id}`,
actor: this.userEntityService.genLocalUserUri(user.id),
@ -582,78 +582,78 @@ export class ApRendererService {
totalItems: poll.votes[i],
},
})),
}, 'question');
};
}
@bindThis
public renderReject(object: string | IObject, user: { id: MiUser['id'] }): IReject {
return markOutgoing({
return {
type: 'Reject',
actor: this.userEntityService.genLocalUserUri(user.id),
object,
}, undefined);
};
}
@bindThis
public renderRemove(user: { id: MiUser['id'] }, target: string | IObject | undefined, object: string | IObject): IRemove {
return markOutgoing({
return {
type: 'Remove',
actor: this.userEntityService.genLocalUserUri(user.id),
target,
object,
}, undefined);
};
}
@bindThis
public renderTombstone(id: string): ITombstone {
return markOutgoing({
return {
id,
type: 'Tombstone',
}, undefined);
};
}
@bindThis
public renderUndo(object: string | IObject, user: { id: MiUser['id'] }): IUndo {
const id = typeof object !== 'string' && typeof object.id === 'string' && this.utilityService.isUriLocal(object.id) ? `${object.id}/undo` : undefined;
return markOutgoing({
return {
type: 'Undo',
...(id ? { id } : {}),
actor: this.userEntityService.genLocalUserUri(user.id),
object,
published: new Date().toISOString(),
}, undefined);
};
}
@bindThis
public renderUpdate(object: string | IObject, user: { id: MiUser['id'] }): IUpdate {
return markOutgoing( {
return {
id: `${this.config.url}/users/${user.id}#updates/${new Date().getTime()}`,
actor: this.userEntityService.genLocalUserUri(user.id),
type: 'Update',
to: ['https://www.w3.org/ns/activitystreams#Public'],
object,
published: new Date().toISOString(),
}, undefined);
};
}
@bindThis
public renderVote(user: { id: MiUser['id'] }, vote: MiPollVote, note: MiNote, poll: MiPoll, pollOwner: MiRemoteUser): ICreate {
return markOutgoing({
return {
id: `${this.config.url}/users/${user.id}#votes/${vote.id}/activity`,
actor: this.userEntityService.genLocalUserUri(user.id),
type: 'Create',
to: [pollOwner.uri],
published: new Date().toISOString(),
object: markOutgoing({
object: {
id: `${this.config.url}/users/${user.id}#votes/${vote.id}`,
type: 'Note',
attributedTo: this.userEntityService.genLocalUserUri(user.id),
to: [pollOwner.uri],
inReplyTo: note.uri,
name: poll.choices[vote.choice],
}, undefined),
}, undefined);
},
};
}
@bindThis

19
packages/backend/src/core/activitypub/ApRequestService.ts
View file

@ -19,7 +19,6 @@ import type Logger from '@/logger.js';
import { validateContentTypeSetAsActivityPub } from '@/core/activitypub/misc/validator.js';
import { assertActivityMatchesUrls, FetchAllowSoftFailMask as FetchAllowSoftFailMask } from '@/core/activitypub/misc/check-against-url.js';
import type { IObject } from './type.js';
import { ActorKeySignerService } from '../ActorKeySignerService.js';
type Request = {
url: string;
@ -40,8 +39,7 @@ type PrivateKey = {
};
export class ApRequestCreator {
static async createSignedPost(signer: ActorKeySignerService, args: { key: PrivateKey, url: string, body: string, digest?: string, additionalHeaders: Record<string, string> }): Promise<Signed> {
static createSignedPost(args: { key: PrivateKey, url: string, body: string, digest?: string, additionalHeaders: Record<string, string> }): Signed {
const u = new URL(args.url);
const digestHeader = args.digest ?? this.createDigest(args.body);
@ -56,7 +54,7 @@ export class ApRequestCreator {
}, args.additionalHeaders),
};
const result = await this.#signToRequest(signer, request, args.key, ['(request-target)', 'date', 'host', 'digest']);
const result = this.#signToRequest(request, args.key, ['(request-target)', 'date', 'host', 'digest']);
return {
request,
@ -70,7 +68,7 @@ export class ApRequestCreator {
return `SHA-256=${crypto.createHash('sha256').update(body).digest('base64')}`;
}
static async createSignedGet(signer: ActorKeySignerService, args: { key: PrivateKey, url: string, additionalHeaders: Record<string, string> }): Promise<Signed> {
static createSignedGet(args: { key: PrivateKey, url: string, additionalHeaders: Record<string, string> }): Signed {
const u = new URL(args.url);
const request: Request = {
@ -83,7 +81,7 @@ export class ApRequestCreator {
}, args.additionalHeaders),
};
const result = await this.#signToRequest(signer, request, args.key, ['(request-target)', 'date', 'host', 'accept']);
const result = this.#signToRequest(request, args.key, ['(request-target)', 'date', 'host', 'accept']);
return {
request,
@ -93,9 +91,9 @@ export class ApRequestCreator {
};
}
static async #signToRequest(signer: ActorKeySignerService, request: Request, key: PrivateKey, includeHeaders: string[]): Promise<Signed> {
static #signToRequest(request: Request, key: PrivateKey, includeHeaders: string[]): Signed {
const signingString = this.#genSigningString(request, includeHeaders);
const signature = await signer.sign(key.keyId, key.privateKeyPem, signingString);
const signature = crypto.sign('sha256', Buffer.from(signingString), key.privateKeyPem).toString('base64');
const signatureHeader = `keyId="${key.keyId}",algorithm="rsa-sha256",headers="${includeHeaders.join(' ')}",signature="${signature}"`;
request.headers = this.#objectAssignWithLcKey(request.headers, {
@ -151,7 +149,6 @@ export class ApRequestService {
private httpRequestService: HttpRequestService,
private loggerService: LoggerService,
private utilityService: UtilityService,
private actorKeySignerService: ActorKeySignerService,
) {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
this.logger = this.loggerService?.getLogger('ap-request'); // なぜか TypeError: Cannot read properties of undefined (reading 'getLogger') と言われる
@ -163,7 +160,7 @@ export class ApRequestService {
const keypair = await this.userKeypairService.getUserKeypair(user.id);
const req = await ApRequestCreator.createSignedPost(this.actorKeySignerService, {
const req = ApRequestCreator.createSignedPost({
key: {
privateKeyPem: keypair.privateKey,
keyId: `${this.config.url}/users/${user.id}#main-key`,
@ -192,7 +189,7 @@ export class ApRequestService {
const _followAlternate = followAlternate ?? true;
const keypair = await this.userKeypairService.getUserKeypair(user.id);
const req = await ApRequestCreator.createSignedGet(this.actorKeySignerService, {
const req = ApRequestCreator.createSignedGet({
key: {
privateKeyPem: keypair.privateKey,
keyId: `${this.config.url}/users/${user.id}#main-key`,

30
packages/backend/src/core/activitypub/ApResolverService.ts
View file

@ -5,7 +5,6 @@
import { Inject, Injectable } from '@nestjs/common';
import { IsNull, Not } from 'typeorm';
import * as Bull from 'bullmq';
import type { MiLocalUser, MiRemoteUser } from '@/models/User.js';
import type { NotesRepository, PollsRepository, NoteReactionsRepository, UsersRepository, FollowRequestsRepository, MiMeta } from '@/models/_.js';
import type { Config } from '@/config.js';
@ -17,13 +16,12 @@ import { LoggerService } from '@/core/LoggerService.js';
import type Logger from '@/logger.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import { IdentifiableError } from '@/misc/identifiable-error.js';
import { isCollectionOrOrderedCollection, yumeNormalizeObject } from './type.js';
import { isCollectionOrOrderedCollection } from './type.js';
import { ApDbResolverService } from './ApDbResolverService.js';
import { ApRendererService } from './ApRendererService.js';
import { ApRequestService } from './ApRequestService.js';
import { FetchAllowSoftFailMask } from './misc/check-against-url.js';
import type { IObject, ICollection, IOrderedCollection, IUnsanitizedObject } from './type.js';
import { yumeAssertAcceptableURL } from './misc/validator.js';
import type { IObject, ICollection, IOrderedCollection } from './type.js';
export class Resolver {
private history: Set<string>;
@ -65,7 +63,7 @@ export class Resolver {
public async resolveCollection(value: string | IObject): Promise<ICollection | IOrderedCollection> {
const collection = typeof value === 'string'
? await this.resolve(value)
: yumeNormalizeObject(value);
: value;
if (isCollectionOrOrderedCollection(collection)) {
return collection;
@ -75,13 +73,11 @@ export class Resolver {
}
@bindThis
private async resolveNotNormalized(value: string | IObject, allowSoftfail: FetchAllowSoftFailMask = FetchAllowSoftFailMask.Strict): Promise<IUnsanitizedObject> {
public async resolve(value: string | IObject, allowSoftfail: FetchAllowSoftFailMask = FetchAllowSoftFailMask.Strict): Promise<IObject> {
if (typeof value !== 'string') {
return value;
}
yumeAssertAcceptableURL(value);
if (value.includes('#')) {
// URLs with fragment parts cannot be resolved correctly because
// the fragment part does not get transmitted over HTTP(S).
@ -94,7 +90,7 @@ export class Resolver {
}
if (this.history.size > this.recursionLimit) {
throw new Bull.UnrecoverableError(`hit recursion limit: ${this.utilityService.extractDbHost(value)}`);
throw new IdentifiableError('d592da9f-822f-4d91-83d7-4ceefabcf3d2', `hit recursion limit: ${this.utilityService.extractDbHost(value)}`);
}
this.history.add(value);
@ -113,8 +109,8 @@ export class Resolver {
}
const object = (this.user
? await this.apRequestService.signedGet(value, this.user, allowSoftfail) as IUnsanitizedObject
: await this.httpRequestService.getActivityJson(value, undefined, allowSoftfail)) as IUnsanitizedObject;
? await this.apRequestService.signedGet(value, this.user, allowSoftfail) as IObject
: await this.httpRequestService.getActivityJson(value, undefined, allowSoftfail)) as IObject;
if (
Array.isArray(object['@context']) ?
@ -127,18 +123,6 @@ export class Resolver {
return object;
}
@bindThis
public async resolve(value: string | IObject, allowSoftfail: FetchAllowSoftFailMask = FetchAllowSoftFailMask.Strict): Promise<IObject> {
try {
const object = await this.resolveNotNormalized(value, allowSoftfail);
return yumeNormalizeObject(object);
} catch (e) {
this.logger.error(`Failed to resolve ${value}: ${e}`);
throw e;
}
}
@bindThis
private resolveLocal(url: string): Promise<IObject> {
const parsed = this.apDbResolverService.parseUri(url);

13
packages/backend/src/core/activitypub/JsonLdService.ts
View file

@ -11,7 +11,6 @@ import { CONTEXT, PRELOADED_CONTEXTS } from './misc/contexts.js';
import { validateContentTypeSetAsJsonLD } from './misc/validator.js';
import type { JsonLdDocument } from 'jsonld';
import type { JsonLd as JsonLdObject, RemoteDocument } from 'jsonld/jsonld-spec.js';
import { ActorKeySignerService } from '../ActorKeySignerService.js';
// RsaSignature2017 implementation is based on https://github.com/transmute-industries/RsaSignature2017
@ -22,7 +21,6 @@ class JsonLd {
constructor(
private httpRequestService: HttpRequestService,
private actorKeySignerService: ActorKeySignerService,
) {
}
@ -47,13 +45,17 @@ class JsonLd {
const toBeSigned = await this.createVerifyData(data, options);
const signature = await this.actorKeySignerService.sign(creator, privateKey, toBeSigned);
const signer = crypto.createSign('sha256');
signer.update(toBeSigned);
signer.end();
const signature = signer.sign(privateKey);
return {
...data,
signature: {
...options,
signatureValue: signature,
signatureValue: signature.toString('base64'),
},
};
}
@ -167,12 +169,11 @@ class JsonLd {
export class JsonLdService {
constructor(
private httpRequestService: HttpRequestService,
private actorKeySignerService: ActorKeySignerService,
) {
}
@bindThis
public use(): JsonLd {
return new JsonLd(this.httpRequestService, this.actorKeySignerService);
return new JsonLd(this.httpRequestService);
}
}

37
packages/backend/src/core/activitypub/misc/check-against-url.ts
View file

@ -1,16 +1,9 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project
* SPDX-FileCopyrightText: dakkar and sharkey-project
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { metricCounter } from '@/server/api/MetricsService.js';
import type { IObject } from '../type.js';
const mApUrlValidationFailed = metricCounter({
name: 'misskey_ap_url_validation_failed',
help: 'AP url validation failed',
labelNames: ['reason', 'id_host', 'request_host', 'candidate_host'],
});
export enum FetchAllowSoftFailMask {
// Allow no softfail flags
Strict = 0,
@ -91,14 +84,8 @@ export function assertActivityMatchesUrls(requestUrl: string | URL, activity: IO
let softfail = 0;
// if the flag is allowed, set the flag on return otherwise throw
const requireSoftfail = (needed: FetchAllowSoftFailMask, reason: string, message: string) => {
const requireSoftfail = (needed: FetchAllowSoftFailMask, message: string) => {
if ((allowSoftfail & needed) !== needed) {
mApUrlValidationFailed?.inc({
reason,
id_host: idParsed.host,
request_host: requestUrlParsed.host,
candidate_host: candidateUrlsParsed.map(it => it.host)[0],
});
throw new Error(message);
}
@ -113,41 +100,27 @@ export function assertActivityMatchesUrls(requestUrl: string | URL, activity: IO
const requestUrlSecure = requestUrlParsed.protocol === 'https:';
const finalUrlSecure = candidateUrlsParsed.every(it => it.protocol === 'https:');
if (requestUrlSecure && !finalUrlSecure) {
mApUrlValidationFailed?.inc({
reason: 'downgrade',
id_host: idParsed.host,
request_host: requestUrlParsed.host,
candidate_host: candidateUrlsParsed.map(it => it.host)[0],
});
throw new Error(`bad Activity: id(${activity.id}) is not allowed to have http:// in the url`);
}
// Compare final URL to the ID
if (!candidateUrlsParsed.some(it => it.href === idParsed.href)) {
requireSoftfail(FetchAllowSoftFailMask.NonCanonicalId, 'non_canonical_id', `bad Activity: id(${activity.id}) does not match response url(${candidateUrlsParsed.map(it => it.toString())})`);
requireSoftfail(FetchAllowSoftFailMask.NonCanonicalId, `bad Activity: id(${activity.id}) does not match response url(${candidateUrlsParsed.map(it => it.toString())})`);
// at lease host need to match exactly (ActivityPub requirement)
if (!candidateUrlsParsed.some(it => idParsed.host === it.host)) {
mApUrlValidationFailed?.inc({
reason: 'not_same_host',
id_host: idParsed.host,
request_host: requestUrlParsed.host,
candidate_host: candidateUrlsParsed.map(it => it.host)[0],
});
throw new Error(`bad Activity: id(${activity.id}) does not match response host(${candidateUrlsParsed.map(it => it.host)})`);
}
}
// Compare request URL to the ID
if (!requestUrlParsed.href.includes(idParsed.href)) {
requireSoftfail(FetchAllowSoftFailMask.NonCanonicalId, 'non_canonical_id', `bad Activity: id(${activity.id}) does not match request url(${requestUrlParsed.toString()})`);
requireSoftfail(FetchAllowSoftFailMask.NonCanonicalId, `bad Activity: id(${activity.id}) does not match request url(${requestUrlParsed.toString()})`);
// if cross-origin lookup is allowed, we can accept some variation between the original request URL to the final object ID (but not between the final URL and the object ID)
const hostResult = hostFuzzyMatch(requestUrlParsed.host, idParsed.host);
requireSoftfail(hostResult, 'not_same_origin', `bad Activity: id(${activity.id}) is valid but is not the same origin as request url(${requestUrlParsed.toString()})`);
requireSoftfail(hostResult, `bad Activity: id(${activity.id}) is valid but is not the same origin as request url(${requestUrlParsed.toString()})`);
}
return softfail;

22
packages/backend/src/core/activitypub/misc/validator.ts
View file

@ -4,28 +4,6 @@
*/
import type { Response } from 'node-fetch';
import * as Bull from 'bullmq';
import { toASCII } from 'node:punycode';
export function yumeAssertAcceptableURL(url: string | URL): URL {
const urlParsed = url instanceof URL ? url : new URL(url);
if (urlParsed.search.length + urlParsed.pathname.length > 1024) {
throw new Bull.UnrecoverableError('URL is too long');
}
if (urlParsed.protocol !== 'https:') {
throw new Bull.UnrecoverableError('URL protocol is not https');
}
if (urlParsed.port && urlParsed.port !== '443') {
throw new Bull.UnrecoverableError('URL port is not 443');
}
urlParsed.hostname = toASCII(urlParsed.hostname);
return urlParsed;
}
export function validateContentTypeSetAsActivityPub(response: Response): void {
const contentType = (response.headers.get('content-type') ?? '').toLowerCase();

15
packages/backend/src/core/activitypub/models/ApNoteService.ts
View file

@ -3,7 +3,6 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import * as Bull from 'bullmq';
import { forwardRef, Inject, Injectable } from '@nestjs/common';
import { In } from 'typeorm';
import { DI } from '@/di-symbols.js';
@ -37,7 +36,6 @@ import { ApQuestionService } from './ApQuestionService.js';
import { ApImageService } from './ApImageService.js';
import type { Resolver } from '../ApResolverService.js';
import type { IObject, IPost } from '../type.js';
import { yumeAssertAcceptableURL } from '../misc/validator.js';
@Injectable()
export class ApNoteService {
@ -156,17 +154,8 @@ export class ApNoteService {
const url = getOneApHrefNullable(note.url);
if (url != null) {
if (!checkHttps(url)) {
throw new Error('unexpected schema of note url: ' + url);
}
const actUrl = yumeAssertAcceptableURL(url);
const noteUrl = yumeAssertAcceptableURL(note.id);
if (noteUrl.host !== actUrl.host) {
throw new Bull.UnrecoverableError(`note url & uri host mismatch: note url: ${url}, note uri: ${note.id}`);
}
if (url && !checkHttps(url)) {
throw new Error('unexpected schema of note url: ' + url);
}
this.logger.info(`Creating the Note: ${note.id}`);

5
packages/backend/src/core/activitypub/models/ApPersonService.ts
View file

@ -304,13 +304,14 @@ export class ApPersonService implements OnModuleInit {
public async createPerson(uri: string, resolver?: Resolver): Promise<MiRemoteUser> {
if (typeof uri !== 'string') throw new Error('uri is not string');
if (resolver == null) resolver = this.apResolverService.createResolver();
const host = this.utilityService.punyHost(uri);
if (host === this.utilityService.toPuny(this.config.host)) {
throw new StatusError('cannot resolve local user', 400, 'cannot resolve local user');
}
// eslint-disable-next-line no-param-reassign
if (resolver == null) resolver = this.apResolverService.createResolver();
const object = await resolver.resolve(uri);
if (object.id == null) throw new Error('invalid object.id: ' + object.id);

11
packages/backend/src/core/activitypub/models/ApQuestionService.ts
View file

@ -17,7 +17,6 @@ import { ApLoggerService } from '../ApLoggerService.js';
import { ApResolverService } from '../ApResolverService.js';
import type { Resolver } from '../ApResolverService.js';
import type { IObject } from '../type.js';
import { yumeAssertAcceptableURL } from '../misc/validator.js';
@Injectable()
export class ApQuestionService {
@ -73,16 +72,14 @@ export class ApQuestionService {
*/
@bindThis
public async updateQuestion(value: string | IObject, actor?: MiRemoteUser, resolver?: Resolver): Promise<boolean> {
const uriIn = typeof value === 'string' ? value : value.id;
if (uriIn == null) throw new Error('uri is null');
const uri = yumeAssertAcceptableURL(uriIn);
const uri = typeof value === 'string' ? value : value.id;
if (uri == null) throw new Error('uri is null');
// URIがこのサーバーを指しているならスキップ
if (this.utilityService.isUriLocal(uri.href)) throw new Error('uri points local');
if (this.utilityService.isUriLocal(uri)) throw new Error('uri points local');
//#region このサーバーに既に登録されているか
const note = await this.notesRepository.findOneBy({ uri: uri.href });
const note = await this.notesRepository.findOneBy({ uri });
if (note == null) throw new Error('Question is not registered');
const poll = await this.pollsRepository.findOneBy({ noteId: note.id });

452
packages/backend/src/core/activitypub/type.ts
View file

@ -3,52 +3,27 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { target } from "happy-dom/lib/PropertySymbol.js";
import { toASCII } from "node:punycode";
import * as bull from "bullmq";
export type Obj = { [x: string]: any };
export type ApObject = IObject | string | (IObject | string)[];
export interface MisskeyVendorKeys {
_misskey_summary: string;
_misskey_followedMessage: string | null;
_misskey_requireSigninToViewContents: boolean;
_misskey_makeNotesFollowersOnlyBefore: number | null;
_misskey_makeNotesHiddenBefore: number | null;
_misskey_quote: string;
_misskey_content: string;
_misskey_reaction: string;
_misskey_votes: number;
}
function extractMisskeyVendorKeys(object: IObject): Partial<MisskeyVendorKeys> {
return {
_misskey_summary: object._misskey_summary,
_misskey_followedMessage: object._misskey_followedMessage,
_misskey_requireSigninToViewContents: object._misskey_requireSigninToViewContents,
_misskey_makeNotesFollowersOnlyBefore: object._misskey_makeNotesFollowersOnlyBefore,
_misskey_makeNotesHiddenBefore: object._misskey_makeNotesHiddenBefore,
_misskey_quote: object._misskey_quote,
_misskey_content: object._misskey_content,
_misskey_reaction: object._misskey_reaction,
_misskey_votes: object._misskey_votes,
};
}
export interface IUnsanitizedObject extends Partial<MisskeyVendorKeys> {
export interface IObject {
'@context'?: string | string[] | Obj | Obj[];
type: string | string[];
id?: string;
name?: string | null;
summary?: string;
_misskey_summary?: string;
_misskey_followedMessage?: string | null;
_misskey_requireSigninToViewContents?: boolean;
_misskey_makeNotesFollowersOnlyBefore?: number | null;
_misskey_makeNotesHiddenBefore?: number | null;
published?: string;
cc?: ApObject;
to?: ApObject;
attributedTo?: ApObject;
attachment?: any[];
inReplyTo?: any;
replies?: ICollection | string;
replies?: ICollection;
content?: string | null;
startTime?: Date;
endTime?: Date;
@ -59,84 +34,6 @@ export interface IUnsanitizedObject extends Partial<MisskeyVendorKeys> {
href?: string;
tag?: IObject | IObject[];
sensitive?: boolean;
visibility?: string;
mentionedUsers?: any[];
visibleUsers?: any[];
}
export interface IObject extends IUnsanitizedObject {
__yume_normalized_object: true | 'outgoing';
};
export interface YumeDowncastSanitizedBadge<L extends 'question' | undefined> {
__yume_normalized_badge: L | 'outgoing';
};
export function markOutgoing<T, L extends 'question' | undefined>(object: T, _badge: L): T & IObject & YumeDowncastSanitizedBadge<L> {
return object as T & IObject & YumeDowncastSanitizedBadge<L>;
}
export function yumeNormalizeURL(url: string): string {
const u = new URL(url);
u.host = toASCII(u.host);
if (u.protocol && u.protocol !== 'https:') {
throw new bull.UnrecoverableError('protocol is not https');
}
u.protocol = 'https:';
if (u.port && u.port !== '443') {
throw new bull.UnrecoverableError('port is not 443');
}
return u.toString();
}
export function yumeNormalizeRecursive<O extends IUnsanitizedObject | string | (IUnsanitizedObject | string)[]>(object: O, depth = 0):
IObject | string | (IObject | string)[] {
if (depth > 16) {
throw new bull.UnrecoverableError('recursion limit exceeded');
}
if (typeof object === 'string') {
return yumeNormalizeURL(object);
}
if (Array.isArray(object)) {
if (object.length > 64) {
throw new bull.UnrecoverableError('array length limit exceeded');
}
return object.flatMap((x) => yumeNormalizeRecursive(x, depth + (object.length + 3 / 4)));
}
return yumeNormalizeObject(object, depth + 1);
}
export function yumeNormalizeObject(object: IUnsanitizedObject, depth = 0): IObject {
if (object.cc) {
object.cc = yumeNormalizeRecursive(object.cc, depth + 1);
}
if (object.id) {
object.id = yumeNormalizeURL(object.id);
}
if (object.url) {
object.url = yumeNormalizeRecursive(object.url, depth + 1);
}
if (object.replies) {
if (typeof object.replies === 'string') {
object.replies = yumeNormalizeURL(object.replies);
} else {
object.replies.first = object.replies.first ?
typeof object.replies.first === 'string' ? yumeNormalizeURL(object.replies.first) : yumeNormalizeObject(object.replies.first, depth + 1) : undefined;
object.replies.items = object.replies.items ?
typeof object.replies.items === 'string' ? yumeNormalizeURL(object.replies.items) : yumeNormalizeRecursive(object.replies.items, depth + 1) : undefined;
}
}
if (object.inReplyTo) {
object.inReplyTo = yumeNormalizeRecursive(object.inReplyTo, depth + 1);
}
return object as IObject;
}
/**
@ -183,7 +80,7 @@ export function getOneApHrefNullable(value: ApObject | undefined): string | unde
}
export function getApHrefNullable(value: string | IObject | undefined): string | undefined {
if (typeof value === 'string') return value;
if (typeof value === 'string') return value;
if (typeof value?.href === 'string') return value.href;
return undefined;
}
@ -204,28 +101,6 @@ export interface IActivity extends IObject {
};
}
export interface SafeList {
id: string;
content: string | null;
tag: IObject | IObject[];
published: string;
visibility: string;
mentionedUsers: any[];
visibleUsers: any[];
}
function extractSafe(object: IObject): Partial<SafeList> {
return {
id: object.id,
content: object.content,
tag: object.tag,
published: object.published,
visibility: object.visibility,
mentionedUsers: object.mentionedUsers,
visibleUsers: object.visibleUsers,
};
}
export interface ICollection extends IObject {
type: 'Collection';
totalItems: number;
@ -258,7 +133,7 @@ export interface IPost extends IObject {
quoteUrl?: string;
}
export interface IUnsanitizedQuestion extends IObject {
export interface IQuestion extends IObject {
type: 'Note' | 'Question';
actor: string;
source?: {
@ -273,25 +148,7 @@ export interface IUnsanitizedQuestion extends IObject {
closed?: Date;
}
export interface IQuestion extends IUnsanitizedQuestion, YumeDowncastSanitizedBadge<'question'> {}
export function yumeSanitizeQuestion(object: IUnsanitizedQuestion): IQuestion {
return {
type: object.type,
actor: yumeNormalizeURL(object.actor),
source: object.source,
_misskey_quote: object._misskey_quote,
quoteUrl: object.quoteUrl ? yumeNormalizeURL(object.quoteUrl) : '',
oneOf: object.oneOf,
anyOf: object.anyOf,
endTime: object.endTime,
closed: object.closed,
__yume_normalized_object: true,
__yume_normalized_badge: 'question',
};
}
export const isQuestion = (object: IObject): object is IUnsanitizedQuestion =>
export const isQuestion = (object: IObject): object is IQuestion =>
getApType(object) === 'Note' || getApType(object) === 'Question';
interface IQuestionChoice {
@ -412,307 +269,88 @@ export const isDocument = (object: IObject): object is IApDocument => {
return type != null && validDocumentTypes.includes(type);
};
export interface IApImage extends IApDocument, Partial<SafeList> {
export interface IApImage extends IApDocument {
type: 'Image';
}
export interface ICreate extends IActivity, Partial<SafeList> {
export interface ICreate extends IActivity {
type: 'Create';
}
export interface IDelete extends IActivity, Partial<SafeList> {
export interface IDelete extends IActivity {
type: 'Delete';
}
export interface IUpdate extends IActivity, Partial<SafeList> {
export interface IUpdate extends IActivity {
type: 'Update';
}
export interface IRead extends IActivity, Partial<SafeList> {
export interface IRead extends IActivity {
type: 'Read';
}
export interface IUndo extends IActivity, Partial<SafeList> {
export interface IUndo extends IActivity {
type: 'Undo';
}
export interface IFollow extends IActivity, Partial<SafeList> {
export interface IFollow extends IActivity {
type: 'Follow';
}
export interface IAccept extends IActivity, Partial<SafeList> {
export interface IAccept extends IActivity {
type: 'Accept';
}
export interface IReject extends IActivity, Partial<SafeList> {
export interface IReject extends IActivity {
type: 'Reject';
}
export interface IAdd extends IActivity, Partial<SafeList> {
export interface IAdd extends IActivity {
type: 'Add';
}
export interface IRemove extends IActivity, Partial<SafeList> {
export interface IRemove extends IActivity {
type: 'Remove';
}
export interface ILike extends IActivity, Partial<SafeList> {
export interface ILike extends IActivity {
type: 'Like' | 'EmojiReaction' | 'EmojiReact';
_misskey_reaction?: string;
}
export interface IAnnounce extends IActivity, Partial<SafeList> {
export interface IAnnounce extends IActivity {
type: 'Announce';
}
export interface IBlock extends IActivity, Partial<SafeList> {
export interface IBlock extends IActivity {
type: 'Block';
}
export interface IFlag extends IActivity, Partial<SafeList> {
export interface IFlag extends IActivity {
type: 'Flag';
}
export interface IMove extends IActivity, Partial<SafeList> {
export interface IMove extends IActivity {
type: 'Move';
target: IObject | string;
}
export function yumeDowncastCreate(object: IObject): ICreate | null {
if (getApType(object) !== 'Create') return null;
const obj = object as ICreate;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Create',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastDelete(object: IObject): IDelete | null {
if (getApType(object) !== 'Delete') return null;
const obj = object as IDelete;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Delete',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastUpdate(object: IObject): IUpdate | null {
if (getApType(object) !== 'Update') return null;
const obj = object as IUpdate;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Update',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastRead(object: IObject): IRead | null {
if (getApType(object) !== 'Read') return null;
const obj = object as IRead;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Read',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastUndo(object: IObject): IUndo | null {
if (getApType(object) !== 'Undo') return null;
const obj = object as IUndo;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Undo',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastFollow(object: IObject): IFollow | null {
if (getApType(object) !== 'Follow') return null;
const obj = object as IFollow;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Follow',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastAccept(object: IObject): IAccept | null {
if (getApType(object) !== 'Accept') return null;
const obj = object as IAccept;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Accept',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastReject(object: IObject): IReject | null {
if (getApType(object) !== 'Reject') return null;
const obj = object as IReject;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Reject',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastAdd(object: IObject): IAdd | null {
if (getApType(object) !== 'Add') return null;
const obj = object as IAdd;
if (!obj.actor || !obj.object ) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Add',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastRemove(object: IObject): IRemove | null {
if (getApType(object) !== 'Remove') return null;
const obj = object as IRemove;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Remove',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastLike(object: IObject): ILike | null {
if (getApType(object) !== 'Like') return null;
const obj = object as ILike;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Like',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastAnnounce(object: IObject): IAnnounce | null {
if (getApType(object) !== 'Announce') return null;
const obj = object as IAnnounce;
if (!obj.actor || !obj.object) return null;
return {
// ...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Announce',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastBlock(object: IObject): IBlock | null {
if (getApType(object) !== 'Block') return null;
const obj = object as IBlock;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Block',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastFlag(object: IObject): IFlag | null {
if (getApType(object) !== 'Flag') return null;
const obj = object as IFlag;
if (!obj.actor || !obj.object) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Flag',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: obj.target ? (typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target)) : undefined,
__yume_normalized_object: true,
};
}
export function yumeDowncastMove(object: IObject): IMove | null {
if (getApType(object) !== 'Move') return null;
const obj = object as IMove;
if (!obj.actor || !obj.object || !obj.target) return null;
return {
...extractMisskeyVendorKeys(object),
...extractSafe(object),
type: 'Move',
actor: typeof obj.actor === 'string' ? yumeNormalizeURL(obj.actor) : yumeNormalizeObject(obj.actor),
object: typeof obj.object === 'string' ? yumeNormalizeURL(obj.object) : yumeNormalizeObject(obj.object),
target: typeof obj.target === 'string' ? yumeNormalizeURL(obj.target) : yumeNormalizeObject(obj.target),
__yume_normalized_object: true,
};
}
export function yumeDowncastMention(object: IObject): IApMention | null {
if (getApType(object) !== 'Mention') {
return null;
}
const href = getApHrefNullable(object);
return {
...object,
type: 'Mention',
href: href ? yumeNormalizeURL(href) : '',
name: object.name ?? '',
};
}
export const isCreate = (object: IObject): object is ICreate => getApType(object) === 'Create';
export const isDelete = (object: IObject): object is IDelete => getApType(object) === 'Delete';
export const isUpdate = (object: IObject): object is IUpdate => getApType(object) === 'Update';
export const isRead = (object: IObject): object is IRead => getApType(object) === 'Read';
export const isUndo = (object: IObject): object is IUndo => getApType(object) === 'Undo';
export const isFollow = (object: IObject): object is IFollow => getApType(object) === 'Follow';
export const isAccept = (object: IObject): object is IAccept => getApType(object) === 'Accept';
export const isReject = (object: IObject): object is IReject => getApType(object) === 'Reject';
export const isAdd = (object: IObject): object is IAdd => getApType(object) === 'Add';
export const isRemove = (object: IObject): object is IRemove => getApType(object) === 'Remove';
export const isLike = (object: IObject): object is ILike => {
const type = getApType(object);
return type != null && ['Like', 'EmojiReaction', 'EmojiReact'].includes(type);
};
export const isAnnounce = (object: IObject): object is IAnnounce => getApType(object) === 'Announce';
export const isBlock = (object: IObject): object is IBlock => getApType(object) === 'Block';
export const isFlag = (object: IObject): object is IFlag => getApType(object) === 'Flag';
export const isMove = (object: IObject): object is IMove => getApType(object) === 'Move';
export const isNote = (object: IObject): object is IPost => getApType(object) === 'Note';

8
packages/backend/src/core/entities/DriveFileEntityService.ts
View file

@ -18,6 +18,7 @@ import { bindThis } from '@/decorators.js';
import { isMimeImage } from '@/misc/is-mime-image.js';
import { IdService } from '@/core/IdService.js';
import { UtilityService } from '../UtilityService.js';
import { VideoProcessingService } from '../VideoProcessingService.js';
import { UserEntityService } from './UserEntityService.js';
import { DriveFolderEntityService } from './DriveFolderEntityService.js';
@ -42,6 +43,7 @@ export class DriveFileEntityService {
private utilityService: UtilityService,
private driveFolderEntityService: DriveFolderEntityService,
private videoProcessingService: VideoProcessingService,
private idService: IdService,
) {
}
@ -84,7 +86,11 @@ export class DriveFileEntityService {
@bindThis
public getThumbnailUrl(file: MiDriveFile): string | null {
if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
if (file.type.startsWith('video')) {
if (file.thumbnailUrl) return file.thumbnailUrl;
return this.videoProcessingService.getExternalVideoThumbnailUrl(file.webpublicUrl ?? file.url);
} else if (file.uri != null && file.userHost != null && this.config.externalMediaProxyEnabled) {
// 動画ではなくリモートかつメディアプロキシ
return this.getProxiedUrl(file.uri, 'static');
}

6
packages/backend/src/core/entities/DriveFolderEntityService.ts
View file

@ -30,8 +30,7 @@ export class DriveFolderEntityService {
public async pack(
src: MiDriveFolder['id'] | MiDriveFolder,
options?: {
detail: boolean,
maxDepth?: number,
detail: boolean
},
): Promise<Packed<'DriveFolder'>> {
const opts = Object.assign({
@ -56,8 +55,7 @@ export class DriveFolderEntityService {
...(folder.parentId ? {
parent: this.pack(folder.parentId, {
detail: (options?.maxDepth ? options.maxDepth > 0 : true),
maxDepth: options?.maxDepth || 32,
detail: true,
}),
} : {}),
} : {}),

112
packages/backend/src/core/entities/EmojiEntityService.ts
View file

@ -10,124 +10,68 @@ import type { EmojisRepository, MiRole, RolesRepository } from '@/models/_.js';
import type { Packed } from '@/misc/json-schema.js';
import type { MiEmoji } from '@/models/Emoji.js';
import { bindThis } from '@/decorators.js';
import type { Config } from '@/config.js';
@Injectable()
export class EmojiEntityService {
constructor(
@Inject(DI.emojisRepository)
private emojisRepository: EmojisRepository,
@Inject(DI.config)
private config: Config,
@Inject(DI.rolesRepository)
private rolesRepository: RolesRepository,
) {
}
private stripProxyIfOrigin(url: string): string {
try {
const u = new URL(url);
let origin = u.origin;
if (u.origin === new URL(this.config.mediaProxy).origin) {
const innerUrl = u.searchParams.get('url');
if (innerUrl) {
origin = new URL(innerUrl).origin;
}
}
if (origin === u.origin) {
return url;
}
} catch (e) {
return url;
}
return url;
}
@bindThis
public packSimpleNoQuery(
emoji: MiEmoji,
): Packed<'EmojiSimple'> {
return {
aliases: emoji.aliases,
name: emoji.name,
category: emoji.category,
// || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
url: this.stripProxyIfOrigin(emoji.publicUrl || emoji.originalUrl),
localOnly: emoji.localOnly ? true : undefined,
isSensitive: emoji.isSensitive ? true : undefined,
roleIdsThatCanBeUsedThisEmojiAsReaction: emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length > 0 ? emoji.roleIdsThatCanBeUsedThisEmojiAsReaction : undefined,
};
}
@bindThis
public async packSimple(
src: MiEmoji['id'] | MiEmoji,
): Promise<Packed<'EmojiSimple'>> {
const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src });
return this.packSimpleNoQuery(emoji);
}
@bindThis
public async packSimpleMany(
emojis: MiEmoji['id'][] | MiEmoji[],
): Promise<Packed<'EmojiSimple'>[]> {
if (emojis.length === 0) {
return [];
}
if (typeof emojis[0] === 'string') {
const res = await this.emojisRepository.findBy({ id: In(emojis as MiEmoji['id'][]) });
return res.map(this.packSimpleNoQuery);
}
return (emojis as MiEmoji[]).map(this.packSimpleNoQuery);
}
@bindThis
public packDetailedNoQuery(
emoji: MiEmoji,
): Packed<'EmojiDetailed'> {
return {
id: emoji.id,
aliases: emoji.aliases,
name: emoji.name,
category: emoji.category,
host: emoji.host,
// || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
url: this.stripProxyIfOrigin(emoji.publicUrl || emoji.originalUrl),
license: emoji.license,
isSensitive: emoji.isSensitive,
localOnly: emoji.localOnly,
roleIdsThatCanBeUsedThisEmojiAsReaction: emoji.roleIdsThatCanBeUsedThisEmojiAsReaction,
url: emoji.publicUrl || emoji.originalUrl,
localOnly: emoji.localOnly ? true : undefined,
isSensitive: emoji.isSensitive ? true : undefined,
roleIdsThatCanBeUsedThisEmojiAsReaction: emoji.roleIdsThatCanBeUsedThisEmojiAsReaction.length > 0 ? emoji.roleIdsThatCanBeUsedThisEmojiAsReaction : undefined,
};
}
@bindThis
public packSimpleMany(
emojis: any[],
) {
return Promise.all(emojis.map(x => this.packSimple(x)));
}
@bindThis
public async packDetailed(
src: MiEmoji['id'] | MiEmoji,
): Promise<Packed<'EmojiDetailed'>> {
const emoji = typeof src === 'object' ? src : await this.emojisRepository.findOneByOrFail({ id: src });
return this.packDetailedNoQuery(emoji);
return {
id: emoji.id,
aliases: emoji.aliases,
name: emoji.name,
category: emoji.category,
host: emoji.host,
// || emoji.originalUrl してるのは後方互換性のため（publicUrlはstringなので??はだめ）
url: emoji.publicUrl || emoji.originalUrl,
license: emoji.license,
isSensitive: emoji.isSensitive,
localOnly: emoji.localOnly,
roleIdsThatCanBeUsedThisEmojiAsReaction: emoji.roleIdsThatCanBeUsedThisEmojiAsReaction,
};
}
@bindThis
public async packDetailedMany(
emojis: MiEmoji['id'][] | MiEmoji[],
) : Promise<Packed<'EmojiDetailed'>[]> {
if (emojis.length === 0) {
return [];
}
if (typeof emojis[0] === 'string') {
const res = await this.emojisRepository.findBy({ id: In(emojis as MiEmoji['id'][]) });
return res.map(this.packDetailedNoQuery);
}
return (emojis as MiEmoji[]).map(this.packDetailedNoQuery);
public packDetailedMany(
emojis: any[],
): Promise<Packed<'EmojiDetailed'>[]> {
return Promise.all(emojis.map(x => this.packDetailed(x)));
}
@bindThis

1
packages/backend/src/core/entities/MetaEntityService.ts
View file

@ -95,6 +95,7 @@ export class MetaEntityService {
enableTurnstile: instance.enableTurnstile,
turnstileSiteKey: instance.turnstileSiteKey,
enableTestcaptcha: instance.enableTestcaptcha,
googleAnalyticsMeasurementId: instance.googleAnalyticsMeasurementId,
swPublickey: instance.swPublicKey,
themeColor: instance.themeColor,
mascotImageUrl: instance.mascotImageUrl ?? '/assets/ai.png',

72
packages/backend/src/misc/log-sanitization.ts
View file

@ -1,72 +0,0 @@
/*
* SPDX-FileCopyrightText: syuilo and misskey-project and yumechi
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { aidRegExp } from "./id/aid.js";
import { aidxRegExp } from "./id/aidx.js";
export function sanitizeRequestURI(uri: string): string {
const vite = /^\/vite\/.+\.([a-z0-9]{1,4})$/;
const embed_vite = /^\/embed_vite\/.+\.([a-z0-9]{1,4})$/;
if (vite.test(uri)) {
return '[vite]';
}
if (embed_vite.test(uri)) {
return '[embed_vite]';
}
if (uri.startsWith('/emoji/')) {
return '/emoji/[emoji]';
}
if (uri.startsWith('/identicon/')) {
return '/identicon/[identicon]';
}
if (uri.startsWith('/tags/')) {
return '/tags/[tag]';
}
if (uri.startsWith('/user-tags/')) {
return '/user-tags/[tag]';
}
if (uri.startsWith('/page/')) {
return '/page/[page]';
}
if (uri.startsWith('/fluent-emoji/')) {
return '/fluent-emoji/[fluent-emoji]';
}
if (uri.startsWith('/twemoji/')) {
return '/twemoji/[twemoji]';
}
if (uri.startsWith('/twemoji-badge/')) {
return '/twemoji-badge/[twemoji-badge]';
}
if (!uri.startsWith('/api/')) {
return '[other]';
}
const uuid = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
const username_local = /\/@\w+(\/|$)/;
const username_remote = /\/@\w+@[a-zA-Z0-9-.]+\.[a-zA-Z]{2,4}(\/|$)/;
const token = /=[0-9a-zA-Z]{16}/g;
const aidx = new RegExp(`/${aidxRegExp.source.replace(/^\^/, '').replace(/\$$/, '')}(\/|$)`, 'g');
const aid = new RegExp(`/${aidRegExp.source.replace(/^\^/, '').replace(/\$$/, '')}(\/|$)`, 'g');
return uri
.replace(aidx, '/[aidx]/')
.replace(aid, '/[aid]/')
.replace(token, '=[token]')
.replace(uuid, '[uuid]')
.replace(username_local, '/[user_local]/')
.replace(username_remote, '/[user_remote]/');
}

2
packages/backend/src/models/DriveFile.ts
View file

@ -61,7 +61,7 @@ export class MiDriveFile {
public size: number;
@Column('varchar', {
length: 32768, nullable: true,
length: 512, nullable: true,
comment: 'The comment of the DriveFile.',
})
public comment: string | null;

3
packages/backend/src/models/Following.ts
View file

@ -3,7 +3,7 @@
* SPDX-License-Identifier: AGPL-3.0-only
*/
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne, ViewEntity } from 'typeorm';
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { id } from './util/id.js';
import { MiUser } from './User.js';
@ -98,4 +98,3 @@ export class MiFollowing {
public followeeSharedInbox: string | null;
//#endregion
}

6
packages/backend/src/models/Meta.ts
View file

@ -658,4 +658,10 @@ export class MiMeta {
default: '{}',
})
public federationHosts: string[];
@Column('varchar', {
length: 64,
nullable: true,
})
public googleAnalyticsMeasurementId: string | null;
}

2
packages/backend/src/models/Note.ts
View file

@ -50,7 +50,6 @@ export class MiNote {
public threadId: string | null;
// TODO: varcharにしたい
@Index() // USING pgroonga
@Column('text', {
nullable: true,
})
@ -61,7 +60,6 @@ export class MiNote {
})
public name: string | null;
@Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2
@Column('varchar', {
length: 512, nullable: true,
})

1
packages/backend/src/models/User.ts
View file

@ -49,7 +49,6 @@ export class MiUser {
})
public usernameLower: string;
@Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2
@Column('varchar', {
length: 128, nullable: true,
comment: 'The name of the User.',

1
packages/backend/src/models/UserProfile.ts
View file

@ -36,7 +36,6 @@ export class MiUserProfile {
})
public birthday: string | null;
@Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2
@Column('varchar', {
length: 2048, nullable: true,
comment: 'The description (bio) of the User.',

4
packages/backend/src/models/json-schema/meta.ts
View file

@ -119,6 +119,10 @@ export const packedMetaLiteSchema = {
type: 'boolean',
optional: false, nullable: false,
},
googleAnalyticsMeasurementId: {
type: 'string',
optional: false, nullable: true,
},
swPublickey: {
type: 'string',
optional: false, nullable: true,

102
packages/backend/src/postgres.ts
View file

@ -7,7 +7,6 @@
import pg from 'pg';
import { DataSource, Logger } from 'typeorm';
import * as highlight from 'cli-highlight';
import { createHash } from 'crypto';
import { entities as charts } from '@/core/chart/entities.js';
import { MiAbuseUserReport } from '@/models/AbuseUserReport.js';
@ -83,8 +82,6 @@ import { MiReversiGame } from '@/models/ReversiGame.js';
import { Config } from '@/config.js';
import MisskeyLogger from '@/logger.js';
import { bindThis } from '@/decorators.js';
import { MemoryKVCache } from './misc/cache.js';
import { metricCounter, metricHistogram } from './server/api/MetricsService.js';
import { MiSystemAccount } from './models/SystemAccount.js';
pg.types.setTypeParser(20, Number);
@ -93,51 +90,6 @@ export const dbLogger = new MisskeyLogger('db');
const sqlLogger = dbLogger.createSubLogger('sql', 'gray');
type QueryTagCache = {
join: string;
from: string;
};
function dedupConsecutive<T>(arr: T[]): T[] {
return arr.filter((v, i, a) => i === 0 || a[i - 1] !== v);
}
function simplifyIdentifiers(sql: string) {
return sql.replace(/"([a-zA-Z_]+)"/g, '$1');
}
function extractQueryTags(query: string): QueryTagCache {
const joins = query.matchAll(/(LEFT|RIGHT|INNER|OUTER)[\s\S]+JOIN[\s\r\n]+([a-zA-Z0-9_"`.]+)/ig);
const froms = query.matchAll(/FROM[\s\r\n]+([a-zA-Z0-9_"`.]+)/ig);
const join = Array.from(joins).map(j => `${j[1]}:${simplifyIdentifiers(j[2])}`).join('|');
const from = dedupConsecutive(Array.from(froms).map(f => simplifyIdentifiers(f[1]))).join('|');
return {
join,
from,
};
}
const mQueryCounter = metricCounter({
name: 'misskey_postgres_query_total',
help: 'Total queries to postgres',
labelNames: ['join', 'from'],
});
const mQueryErrorCounter = metricCounter({
name: 'misskey_postgres_query_error_total',
help: 'Total errors in queries to postgres',
labelNames: ['join', 'from'],
});
const mSlowQueryHisto = metricHistogram({
name: 'misskey_postgres_query_slow_duration_seconds',
help: 'Duration of slow queries to postgres',
labelNames: ['join', 'from'],
buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 300],
});
export type LoggerProps = {
disableQueryTruncation?: boolean;
enableQueryParamLogging?: boolean;
@ -162,11 +114,9 @@ function stringifyParameter(param: any) {
}
class MyCustomLogger implements Logger {
constructor(private metricOnly = true, private props: LoggerProps = {}) {
constructor(private props: LoggerProps = {}) {
}
private queryHashCache = new MemoryKVCache<QueryTagCache>(1000 * 60 * 5); // 5m
@bindThis
private transformQueryLog(sql: string) {
let modded = sql;
@ -186,69 +136,33 @@ class MyCustomLogger implements Logger {
return undefined;
}
@bindThis
private getQueryTags(query: string): QueryTagCache {
const existing = this.queryHashCache.get(query);
if (existing) {
return existing;
}
const result = extractQueryTags(query);
this.queryHashCache.set(query, result);
return result;
}
@bindThis
public logQuery(query: string, parameters?: any[]) {
mQueryCounter?.inc(this.getQueryTags(query));
if (this.metricOnly) {
return;
}
sqlLogger.info(this.transformQueryLog(query), this.transformParameters(parameters));
}
@bindThis
public logQueryError(error: string, query: string, parameters?: any[]) {
mQueryErrorCounter?.inc(this.getQueryTags(query));
if (this.metricOnly) {
return;
}
sqlLogger.error(this.transformQueryLog(query), this.transformParameters(parameters));
}
@bindThis
public logQuerySlow(time: number, query: string, parameters?: any[]) {
mSlowQueryHisto?.observe(this.getQueryTags(query), time);
if (this.metricOnly) {
return;
}
sqlLogger.warn(this.transformQueryLog(query), this.transformParameters(parameters));
}
@bindThis
public logSchemaBuild(message: string) {
if (this.metricOnly) {
return;
}
sqlLogger.info(message);
}
@bindThis
public log(message: string) {
if (this.metricOnly) {
return;
}
sqlLogger.info(message);
}
@bindThis
public logMigration(message: string) {
if (this.metricOnly) {
return;
}
sqlLogger.info(message);
}
}
@ -329,7 +243,7 @@ export const entities = [
const log = process.env.NODE_ENV !== 'production';
export function createPostgresDataSource(config: Config, isMain = false) {
export function createPostgresDataSource(config: Config) {
return new DataSource({
type: 'postgres',
host: config.db.host,
@ -372,11 +286,13 @@ export function createPostgresDataSource(config: Config, isMain = false) {
db: config.redis.db ?? 0,
},
} : false,
logging: true,
logger: new MyCustomLogger(!log, {
disableQueryTruncation: config.logging?.sql?.disableQueryTruncation,
enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging,
}),
logging: log,
logger: log
? new MyCustomLogger({
disableQueryTruncation: config.logging?.sql?.disableQueryTruncation,
enableQueryParamLogging: config.logging?.sql?.enableQueryParamLogging,
})
: undefined,
maxQueryExecutionTime: 300,
entities: entities,
migrations: ['../../migration/*.js'],

41
packages/backend/src/queue/QueueProcessorService.ts
View file

@ -45,7 +45,6 @@ import { CleanProcessorService } from './processors/CleanProcessorService.js';
import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js';
import { QueueLoggerService } from './QueueLoggerService.js';
import { QUEUE, baseQueueOptions } from './const.js';
import { mStalledWorkerCounter } from './metrics.js';
// ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019
function httpRelatedBackoff(attemptsMade: number) {
@ -195,10 +194,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -255,10 +251,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.DB });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -298,10 +291,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.DELIVER });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -341,10 +331,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.INBOX });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -384,10 +371,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.USER_WEBHOOK_DELIVER });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -427,10 +411,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.SYSTEM_WEBHOOK_DELIVER });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -477,10 +458,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.RELATIONSHIP });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion
@ -521,10 +499,7 @@ export class QueueProcessorService implements OnApplicationShutdown {
}
})
.on('error', (err: Error) => logger.error(`error ${err.name}: ${err.message}`, { e: renderError(err) }))
.on('stalled', (jobId) => {
mStalledWorkerCounter?.inc({ queue: QUEUE.OBJECT_STORAGE });
logger.warn(`stalled id=${jobId}`);
});
.on('stalled', (jobId) => logger.warn(`stalled id=${jobId}`));
}
//#endregion

37
packages/backend/src/queue/metrics.ts
View file

@ -1,37 +0,0 @@
import { metricCounter, metricGauge } from '@/server/api/MetricsService.js';
export const mJobBlockedCounter = metricCounter({
name: 'misskey_queue_jobs_blocked_total',
help: 'Total number of jobs waiting for a worker',
labelNames: ['queue'],
});
export const mActiveJobs = metricGauge({
name: 'misskey_queue_active_jobs',
help: 'Number of active jobs in queue',
labelNames: ['queue'],
});
export const mDelayedJobs = metricGauge({
name: 'misskey_queue_delayed_jobs',
help: 'Number of delayed jobs in queue',
labelNames: ['queue'],
});
export const mWaitingJobs = metricGauge({
name: 'misskey_queue_waiting_jobs',
help: 'Number of waiting jobs in queue',
labelNames: ['queue'],
});
export const mFailedJobs = metricGauge({
name: 'misskey_queue_failed_jobs',
help: 'Total number of failed jobs',
labelNames: ['queue'],
});
export const mStalledWorkerCounter = metricCounter({
name: 'misskey_queue_stalled_workers_total',
help: 'Total number of stalled workers',
labelNames: ['queue'],
});

62
packages/backend/src/queue/processors/DeleteAccountProcessorService.ts
View file

@ -4,9 +4,9 @@
*/
import { Inject, Injectable } from '@nestjs/common';
import { DataSource, MoreThan, QueryFailedError, TypeORMError } from 'typeorm';
import { MoreThan } from 'typeorm';
import { DI } from '@/di-symbols.js';
import { MiUser, type DriveFilesRepository, type NotesRepository, type UserProfilesRepository, type UsersRepository } from '@/models/_.js';
import type { DriveFilesRepository, NotesRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js';
import type Logger from '@/logger.js';
import { DriveService } from '@/core/DriveService.js';
import type { MiDriveFile } from '@/models/DriveFile.js';
@ -26,9 +26,6 @@ export class DeleteAccountProcessorService {
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.db)
private db: DataSource,
@Inject(DI.userProfilesRepository)
private userProfilesRepository: UserProfilesRepository,
@ -55,14 +52,6 @@ export class DeleteAccountProcessorService {
return;
}
if (!user.isDeleted) {
this.logger.warn('User is not pre-marked as deleted, this is likely a bug');
if (process.env.NODE_ENV !== 'production') {
throw new Error('User is not pre-marked as deleted'); // make some noise to make sure tests fail
}
await this.usersRepository.update({ id: user.id }, { isDeleted: true });
}
{ // Delete notes
let cursor: MiNote['id'] | null = null;
@ -132,46 +121,13 @@ export class DeleteAccountProcessorService {
}
}
// Deadlockが発生した場合にリトライする
for (let remaining = 3; remaining > 0; remaining--) {
try {
// soft指定されている場合は物理削除しない
await this.db.transaction(async txn => {
// soft指定してもデータをすべで削除する
await txn.delete(MiUser, user.id);
if (job.data.soft) {
await txn.insert(MiUser, {
...user,
isRoot: false,
updatedAt: new Date(),
emojis: [],
hideOnlineStatus: true,
followersCount: 0,
followingCount: 0,
avatarUrl: null,
avatarId: null,
notesCount: 0,
inbox: null,
sharedInbox: null,
featured: null,
uri: null,
followersUri: null,
token: null,
isDeleted: true,
});
}
});
return 'Account deleted';
} catch (e) {
// 40P01 = deadlock_detected
// https://www.postgresql.org/docs/current/errcodes-appendix.html
if (remaining > 0 && e instanceof QueryFailedError && e.driverError.code === '40P01') {
this.logger.warn(`Deadlock occurred, retrying after 1s... [${remaining - 1} remaining]`);
await new Promise(resolve => setTimeout(resolve, 1000));
continue;
}
throw e;
}
// soft指定されている場合は物理削除しない
if (job.data.soft) {
// nop
} else {
await this.usersRepository.delete(job.data.user.id);
}
return 'Account deleted';
}
}

79
packages/backend/src/queue/processors/InboxProcessorService.ts
View file

@ -7,7 +7,6 @@ import { URL } from 'node:url';
import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
import httpSignature from '@peertube/http-signature';
import * as Bull from 'bullmq';
import * as prom from 'prom-client';
import type Logger from '@/logger.js';
import { FederatedInstanceService } from '@/core/FederatedInstanceService.js';
import { FetchInstanceMetadataService } from '@/core/FetchInstanceMetadataService.js';
@ -30,7 +29,6 @@ import { CollapsedQueue } from '@/misc/collapsed-queue.js';
import { MiNote } from '@/models/Note.js';
import { MiMeta } from '@/models/Meta.js';
import { DI } from '@/di-symbols.js';
import { metricCounter, metricHistogram } from '@/server/api/MetricsService.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type { InboxJobData } from '../types.js';
@ -39,37 +37,6 @@ type UpdateInstanceJob = {
shouldUnsuspend: boolean,
};
const mIncomingApProcessingTime = metricHistogram({
name: 'misskey_incoming_ap_processing_time',
help: 'Incoming AP processing time in seconds',
labelNames: ['incoming_host', 'incoming_type', 'success'],
buckets: [2, 10, 60, 300],
});
const mIncomingApEvent = metricCounter({
name: 'misskey_incoming_ap_event',
help: 'Incoming AP event',
labelNames: ['incoming_host', 'incoming_type'],
});
const mIncomingApEventAccepted = metricCounter({
name: 'misskey_incoming_ap_event_accepted',
help: 'Incoming AP event accepted',
labelNames: ['incoming_host', 'incoming_type'],
});
const mIncomingApReject = metricCounter({
name: 'misskey_incoming_ap_reject',
help: 'Incoming AP reject',
labelNames: ['incoming_host', 'incoming_type', 'reason'],
});
const mincomingApProcessingError = metricCounter({
name: 'misskey_incoming_ap_processing_error',
help: 'Incoming AP processing error',
labelNames: ['incoming_host', 'incoming_type'],
});
@Injectable()
export class InboxProcessorService implements OnApplicationShutdown {
private logger: Logger;
@ -99,6 +66,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
public async process(job: Bull.Job<InboxJobData>): Promise<string> {
const signature = job.data.signature; // HTTP-signature
let activity = job.data.activity;
//#region Log
const info = Object.assign({}, activity);
delete info['@context'];
@ -107,34 +75,12 @@ export class InboxProcessorService implements OnApplicationShutdown {
const host = this.utilityService.toPuny(new URL(signature.keyId).hostname);
const incCounter = <T extends 'incoming_host' | 'incoming_type', U>(counter: prom.Counter<T> | null, addn_labels: U) => {
if (Array.isArray(activity.type)) {
for (const t of activity.type) {
counter?.inc({ incoming_host: host.toString(), incoming_type: t, ...addn_labels });
}
} else {
counter?.inc({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels });
}
};
const observeHistogram = <T extends 'incoming_host' | 'incoming_type', U>(histogram: prom.Histogram<T> | null, addn_labels: U, value: number) => {
if (Array.isArray(activity.type)) {
for (const t of activity.type) {
histogram?.observe({ incoming_host: host.toString(), incoming_type: t, ...addn_labels }, value);
}
} else {
histogram?.observe({ incoming_host: host.toString(), incoming_type: activity.type ?? 'unknown', ...addn_labels }, value);
}
};
if (!this.utilityService.isFederationAllowedHost(host)) {
incCounter(mIncomingApReject, { reason: 'host_not_allowed' });
return `Blocked request: ${host}`;
}
const keyIdLower = signature.keyId.toLowerCase();
if (keyIdLower.startsWith('acct:')) {
incCounter(mIncomingApReject, { reason: 'keyid_acct' });
return `Old keyId is no longer supported. ${keyIdLower}`;
}
@ -154,7 +100,6 @@ export class InboxProcessorService implements OnApplicationShutdown {
if (!err.isRetryable) {
throw new Bull.UnrecoverableError(`skip: Ignored deleted actors on both ends ${activity.actor} - ${err.statusCode}`);
}
incCounter(mIncomingApReject, { reason: 'actor_key_unresolvable' });
throw new Error(`Error in actor ${activity.actor} - ${err.statusCode}`);
}
}
@ -162,13 +107,11 @@ export class InboxProcessorService implements OnApplicationShutdown {
// それでもわからなければ終了
if (authUser == null) {
incCounter(mIncomingApReject, { reason: 'actor_unresolvable' });
throw new Bull.UnrecoverableError(`skip: failed to resolve user ${getApId(activity.actor)}`);
}
// publicKey がなくても終了
if (authUser.key == null) {
incCounter(mIncomingApReject, { reason: 'publickey_unresolvable' });
throw new Bull.UnrecoverableError(`skip: failed to resolve user publicKey ${getApId(activity.actor)}`);
}
@ -181,7 +124,6 @@ export class InboxProcessorService implements OnApplicationShutdown {
const ldSignature = activity.signature;
if (ldSignature) {
if (ldSignature.type !== 'RsaSignature2017') {
incCounter(mIncomingApReject, { reason: 'ld_signature_unsupported' });
throw new Bull.UnrecoverableError(`skip: unsupported LD-signature type ${ldSignature.type}`);
}
@ -195,12 +137,10 @@ export class InboxProcessorService implements OnApplicationShutdown {
// keyIdからLD-Signatureのユーザーを取得
authUser = await this.apDbResolverService.getAuthUserFromKeyId(ldSignature.creator);
if (authUser == null) {
incCounter(mIncomingApReject, { reason: 'ld_signature_user_unresolvable' });
throw new Bull.UnrecoverableError('skip: LD-Signatureのユーザーが取得できませんでした');
}
if (authUser.key == null) {
incCounter(mIncomingApReject, { reason: 'ld_signature_publickey_unavailable' });
throw new Bull.UnrecoverableError('skip: LD-SignatureのユーザーはpublicKeyを持っていませんでした');
}
@ -209,7 +149,6 @@ export class InboxProcessorService implements OnApplicationShutdown {
// LD-Signature検証
const verified = await jsonLd.verifyRsaSignature2017(activity, authUser.key.keyPem).catch(() => false);
if (!verified) {
incCounter(mIncomingApReject, { reason: 'ld_signature_verification_failed' });
throw new Bull.UnrecoverableError('skip: LD-Signatureの検証に失敗しました');
}
@ -232,17 +171,14 @@ export class InboxProcessorService implements OnApplicationShutdown {
// もう一度actorチェック
if (authUser.user.uri !== activity.actor) {
incCounter(mIncomingApReject, { reason: 'ld_signature_actor_mismatch' });
throw new Bull.UnrecoverableError(`skip: LD-Signature user(${authUser.user.uri}) !== activity.actor(${activity.actor})`);
}
const ldHost = this.utilityService.extractDbHost(authUser.user.uri);
if (!this.utilityService.isFederationAllowedHost(ldHost)) {
incCounter(mIncomingApReject, { reason: 'fed_host_not_allowed' });
throw new Bull.UnrecoverableError(`Blocked request: ${ldHost}`);
}
} else {
incCounter(mIncomingApReject, { reason: 'ld_signature_unavailable' });
throw new Bull.UnrecoverableError(`skip: http-signature verification failed and no LD-Signature. keyId=${signature.keyId}`);
}
}
@ -252,7 +188,6 @@ export class InboxProcessorService implements OnApplicationShutdown {
const signerHost = this.utilityService.extractDbHost(authUser.user.uri!);
const activityIdHost = this.utilityService.extractDbHost(activity.id);
if (signerHost !== activityIdHost) {
incCounter(mIncomingApReject, 'host_signature_mismatch');
throw new Bull.UnrecoverableError(`skip: signerHost(${signerHost}) !== activity.id host(${activityIdHost}`);
}
} else {
@ -282,10 +217,7 @@ export class InboxProcessorService implements OnApplicationShutdown {
this.fetchInstanceMetadataService.fetchInstanceMetadata(i);
});
incCounter(mIncomingApEvent, {});
// アクティビティを処理
const begin = +new Date();
try {
const result = await this.apInboxService.performActivity(authUser.user, activity);
if (result && !result.startsWith('ok')) {
@ -295,26 +227,17 @@ export class InboxProcessorService implements OnApplicationShutdown {
} catch (e) {
if (e instanceof IdentifiableError) {
if (e.id === '689ee33f-f97c-479a-ac49-1b9f8140af99') {
incCounter(mIncomingApReject, { reason: 'blocked_notes_with_prohibited_words' });
return 'blocked notes with prohibited words';
}
if (e.id === '85ab9bd7-3a41-4530-959d-f07073900109') {
incCounter(mIncomingApReject, { reason: 'actor_suspended' });
return 'actor has been suspended';
}
if (e.id === 'd450b8a9-48e4-4dab-ae36-f4db763fda7c') { // invalid Note
incCounter(mIncomingApReject, { reason: 'invalid_note' });
return e.message;
}
}
const end = +new Date();
observeHistogram(mIncomingApProcessingTime, { success: 'false' }, (end - begin) / 1000);
incCounter(mincomingApProcessingError, {});
throw e;
}
observeHistogram(mIncomingApProcessingTime, { success: 'true' }, (+new Date() - begin) / 1000);
incCounter(mIncomingApEventAccepted, {});
return 'ok';
}

453
packages/backend/src/server/FileServerService.ts
View file

@ -8,19 +8,27 @@ import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import { Inject, Injectable } from '@nestjs/common';
import rename from 'rename';
import sharp from 'sharp';
import { sharpBmp } from '@misskey-dev/sharp-read-bmp';
import type { Config } from '@/config.js';
import type { MiDriveFile, DriveFilesRepository } from '@/models/_.js';
import { DI } from '@/di-symbols.js';
import { createTemp } from '@/misc/create-temp.js';
import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { StatusError } from '@/misc/status-error.js';
import type Logger from '@/logger.js';
import { DownloadService } from '@/core/DownloadService.js';
import { IImageStreamable, ImageProcessingService, webpDefault } from '@/core/ImageProcessingService.js';
import { VideoProcessingService } from '@/core/VideoProcessingService.js';
import { InternalStorageService } from '@/core/InternalStorageService.js';
import { contentDisposition } from '@/misc/content-disposition.js';
import { FileInfoService } from '@/core/FileInfoService.js';
import { LoggerService } from '@/core/LoggerService.js';
import { bindThis } from '@/decorators.js';
import { isMimeImage } from '@/misc/is-mime-image.js';
import { correctFilename } from '@/misc/correct-filename.js';
import { handleRequestRedirectToOmitSearch } from '@/misc/fastify-hook-handlers.js';
import type { FastifyInstance, FastifyRequest, FastifyReply, FastifyPluginOptions } from 'fastify';
import { InternalStorageService } from '@/core/InternalStorageService.js';
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@ -38,8 +46,11 @@ export class FileServerService {
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
private internalStorageService: InternalStorageService,
private fileInfoService: FileInfoService,
private downloadService: DownloadService,
private imageProcessingService: ImageProcessingService,
private videoProcessingService: VideoProcessingService,
private internalStorageService: InternalStorageService,
private loggerService: LoggerService,
) {
this.logger = this.loggerService.getLogger('server', 'gray');
@ -123,72 +134,159 @@ export class FileServerService {
return;
}
if (file.state === 'remote') {
const url = new URL(`${this.config.mediaProxy}/`);
try {
if (file.state === 'remote') {
let image: IImageStreamable | null = null;
url.searchParams.set('url', file.url);
if (file.fileRole === 'thumbnail') {
if (isMimeImage(file.mime, 'sharp-convertible-image-with-bmp')) {
reply.header('Cache-Control', 'max-age=31536000, immutable');
return await reply.redirect(url.toString(), 301);
}
const url = new URL(`${this.config.mediaProxy}/static.webp`);
url.searchParams.set('url', file.url);
url.searchParams.set('static', '1');
if (file.fileRole !== 'original') {
const filename = rename(file.filename, {
suffix: file.fileRole === 'thumbnail' ? '-thumb' : '-web',
extname: file.ext ? `.${file.ext}` : '.unknown',
}).toString();
file.cleanup();
return await reply.redirect(url.toString(), 301);
} else if (file.mime.startsWith('video/')) {
const externalThumbnail = this.videoProcessingService.getExternalVideoThumbnailUrl(file.url);
if (externalThumbnail) {
file.cleanup();
return await reply.redirect(externalThumbnail, 301);
}
reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(file.mime) ? file.mime : 'application/octet-stream');
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition', contentDisposition('inline', filename));
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
image = await this.videoProcessingService.generateVideoThumbnail(file.path);
}
}
const chunksize = end - start + 1;
const fileStream = fs.createReadStream(file.path, {
start,
end,
});
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
return fileStream;
if (file.fileRole === 'webpublic') {
if (['image/svg+xml'].includes(file.mime)) {
reply.header('Cache-Control', 'max-age=31536000, immutable');
const url = new URL(`${this.config.mediaProxy}/svg.webp`);
url.searchParams.set('url', file.url);
file.cleanup();
return await reply.redirect(url.toString(), 301);
}
}
if (!image) {
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
image = {
data: fs.createReadStream(file.path, {
start,
end,
}),
ext: file.ext,
type: file.mime,
};
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
} else {
image = {
data: fs.createReadStream(file.path),
ext: file.ext,
type: file.mime,
};
}
}
if ('pipe' in image.data && typeof image.data.pipe === 'function') {
// image.dataがstreamなら、stream終了後にcleanup
image.data.on('end', file.cleanup);
image.data.on('close', file.cleanup);
} else {
// image.dataがstreamでないなら直ちにcleanup
file.cleanup();
}
reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(image.type) ? image.type : 'application/octet-stream');
reply.header('Content-Length', file.file.size);
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition',
contentDisposition(
'inline',
correctFilename(file.filename, image.ext),
),
);
return image.data;
}
return fs.createReadStream(file.path);
} else {
reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(file.file.type) ? file.file.type : 'application/octet-stream');
reply.header('Content-Length', file.file.size);
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition', contentDisposition('inline', file.filename));
if (file.fileRole !== 'original') {
const filename = rename(file.filename, {
suffix: file.fileRole === 'thumbnail' ? '-thumb' : '-web',
extname: file.ext ? `.${file.ext}` : '.unknown',
}).toString();
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(file.mime) ? file.mime : 'application/octet-stream');
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition', contentDisposition('inline', filename));
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
const fileStream = fs.createReadStream(file.path, {
start,
end,
});
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
return fileStream;
}
const chunksize = end - start + 1;
const fileStream = fs.createReadStream(file.path, {
start,
end,
});
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
return fileStream;
}
return fs.createReadStream(file.path);
return fs.createReadStream(file.path);
} else {
reply.header('Content-Type', FILE_TYPE_BROWSERSAFE.includes(file.file.type) ? file.file.type : 'application/octet-stream');
reply.header('Content-Length', file.file.size);
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition', contentDisposition('inline', file.filename));
if (request.headers.range && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
const fileStream = fs.createReadStream(file.path, {
start,
end,
});
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
return fileStream;
}
return fs.createReadStream(file.path);
}
} catch (e) {
if ('cleanup' in file) file.cleanup();
throw e;
}
}
@ -204,56 +302,224 @@ export class FileServerService {
// アバタークロップなど、どうしてもオリジンである必要がある場合
const mustOrigin = 'origin' in request.query;
if (!this.config.mediaProxy) {
reply.code(501);
}
if (this.config.externalMediaProxyEnabled && !mustOrigin) {
// 外部のメディアプロキシが有効なら、そちらにリダイレクト
const proxiedURL = new URL(`${this.config.mediaProxy}/?url=${encodeURIComponent(url)}`);
reply.header('Cache-Control', 'public, max-age=259200'); // 3 days
for (const [key, value] of Object.entries(request.query)) {
if (key.toLowerCase() === 'url') continue;
proxiedURL.searchParams.append(key, value);
}
const url = new URL(`${this.config.mediaProxy}/${request.params.url || ''}`);
for (const [key, value] of Object.entries(request.query)) {
url.searchParams.append(key, value);
}
if (!mustOrigin) {
return await reply.redirect(
proxiedURL.toString(),
url.toString(),
301,
);
}
reply.header('Cache-Control', 'public, max-age=259200'); // 3 days
if (!request.headers['user-agent']) {
throw new StatusError('User-Agent is required', 400, 'User-Agent is required');
} else if (request.headers['user-agent'].toLowerCase().indexOf('misskey/') !== -1) {
throw new StatusError('Refusing to proxy a request from another proxy', 403, 'Proxy is recursive');
}
if (!request.headers['user-agent']) {
throw new StatusError('User-Agent is required', 400, 'User-Agent is required');
} else if (request.headers['user-agent'].toLowerCase().indexOf('misskey/') !== -1) {
throw new StatusError('Refusing to proxy a request from another proxy', 403, 'Proxy is recursive');
// Create temp file
const file = await this.getStreamAndTypeFromUrl(url);
if (file === '404') {
reply.code(404);
reply.header('Cache-Control', 'max-age=86400');
return reply.sendFile('/dummy.png', assets);
}
// directly proxy request through
const res = await fetch(proxiedURL, {
headers: {
'X-Forwarded-For': request.headers['x-forwarded-for']?.at(0) ?? request.ip,
'User-Agent': request.headers['user-agent'],
},
});
reply.code(res.status);
for (const [key, value] of res.headers.entries()) {
reply.header(key, value);
if (file === '204') {
reply.code(204);
reply.header('Cache-Control', 'max-age=86400');
return;
}
try {
const isConvertibleImage = isMimeImage(file.mime, 'sharp-convertible-image-with-bmp');
const isAnimationConvertibleImage = isMimeImage(file.mime, 'sharp-animation-convertible-image-with-bmp');
if (
'emoji' in request.query ||
'avatar' in request.query ||
'static' in request.query ||
'preview' in request.query ||
'badge' in request.query
) {
if (!isConvertibleImage) {
// 画像でないなら404でお茶を濁す
throw new StatusError('Unexpected mime', 404);
}
}
let image: IImageStreamable | null = null;
if ('emoji' in request.query || 'avatar' in request.query) {
if (!isAnimationConvertibleImage && !('static' in request.query)) {
image = {
data: fs.createReadStream(file.path),
ext: file.ext,
type: file.mime,
};
} else {
const data = (await sharpBmp(file.path, file.mime, { animated: !('static' in request.query) }))
.resize({
height: 'emoji' in request.query ? 128 : 320,
withoutEnlargement: true,
})
.webp(webpDefault);
image = {
data,
ext: 'webp',
type: 'image/webp',
};
}
} else if ('static' in request.query) {
image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 498, 422);
} else if ('preview' in request.query) {
image = this.imageProcessingService.convertSharpToWebpStream(await sharpBmp(file.path, file.mime), 200, 200);
} else if ('badge' in request.query) {
const mask = (await sharpBmp(file.path, file.mime))
.resize(96, 96, {
fit: 'contain',
position: 'centre',
withoutEnlargement: false,
})
.greyscale()
.normalise()
.linear(1.75, -(128 * 1.75) + 128) // 1.75x contrast
.flatten({ background: '#000' })
.toColorspace('b-w');
const stats = await mask.clone().stats();
if (stats.entropy < 0.1) {
// エントロピーがあまりない場合は404にする
throw new StatusError('Skip to provide badge', 404);
}
const data = sharp({
create: { width: 96, height: 96, channels: 4, background: { r: 0, g: 0, b: 0, alpha: 0 } },
})
.pipelineColorspace('b-w')
.boolean(await mask.png().toBuffer(), 'eor');
image = {
data: await data.png().toBuffer(),
ext: 'png',
type: 'image/png',
};
} else if (file.mime === 'image/svg+xml') {
image = this.imageProcessingService.convertToWebpStream(file.path, 2048, 2048);
} else if (!file.mime.startsWith('image/') || !FILE_TYPE_BROWSERSAFE.includes(file.mime)) {
throw new StatusError('Rejected type', 403, 'Rejected type');
}
if (!image) {
if (request.headers.range && file.file && file.file.size > 0) {
const range = request.headers.range as string;
const parts = range.replace(/bytes=/, '').split('-');
const start = parseInt(parts[0], 10);
let end = parts[1] ? parseInt(parts[1], 10) : file.file.size - 1;
if (end > file.file.size) {
end = file.file.size - 1;
}
const chunksize = end - start + 1;
image = {
data: fs.createReadStream(file.path, {
start,
end,
}),
ext: file.ext,
type: file.mime,
};
reply.header('Content-Range', `bytes ${start}-${end}/${file.file.size}`);
reply.header('Accept-Ranges', 'bytes');
reply.header('Content-Length', chunksize);
reply.code(206);
} else {
image = {
data: fs.createReadStream(file.path),
ext: file.ext,
type: file.mime,
};
}
}
if ('cleanup' in file) {
if ('pipe' in image.data && typeof image.data.pipe === 'function') {
// image.dataがstreamなら、stream終了後にcleanup
image.data.on('end', file.cleanup);
image.data.on('close', file.cleanup);
} else {
// image.dataがstreamでないなら直ちにcleanup
file.cleanup();
}
}
reply.header('Content-Type', image.type);
reply.header('Cache-Control', 'max-age=31536000, immutable');
reply.header('Content-Disposition',
contentDisposition(
'inline',
correctFilename(file.filename, image.ext),
),
);
return image.data;
} catch (e) {
if ('cleanup' in file) file.cleanup();
throw e;
}
}
@bindThis
private async getStreamAndTypeFromUrl(url: string): Promise<
{ state: 'remote'; fileRole?: 'thumbnail' | 'webpublic' | 'original'; file?: MiDriveFile; mime: string; ext: string | null; path: string; cleanup: () => void; filename: string; }
| { state: 'stored_internal'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: MiDriveFile; filename: string; mime: string; ext: string | null; path: string; }
| '404'
| '204'
> {
if (url.startsWith(`${this.config.url}/files/`)) {
const key = url.replace(`${this.config.url}/files/`, '').split('/').shift();
if (!key) throw new StatusError('Invalid File Key', 400, 'Invalid File Key');
return await this.getFileFromKey(key);
}
return await this.downloadAndDetectTypeFromUrl(url);
}
@bindThis
private async downloadAndDetectTypeFromUrl(url: string): Promise<
{ state: 'remote' ; mime: string; ext: string | null; path: string; cleanup: () => void; filename: string; }
> {
const [path, cleanup] = await createTemp();
try {
const { filename } = await this.downloadService.downloadUrl(url, path);
const { mime, ext } = await this.fileInfoService.detectType(path);
return {
state: 'remote',
mime, ext,
path, cleanup,
filename,
};
} catch (e) {
cleanup();
throw e;
}
reply.send(res.body);
}
@bindThis
private async getFileFromKey(key: string): Promise<
{ state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; filename: string; url: string; }
{ state: 'remote'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: MiDriveFile; filename: string; url: string; mime: string; ext: string | null; path: string; cleanup: () => void; }
| { state: 'stored_internal'; fileRole: 'thumbnail' | 'webpublic' | 'original'; file: MiDriveFile; filename: string; mime: string; ext: string | null; path: string; }
| '404'
| '204'
@ -272,10 +538,15 @@ export class FileServerService {
if (!file.storedInternal) {
if (!(file.isLink && file.uri)) return '204';
return { state: 'remote',
fileRole: isThumbnail ? 'thumbnail' : isWebpublic ? 'webpublic' : 'original',
filename: file.name
, url: file.uri };
const result = await this.downloadAndDetectTypeFromUrl(file.uri);
file.size = (await fs.promises.stat(result.path)).size; // DB file.sizeは正確とは限らないので
return {
...result,
url: file.uri,
fileRole: isThumbnail ? 'thumbnail' : isWebpublic ? 'webpublic' : 'original',
file,
filename: file.name,
};
}
const path = this.internalStorageService.resolvePath(key);

24
packages/backend/src/server/NodeinfoServerService.ts
View file

@ -15,8 +15,6 @@ import UsersChart from '@/core/chart/charts/users.js';
import { DEFAULT_POLICIES } from '@/core/RoleService.js';
import { SystemAccountService } from '@/core/SystemAccountService.js';
import type { FastifyInstance, FastifyPluginOptions } from 'fastify';
import { IsNull, MoreThan, Not } from 'typeorm';
import type { NotesRepository, UsersRepository } from '@/models/_.js';
const nodeinfo2_1path = '/nodeinfo/2.1';
const nodeinfo2_0path = '/nodeinfo/2.0';
@ -27,10 +25,6 @@ export class NodeinfoServerService {
constructor(
@Inject(DI.config)
private config: Config,
@Inject(DI.usersRepository)
private usersRepository: UsersRepository,
@Inject(DI.notesRepository)
private notesRepository: NotesRepository,
private systemAccountService: SystemAccountService,
private metaService: MetaService,
@ -64,16 +58,18 @@ export class NodeinfoServerService {
const [
meta,
activeHalfyear,
activeMonth,
localComments,
//activeHalfyear,
//activeMonth,
] = await Promise.all([
this.metaService.fetch(true),
this.usersRepository.count({ where: { host: IsNull(), lastActiveDate: MoreThan(new Date(now - 15552000000)) } }),
this.usersRepository.count({ where: { host: IsNull(), lastActiveDate: MoreThan(new Date(now - 2592000000)) } }),
this.notesRepository.count({ where: { userHost: IsNull(), replyId: Not(IsNull()) } }),
// 重い
//this.usersRepository.count({ where: { host: IsNull(), lastActiveDate: MoreThan(new Date(now - 15552000000)) } }),
//this.usersRepository.count({ where: { host: IsNull(), lastActiveDate: MoreThan(new Date(now - 2592000000)) } }),
]);
const activeHalfyear = null;
const activeMonth = null;
const proxyAccount = await this.systemAccountService.fetch('proxy');
const basePolicies = { ...DEFAULT_POLICIES, ...meta.policies };
@ -95,7 +91,7 @@ export class NodeinfoServerService {
usage: {
users: { total, activeHalfyear, activeMonth },
localPosts,
localComments,
localComments: 0,
},
metadata: {
nodeName: meta.name,
@ -109,8 +105,6 @@ export class NodeinfoServerService {
name: meta.maintainerName,
email: meta.maintainerEmail,
},
gitCommit: this.config.gitCommit,
gitDescribe: this.config.gitDescribe,
langs: meta.langs,
tosUrl: meta.termsOfServiceUrl,
privacyPolicyUrl: meta.privacyPolicyUrl,

4
packages/backend/src/server/ServerModule.ts
View file

@ -47,8 +47,6 @@ import { RoleTimelineChannelService } from './api/stream/channels/role-timeline.
import { ReversiChannelService } from './api/stream/channels/reversi.js';
import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js';
import { SigninWithPasskeyApiService } from './api/SigninWithPasskeyApiService.js';
import { MetricsService } from './api/MetricsService.js';
import { ActorKeySignerService } from '@/core/ActorKeySignerService.js';
@Module({
imports: [
@ -61,7 +59,6 @@ import { ActorKeySignerService } from '@/core/ActorKeySignerService.js';
FeedService,
HealthServerService,
UrlPreviewService,
ActorKeySignerService,
ActivityPubServerService,
FileServerService,
NodeinfoServerService,
@ -97,7 +94,6 @@ import { ActorKeySignerService } from '@/core/ActorKeySignerService.js';
UserListChannelService,
OpenApiServerService,
OAuth2ProviderService,
MetricsService,
],
exports: [
ServerService,

Some files were not shown because too many files have changed in this diff Show more