From 514e7683c0913c6c2bf7b6af5b24e97a326c4237 Mon Sep 17 00:00:00 2001 From: fly_mc Date: Sun, 29 Sep 2024 01:07:46 +0800 Subject: [PATCH 1/5] use PGroonga --- .../migration/1727542814599-Pgroonga.js | 18 ++++++++++++++++++ packages/backend/src/core/SearchService.ts | 2 +- packages/backend/src/models/Note.ts | 2 ++ packages/backend/src/models/User.ts | 1 + packages/backend/src/models/UserProfile.ts | 1 + .../src/server/api/endpoints/notes/search.ts | 6 ++++++ .../src/server/api/endpoints/users/search.ts | 8 ++++---- 7 files changed, 33 insertions(+), 5 deletions(-) create mode 100644 packages/backend/migration/1727542814599-Pgroonga.js diff --git a/packages/backend/migration/1727542814599-Pgroonga.js b/packages/backend/migration/1727542814599-Pgroonga.js new file mode 100644 index 0000000000..1a0dab1104 --- /dev/null +++ b/packages/backend/migration/1727542814599-Pgroonga.js @@ -0,0 +1,18 @@ +export class Pgroonga1727542814599 { + name = 'Pgroonga1727542814599' + + async up(queryRunner) { + await queryRunner.query(`CREATE INDEX "IDX_f27f5d88941e57442be75ba9c8" ON "note" USING "pgroonga" ("text")`); + await queryRunner.query(`CREATE INDEX "IDX_7cc8d9b0ee7861b4e5dc86ad85" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2)`); + await queryRunner.query(`CREATE INDEX "IDX_065d4d8f3b5adb4a08841eae3c" ON "user" USING "pgroonga" ("name" pgroonga_varchar_full_text_search_ops_v2)`); + await queryRunner.query(`CREATE INDEX "IDX_fcb770976ff8240af5799e3ffc" ON "user_profile" USING "pgroonga" ("description" pgroonga_varchar_full_text_search_ops_v2) `); + + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "public"."IDX_f27f5d88941e57442be75ba9c8"`); + await queryRunner.query(`DROP INDEX "public"."IDX_7cc8d9b0ee7861b4e5dc86ad85"`); + await queryRunner.query(`DROP INDEX "public"."IDX_065d4d8f3b5adb4a08841eae3c"`); + await queryRunner.query(`DROP INDEX "public"."IDX_fcb770976ff8240af5799e3ffc"`); + } +} \ No newline at end of file diff --git a/packages/backend/src/core/SearchService.ts b/packages/backend/src/core/SearchService.ts index edfc470375..42a32220db 100644 --- a/packages/backend/src/core/SearchService.ts +++ b/packages/backend/src/core/SearchService.ts @@ -215,7 +215,7 @@ export class SearchService { } query - .andWhere('note.text ILIKE :q', { q: `%${ sqlLikeEscape(q) }%` }) + .andWhere('note.text &@~ :q', { q: sqlLikeEscape(q) }) .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index 9a95c6faab..1de356678b 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -50,6 +50,7 @@ export class MiNote { public threadId: string | null; // TODO: varcharにしたい + @Index() // USING pgroonga @Column('text', { nullable: true, }) @@ -60,6 +61,7 @@ export class MiNote { }) public name: string | null; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column('varchar', { length: 512, nullable: true, }) diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 96de30c4c2..5328131b93 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -49,6 +49,7 @@ export class MiUser { }) public usernameLower: string; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column('varchar', { length: 128, nullable: true, comment: 'The name of the User.', diff --git a/packages/backend/src/models/UserProfile.ts b/packages/backend/src/models/UserProfile.ts index 5544555296..5f1a7a7d12 100644 --- a/packages/backend/src/models/UserProfile.ts +++ b/packages/backend/src/models/UserProfile.ts @@ -36,6 +36,7 @@ export class MiUserProfile { }) public birthday: string | null; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column('varchar', { length: 2048, nullable: true, comment: 'The description (bio) of the User.', diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 3fe19806e3..a26c49c946 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -31,6 +31,11 @@ export const meta = { code: 'UNAVAILABLE', id: '0b44998d-77aa-4427-80d0-d2c9b8523011', }, + noSuchNote: { + message: 'Query is empty.', + code: 'QUERY_IS_EMPTY', + id: 'd0410b51-f409-4667-8118-cfe999e453c3', + }, }, } as const; @@ -62,6 +67,7 @@ export default class extends Endpoint { // eslint- private roleService: RoleService, ) { super(meta, paramDef, async (ps, me) => { + if (ps.query.trim().length === 0) throw new ApiError(meta.errors.noSuchNote); const policies = await this.roleService.getUserPolicies(me ? me.id : null); if (!policies.canSearchNotes) { throw new ApiError(meta.errors.unavailable); diff --git a/packages/backend/src/server/api/endpoints/users/search.ts b/packages/backend/src/server/api/endpoints/users/search.ts index 0b0136066d..6c686a8eb7 100644 --- a/packages/backend/src/server/api/endpoints/users/search.ts +++ b/packages/backend/src/server/api/endpoints/users/search.ts @@ -63,12 +63,12 @@ export default class extends Endpoint { // eslint- const nameQuery = this.usersRepository.createQueryBuilder('user') .where(new Brackets(qb => { - qb.where('user.name ILIKE :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }); + qb.where('user.name &@~ :query', { query: ps.query }); if (isUsername) { qb.orWhere('user.usernameLower LIKE :username', { username: sqlLikeEscape(ps.query.replace('@', '').toLowerCase()) + '%' }); - } else if (this.userEntityService.validateLocalUsername(ps.query)) { // Also search username if it qualifies as username - qb.orWhere('user.usernameLower LIKE :username', { username: '%' + sqlLikeEscape(ps.query.toLowerCase()) + '%' }); + } else if (userEntityService.validateLocalUsername(ps.query)) { // Also search username if it qualifies as username + qb.orWhere('user.usernameLower LIKE :username', { username: ps.query.toLowerCase() + '%' }); } })) .andWhere(new Brackets(qb => { @@ -93,7 +93,7 @@ export default class extends Endpoint { // eslint- if (users.length < ps.limit) { const profQuery = this.userProfilesRepository.createQueryBuilder('prof') .select('prof.userId') - .where('prof.description ILIKE :query', { query: '%' + sqlLikeEscape(ps.query) + '%' }); + .where('prof.description &@~ :query', { query: ps.query }); if (ps.origin === 'local') { profQuery.andWhere('prof.userHost IS NULL'); -- 2.45.2 From 6411e65989cb120f19bf8ad06f078cc0052763b4 Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Wed, 6 Nov 2024 18:06:43 -0600 Subject: [PATCH 2/5] bump Pgroonga migration timestamp Signed-off-by: eternal-flame-AD --- .../{1727542814599-Pgroonga.js => 1730937958242-Pgroonga.js} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename packages/backend/migration/{1727542814599-Pgroonga.js => 1730937958242-Pgroonga.js} (93%) diff --git a/packages/backend/migration/1727542814599-Pgroonga.js b/packages/backend/migration/1730937958242-Pgroonga.js similarity index 93% rename from packages/backend/migration/1727542814599-Pgroonga.js rename to packages/backend/migration/1730937958242-Pgroonga.js index 1a0dab1104..263473c56c 100644 --- a/packages/backend/migration/1727542814599-Pgroonga.js +++ b/packages/backend/migration/1730937958242-Pgroonga.js @@ -1,5 +1,5 @@ -export class Pgroonga1727542814599 { - name = 'Pgroonga1727542814599' +export class Pgroonga1730937958242 { + name = 'Pgroonga1730937958242' async up(queryRunner) { await queryRunner.query(`CREATE INDEX "IDX_f27f5d88941e57442be75ba9c8" ON "note" USING "pgroonga" ("text")`); -- 2.45.2 From eb4ccbb66255c388c61350d4a1921458f77a665f Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Wed, 6 Nov 2024 18:09:41 -0600 Subject: [PATCH 3/5] update docker image Signed-off-by: eternal-flame-AD --- compose_example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose_example.yml b/compose_example.yml index 33c4a78760..db924f90c3 100644 --- a/compose_example.yml +++ b/compose_example.yml @@ -62,7 +62,7 @@ services: db: restart: always - image: postgres:15-alpine + image: l1drm/postgres-pgroonga:alpine-17-znver4 user: "${MISSKEY_UID}:${MISSKEY_GID}" networks: - internal_network -- 2.45.2 From 7f071473b98b2a7cabc9fbd4d7f5789141b24560 Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Wed, 6 Nov 2024 18:26:33 -0600 Subject: [PATCH 4/5] set CREATE INDEX CONCURRENTLY Signed-off-by: eternal-flame-AD --- packages/backend/migration/1730937958242-Pgroonga.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/backend/migration/1730937958242-Pgroonga.js b/packages/backend/migration/1730937958242-Pgroonga.js index 263473c56c..7029306e44 100644 --- a/packages/backend/migration/1730937958242-Pgroonga.js +++ b/packages/backend/migration/1730937958242-Pgroonga.js @@ -2,10 +2,10 @@ export class Pgroonga1730937958242 { name = 'Pgroonga1730937958242' async up(queryRunner) { - await queryRunner.query(`CREATE INDEX "IDX_f27f5d88941e57442be75ba9c8" ON "note" USING "pgroonga" ("text")`); - await queryRunner.query(`CREATE INDEX "IDX_7cc8d9b0ee7861b4e5dc86ad85" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2)`); - await queryRunner.query(`CREATE INDEX "IDX_065d4d8f3b5adb4a08841eae3c" ON "user" USING "pgroonga" ("name" pgroonga_varchar_full_text_search_ops_v2)`); - await queryRunner.query(`CREATE INDEX "IDX_fcb770976ff8240af5799e3ffc" ON "user_profile" USING "pgroonga" ("description" pgroonga_varchar_full_text_search_ops_v2) `); + await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_f27f5d88941e57442be75ba9c8" ON "note" USING "pgroonga" ("text")`); + await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_7cc8d9b0ee7861b4e5dc86ad85" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2)`); + await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_065d4d8f3b5adb4a08841eae3c" ON "user" USING "pgroonga" ("name" pgroonga_varchar_full_text_search_ops_v2)`); + await queryRunner.query(`CREATE INDEX CONCURRENTLY "IDX_fcb770976ff8240af5799e3ffc" ON "user_profile" USING "pgroonga" ("description" pgroonga_varchar_full_text_search_ops_v2) `); } -- 2.45.2 From 6dbc02af9a178836d84ef18279f30e3ca04dc063 Mon Sep 17 00:00:00 2001 From: eternal-flame-AD Date: Wed, 6 Nov 2024 18:27:55 -0600 Subject: [PATCH 5/5] fixup! update docker image --- compose.local-db.yml | 2 +- compose_example.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose.local-db.yml b/compose.local-db.yml index ff202ae3e5..e7db6a3e72 100644 --- a/compose.local-db.yml +++ b/compose.local-db.yml @@ -15,7 +15,7 @@ services: db: restart: always - image: postgres:15-alpine + image: l1drm/postgres-pgroonga:alpine-15-znver4 ports: - "5432:5432" env_file: diff --git a/compose_example.yml b/compose_example.yml index db924f90c3..07e273762c 100644 --- a/compose_example.yml +++ b/compose_example.yml @@ -62,7 +62,7 @@ services: db: restart: always - image: l1drm/postgres-pgroonga:alpine-17-znver4 + image: l1drm/postgres-pgroonga:alpine-15-znver4 user: "${MISSKEY_UID}:${MISSKEY_GID}" networks: - internal_network -- 2.45.2