From b30e68c98ccda393b3e065ec4cda593f0b7e870a Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 1 Mar 2024 22:17:02 +0900 Subject: [PATCH] feat: post search filters Co-authored-by: sup39 --- docs/api-change.md | 7 +- docs/changelog.md | 2 +- docs/downgrade.sql | 5 + locales/en-US.yml | 8 + locales/ja-JP.yml | 8 + .../1708872574733-index-alt-text-and-cw.js | 17 ++ .../src/server/api/endpoints/notes/search.ts | 61 ++++- packages/client/package.json | 1 + packages/client/src/components/MkDialog.vue | 120 --------- .../client/src/components/MkSearchBox.vue | 248 ++++++++++++++++++ .../src/components/MkSimpleTextWindow.vue | 43 +++ packages/client/src/pages/search.vue | 44 +++- packages/client/src/router.ts | 6 + packages/client/src/scripts/search.ts | 128 +++++---- pnpm-lock.yaml | 7 + 15 files changed, 531 insertions(+), 174 deletions(-) create mode 100644 packages/backend/migration/1708872574733-index-alt-text-and-cw.js create mode 100644 packages/client/src/components/MkSearchBox.vue create mode 100644 packages/client/src/components/MkSimpleTextWindow.vue diff --git a/docs/api-change.md b/docs/api-change.md index cf9799f00b..c024b00d90 100644 --- a/docs/api-change.md +++ b/docs/api-change.md @@ -4,7 +4,12 @@ Breaking changes are indicated by the :warning: icon. ## Unreleased -- `admin/search/index-all` is removed since posts are now indexed automatically. +- :warning: `admin/search/index-all` is removed since posts are now indexed automatically. +- New optional parameters are added to `notes/search` endpoint: + - `sinceDate` + - `untilDate` + - `withFiles` + - `searchCwAndAlt` ## v20240301 diff --git a/docs/changelog.md b/docs/changelog.md index 049e5e0ed2..197db641b2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,7 +4,7 @@ Critical security updates are indicated by the :warning: icon. ## Unreleased -- Introduce new full-text search engine +- Introduce new full-text search engine and post search filters ## v20240301 diff --git a/docs/downgrade.sql b/docs/downgrade.sql index b1fc95eab7..7f0921edb6 100644 --- a/docs/downgrade.sql +++ b/docs/downgrade.sql @@ -1,6 +1,7 @@ BEGIN; DELETE FROM "migrations" WHERE name IN ( + 'IndexAltTextAndCw1708872574733', 'Pgroonga1698420787202', 'ChangeDefaultConfigs1709251460718', 'AddReplyMuting1704851359889', @@ -13,6 +14,10 @@ DELETE FROM "migrations" WHERE name IN ( 'RemoveNativeUtilsMigration1705877093218' ); +-- index-alt-text-and-cw +DROP INDEX "IDX_f4f7b93d05958527300d79ac82"; +DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f"; + -- pgroonga DROP INDEX "IDX_f27f5d88941e57442be75ba9c8"; DROP INDEX "IDX_065d4d8f3b5adb4a08841eae3c"; diff --git a/locales/en-US.yml b/locales/en-US.yml index 476b340492..1c2d210512 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1181,6 +1181,14 @@ pullDownToReload: "Pull down to reload" releaseToReload: "Release to reload" reloading: "Reloading" enableTimelineStreaming: "Update timelines automatically" +searchWords: "Words to search / ID or URL to lookup" +searchWordsDescription: "To search for posts, enter the search term. Separate words with a space for an AND search, or 'OR' (without quotes) between words for an OR search.\nFor example, 'morning night' will find posts that contain both 'morning' and 'night', and 'morning OR night' will find posts that contain either 'morning' or 'night' (or both).\nYou can also combine AND/OR conditions like '(morning OR night) sleepy'.\n\nIf you want to go to a specific user page or post page, enter the ID or URL in this field and click the 'Lookup' button. Clicking 'Search' will search for posts that literally contain the ID/URL." +searchUsers: "Posted by (optional)" +searchUsersDescription: "To search for posts by a specific user/server, enter the ID (@user@example.com, or @user for a local user) or domain name (example.com).\n\nIf you enter 'me' (without quotes), all of your posts (including unlisted, followers-only, direct, and secret posts) will be searched.\n\nIf you enter 'local' (without quotes), the results will be filtered to include only posts from this server." +searchRange: "Posted within (optional)" +searchRangeDescription: "If you want to filter the time period, enter it in this format: 20220615-20231031\n\nIf you leave out the year (like 0105-0106 or 20231105-0110), it's interpreted as the current year.\n\nYou can also omit either the start or end date. For example, -0102 will filter the search results to show only posts made before 2 January this year, and 20231026- will filter the results to show only posts made after 26 October 2023." +searchPostsWithFiles: "Only posts with files" +searchCwAndAlt: "Include content warnings and file descriptions" _emojiModPerm: unauthorized: "None" diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 64626b5fe9..8ddf99aa21 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1001,6 +1001,14 @@ pullDownToReload: "下に引っ張って再読み込み" releaseToReload: "離して再読み込み" reloading: "読み込み中" enableTimelineStreaming: "タイムラインを自動で更新する" +searchWords: "検索語句・照会するIDやURL" +searchWordsDescription: "投稿を検索するには、ここに検索語句を入力してください。空白区切りでAND検索になり、ORを挟むとOR検索になります。\n例えば「朝 夜」と入力すると「朝」と「夜」が両方含まれた投稿を検索し、「朝 OR 夜」と入力すると「朝」または「夜」（または両方）が含まれた投稿を検索します。\n「(朝 OR 夜) 眠い」のように、AND検索とOR検索を同時に行うこともできます。\n\n特定のユーザーや投稿のページに飛びたい場合には、この欄にID (@user@example.com) や投稿のURLを入力し「照会」を押してください。「検索」を押すとそのIDやURLが文字通り含まれる投稿を検索します。" +searchUsers: "投稿元（オプション）" +searchUsersDescription: "投稿検索で投稿者を絞りたい場合、@user@example.com（ローカルユーザーなら @user）の形式で投稿者のIDを入力してください。ユーザーIDではなくドメイン名 (example.com) を指定すると、そのサーバーの投稿を検索します。\n\nme とだけ入力すると、自分の投稿を検索します。この検索結果には未収載・フォロワー限定・ダイレクト・秘密を含む全ての投稿が含まれます。\n\nlocal とだけ入力すると、ローカルサーバーの投稿を検索します。" +searchRange: "投稿期間（オプション）" +searchRangeDescription: "投稿検索で投稿期間を絞りたい場合、20220615-20231031 のような形式で投稿期間を入力してください。今年の日付を指定する場合には年の指定を省略できます（0105-0106 や 20231105-0110 のように）。\n\n開始日と終了日のどちらか一方は省略可能です。例えば -0102 とすると今年1月2日までの投稿のみを、20231026- とすると2023年10月26日以降の投稿のみを検索します。" +searchPostsWithFiles: "添付ファイルのある投稿のみ" +searchCwAndAlt: "閲覧注意の注釈と添付ファイルの代替テキストも検索する" _sensitiveMediaDetection: description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てられます。サーバーの負荷が少し増えます。" diff --git a/packages/backend/migration/1708872574733-index-alt-text-and-cw.js b/packages/backend/migration/1708872574733-index-alt-text-and-cw.js new file mode 100644 index 0000000000..dbe4ee3c13 --- /dev/null +++ b/packages/backend/migration/1708872574733-index-alt-text-and-cw.js @@ -0,0 +1,17 @@ +export class IndexAltTextAndCw1708872574733 { + name = "IndexAltTextAndCw1708872574733"; + + async up(queryRunner) { + await queryRunner.query( + `CREATE INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_f4f7b93d05958527300d79ac82" ON "drive_file" USING "pgroonga" ("comment" pgroonga_varchar_full_text_search_ops_v2)`, + ); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_f4f7b93d05958527300d79ac82"`); + await queryRunner.query(`DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f"`); + } +} diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 6846a7dd14..0bc70d37f9 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -1,3 +1,4 @@ +import { Brackets } from "typeorm"; import { Notes } from "@/models/index.js"; import { Note } from "@/models/entities/note.js"; import define from "@/server/api/define.js"; @@ -34,6 +35,8 @@ export const paramDef = { query: { type: "string" }, sinceId: { type: "string", format: "misskey:id" }, untilId: { type: "string", format: "misskey:id" }, + sinceDate: { type: "number", nullable: true }, + untilDate: { type: "number", nullable: true }, limit: { type: "integer", minimum: 1, maximum: 100, default: 10 }, offset: { type: "integer", default: 0 }, host: { @@ -47,6 +50,8 @@ export const paramDef = { nullable: true, default: null, }, + withFiles: { type: "boolean", nullable: true }, + searchCwAndAlt: { type: "boolean", nullable: true }, channelId: { type: "string", format: "misskey:id", @@ -68,6 +73,8 @@ export default define(meta, paramDef, async (ps, me) => { Notes.createQueryBuilder("note"), ps.sinceId, ps.untilId, + ps.sinceDate ?? undefined, + ps.untilDate ?? undefined, ); if (ps.userId != null) { @@ -80,11 +87,57 @@ export default define(meta, paramDef, async (ps, me) => { }); } + if (ps.query != null) { + const q = sqlLikeEscape(ps.query); + + if (ps.searchCwAndAlt) { + query.andWhere( + new Brackets((qb) => { + qb.where("note.text &@~ :q", { q }) + .orWhere("note.cw &@~ :q", { q }) + .orWhere( + `EXISTS ( + SELECT FROM "drive_file" + WHERE + comment &@~ :q + AND + drive_file."id" = ANY(note."fileIds") + )`, + { q }, + ); + }), + ); + } else { + query.andWhere("note.text &@~ :q", { q }); + } + } + + query.innerJoinAndSelect("note.user", "user"); + + // "from: me": search all (public, home, followers, specified) my posts + // otherwise: search public indexable posts only + if (ps.userId == null || ps.userId !== me?.id) { + query + .andWhere("note.visibility = 'public'") + .andWhere("user.isIndexable = TRUE"); + } + + if (ps.userId != null) { + query.andWhere("note.userId = :userId", { userId: ps.userId }); + } + + if (ps.host === null) { + query.andWhere("note.userHost IS NULL"); + } + if (ps.host != null) { + query.andWhere("note.userHost = :userHost", { userHost: ps.host }); + } + + if (ps.withFiles === true) { + query.andWhere("note.fileIds != '{}'"); + } + query - .andWhere("note.text &@~ :q", { q: `${sqlLikeEscape(ps.query)}` }) - .andWhere("note.visibility = 'public'") - .innerJoinAndSelect("note.user", "user") - .andWhere("user.isIndexable = TRUE") .leftJoinAndSelect("user.avatar", "avatar") .leftJoinAndSelect("user.banner", "banner") .leftJoinAndSelect("note.reply", "reply") diff --git a/packages/client/package.json b/packages/client/package.json index 93d1e68a50..9ff60dda54 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -65,6 +65,7 @@ "libopenmpt-wasm": "github:TheEssem/libopenmpt-packaging#build", "matter-js": "0.19.0", "mfm-js": "0.24.0", + "moment": "2.30.1", "photoswipe": "5.4.3", "prettier": "3.2.5", "prismjs": "1.29.0", diff --git a/packages/client/src/components/MkDialog.vue b/packages/client/src/components/MkDialog.vue index f93e164997..3c9e0e3a8c 100644 --- a/packages/client/src/components/MkDialog.vue +++ b/packages/client/src/components/MkDialog.vue @@ -101,15 +101,6 @@ " /> - { - os.selectUser().then((user) => { - inputValue.value = appendFilter( - "from:@" + acct.toString(user), - ); - }); - }, - }, - { - type: "parent", - text: i18n.ts._filters.withFile, - icon: `${iconClass("ph-paperclip")}`, - children: [ - { - text: i18n.ts.image, - icon: `${iconClass("ph-image-square")}`, - action: () => { - inputValue.value = appendFilter("has:image"); - }, - }, - { - text: i18n.ts.video, - icon: `${iconClass("ph-video-camera")}`, - action: () => { - inputValue.value = appendFilter("has:video"); - }, - }, - { - text: i18n.ts.audio, - icon: `${iconClass("ph-music-note")}`, - action: () => { - inputValue.value = appendFilter("has:audio"); - }, - }, - { - text: i18n.ts.file, - icon: `${iconClass("ph-file")}`, - action: () => { - inputValue.value = appendFilter("has:file"); - }, - }, - ], - }, - { - icon: `${iconClass("ph-link")}`, - text: i18n.ts._filters.fromDomain, - action: () => { - inputValue.value = appendFilter("domain:"); - }, - }, - { - icon: `${iconClass("ph-calendar-blank")}`, - text: i18n.ts._filters.notesBefore, - action: () => { - os.inputDate({ - title: i18n.ts._filters.notesBefore, - }).then((res) => { - if (res.canceled) return; - inputValue.value = appendFilter( - "before:" + formatDateToYYYYMMDD(res.result), - ); - }); - }, - }, - { - icon: `${iconClass("ph-calendar-blank")}`, - text: i18n.ts._filters.notesAfter, - action: () => { - os.inputDate({ - title: i18n.ts._filters.notesAfter, - }).then((res) => { - if (res.canceled) return; - inputValue.value = appendFilter( - "after:" + formatDateToYYYYMMDD(res.result), - ); - }); - }, - }, - { - icon: `${iconClass("ph-eye")}`, - text: i18n.ts._filters.followingOnly, - action: () => { - inputValue.value = appendFilter("filter:following"); - }, - }, - { - icon: `${iconClass("ph-users-three")}`, - text: i18n.ts._filters.followersOnly, - action: () => { - inputValue.value = appendFilter("filter:followers"); - }, - }, - ], - ev.target, - { noReturnFocus: true }, - ); - inputEl.value?.focus(); - if (typeof inputValue.value === "string") { - inputEl.value?.selectRange( - inputValue.value.length, - inputValue.value.length, - ); // cursor at end - } -} - onMounted(() => { document.addEventListener("keydown", onKeydown); }); diff --git a/packages/client/src/components/MkSearchBox.vue b/packages/client/src/components/MkSearchBox.vue new file mode 100644 index 0000000000..0a1e7315af --- /dev/null +++ b/packages/client/src/components/MkSearchBox.vue @@ -0,0 +1,248 @@ + + + + + diff --git a/packages/client/src/components/MkSimpleTextWindow.vue b/packages/client/src/components/MkSimpleTextWindow.vue new file mode 100644 index 0000000000..e8766476e7 --- /dev/null +++ b/packages/client/src/components/MkSimpleTextWindow.vue @@ -0,0 +1,43 @@ + + + + + diff --git a/packages/client/src/pages/search.vue b/packages/client/src/pages/search.vue index 61f211bcf3..5967b4e545 100644 --- a/packages/client/src/pages/search.vue +++ b/packages/client/src/pages/search.vue @@ -43,6 +43,7 @@ import { computed, onMounted, ref, watch } from "vue"; import { Virtual } from "swiper/modules"; import { Swiper, SwiperSlide } from "swiper/vue"; +import moment from "moment"; import XNotes from "@/components/MkNotes.vue"; import XUserList from "@/components/MkUserList.vue"; import { i18n } from "@/i18n"; @@ -50,19 +51,37 @@ import { definePageMetadata } from "@/scripts/page-metadata"; import { defaultStore } from "@/store"; import { deviceKind } from "@/scripts/device-kind"; import icon from "@/scripts/icon"; +import { $i } from "@/reactiveAccount"; import "swiper/scss"; import "swiper/scss/virtual"; +import { api } from "@/os"; const props = defineProps<{ - query: string; + query?: string; + user?: string; + host?: string; + since?: string; + until?: string; channel?: string; + withFiles: "0" | "1"; + searchCwAndAlt: "0" | "1"; }>(); +const userId = props.user == null ? undefined : await getUserId(props.user); + const notesPagination = { endpoint: "notes/search" as const, limit: 10, params: computed(() => ({ - query: props.query, + query: props.query ?? undefined, + userId, + host: props.host == null ? undefined : getHost(props.host), + sinceDate: + props.since == null ? undefined : getUnixTime(props.since, false), + untilDate: + props.until == null ? undefined : getUnixTime(props.until, true), + withFiles: props.withFiles === "1", + searchCwAndAlt: props.searchCwAndAlt === "1", channelId: props.channel, })), }; @@ -76,6 +95,27 @@ const usersPagination = { })), }; +async function getUserId(user: string): Promise { + if (user === "me") return $i!.id; + + const split = (user.startsWith("@") ? user.slice(1) : user).split("@"); + const username = split[0]; + const host = split.length === 1 ? undefined : split[1]; + + return (await api("users/show", { username, host })).id; +} + +function getHost(host: string): string | null { + if (host === "local") return null; + return host; +} + +function getUnixTime(date: string, nextDay: boolean): number { + return moment(date, date.length === 4 ? "MMDD" : "YYYYMMDD") + .add(nextDay ? 1 : 0, "days") + .valueOf(); +} + const tabs = ["notes", "users"]; const tab = ref(tabs[0]); watch(tab, () => syncSlide(tabs.indexOf(tab.value))); diff --git a/packages/client/src/router.ts b/packages/client/src/router.ts index 3dae7e16bd..deca773d60 100644 --- a/packages/client/src/router.ts +++ b/packages/client/src/router.ts @@ -306,7 +306,13 @@ export const routes = [ component: page(() => import("./pages/search.vue")), query: { q: "query", + user: "user", + host: "host", + since: "since", + until: "until", + withFiles: "withFiles", channel: "channel", + detailed: "searchCwAndAlt", }, }, { diff --git a/packages/client/src/scripts/search.ts b/packages/client/src/scripts/search.ts index 7b75771358..552b87f7b8 100644 --- a/packages/client/src/scripts/search.ts +++ b/packages/client/src/scripts/search.ts @@ -1,65 +1,101 @@ -import * as os from "@/os"; import { i18n } from "@/i18n"; +import { api, popup, promiseDialog } from "@/os"; import { mainRouter } from "@/router"; -import { instance } from "@/instance"; +import MkSearchBox from "@/components/MkSearchBox.vue"; export async function search() { - const { canceled, result: query } = await os.inputText({ - type: instance.features.searchFilters ? "search" : "text", - title: i18n.ts.search, - placeholder: i18n.ts.searchPlaceholder, + const { canceled, result } = await new Promise< + | { canceled: true; result: undefined } + | { + canceled: false; + result: { + action: "lookup"; + query: string; + }; + } + | { + canceled: false; + result: { + action: "search"; + query?: string; + from?: string; + range?: string; + withFiles: boolean; + searchCwAndAlt: boolean; + }; + } + >((resolve, _) => { + popup( + MkSearchBox, + {}, + { + done: (result) => { + resolve(result ?? { canceled: true }); + }, + }, + "closed", + ); }); - if (canceled || query == null || query === "") return; - const q = query.trim(); + if (canceled || result == null || result.query === "") return; - if (q.startsWith("@") && !q.includes(" ")) { - mainRouter.push(`/${q}`); - return; - } + if (result.action === "lookup") { + if (result.query.startsWith("#")) { + mainRouter.push(`/tags/${encodeURIComponent(result.query.slice(1))}`); + return; + } + if (result.query.startsWith("@")) { + mainRouter.push(`/${result.query}`); + return; + } + if (result.query.startsWith("https://")) { + const promise = api("ap/show", { + uri: result.query, + }); - if (q.startsWith("#")) { - mainRouter.push(`/tags/${encodeURIComponent(q.slice(1))}`); - return; - } + promiseDialog(promise, null, null, i18n.ts.fetchingAsApObject); + const res = await promise; - // like 2018/03/12 - if (/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}/.test(q.replace(/-/g, "/"))) { - const date = new Date(q.replace(/-/g, "/")); + if (res.type === "User") { + mainRouter.push(`/@${res.object.username}@${res.object.host}`); + } else if (res.type === "Note") { + mainRouter.push(`/notes/${res.object.id}`); + } - // 日付しか指定されてない場合、例えば 2018/03/12 ならユーザーは - // 2018/03/12 のコンテンツを「含む」結果になることを期待するはずなので - // 23時間59分進める(そのままだと 2018/03/12 00:00:00 「まで」の - // 結果になってしまい、2018/03/12 のコンテンツは含まれない) - if (q.replace(/-/g, "/").match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$/)) { - date.setHours(23, 59, 59, 999); + return; } - // TODO - // v.$root.$emit('warp', date); - os.alert({ - type: "waiting", - }); - return; + // fallback + mainRouter.push(`/search?q=${encodeURIComponent(result.query)}`); } - if (q.startsWith("https://")) { - const promise = os.api("ap/show", { - uri: q, - }); + if (result.action === "search") { + const params = new URLSearchParams(); - os.promiseDialog(promise, null, null, i18n.ts.fetchingAsApObject); - - const res = await promise; - - if (res.type === "User") { - mainRouter.push(`/@${res.object.username}@${res.object.host}`); - } else if (res.type === "Note") { - mainRouter.push(`/notes/${res.object.id}`); + if (result.query != null) { + params.append("q", result.query); } - return; - } + if (result.from != null) { + if (result.from === "me" || result.from.includes("@")) + params.append("user", result.from); + else params.append("host", result.from); + } - mainRouter.push(`/search?q=${encodeURIComponent(q)}`); + if (result.range != null) { + const split = result.range.split("-"); + if (split.length === 1) { + params.append("since", result.range); + params.append("until", result.range); + } else { + if (split[0] !== "") params.append("since", split[0]); + if (split[1] !== "") params.append("until", split[1]); + } + } + + params.append("detailed", result.searchCwAndAlt ? "1" : "0"); + params.append("withFiles", result.withFiles ? "1" : "0"); + + mainRouter.push(`/search?${params.toString()}`); + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3715d96ad7..8313fdaeb3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -731,6 +731,9 @@ importers: mfm-js: specifier: 0.24.0 version: 0.24.0 + moment: + specifier: 2.30.1 + version: 2.30.1 photoswipe: specifier: 5.4.3 version: 5.4.3 @@ -12809,6 +12812,10 @@ packages: resolution: {integrity: sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==} dev: false + /moment@2.30.1: + resolution: {integrity: sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==} + dev: true + /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} dev: false