From 48e5d9de718b037de82492c9a82e626a71c6a853 Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 1 Mar 2024 21:39:13 +0900 Subject: [PATCH 001/191] refactor: use PGroonga for full-text search and remove support of other engines Co-authored-by: tamaina Co-authored-by: sup39 --- .config/example.yml | 22 - README.md | 4 - dev/docker-compose.yml | 4 +- dev/install.sql | 1 + docker-compose.example.yml | 32 +- docs/api-change.md | 4 + docs/changelog.md | 4 + docs/downgrade.sql | 7 + docs/notice-for-admins.md | 94 ++++ locales/ca-ES.yml | 5 - locales/de-DE.yml | 8 - locales/en-US.yml | 8 - locales/es-ES.yml | 8 - locales/fr-FR.yml | 8 - locales/id-ID.yml | 8 - locales/it-IT.yml | 8 - locales/ja-JP.yml | 7 - locales/ko-KR.yml | 7 - locales/no-NO.yml | 8 - locales/pl-PL.yml | 5 - locales/ru-RU.yml | 8 - locales/th-TH.yml | 4 - locales/tr-TR.yml | 8 - locales/uk-UA.yml | 8 - locales/vi-VN.yml | 8 - locales/zh-CN.yml | 7 - locales/zh-TW.yml | 7 - .../migration/1698420787202-pgroonga.js | 21 + packages/backend/package.json | 3 - packages/backend/src/config/types.ts | 21 - packages/backend/src/daemons/server-stats.ts | 16 - packages/backend/src/db/elasticsearch.ts | 65 --- packages/backend/src/db/meilisearch.ts | 451 ------------------ packages/backend/src/db/sonic.ts | 51 -- .../backend/src/models/entities/drive-file.ts | 1 + packages/backend/src/models/entities/note.ts | 2 + .../src/models/entities/user-profile.ts | 1 + packages/backend/src/models/entities/user.ts | 1 + packages/backend/src/queue/index.ts | 10 - .../processors/background/index-all-notes.ts | 89 ---- .../src/queue/processors/background/index.ts | 12 - .../src/queue/processors/db/delete-account.ts | 4 - packages/backend/src/server/api/endpoints.ts | 2 - .../api/endpoints/admin/search/index-all.ts | 28 -- .../src/server/api/endpoints/notes/edit.ts | 3 - .../src/server/api/endpoints/notes/search.ts | 318 ++---------- .../src/server/api/endpoints/server-info.ts | 13 - .../src/server/api/endpoints/users/search.ts | 8 +- packages/backend/src/server/nodeinfo.ts | 1 - packages/backend/src/services/note/create.ts | 43 -- packages/backend/src/services/note/delete.ts | 5 - packages/client/src/pages/admin/database.vue | 4 - packages/client/src/pages/admin/index.vue | 11 - .../src/pages/admin/overview.metrics.vue | 29 -- packages/client/src/scripts/index-posts.ts | 26 - .../src/widgets/server-metric/index.vue | 8 - .../src/widgets/server-metric/meilisearch.vue | 86 ---- pnpm-lock.yaml | 51 +- 58 files changed, 182 insertions(+), 1504 deletions(-) create mode 100644 dev/install.sql create mode 100644 packages/backend/migration/1698420787202-pgroonga.js delete mode 100644 packages/backend/src/db/elasticsearch.ts delete mode 100644 packages/backend/src/db/meilisearch.ts delete mode 100644 packages/backend/src/db/sonic.ts delete mode 100644 packages/backend/src/queue/processors/background/index-all-notes.ts delete mode 100644 packages/backend/src/queue/processors/background/index.ts delete mode 100644 packages/backend/src/server/api/endpoints/admin/search/index-all.ts delete mode 100644 packages/client/src/scripts/index-posts.ts delete mode 100644 packages/client/src/widgets/server-metric/meilisearch.vue diff --git a/.config/example.yml b/.config/example.yml index 02c0296172..43f6d7e923 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -85,28 +85,6 @@ redis: #prefix: example-prefix #db: 1 -# Please configure either MeiliSearch *or* Sonic. -# If both MeiliSearch and Sonic configurations are present, MeiliSearch will take precedence. - -# ┌───────────────────────────┐ -#───┘ MeiliSearch configuration └───────────────────────────────────── -#meilisearch: -# host: meilisearch -# port: 7700 -# ssl: false -# apiKey: - -# ┌─────────────────────┐ -#───┘ Sonic configuration └───────────────────────────────────── - -#sonic: -# host: localhost -# port: 1491 -# auth: SecretPassword -# collection: notes -# bucket: default - - # ┌───────────────┐ #───┘ ID generation └─────────────────────────────────────────── diff --git a/README.md b/README.md index b7ba3cc6fe..6edbd5efbd 100644 --- a/README.md +++ b/README.md @@ -59,10 +59,6 @@ If you have access to a server that supports one of the sources below, I recomme ### Optional dependencies - [FFmpeg](https://ffmpeg.org/) for video transcoding -- Full text search (one of the following) - - [Sonic](https://crates.io/crates/sonic-server) - - [MeiliSearch](https://www.meilisearch.com/) - - [ElasticSearch](https://www.elastic.co/elasticsearch/) - Caching server (one of the following) - [DragonflyDB](https://www.dragonflydb.io/) (recommended) - [KeyDB](https://keydb.dev/) diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 184c0366c9..d1037fd593 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -6,10 +6,12 @@ services: ports: - "26379:6379" db: - image: docker.io/postgres:16-alpine + image: docker.io/groonga/pgroonga:latest-alpine-16-slim environment: - "POSTGRES_PASSWORD=password" - "POSTGRES_USER=firefish" - "POSTGRES_DB=firefish_db" ports: - "25432:5432" + volumes: + - "./install.sql:/docker-entrypoint-initdb.d/install.sql:ro" diff --git a/dev/install.sql b/dev/install.sql new file mode 100644 index 0000000000..11418be32b --- /dev/null +++ b/dev/install.sql @@ -0,0 +1 @@ +CREATE EXTENSION pgroonga; diff --git a/docker-compose.example.yml b/docker-compose.example.yml index a5fd4c315e..c2c60786d9 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -8,9 +8,6 @@ services: depends_on: - db - redis -### Uncomment one of the following to use a search engine -# - meilisearch -# - sonic ports: - "3000:3000" networks: @@ -34,7 +31,7 @@ services: db: restart: unless-stopped - image: docker.io/postgres:16-alpine + image: docker.io/groonga/pgroonga:latest-alpine-16-slim container_name: firefish_db networks: - calcnet @@ -43,33 +40,6 @@ services: volumes: - ./db:/var/lib/postgresql/data -### Only one of the below should be used. -### Meilisearch is better overall, but resource-intensive. Sonic is a very light full text search engine. - -# meilisearch: -# container_name: meilisearch -# image: getmeili/meilisearch:v1.1.1 -# environment: -# - MEILI_ENV=${MEILI_ENV:-development} -# ports: -# - "7700:7700" -# networks: -# - calcnet -# volumes: -# - ./meili_data:/meili_data -# restart: unless-stopped - -# sonic: -# restart: unless-stopped -# image: docker.io/valeriansaliou/sonic:v1.4.0 -# logging: -# driver: none -# networks: -# - calcnet -# volumes: -# - ./sonic:/var/lib/sonic/store -# - ./sonic/config.cfg:/etc/sonic.cfg - networks: calcnet: # web: diff --git a/docs/api-change.md b/docs/api-change.md index 4046e5c48c..cf9799f00b 100644 --- a/docs/api-change.md +++ b/docs/api-change.md @@ -2,6 +2,10 @@ Breaking changes are indicated by the :warning: icon. +## Unreleased + +- `admin/search/index-all` is removed since posts are now indexed automatically. + ## v20240301 - With the addition of new features, the following endpoints are added: diff --git a/docs/changelog.md b/docs/changelog.md index 71ba30951e..049e5e0ed2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ Critical security updates are indicated by the :warning: icon. +## Unreleased + +- Introduce new full-text search engine + ## v20240301 - Add a page (`/my/follow-requests/sent`) to check your follow requests that haven't been approved diff --git a/docs/downgrade.sql b/docs/downgrade.sql index 8a7769dcb9..b1fc95eab7 100644 --- a/docs/downgrade.sql +++ b/docs/downgrade.sql @@ -1,6 +1,7 @@ BEGIN; DELETE FROM "migrations" WHERE name IN ( + 'Pgroonga1698420787202', 'ChangeDefaultConfigs1709251460718', 'AddReplyMuting1704851359889', 'FixNoteUrlIndex1709129810501', @@ -12,6 +13,12 @@ DELETE FROM "migrations" WHERE name IN ( 'RemoveNativeUtilsMigration1705877093218' ); +-- pgroonga +DROP INDEX "IDX_f27f5d88941e57442be75ba9c8"; +DROP INDEX "IDX_065d4d8f3b5adb4a08841eae3c"; +DROP INDEX "IDX_fcb770976ff8240af5799e3ffc"; +DROP EXTENSION pgroonga CASCADE; + -- change-default-configs ALTER TABLE "user_profile" ALTER COLUMN "noCrawle" SET DEFAULT false; ALTER TABLE "user_profile" ALTER COLUMN "publicReactions" SET DEFAULT false; diff --git a/docs/notice-for-admins.md b/docs/notice-for-admins.md index 1edb5ecca2..276ffea7b8 100644 --- a/docs/notice-for-admins.md +++ b/docs/notice-for-admins.md @@ -1,3 +1,97 @@ +# Unreleased + +The full-text search engine in Firefish has been changed to [PGroonga](https://pgroonga.github.io/). This is no longer an optional feature, so please enable PGroonga on your system. If you are using Sonic, Meilisearch, or Elasticsearch, you can also uninstall it from your system and remove the settings from `.config/default.yml`. + +## For systemd/pm2 users + +### 1. Install PGroonga + +Please execute `psql --version` to check your PostgreSQL major version. This will print a message like this: + +```text +psql (PostgreSQL) 16.1 +``` + +In this case, your PostgreSQL major version is `16`. + +There are official installation instructions for many operating systems on , so please follow the instructions on this page. However, since many users are using Ubuntu, and there are no instructions for Arch Linux, we explicitly list the instructions for Ubuntu and Arch Linux here. Please keep in mind that this is not official information and the procedures may change. + +#### Ubuntu + +1. Add apt repository + ```sh + sudo apt install -y software-properties-common + sudo add-apt-repository -y universe + sudo add-apt-repository -y ppa:groonga/ppa + sudo apt install -y wget lsb-release + wget https://packages.groonga.org/ubuntu/groonga-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./groonga-apt-source-latest-$(lsb_release --codename --short).deb + echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release --codename --short)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + sudo apt update + ``` +2. Install PGroonga + ```sh + # Please replace "16" with your PostgreSQL major version + sudo apt install postgresql-16-pgdg-pgroonga + ``` + +#### Arch Linux + +You can install PGroonga from the Arch User Repository. + +```sh +git clone https://aur.archlinux.org/pgroonga.git && cd pgroonga && makepkg -si +# or paru -S pgroonga +# or yay -S pgroonga +``` + +### 2. Enable PGroonga + +After the instllation, please execute this command to enable PGroonga: + +```sh +sudo --user=postgres psql --dbname=your_database_name --command='CREATE EXTENSION pgroonga;' +``` + +The database name can be found in `.config/default.yml`. +```yaml +db: + port: 5432 + db: database_name # substitute your_database_name with this + user: firefish + pass: password +``` + +## For Docker/Podman users + +Please edit your `docker-compose.yml` to replace the database container image from `docker.io/postgres` to `docker.io/groonga/pgroonga`. + +Please make sure to use the same PostgreSQL version. If you are using `docker.io/postgres:16-alpine` (PostgreSQL v16), the corresponding image tag is `docker.io/groonga/pgroonga:latest-alpine-16` (or `docker.io/groonga/pgroonga:latest-alpine-16-slim`). + +The list of tags can be found on . + +```yaml +db: + restart: unless-stopped + image: docker.io/groonga/pgroonga:latest-alpine-16-slim # change here + container_name: firefish_db +``` + +After that, execute this command to enable PGroonga: + +```sh +docker-compose up db --detach && docker-compose exec db sh -c 'psql --user="${POSTGRES_USER}" --dbname="${POSTGRES_DB}" --command="CREATE EXTENSION pgroonga;"' +# or podman-compose up db --detach && podman-compose exec db sh -c 'psql --user="${POSTGRES_USER}" --dbname="${POSTGRES_DB}" --command="CREATE EXTENSION pgroonga;"' +``` + +Once this is done, you can start Firefish as usual. + +```sh +docker pull registry.firefish.dev/firefish/firefish && docker-compose up --detach +# or podman pull registry.firefish.dev/firefish/firefish && podman-compose up --detach +``` + # v20240301 ## For all users diff --git a/locales/ca-ES.yml b/locales/ca-ES.yml index 7b66b3bd98..f0b3a9343d 100644 --- a/locales/ca-ES.yml +++ b/locales/ca-ES.yml @@ -1831,11 +1831,6 @@ pushNotificationAlreadySubscribed: Les notificacions push ja estan activades pushNotificationNotSupported: El vostre navegador o servidor no admet notificacions push license: Llicència -indexPosts: Índex de publicacions -indexFrom: Índex a partir de l'ID de Publicacions -indexFromDescription: Deixeu en blanc per indexar cada publicació -indexNotice: Ara indexant. Això probablement trigarà una estona, si us plau, no reinicieu - el servidor durant almenys una hora. _instanceTicker: none: No mostrar mai remote: Mostra per a usuaris remots diff --git a/locales/de-DE.yml b/locales/de-DE.yml index 53024bf845..42228695a4 100644 --- a/locales/de-DE.yml +++ b/locales/de-DE.yml @@ -1500,9 +1500,6 @@ _widgets: chooseList: Wählen Sie eine Liste aus userList: Benutzerliste serverInfo: Server-Infos - meiliStatus: Server-Status - meiliSize: Indexgröße - meiliIndexCount: Indexierte Beiträge _cw: hide: "Verbergen" show: "Inhalt anzeigen" @@ -2083,7 +2080,6 @@ preventAiLearning: KI gestütztes bot-scraping unterdrücken preventAiLearningDescription: Fordern Sie KI-Sprachmodelle von Drittanbietern auf, die von Ihnen hochgeladenen Inhalte, wie z. B. Beiträge und Bilder, nicht zu untersuchen. license: Lizenz -indexPosts: Gelistete Beiträge migrationConfirm: "Sind Sie absolut sicher, dass Sie Ihr Nutzerkonto zu diesem {account} umziehen möchten? Sobald Sie dies bestätigt haben, kann dies nicht mehr rückgängig gemacht werden und Ihr Nutzerkonto kann nicht mehr von ihnen genutzt werden.\nStellen @@ -2111,9 +2107,6 @@ _experiments: kann es zu Verlangsamungen beim Laden während des Imports kommen. noGraze: Bitte deaktivieren Sie die Browsererweiterung "Graze for Mastodon", da sie die Funktion von Firefish stört. -indexFrom: Indexieren ab Beitragskennung aufwärts -indexNotice: Wird jetzt indexiert. Dies wird wahrscheinlich eine Weile dauern, bitte - starten Sie Ihren Server für mindestens eine Stunde nicht neu. customKaTeXMacroDescription: "Richten Sie Makros ein, um mathematische Ausdrücke einfach zu schreiben! Die Notation entspricht den LaTeX-Befehlsdefinitionen und wird als \\newcommand{\\name}{content} oder \\newcommand{\\name}[number of arguments]{content} @@ -2132,7 +2125,6 @@ expandOnNoteClick: Beitrag bei Klick öffnen image: Bild video: Video audio: Audio -indexFromDescription: Leer lassen, um jeden Beitrag zu indexieren _filters: fromUser: Von Benutzer notesAfter: Beiträge nach diff --git a/locales/en-US.yml b/locales/en-US.yml index 0c40a340fb..476b340492 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1092,11 +1092,6 @@ migrationConfirm: "Are you absolutely sure you want to migrate your account to { as the account you're moving from." defaultReaction: "Default emoji reaction for outgoing and incoming posts" license: "License" -indexPosts: "Index Posts" -indexFrom: "Index from Post ID onwards" -indexFromDescription: "Leave blank to index every post" -indexNotice: "Now indexing. This will probably take a while, please don't restart - your server for at least an hour." customKaTeXMacro: "Custom KaTeX macros" customKaTeXMacroDescription: "Set up macros to write mathematical expressions easily! The notation conforms to the LaTeX command definitions and is written as \\newcommand{\\ @@ -1690,9 +1685,6 @@ _widgets: serverInfo: "Server Info" _userList: chooseList: "Select a list" - meiliStatus: "Server Status" - meiliSize: "Index size" - meiliIndexCount: "Indexed posts" _cw: hide: "Hide" diff --git a/locales/es-ES.yml b/locales/es-ES.yml index 328e53a10b..b0b1fc28e1 100644 --- a/locales/es-ES.yml +++ b/locales/es-ES.yml @@ -1474,9 +1474,6 @@ _widgets: _userList: chooseList: Seleccione una lista serverInfo: Información del servidor - meiliStatus: Estado del servidor - meiliSize: Tamaño del índice - meiliIndexCount: Publicaciones indizadas _cw: hide: "Ocultar" show: "Ver más" @@ -2123,13 +2120,10 @@ moveFromDescription: 'Esto pondrá un alias en tu cuenta antigua para así poder ingresa la etiqueta de la cuenta con el formato siguiente: @persona@servidor.tld' defaultReaction: Emoji por defecto para reaccionar a las publicaciones entrantes y salientes -indexFromDescription: Deja en blanco para indizar todas las publicaciones deletePasskeys: Borrar claves de paso deletePasskeysConfirm: Esto borrará irreversiblemente todas las claves de paso y de seguridad en esta cuenta, ¿Proceder? inputNotMatch: Las entradas no coinciden -indexFrom: Indizar desde la ID de la publicación en adelante -indexPosts: Indizar publicaciones isModerator: Moderador isAdmin: Administrador isPatron: Mecenas de Firefish @@ -2139,8 +2133,6 @@ migrationConfirm: "¿Estás absolutamente seguro de que quieres migrar a tu cuen {account}? Una vez hecho esto, no podrás revertir el cambio, ni tampoco usar tu cuenta normalmente.\nTambién, asegúrate de que has configurado ésta cuenta como la cuenta desde la cual estás migrando." -indexNotice: Indizando ahora. Esto puede llevar bastante tiempo, por favor, no reinicies - el servidor por lo menos hasta dentro de una hora. customKaTeXMacro: Macros KaTeX personalizadas customKaTeXMacroDescription: '¡Configura macros para escribir expresiones matemáticas fácilmente! La notación es conforme la las definiciones de comandos LaTeX y puede diff --git a/locales/fr-FR.yml b/locales/fr-FR.yml index 0b1043790c..a2500f7798 100644 --- a/locales/fr-FR.yml +++ b/locales/fr-FR.yml @@ -1400,10 +1400,7 @@ _widgets: _userList: chooseList: Sélectionner une liste unixClock: Horloge UNIX - meiliIndexCount: Publications indexées serverInfo: Info serveur - meiliStatus: État du serveur - meiliSize: Taille de l'index instanceCloud: Nuage de serveurs rssTicker: Bandeau RSS _cw: @@ -2061,9 +2058,6 @@ moveToLabel: 'Compte vers lequel vous migrez :' moveFrom: Migrer vers ce compte depuis un ancien compte defaultReaction: Émoji de réaction par défaut pour les publications entrantes et sortantes license: Licence -indexPosts: Indexer les publications -indexNotice: Indexation en cours. Cela prendra certainement du temps, veuillez ne - pas redémarrer votre serveur pour au moins une heure. customKaTeXMacro: Macros KaTeX personnalisées enableCustomKaTeXMacro: Activer les macros KaTeX personnalisées noteId: ID des publications @@ -2110,7 +2104,6 @@ expandOnNoteClick: Ouvrir la publications en cliquant preventAiLearning: Empêcher le récupération de données par des IA listsDesc: Les listes vous laissent créer des fils personnalisés avec des utilisateur·rice·s spécifié·e·s. Elles sont accessibles depuis la page des fils. -indexFromDescription: Laisser vide pour indexer toutes les publications _feeds: jsonFeed: flux JSON atom: Atom @@ -2120,7 +2113,6 @@ alt: ALT swipeOnMobile: Permettre le balayage entre les pages expandOnNoteClickDesc: Si désactivé, vous pourrez toujours ouvrir les publications dans le menu du clic droit et en cliquant sur l'horodatage. -indexFrom: Indexer à partir de l'ID des publications older: ancien newer: récent accessibility: Accessibilité diff --git a/locales/id-ID.yml b/locales/id-ID.yml index ee902f2be3..e3dffa689e 100644 --- a/locales/id-ID.yml +++ b/locales/id-ID.yml @@ -1390,14 +1390,11 @@ _widgets: aiscript: "Konsol AiScript" aichan: "Ai" rssTicker: Telegraf RSS - meiliIndexCount: Postingan yang terindeks userList: Daftar Pengguna instanceCloud: Server Awan unixClock: Jam UNIX - meiliSize: Ukuran indeks _userList: chooseList: Pilih daftar - meiliStatus: Status Server serverInfo: Info Server _cw: hide: "Sembunyikan" @@ -1984,8 +1981,6 @@ speed: Kecepatan slow: Pelan remoteOnly: Jarak jauh saja moveFrom: Dari akun lama pindahkan ke akun ini -indexNotice: Sedang mengindeks. Ini memerlukan beberapa waktu, mohon jangan mulai - ulang server setidaknya satu jam. sendPushNotificationReadMessage: Hapus pemberitahuan dorong saat pemberitahuan atau pesan relevan sudah dibaca moveAccountDescription: Proses ini permanen. Pastikan kamu sudah mengatur alias dari @@ -2010,7 +2005,6 @@ showAds: Tampilkan spanduk komunitas enterSendsMessage: Tekan Enter pada Pesan untuk mengirim pesan (matikan dengan Ctrl + Enter) showAdminUpdates: Indikasi versi Firefish baru tersedia (hanya admin) -indexFrom: Indeks dari Post ID berikutnya noteId: ID Postingan findOtherInstance: Cari server lain caption: Deskripsi itomatis @@ -2022,7 +2016,6 @@ moveFromDescription: Ini akan mengatur alias akun lamamu jadi kamu dapat pindah akun tersebut ke akun sekarang. Lakukan ini SEBELUM memindahkan akun lama. Silakan masukkan tag akun dengan format seperti @orang@server.com defaultReaction: Reaksi emoji bawaan untuk postingan keluar dan masuk -indexPosts: Indeks Postingan preventAiLearning: Cegah scraping bot AI customKaTeXMacro: Makro KaTeX khusus sendPushNotificationReadMessageCaption: Pemberitahuan yang berisi teks "{emptyPushNotificationMessage}" @@ -2052,7 +2045,6 @@ migrationConfirm: "Kamu sangat yakin ingin memindahkan akunmu ke {account}? Seka lagi secara normal. \nDan juga, harap pastikan kamu sudah mengatur akun sekarang sebagai akun yang dipindahkan." license: Lisensi -indexFromDescription: Kosongkan untuk mengindeks setiap postingan noGraze: Harap nonaktifkan ekstensi peramban "Graze for Mastodon", karena akan menganggu Firefish. silencedWarning: Halaman ini tampil karena pengguna ini datang dari server yang dibisukan diff --git a/locales/it-IT.yml b/locales/it-IT.yml index 7c3f130928..48242989da 100644 --- a/locales/it-IT.yml +++ b/locales/it-IT.yml @@ -1324,9 +1324,6 @@ _widgets: instanceCloud: Cloud del server unixClock: Orologio UNIX serverInfo: Informazioni sul server - meiliIndexCount: Post indicizzati - meiliStatus: Stato del server - meiliSize: Dimensione indice userList: Elenco utenti _cw: hide: "Nascondi" @@ -1838,9 +1835,7 @@ customSplashIconsDescription: Elenco degli URL di icone personalizzate da mostra le immagini siano su un URL statico, preferibilmente di dimensioni 192x192. swipeOnDesktop: Permetti lo swipe su desktop simile alla versione mobile logoImageUrl: URL del logo -indexFrom: Indicizza dal post ID customKaTeXMacro: Macro KaTeX personalizzate -indexPosts: Crea indice dei post signupsDisabled: Le iscrizioni su questo server al momento non sono possibili, ma puoi sempre iscriverti su un altro server! Se invece hai un codice di invito per questo server, inseriscilo qua sotto. @@ -1919,9 +1914,6 @@ lastActiveDate: Ultimo utilizzo enterSendsMessage: Premi "Invio" nei messaggi per inviare (altrimenti è "Ctrl + Invio") customMOTD: Messaggi di caricamento personalizzati (splash screen) replayTutorial: Ripeti il tutorial -indexFromDescription: Lascia vuoto per indicizzare tutti i post -indexNotice: Creazione indice in corso. Sarà necessario del tempo, fai attenzione - a non riavviare il server per almeno un'ora. enableCustomKaTeXMacro: Abilita le macro KaTeX personalizzate preventAiLearningDescription: Richiedi ai bot di intelligenza artificiale di terze parti di non studiare e acquisire il contenuto che carichi, come post e immagini. diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 1117231c18..64626b5fe9 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -974,10 +974,6 @@ migrationConfirm: "本当にこのアカウントを {account} に引っ越し この操作を行う前に引っ越し先のアカウントでエイリアスを作成する必要があります。エイリアスが作成されているか、必ず確認してください。" defaultReaction: "リモートとローカルの投稿に対するデフォルトの絵文字リアクション" license: "ライセンス" -indexPosts: "投稿をインデックス" -indexFrom: "この投稿ID以降をインデックスする" -indexFromDescription: "空白で全ての投稿を指定します" -indexNotice: "インデックスを開始しました。完了まで時間がかかる場合があるため、少なくとも1時間はサーバーを再起動しないでください。" customKaTeXMacro: "カスタムKaTeXマクロ" customKaTeXMacroDescription: "数式入力を楽にするためのマクロを設定しましょう！記法はLaTeXにおけるコマンドの定義と同様に \\newcommand{\\ name}{content} または \\newcommand{\\add}[2]{#1 + #2} のように記述します。後者の例では \\add{3}{foo} @@ -1453,10 +1449,7 @@ _widgets: userList: "ユーザーリスト" _userList: chooseList: "リストを選択" - meiliStatus: サーバーステータス serverInfo: サーバー情報 - meiliSize: インデックスサイズ - meiliIndexCount: インデックス済みの投稿 _cw: hide: "隠す" show: "もっと見る" diff --git a/locales/ko-KR.yml b/locales/ko-KR.yml index 722d48abf8..45a6c09c98 100644 --- a/locales/ko-KR.yml +++ b/locales/ko-KR.yml @@ -1331,10 +1331,7 @@ _widgets: serverInfo: 서버 정보 _userList: chooseList: 리스트 선택 - meiliStatus: 서버 정보 userList: 유저 목록 - meiliSize: 인덱스 크기 - meiliIndexCount: 인덱싱 완료된 게시물 rssTicker: RSS Ticker _cw: hide: "숨기기" @@ -1840,7 +1837,6 @@ customSplashIconsDescription: 유저가 페이지를 로딩/새로고침할 때 이미지는 되도록 정적 URL으로 구성하고, 192x192 해상도로 조정하여 주십시오. moveFromDescription: '이전 계정에 대한 별칭을 작성하여, 이 계정으로 옮길 수 있도록 합니다. 반드시 계정을 이전하기 전에 수행해야 합니다. 이전 계정을 다음과 같은 형식으로 입력하여 주십시오: @person@server.com' -indexFromDescription: 빈 칸으로 두면 모든 게시물을 인덱싱합니다 customKaTeXMacroDescription: 'KaTeX 매크로를 지정하여 수식을 더욱 편리하게 입력하세요! LaTeX의 커맨드 정의와 동일하게 \newcommand{\ 이름}{내용} 또는 \newcommand{\이름}[인수 갯수]{내용} 와 같이 입력하십시오. 예를 들어 \newcommand{\add}[2]{#1 + #2} 와 같이 정의한 경우 \add{3}{foo} 를 입력하면 3 + foo 으로 치환됩니다.매크로의 이름을 감싸는 중괄호를 소괄호() 또는 @@ -1895,7 +1891,6 @@ accessibility: 접근성 userSaysSomethingReasonReply: '{name} 님이 {reason} 을 포함하는 게시물에 답글했습니다' userSaysSomethingReasonRenote: '{name} 님이 {reason} 을 포함하는 게시물을 부스트했습니다' breakFollowConfirm: 팔로워를 해제하시겠습니까? -indexFrom: 이 게시물 ID부터 인덱싱하기 noThankYou: 괜찮습니다 hiddenTags: 숨길 해시태그 image: 이미지 @@ -1927,8 +1922,6 @@ removeMember: 멤버를 삭제 license: 라이선스 migrationConfirm: "정말로 이 계정을 {account}로 이사하시겠습니까? 한 번 이사하면, 현재 이 계정은 두 번 다시 사용할 수 없게 됩니다.\n또한, 이사 갈 계정에 현재 사용 중인 계정의 별칭을 올바르게 작성하였는지 다시 한 번 확인하십시오." -indexPosts: 게시물을 인덱싱 -indexNotice: 인덱싱을 시작했습니다. 이 작업은 시간이 많이 소요되므로, 최소 1시간 이내에 서버를 재시작하지 마십시오. noteId: 게시물 ID signupsDisabled: 현재 이 서버에서는 신규 등록을 받고 있지 않습니다. 초대 코드를 가지고 계신 경우 아래 칸에 입력해 주십시오. 초대 코드를 가지고 있지 않더라도, 신규 등록이 열려 있는 다른 서버에 등록하실 수 있습니다! diff --git a/locales/no-NO.yml b/locales/no-NO.yml index c817d46c24..d8b80e0e2c 100644 --- a/locales/no-NO.yml +++ b/locales/no-NO.yml @@ -939,7 +939,6 @@ allowedInstancesDescription: Tjenernavn for tjenere som skal hvitelistes. En per (Vil bare bli brukt i privat modus). previewNoteText: Forhåndsvisning recentNDays: Siste {n} dager -indexPosts: Indekser poster objectStorageUseProxy: Koble til gjennom en mellomtjener objectStorageUseProxyDesc: Skru av dette dersom du ikke vil bruke mellomtjenere for API-oppkoblinger @@ -1185,10 +1184,6 @@ moveFromDescription: Dette vil sette opp et alias for din gamle kontoen slik at kan flytte fra den gamle kontoen til denne. Gjør dette FØR du flytter fra den gamle kontoen. Skriv inn den gamle kontoen på formen @person@server.com defaultReaction: Standard emoji-reaksjon for utgående og innkommende poster -indexFrom: Indekser poster fra post-id og fremover -indexNotice: Indekserer. Dette vil sannsynligvis ta litt tid, ikke restart tjeneren - før det har gått minst en time. -indexFromDescription: La stå tom for å indeksere alle poster customKaTeXMacroDescription: 'Sett opp makroer for å skrive matematiske uttrykk enkelt. Notasjonen følger LaTeX-kommandoer og er skrevet som \newcommand{\ navn}{uttrykk} eller \newcommand{\navn}{antall argumenter}{uttrykk}. For eksempel vil \newcommand{\add}{2}{#1 @@ -1628,14 +1623,12 @@ _antennaSources: instances: Poster fra alle brukerne på denne tjeneren _widgets: timeline: Tidslinje - meiliSize: Indeks-størrelse instanceCloud: Tjenersky onlineUsers: Påloggede brukere clock: Klokke userList: Brukerliste rss: RSS-leser serverMetric: Tjenermetrikker - meiliIndexCount: Indekserte poster button: Knapp unixClock: Unix-klokke calendar: Kalender @@ -1647,7 +1640,6 @@ _widgets: photos: Bilder rssTicker: RSS-rulletekst aiscript: AiScript-konsoll - meiliStatus: Tjenerstatus memo: Notatlapp notifications: Varsler postForm: Ny post diff --git a/locales/pl-PL.yml b/locales/pl-PL.yml index 1369babee2..73b3939abb 100644 --- a/locales/pl-PL.yml +++ b/locales/pl-PL.yml @@ -1892,11 +1892,6 @@ sendPushNotificationReadMessageCaption: Powiadomienie zawierające tekst "{empty baterii Twojego urządzenia. defaultReaction: Domyślna reakcja emoji dla wychodzących i przychodzących wpisów license: Licencja -indexPosts: Indeksuj wpisy -indexFrom: Indeksuj wpisy od ID -indexFromDescription: Zostaw puste dla indeksowania wszystkich wpisów -indexNotice: Indeksuję. Zapewne zajmie to chwilę, nie restartuj serwera przez co najmniej - godzinę. customKaTeXMacro: Niestandardowe makra KaTeX enableCustomKaTeXMacro: Włącz niestandardowe makra KaTeX noteId: ID wpisu diff --git a/locales/ru-RU.yml b/locales/ru-RU.yml index 9de2c135b9..64742e5479 100644 --- a/locales/ru-RU.yml +++ b/locales/ru-RU.yml @@ -1375,9 +1375,6 @@ _widgets: userList: Список пользователей _userList: chooseList: Выберите список - meiliStatus: Состояние сервера - meiliSize: Размер индекса - meiliIndexCount: Индексированные посты serverInfo: Информация о сервере _cw: hide: "Спрятать" @@ -1951,11 +1948,6 @@ showUpdates: Показывать всплывающее окно при обн recommendedInstances: Рекомендованные серверы defaultReaction: Эмодзи реакция по умолчанию для выходящих и исходящих постов license: Лицензия -indexPosts: Индексировать посты -indexFrom: Индексировать начиная с идентификатора поста и далее -indexFromDescription: оставьте пустым для индексации каждого поста -indexNotice: Теперь индексирование. Вероятно, это займет некоторое время, пожалуйста, - не перезагружайте свой сервер по крайней мере в течение часа. customKaTeXMacro: Кастомные KaTex макросы enableCustomKaTeXMacro: Включить кастомные KaTeX макросы noteId: Идентификатор поста diff --git a/locales/th-TH.yml b/locales/th-TH.yml index bcd97e9f64..1302f86726 100644 --- a/locales/th-TH.yml +++ b/locales/th-TH.yml @@ -1316,8 +1316,6 @@ customMOTDDescription: ข้อความหน้าจอเริ่มต คั่นด้วยการขึ้นบรรทัดใหม่เพื่อแสดงแบบสุ่มทุกครั้งที่ผู้ใช้โหลดเว็บหรือโหลดหน้าเว็บซ้ำ caption: คำอธิบายโดยอัตโนมัติ moveToLabel: 'บัญชีที่คุณจะย้ายไปยัง:' -indexFromDescription: เว้นว่างไว้เพื่อสร้างดัชนีทุกโพสต์ -indexNotice: ตอนนี้กำลังจัดทำดัชนี การดำเนินการนี้อาจใช้เวลาสักครู่ โปรดอย่ารีสตาร์ทเซิร์ฟเวอร์เป็นเวลาอย่างน้อยหนึ่งชั่วโมง noteId: โพสต์ ID apps: แอป enableRecommendedTimeline: เปิดใช้งาน ไทม์ไลน์ที่แนะนำ @@ -1371,9 +1369,7 @@ moveFromDescription: การดำเนินการนี้จะตั migrationConfirm: "คุณแน่ใจหรือไม่ว่าคุณต้องการย้ายบัญชีของคุณไปยัง {account} เมื่อคุณทำเช่นนี้ คุณจะไม่สามารถกู้คืนมาได้ และคุณจะไม่สามารถใช้บัญชีของคุณได้ตามปกติอีก\nนอกจากนี้ โปรดตรวจสอบให้แน่ใจว่าคุณได้ตั้งบัญชีปัจจุบันนี้เป็นบัญชีที่คุณจะย้ายออก" -indexFrom: จัดทำดัชนีตั้งแต่ Post ID เป็นต้นไป license: ใบอนุญาต -indexPosts: ดัชนีโพสต์ signupsDisabled: การลงชื่อสมัครใช้บนเซิร์ฟเวอร์นี้ถูกปิดใช้งานอยู่ในขณะนี้ แต่คุณสามารถสมัครที่เซิร์ฟเวอร์อื่นได้ตลอดเวลา หากคุณมีรหัสเชิญสำหรับเซิร์ฟเวอร์นี้ โปรดป้อนรหัสด้านล่าง customKaTeXMacroDescription: 'ตั้งค่ามาโครเพื่อเขียนนิพจน์ทางคณิตศาสตร์ได้อย่างง่ายดาย diff --git a/locales/tr-TR.yml b/locales/tr-TR.yml index fc0c236e0a..16206614b3 100644 --- a/locales/tr-TR.yml +++ b/locales/tr-TR.yml @@ -158,7 +158,6 @@ _widgets: activity: Aktivite digitalClock: Dijital Saat unixClock: UNIX Saati - meiliIndexCount: Indexlenmiş gönderiler calendar: Takvim trends: Popüler memo: Yapışkan Notlar @@ -166,13 +165,11 @@ _widgets: federation: Federasyon instanceCloud: Sunucu Bulutu postForm: Gönderi Formu - meiliSize: Index boyutu slideshow: Slayt Gösterisi button: Düğme clock: Saat rss: RSS Okuyucu serverInfo: Sunucu Bilgisi - meiliStatus: Sunucu Durumu jobQueue: İş Sırası serverMetric: Sunucu Bilgileri _profile: @@ -473,7 +470,6 @@ activeEmailValidationDescription: Tek kullanımlık adreslerin kontrol edilmesi sağlar. İşaretlenmediğinde, yalnızca e-postanın biçimi doğrulanır. move: Taşı defaultReaction: Giden ve gelen gönderiler için varsayılan emoji tepkisi -indexPosts: Dizin Gönderileri youGotNewFollower: takip etti receiveFollowRequest: Takip isteği alındı followRequestAccepted: Takip isteği onaylandı @@ -1084,7 +1080,6 @@ check: Kontrol Et driveCapOverrideLabel: Bu kullanıcı için drive kapasitesini değiştirin numberOfPageCache: Önbelleğe alınan sayfa sayısı license: Lisans -indexFrom: Post ID'den itibaren dizin xl: XL notificationSetting: Bildirim ayarları fillAbuseReportDescription: Lütfen bu raporla ilgili ayrıntıları doldurun. Belirli @@ -1159,9 +1154,6 @@ migrationConfirm: "Hesabınızı {account} hesabına taşımak istediğinizden k emin misiniz? Bunu yaptığınızda, geri alamazsınız ve hesabınızı bir daha normal şekilde kullanamazsınız.\nAyrıca, lütfen bu cari hesabı, taşındığınız hesap olarak ayarladığınızdan emin olun." -indexFromDescription: Her gönderiyi dizine eklemek için boş bırakın -indexNotice: Şimdi indeksleniyor. Bu muhtemelen biraz zaman alacaktır, lütfen sunucunuzu - en az bir saat yeniden başlatmayın. customKaTeXMacro: Özel KaTeX makroları directNotes: Özel Mesajlar import: İçeri Aktar diff --git a/locales/uk-UA.yml b/locales/uk-UA.yml index 0905848e6a..f0b7296b78 100644 --- a/locales/uk-UA.yml +++ b/locales/uk-UA.yml @@ -1199,14 +1199,11 @@ _widgets: aiscript: "Консоль AiScript" _userList: chooseList: Оберіть список - meiliStatus: Стан сервера - meiliSize: Розмір індексу rssTicker: RSS-тікер instanceCloud: Хмара серверів unixClock: Годинник UNIX userList: Список користувачів serverInfo: Інформація про сервер - meiliIndexCount: Індексовані записи _cw: hide: "Сховати" show: "Показати більше" @@ -1977,11 +1974,6 @@ caption: Автоматичний опис showAdminUpdates: Вказати, що доступна нова версія Firefish (тільки для адміністратора) defaultReaction: Емодзі реакція за замовчуванням для вихідних і вхідних записів license: Ліцензія -indexPosts: Індексувати пости -indexFrom: Індексувати записи з ID -indexFromDescription: Залиште порожнім, щоб індексувати кожен запис -indexNotice: Зараз відбувається індексація. Це, ймовірно, займе деякий час, будь ласка, - не перезавантажуйте сервер принаймні годину. signupsDisabled: Реєстрація на цьому сервері наразі відключена, але ви завжди можете зареєструватися на іншому сервері! Якщо у вас є код запрошення на цей сервер, будь ласка, введіть його нижче. diff --git a/locales/vi-VN.yml b/locales/vi-VN.yml index 2c7790d7c8..119c745ac3 100644 --- a/locales/vi-VN.yml +++ b/locales/vi-VN.yml @@ -1437,9 +1437,6 @@ _widgets: userList: Danh sách người dùng _userList: chooseList: Chọn một danh sách - meiliSize: Kích cỡ chỉ mục - meiliIndexCount: Tút đã lập chỉ mục - meiliStatus: Trạng thái máy chủ serverInfo: Thông tin máy chủ _cw: hide: "Ẩn" @@ -2048,8 +2045,6 @@ hiddenTagsDescription: 'Liệt kê các hashtag (không có #) mà bạn muốn noInstances: Không có máy chủ nào manageGroups: Quản lý nhóm accessibility: Khả năng tiếp cận -indexNotice: Đang lập chỉ mục. Quá trình này có thể mất một lúc, vui lòng không khởi - động lại máy chủ của bạn sau ít nhất một giờ. breakFollowConfirm: Bạn có chắc muốn xóa người theo dõi? caption: Caption tự động objectStorageS3ForcePathStyle: Sử dụng URL điểm cuối dựa trên đường dẫn @@ -2069,7 +2064,6 @@ updateAvailable: Có bản cập nhật mới! swipeOnDesktop: Cho phép vuốt kiểu điện thoại trên máy tính moveFromLabel: 'Tài khoản cũ của bạn:' defaultReaction: Biểu cảm mặc định cho những tút đã đăng và sắp đăng -indexFromDescription: Để trống để lập chỉ mục toàn bộ donationLink: Liên kết tới trang tài trợ deletePasskeys: Xóa passkey delete2faConfirm: Thao tác này sẽ xóa 2FA trên tài khoản này một cách không thể phục @@ -2086,8 +2080,6 @@ audio: Âm thanh selectInstance: Chọn máy chủ userSaysSomethingReason: '{name} cho biết {reason}' pushNotification: Thông báo đẩy -indexPosts: Chỉ mục tút -indexFrom: Chỉ mục từ Post ID customKaTeXMacro: Tùy chỉnh macro KaTeX license: Giấy phép cw: Nội dung ẩn diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 1aa187e0a9..a53e0767e9 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -1350,9 +1350,6 @@ _widgets: aiscript: "AiScript 控制台" aichan: "小蓝" userList: 用户列表 - meiliStatus: 服务器状态 - meiliIndexCount: 已索引的帖子 - meiliSize: 索引大小 serverInfo: 服务器信息 _userList: chooseList: 选择列表 @@ -1917,7 +1914,6 @@ isAdmin: 管理员 findOtherInstance: 寻找其它服务器 moveFromDescription: 这将为您的旧账号设置一个别名，以便您可以从该旧账号迁移到当前账号。在从旧账号迁移之前执行此操作。请输入格式如 @person@server.com 的账号标签 -indexPosts: 索引帖子 signupsDisabled: 该服务器目前关闭注册，但您随时可以在另一台服务器上注册！如果您有该服务器的邀请码，请在下面输入。 silencedWarning: 显示这个页面是因为这些用户来自您的管理员设置的禁言服务器，所以他们有可能是垃圾信息。 isBot: 这个账号是一个机器人 @@ -1931,12 +1927,9 @@ moveTo: 将当前账号迁移至新账号 moveToLabel: 您要迁移到的目标账号： moveAccount: 迁移账号！ migrationConfirm: "您确实确定要将账号迁移到 {account} 吗？此操作无法撤消，并且您将无法再次正常使用旧账号。\n另外，请确保您已将此当前账号设置为要移出的账号。" -indexFromDescription: 留空以索引每个帖子 noteId: 帖子 ID moveFrom: 从旧账号迁移至此账号 defaultReaction: 发出和收到帖子的默认表情符号反应 -indexNotice: 现在开始索引。这可能需要一段时间，请至少一个小时内不要重新启动服务器。 -indexFrom: 从帖子 ID 开始的索引 sendModMail: 发送审核通知 isLocked: 该账号设置了关注请求 _filters: diff --git a/locales/zh-TW.yml b/locales/zh-TW.yml index ae89e38ae8..da573fd82b 100644 --- a/locales/zh-TW.yml +++ b/locales/zh-TW.yml @@ -1349,9 +1349,6 @@ _widgets: userList: 使用者列表 _userList: chooseList: 選擇一個清單 - meiliIndexCount: 編入索引的帖子 - meiliStatus: 伺服器狀態 - meiliSize: 索引大小 _cw: hide: "隱藏" show: "瀏覽更多" @@ -1879,8 +1876,6 @@ pushNotificationNotSupported: 你的瀏覽器或伺服器不支援推送通知 accessibility: 輔助功能 userSaysSomethingReasonReply: '{name} 回覆了包含 {reason} 的貼文' hiddenTags: 隱藏主題標籤 -indexPosts: 索引貼文 -indexNotice: 現在開始索引。 這可能需要一段時間，請不要在一個小時內重啟你的伺服器。 deleted: 已刪除 editNote: 編輯貼文 edited: '於 {date} {time} 編輯' @@ -1925,10 +1920,8 @@ sendModMail: 發送審核通知 enableIdenticonGeneration: 啟用Identicon生成 enableServerMachineStats: 啟用伺服器硬體統計資訊 reactionPickerSkinTone: 首選表情符號膚色 -indexFromDescription: 留空以索引每個貼文 preventAiLearning: 防止 AI 機器人抓取 preventAiLearningDescription: 請求第三方 AI 語言模型不要研究您上傳的內容，例如貼文和圖像。 -indexFrom: 建立此貼文ID以後的索引 isLocked: 該帳戶已獲得以下批准 isModerator: 板主 isAdmin: 管理員 diff --git a/packages/backend/migration/1698420787202-pgroonga.js b/packages/backend/migration/1698420787202-pgroonga.js new file mode 100644 index 0000000000..99638b4036 --- /dev/null +++ b/packages/backend/migration/1698420787202-pgroonga.js @@ -0,0 +1,21 @@ +export class Pgroonga1698420787202 { + name = "Pgroonga1698420787202"; + + async up(queryRunner) { + await queryRunner.query( + `CREATE INDEX "IDX_f27f5d88941e57442be75ba9c8" ON "note" USING "pgroonga" ("text")`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_065d4d8f3b5adb4a08841eae3c" ON "user" USING "pgroonga" ("name" pgroonga_varchar_full_text_search_ops_v2)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_fcb770976ff8240af5799e3ffc" ON "user_profile" USING "pgroonga" ("description" pgroonga_varchar_full_text_search_ops_v2) `, + ); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_fcb770976ff8240af5799e3ffc"`); + await queryRunner.query(`DROP INDEX "IDX_065d4d8f3b5adb4a08841eae3c"`); + await queryRunner.query(`DROP INDEX "IDX_f27f5d88941e57442be75ba9c8"`); + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index d5786f16ec..ecb9f80da3 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -25,7 +25,6 @@ "@bull-board/koa": "5.14.2", "@bull-board/ui": "5.14.2", "@discordapp/twemoji": "^15.0.2", - "@elastic/elasticsearch": "8.12.2", "@koa/cors": "5.0.0", "@koa/multer": "3.0.2", "@koa/router": "12.0.1", @@ -81,7 +80,6 @@ "koa-send": "5.0.1", "koa-slow": "2.1.0", "megalodon": "workspace:*", - "meilisearch": "0.37.0", "mfm-js": "0.24.0", "mime-types": "2.1.35", "msgpackr": "^1.10.1", @@ -112,7 +110,6 @@ "sanitize-html": "2.12.1", "semver": "7.6.0", "sharp": "0.33.2", - "sonic-channel": "^1.3.1", "stringz": "2.1.0", "summaly": "2.7.0", "syslog-pro": "1.0.0", diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 0389bf83fb..882e429c79 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -37,27 +37,6 @@ export type Source = { user?: string; tls?: { [z: string]: string }; }; - elasticsearch: { - host: string; - port: number; - ssl?: boolean; - user?: string; - pass?: string; - index?: string; - }; - sonic: { - host: string; - port: number; - auth?: string; - collection?: string; - bucket?: string; - }; - meilisearch: { - host: string; - port: number; - apiKey?: string; - ssl: boolean; - }; proxy?: string; proxySmtp?: string; diff --git a/packages/backend/src/daemons/server-stats.ts b/packages/backend/src/daemons/server-stats.ts index dc7493381c..58a9b1491b 100644 --- a/packages/backend/src/daemons/server-stats.ts +++ b/packages/backend/src/daemons/server-stats.ts @@ -2,7 +2,6 @@ import si from "systeminformation"; import Xev from "xev"; import * as osUtils from "os-utils"; import { fetchMeta } from "@/misc/fetch-meta.js"; -import meilisearch from "@/db/meilisearch.js"; const ev = new Xev(); @@ -30,7 +29,6 @@ export default function () { const memStats = await mem(); const netStats = await net(); const fsStats = await fs(); - const meilisearchStats = await meilisearchStatus(); const stats = { cpu: roundCpu(cpu), @@ -47,7 +45,6 @@ export default function () { r: round(Math.max(0, fsStats.rIO_sec ?? 0)), w: round(Math.max(0, fsStats.wIO_sec ?? 0)), }, - meilisearch: meilisearchStats, }; ev.emit("serverStats", stats); log.unshift(stats); @@ -86,16 +83,3 @@ async function fs() { const data = await si.disksIO().catch(() => ({ rIO_sec: 0, wIO_sec: 0 })); return data || { rIO_sec: 0, wIO_sec: 0 }; } - -// MEILI STAT -async function meilisearchStatus() { - if (meilisearch) { - return meilisearch.serverStats(); - } else { - return { - health: "unconfigured", - size: 0, - indexed_count: 0, - }; - } -} diff --git a/packages/backend/src/db/elasticsearch.ts b/packages/backend/src/db/elasticsearch.ts deleted file mode 100644 index 2640e7f918..0000000000 --- a/packages/backend/src/db/elasticsearch.ts +++ /dev/null @@ -1,65 +0,0 @@ -import * as elasticsearch from "@elastic/elasticsearch"; -import config from "@/config/index.js"; - -const index = { - settings: { - analysis: { - analyzer: { - ngram: { - tokenizer: "ngram", - }, - }, - }, - }, - mappings: { - properties: { - text: { - type: "text", - index: true, - analyzer: "ngram", - }, - userId: { - type: "keyword", - index: true, - }, - userHost: { - type: "keyword", - index: true, - }, - }, - }, -}; - -// Init ElasticSearch connection -const client = config.elasticsearch - ? new elasticsearch.Client({ - node: `${config.elasticsearch.ssl ? "https://" : "http://"}${ - config.elasticsearch.host - }:${config.elasticsearch.port}`, - auth: - config.elasticsearch.user && config.elasticsearch.pass - ? { - username: config.elasticsearch.user, - password: config.elasticsearch.pass, - } - : undefined, - pingTimeout: 30000, - }) - : null; - -if (client) { - client.indices - .exists({ - index: config.elasticsearch.index || "misskey_note", - }) - .then((exist) => { - if (!exist.body) { - client.indices.create({ - index: config.elasticsearch.index || "misskey_note", - body: index, - }); - } - }); -} - -export default client; diff --git a/packages/backend/src/db/meilisearch.ts b/packages/backend/src/db/meilisearch.ts deleted file mode 100644 index 1f294058e5..0000000000 --- a/packages/backend/src/db/meilisearch.ts +++ /dev/null @@ -1,451 +0,0 @@ -import { Health, Index, MeiliSearch, Stats } from "meilisearch"; -import { dbLogger } from "./logger.js"; - -import config from "@/config/index.js"; -import { Note } from "@/models/entities/note.js"; -import * as url from "url"; -import { ILocalUser } from "@/models/entities/user.js"; -import { Followings, Users } from "@/models/index.js"; - -const logger = dbLogger.createSubLogger("meilisearch", "gray", false); - -let posts: Index; -let client: MeiliSearch; - -const hasConfig = - config.meilisearch && - (config.meilisearch.host || - config.meilisearch.port || - config.meilisearch.apiKey); - -if (hasConfig) { - const host = hasConfig ? config.meilisearch.host ?? "localhost" : ""; - const port = hasConfig ? config.meilisearch.port ?? 7700 : 0; - const auth = hasConfig ? config.meilisearch.apiKey ?? "" : ""; - const ssl = hasConfig ? config.meilisearch.ssl ?? false : false; - - logger.info("Connecting to MeiliSearch"); - - client = new MeiliSearch({ - host: `${ssl ? "https" : "http"}://${host}:${port}`, - apiKey: auth, - }); - - posts = client.index("posts"); - - posts - .updateSearchableAttributes(["text"]) - .catch((e) => - logger.error(`Setting searchable attr failed, searches won't work: ${e}`), - ); - - posts - .updateFilterableAttributes([ - "userName", - "userHost", - "mediaAttachment", - "createdAt", - "userId", - ]) - .catch((e) => - logger.error( - `Setting filterable attr failed, advanced searches won't work: ${e}`, - ), - ); - - posts - .updateSortableAttributes(["createdAt"]) - .catch((e) => - logger.error( - `Setting sortable attr failed, placeholder searches won't sort properly: ${e}`, - ), - ); - - posts - .updateStopWords([ - "the", - "a", - "as", - "be", - "of", - "they", - "these", - "is", - "are", - "これ", - "それ", - "あれ", - "この", - "その", - "あの", - "ここ", - "そこ", - "あそこ", - "こちら", - "どこ", - "私", - "僕", - "俺", - "君", - "あなた", - "我々", - "私達", - "彼女", - "彼", - "です", - "ます", - "は", - "が", - "の", - "に", - "を", - "で", - "へ", - "から", - "まで", - "より", - "も", - "どの", - "と", - "それで", - "しかし", - ]) - .catch((e) => - logger.error( - `Failed to set Meilisearch stop words, database size will be larger: ${e}`, - ), - ); - - posts - .updateRankingRules([ - "sort", - "words", - "typo", - "proximity", - "attribute", - "exactness", - ]) - .catch((e) => { - logger.error("Failed to set ranking rules, sorting won't work properly."); - }); - - logger.info("Connected to MeiliSearch"); -} - -export type MeilisearchNote = { - id: string; - text: string; - userId: string; - userHost: string; - userName: string; - channelId: string; - mediaAttachment: string; - createdAt: number; -}; - -function timestampToUnix(timestamp: string) { - let unix = 0; - - // Only contains numbers => UNIX timestamp - if (/^\d+$/.test(timestamp)) { - unix = Number.parseInt(timestamp); - } - - if (unix === 0) { - // Try to parse the timestamp as JavaScript Date - const date = Date.parse(timestamp); - if (Number.isNaN(date)) return 0; - unix = date / 1000; - } - - return unix; -} - -export default hasConfig - ? { - search: async ( - query: string, - limit: number, - offset: number, - userCtx: ILocalUser | null, - overrideSort: string | null, - ) => { - /// Advanced search syntax - /// from:user => filter by user + optional domain - /// has:image/video/audio/text/file => filter by attachment types - /// domain:domain.com => filter by domain - /// before:Date => show posts made before Date - /// after: Date => show posts made after Date - /// "text" => get posts with exact text between quotes - /// filter:following => show results only from users you follow - /// filter:followers => show results only from followers - /// order:desc/asc => order results ascending or descending - - const constructedFilters: string[] = []; - let sortRules: string[] = []; - - const splitSearch = query.split(" "); - - // Detect search operators and remove them from the actual query - const filteredSearchTerms = ( - await Promise.all( - splitSearch.map(async (term) => { - if (term.startsWith("has:")) { - const fileType = term.slice(4); - constructedFilters.push(`mediaAttachment = "${fileType}"`); - return null; - } else if (term.startsWith("from:")) { - let user = term.slice(5); - - if (user.length === 0) return null; - - // Cut off leading @, those aren't saved in the DB - if (user.charAt(0) === "@") { - user = user.slice(1); - } - - // Determine if we got a webfinger address or a single username - if (user.split("@").length > 1) { - const splitUser = user.split("@"); - - const domain = splitUser.pop(); - user = splitUser.join("@"); - - constructedFilters.push( - `userName = ${user} AND userHost = ${domain}`, - ); - } else { - constructedFilters.push(`userName = ${user}`); - } - - return null; - } else if (term.startsWith("domain:")) { - const domain = term.slice(7); - if ( - domain.length === 0 || - domain === "local" || - domain === config.hostname - ) { - constructedFilters.push("userHost NOT EXISTS"); - return null; - } - constructedFilters.push(`userHost = ${domain}`); - return null; - } else if (term.startsWith("after:")) { - const timestamp = term.slice(6); - - const unix = timestampToUnix(timestamp); - - if (unix !== 0) constructedFilters.push(`createdAt > ${unix}`); - - return null; - } else if (term.startsWith("before:")) { - const timestamp = term.slice(7); - - const unix = timestampToUnix(timestamp); - if (unix !== 0) constructedFilters.push(`createdAt < ${unix}`); - - return null; - } else if (term.startsWith("filter:following")) { - // Check if we got a context user - if (userCtx) { - // Fetch user follows from DB - const followedUsers = await Followings.find({ - where: { - followerId: userCtx.id, - }, - select: { - followeeId: true, - }, - }); - const followIDs = followedUsers.map( - (user) => user.followeeId, - ); - - if (followIDs.length === 0) return null; - - constructedFilters.push(`userId IN [${followIDs.join(",")}]`); - } else { - logger.warn( - "search filtered to follows called without user context", - ); - } - - return null; - } else if (term.startsWith("filter:followers")) { - // Check if we got a context user - if (userCtx) { - // Fetch users follows from DB - const followedUsers = await Followings.find({ - where: { - followeeId: userCtx.id, - }, - select: { - followerId: true, - }, - }); - const followIDs = followedUsers.map( - (user) => user.followerId, - ); - - if (followIDs.length === 0) return null; - - constructedFilters.push(`userId IN [${followIDs.join(",")}]`); - } else { - logger.warn( - "search filtered to followers called without user context", - ); - } - - return null; - } else if (term.startsWith("order:desc")) { - sortRules.push("createdAt:desc"); - - return null; - } else if (term.startsWith("order:asc")) { - sortRules.push("createdAt:asc"); - - return null; - } - - return term; - }), - ) - ).filter((term) => term !== null); - - // An empty search term with defined filters means we have a placeholder search => https://www.meilisearch.com/docs/reference/api/search#placeholder-search - // These have to be ordered manually, otherwise the *oldest* posts are returned first, which we don't want - // If the user has defined a sort rule, don't mess with it - if ( - filteredSearchTerms.length === 0 && - constructedFilters.length > 0 && - sortRules.length === 0 - ) { - sortRules.push("createdAt:desc"); - } - - // More than one sorting rule doesn't make sense. We only keep the first one, otherwise weird stuff may happen. - if (sortRules.length > 1) { - sortRules = [sortRules[0]]; - } - - // An override sort takes precedence, user sorting is ignored here - if (overrideSort) { - sortRules = [overrideSort]; - } - - logger.info(`Searching for ${filteredSearchTerms.join(" ")}`); - logger.info(`Limit: ${limit}`); - logger.info(`Offset: ${offset}`); - logger.info(`Filters: ${constructedFilters}`); - logger.info(`Ordering: ${sortRules}`); - - return posts.search(filteredSearchTerms.join(" "), { - limit: limit, - offset: offset, - filter: constructedFilters, - sort: sortRules, - }); - }, - ingestNote: async (ingestNotes: Note | Note[]) => { - if (ingestNotes instanceof Note) { - ingestNotes = [ingestNotes]; - } - - const indexingBatch: MeilisearchNote[] = []; - - for (const note of ingestNotes) { - if (note.user === undefined) { - note.user = await Users.findOne({ - where: { - id: note.userId, - }, - }); - } - - let attachmentType = ""; - if (note.attachedFileTypes.length > 0) { - attachmentType = note.attachedFileTypes[0].split("/")[0]; - switch (attachmentType) { - case "image": - case "video": - case "audio": - case "text": - break; - default: - attachmentType = "file"; - break; - } - } - - indexingBatch.push({ - id: note.id.toString(), - text: note.text ? note.text : "", - userId: note.userId, - userHost: - note.userHost !== "" - ? note.userHost - : url.parse(config.host).host, - channelId: note.channelId ? note.channelId : "", - mediaAttachment: attachmentType, - userName: note.user?.username ?? "UNKNOWN", - createdAt: note.createdAt.getTime() / 1000, // division by 1000 is necessary because Node returns in ms-accuracy - }); - } - - return posts - .addDocuments(indexingBatch, { - primaryKey: "id", - }) - .then(() => - logger.info(`sent ${indexingBatch.length} posts for indexing`), - ); - }, - serverStats: async () => { - const health: Health = await client.health(); - const stats: Stats = await client.getStats(); - - return { - health: health.status, - size: stats.databaseSize, - indexed_count: stats.indexes["posts"].numberOfDocuments, - }; - }, - deleteNotes: async (note: Note | Note[] | string | string[]) => { - if (note instanceof Note) { - note = [note]; - } - if (typeof note === "string") { - note = [note]; - } - - const deletionBatch = note - .map((n) => { - if (n instanceof Note) { - return n.id; - } - - if (n.length > 0) return n; - - logger.error( - `Failed to delete note from Meilisearch, invalid post ID: ${JSON.stringify( - n, - )}`, - ); - - throw new Error( - `Invalid note ID passed to meilisearch deleteNote: ${JSON.stringify( - n, - )}`, - ); - }) - .filter((el) => el !== null); - - await posts.deleteDocuments(deletionBatch as string[]).then(() => { - logger.info( - `submitted ${deletionBatch.length} large batch for deletion`, - ); - }); - }, - } - : null; diff --git a/packages/backend/src/db/sonic.ts b/packages/backend/src/db/sonic.ts deleted file mode 100644 index 032982f083..0000000000 --- a/packages/backend/src/db/sonic.ts +++ /dev/null @@ -1,51 +0,0 @@ -import * as SonicChannel from "sonic-channel"; -import { dbLogger } from "./logger.js"; - -import config from "@/config/index.js"; - -const logger = dbLogger.createSubLogger("sonic", "gray", false); - -const handlers = (type: string): SonicChannel.Handlers => ({ - connected: () => { - logger.succ(`Connected to Sonic ${type}`); - }, - disconnected: (error) => { - logger.warn(`Disconnected from Sonic ${type}, error: ${error}`); - }, - error: (error) => { - logger.warn(`Sonic ${type} error: ${error}`); - }, - retrying: () => { - logger.info(`Sonic ${type} retrying`); - }, - timeout: () => { - logger.warn(`Sonic ${type} timeout`); - }, -}); - -const hasConfig = - config.sonic && (config.sonic.host || config.sonic.port || config.sonic.auth); - -if (hasConfig) { - logger.info("Connecting to Sonic"); -} - -const host = hasConfig ? config.sonic.host ?? "localhost" : ""; -const port = hasConfig ? config.sonic.port ?? 1491 : 0; -const auth = hasConfig ? config.sonic.auth ?? "SecretPassword" : ""; -const collection = hasConfig ? config.sonic.collection ?? "main" : ""; -const bucket = hasConfig ? config.sonic.bucket ?? "default" : ""; - -export default hasConfig - ? { - search: new SonicChannel.Search({ host, port, auth }).connect( - handlers("search"), - ), - ingest: new SonicChannel.Ingest({ host, port, auth }).connect( - handlers("ingest"), - ), - - collection, - bucket, - } - : null; diff --git a/packages/backend/src/models/entities/drive-file.ts b/packages/backend/src/models/entities/drive-file.ts index 54f5a8ca28..5b31a7e336 100644 --- a/packages/backend/src/models/entities/drive-file.ts +++ b/packages/backend/src/models/entities/drive-file.ts @@ -70,6 +70,7 @@ export class DriveFile { }) public size: number; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: DB_MAX_IMAGE_COMMENT_LENGTH, nullable: true, diff --git a/packages/backend/src/models/entities/note.ts b/packages/backend/src/models/entities/note.ts index 8f7958e83d..e45fb02466 100644 --- a/packages/backend/src/models/entities/note.ts +++ b/packages/backend/src/models/entities/note.ts @@ -61,6 +61,7 @@ export class Note { }) public threadId: string | null; + @Index() // USING pgroonga @Column("text", { nullable: true, }) @@ -78,6 +79,7 @@ export class Note { }) public name: string | null; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: 512, nullable: true, diff --git a/packages/backend/src/models/entities/user-profile.ts b/packages/backend/src/models/entities/user-profile.ts index 56887035ce..0cdf4e8393 100644 --- a/packages/backend/src/models/entities/user-profile.ts +++ b/packages/backend/src/models/entities/user-profile.ts @@ -38,6 +38,7 @@ export class UserProfile { }) public birthday: string | null; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: 2048, nullable: true, diff --git a/packages/backend/src/models/entities/user.ts b/packages/backend/src/models/entities/user.ts index 07aba7badf..cb99fe53ca 100644 --- a/packages/backend/src/models/entities/user.ts +++ b/packages/backend/src/models/entities/user.ts @@ -58,6 +58,7 @@ export class User { }) public usernameLower: string; + @Index() // USING pgroonga pgroonga_varchar_full_text_search_ops_v2 @Column("varchar", { length: 128, nullable: true, diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index ced08fee3b..58a0ae7486 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -13,7 +13,6 @@ import processDb from "./processors/db/index.js"; import processObjectStorage from "./processors/object-storage/index.js"; import processSystemQueue from "./processors/system/index.js"; import processWebhookDeliver from "./processors/webhook-deliver.js"; -import processBackground from "./processors/background/index.js"; import { endedPollNotification } from "./processors/ended-poll-notification.js"; import { queueLogger } from "./logger.js"; import { getJobInfo } from "./get-job-info.js"; @@ -482,14 +481,6 @@ export function createCleanRemoteFilesJob() { ); } -export function createIndexAllNotesJob(data = {}) { - return backgroundQueue.add("indexAllNotes", data, { - removeOnComplete: true, - removeOnFail: false, - timeout: 1000 * 60 * 60 * 24, - }); -} - export function webhookDeliver( webhook: Webhook, type: (typeof webhookEventTypes)[number], @@ -526,7 +517,6 @@ export default function () { webhookDeliverQueue.process(64, processWebhookDeliver); processDb(dbQueue); processObjectStorage(objectStorageQueue); - processBackground(backgroundQueue); systemQueue.add( "cleanCharts", diff --git a/packages/backend/src/queue/processors/background/index-all-notes.ts b/packages/backend/src/queue/processors/background/index-all-notes.ts deleted file mode 100644 index 115f2147b6..0000000000 --- a/packages/backend/src/queue/processors/background/index-all-notes.ts +++ /dev/null @@ -1,89 +0,0 @@ -import type Bull from "bull"; -import type { DoneCallback } from "bull"; - -import { queueLogger } from "../../logger.js"; -import { Notes } from "@/models/index.js"; -import { MoreThan } from "typeorm"; -import { index } from "@/services/note/create.js"; -import { Note } from "@/models/entities/note.js"; -import meilisearch from "@/db/meilisearch.js"; -import { inspect } from "node:util"; - -const logger = queueLogger.createSubLogger("index-all-notes"); - -export default async function indexAllNotes( - job: Bull.Job>, - done: DoneCallback, -): Promise { - logger.info("Indexing all notes..."); - - let cursor: string | null = (job.data.cursor as string) ?? null; - let indexedCount: number = (job.data.indexedCount as number) ?? 0; - let total: number = (job.data.total as number) ?? 0; - - let running = true; - const take = 10000; - const batch = 100; - while (running) { - logger.info( - `Querying for ${take} notes ${indexedCount}/${ - total ? total : "?" - } at ${cursor}`, - ); - - let notes: Note[] = []; - try { - notes = await Notes.find({ - where: { - ...(cursor ? { id: MoreThan(cursor) } : {}), - }, - take: take, - order: { - id: 1, - }, - relations: ["user"], - }); - } catch (e: any) { - logger.error(`Failed to query notes:\n${inspect(e)}`); - done(e); - break; - } - - if (notes.length === 0) { - await job.progress(100); - running = false; - break; - } - - try { - const count = await Notes.count(); - total = count; - await job.update({ indexedCount, cursor, total }); - } catch (e) {} - - for (let i = 0; i < notes.length; i += batch) { - const chunk = notes.slice(i, i + batch); - - if (meilisearch) { - await meilisearch.ingestNote(chunk); - } - - await Promise.all(chunk.map((note) => index(note, true))); - - indexedCount += chunk.length; - const pct = (indexedCount / total) * 100; - await job.update({ indexedCount, cursor, total }); - await job.progress(+pct.toFixed(1)); - logger.info(`Indexed notes ${indexedCount}/${total ? total : "?"}`); - } - cursor = notes[notes.length - 1].id; - await job.update({ indexedCount, cursor, total }); - - if (notes.length < take) { - running = false; - } - } - - done(); - logger.info("All notes have been indexed."); -} diff --git a/packages/backend/src/queue/processors/background/index.ts b/packages/backend/src/queue/processors/background/index.ts deleted file mode 100644 index 6674f954b0..0000000000 --- a/packages/backend/src/queue/processors/background/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type Bull from "bull"; -import indexAllNotes from "./index-all-notes.js"; - -const jobs = { - indexAllNotes, -} as Record>>; - -export default function (q: Bull.Queue) { - for (const [k, v] of Object.entries(jobs)) { - q.process(k, 16, v); - } -} diff --git a/packages/backend/src/queue/processors/db/delete-account.ts b/packages/backend/src/queue/processors/db/delete-account.ts index adb93484bd..b43cdd137c 100644 --- a/packages/backend/src/queue/processors/db/delete-account.ts +++ b/packages/backend/src/queue/processors/db/delete-account.ts @@ -7,7 +7,6 @@ import type { DriveFile } from "@/models/entities/drive-file.js"; import { MoreThan } from "typeorm"; import { deleteFileSync } from "@/services/drive/delete-file.js"; import { sendEmail } from "@/services/send-email.js"; -import meilisearch from "@/db/meilisearch.js"; const logger = queueLogger.createSubLogger("delete-account"); @@ -42,9 +41,6 @@ export async function deleteAccount( cursor = notes[notes.length - 1].id; await Notes.delete(notes.map((note) => note.id)); - if (meilisearch) { - await meilisearch.deleteNotes(notes); - } } logger.succ("All of notes deleted"); diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 8184df5d23..fa029c1f4a 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -51,7 +51,6 @@ import * as ep___admin_relays_list from "./endpoints/admin/relays/list.js"; import * as ep___admin_relays_remove from "./endpoints/admin/relays/remove.js"; import * as ep___admin_resetPassword from "./endpoints/admin/reset-password.js"; import * as ep___admin_resolveAbuseUserReport from "./endpoints/admin/resolve-abuse-user-report.js"; -import * as ep___admin_search_indexAll from "./endpoints/admin/search/index-all.js"; import * as ep___admin_sendEmail from "./endpoints/admin/send-email.js"; import * as ep___admin_sendModMail from "./endpoints/admin/send-mod-mail.js"; import * as ep___admin_serverInfo from "./endpoints/admin/server-info.js"; @@ -400,7 +399,6 @@ const eps = [ ["admin/relays/remove", ep___admin_relays_remove], ["admin/reset-password", ep___admin_resetPassword], ["admin/resolve-abuse-user-report", ep___admin_resolveAbuseUserReport], - ["admin/search/index-all", ep___admin_search_indexAll], ["admin/send-email", ep___admin_sendEmail], ["admin/send-mod-mail", ep___admin_sendModMail], ["admin/server-info", ep___admin_serverInfo], diff --git a/packages/backend/src/server/api/endpoints/admin/search/index-all.ts b/packages/backend/src/server/api/endpoints/admin/search/index-all.ts deleted file mode 100644 index 531428849b..0000000000 --- a/packages/backend/src/server/api/endpoints/admin/search/index-all.ts +++ /dev/null @@ -1,28 +0,0 @@ -import define from "@/server/api/define.js"; -import { createIndexAllNotesJob } from "@/queue/index.js"; - -export const meta = { - tags: ["admin"], - - requireCredential: true, - requireModerator: true, -} as const; - -export const paramDef = { - type: "object", - properties: { - cursor: { - type: "string", - format: "misskey:id", - nullable: true, - default: null, - }, - }, - required: [], -} as const; - -export default define(meta, paramDef, async (ps, _me) => { - createIndexAllNotesJob({ - cursor: ps.cursor ?? undefined, - }); -}); diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index 8e818745fc..f8a7a626b7 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -1,5 +1,4 @@ import { In } from "typeorm"; -import { index } from "@/services/note/create.js"; import type { IRemoteUser, User } from "@/models/entities/user.js"; import { Users, @@ -626,8 +625,6 @@ export default define(meta, paramDef, async (ps, user) => { } if (publishing && user.isIndexable) { - index(note, true); - // Publish update event for the updated note details publishNoteStream(note.id, "updated", { updatedAt: update.updatedAt, diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 9374e0bac7..6846a7dd14 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -1,10 +1,5 @@ -import { FindManyOptions, In } from "typeorm"; import { Notes } from "@/models/index.js"; import { Note } from "@/models/entities/note.js"; -import config from "@/config/index.js"; -import es from "@/db/elasticsearch.js"; -import sonic from "@/db/sonic.js"; -import meilisearch, { MeilisearchNote } from "@/db/meilisearch.js"; import define from "@/server/api/define.js"; import { makePaginationQuery } from "@/server/api/common/make-pagination-query.js"; import { generateVisibilityQuery } from "@/server/api/common/generate-visibility-query.js"; @@ -69,282 +64,43 @@ export const paramDef = { } as const; export default define(meta, paramDef, async (ps, me) => { - if (es == null && sonic == null && meilisearch == null) { - const query = makePaginationQuery( - Notes.createQueryBuilder("note"), - ps.sinceId, - ps.untilId, - ); + const query = makePaginationQuery( + Notes.createQueryBuilder("note"), + ps.sinceId, + ps.untilId, + ); - if (ps.userId != null) { - query.andWhere("note.userId = :userId", { userId: ps.userId }); - } - - if (ps.channelId != null) { - query.andWhere("note.channelId = :channelId", { - channelId: ps.channelId, - }); - } - - query - .andWhere("note.text ILIKE :q", { q: `%${sqlLikeEscape(ps.query)}%` }) - .andWhere("note.visibility = 'public'") - .innerJoinAndSelect("note.user", "user") - .andWhere("user.isIndexable = TRUE") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") - .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); - - generateVisibilityQuery(query, me); - if (me) generateMutedUserQuery(query, me); - if (me) generateBlockedUserQuery(query, me); - - const notes: Note[] = await query.take(ps.limit).getMany(); - - return await Notes.packMany(notes, me); - } else if (sonic) { - let start = 0; - const chunkSize = 100; - - // Use sonic to fetch and step through all search results that could match the requirements - const ids = []; - while (true) { - const results = await sonic.search.query( - sonic.collection, - sonic.bucket, - ps.query, - { - limit: chunkSize, - offset: start, - }, - ); - - start += chunkSize; - - if (results.length === 0) { - break; - } - - const res = results - .map((k) => JSON.parse(k)) - .filter((key) => { - if (ps.userId && key.userId !== ps.userId) { - return false; - } - if (ps.channelId && key.channelId !== ps.channelId) { - return false; - } - if (ps.sinceId && key.id <= ps.sinceId) { - return false; - } - if (ps.untilId && key.id >= ps.untilId) { - return false; - } - return true; - }) - .map((key) => key.id); - - ids.push(...res); - } - - // Sort all the results by note id DESC (newest first) - ids.sort((a, b) => b - a); - - // Fetch the notes from the database until we have enough to satisfy the limit - start = 0; - const found = []; - while (found.length < ps.limit && start < ids.length) { - const chunk = ids.slice(start, start + chunkSize); - const notes: Note[] = await Notes.find({ - where: { - id: In(chunk), - }, - }); - - // The notes are checked for visibility and muted/blocked users when packed - found.push(...(await Notes.packMany(notes, me))); - start += chunkSize; - } - - // If we have more results than the limit, trim them - if (found.length > ps.limit) { - found.length = ps.limit; - } - - return found; - } else if (meilisearch) { - let start = 0; - const chunkSize = 100; - const sortByDate = ps.order !== "relevancy"; - - type NoteResult = { - id: string; - createdAt: number; - }; - const extractedNotes: NoteResult[] = []; - - while (true) { - const searchRes = await meilisearch.search( - ps.query, - chunkSize, - start, - me, - sortByDate ? "createdAt:desc" : null, - ); - const results: MeilisearchNote[] = searchRes.hits as MeilisearchNote[]; - - start += chunkSize; - - if (results.length === 0) { - break; - } - - const res = results - .filter((key: MeilisearchNote) => { - if (ps.userId && key.userId !== ps.userId) { - return false; - } - if (ps.channelId && key.channelId !== ps.channelId) { - return false; - } - if (ps.sinceId && key.id <= ps.sinceId) { - return false; - } - if (ps.untilId && key.id >= ps.untilId) { - return false; - } - return true; - }) - .map((key) => { - return { - id: key.id, - createdAt: key.createdAt, - }; - }); - - extractedNotes.push(...res); - } - - // Fetch the notes from the database until we have enough to satisfy the limit - start = 0; - const found = []; - const noteIDs = extractedNotes.map((note) => note.id); - - // Index the ID => index number into a map, so we can restore the array ordering efficiently later - const idIndexMap = new Map(noteIDs.map((id, index) => [id, index])); - - while (found.length < ps.limit && start < noteIDs.length) { - const chunk = noteIDs.slice(start, start + chunkSize); - - let query: FindManyOptions = { - where: { - id: In(chunk), - }, - }; - - const notes: Note[] = await Notes.find(query); - - // Re-order the note result according to the noteIDs array (cannot be undefined, we map this earlier) - // @ts-ignore - notes.sort((a, b) => idIndexMap.get(a.id) - idIndexMap.get(b.id)); - - // The notes are checked for visibility and muted/blocked users when packed - found.push(...(await Notes.packMany(notes, me))); - start += chunkSize; - } - - // If we have more results than the limit, trim the results down - if (found.length > ps.limit) { - found.length = ps.limit; - } - - return found; - } else { - const userQuery = - ps.userId != null - ? [ - { - term: { - userId: ps.userId, - }, - }, - ] - : []; - - const hostQuery = - ps.userId == null - ? ps.host === null - ? [ - { - bool: { - must_not: { - exists: { - field: "userHost", - }, - }, - }, - }, - ] - : ps.host !== undefined - ? [ - { - term: { - userHost: ps.host, - }, - }, - ] - : [] - : []; - - const result = await es.search({ - index: config.elasticsearch.index || "misskey_note", - body: { - size: ps.limit, - from: ps.offset, - query: { - bool: { - must: [ - { - simple_query_string: { - fields: ["text"], - query: ps.query.toLowerCase(), - default_operator: "and", - }, - }, - ...hostQuery, - ...userQuery, - ], - }, - }, - sort: [ - { - _doc: "desc", - }, - ], - }, - }); - - const hits = result.body.hits.hits.map((hit: any) => hit._id); - - if (hits.length === 0) return []; - - // Fetch found notes - const notes = await Notes.find({ - where: { - id: In(hits), - }, - order: { - id: -1, - }, - }); - - return await Notes.packMany(notes, me); + if (ps.userId != null) { + query.andWhere("note.userId = :userId", { userId: ps.userId }); } + + if (ps.channelId != null) { + query.andWhere("note.channelId = :channelId", { + channelId: ps.channelId, + }); + } + + query + .andWhere("note.text &@~ :q", { q: `${sqlLikeEscape(ps.query)}` }) + .andWhere("note.visibility = 'public'") + .innerJoinAndSelect("note.user", "user") + .andWhere("user.isIndexable = TRUE") + .leftJoinAndSelect("user.avatar", "avatar") + .leftJoinAndSelect("user.banner", "banner") + .leftJoinAndSelect("note.reply", "reply") + .leftJoinAndSelect("note.renote", "renote") + .leftJoinAndSelect("reply.user", "replyUser") + .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") + .leftJoinAndSelect("replyUser.banner", "replyUserBanner") + .leftJoinAndSelect("renote.user", "renoteUser") + .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") + .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); + + generateVisibilityQuery(query, me); + if (me) generateMutedUserQuery(query, me); + if (me) generateBlockedUserQuery(query, me); + + const notes: Note[] = await query.take(ps.limit).getMany(); + + return await Notes.packMany(notes, me); }); diff --git a/packages/backend/src/server/api/endpoints/server-info.ts b/packages/backend/src/server/api/endpoints/server-info.ts index ff3bfcd100..d3b6a08074 100644 --- a/packages/backend/src/server/api/endpoints/server-info.ts +++ b/packages/backend/src/server/api/endpoints/server-info.ts @@ -1,7 +1,6 @@ import * as os from "node:os"; import si from "systeminformation"; import define from "@/server/api/define.js"; -import meilisearch from "@/db/meilisearch.js"; import { fetchMeta } from "@/misc/fetch-meta.js"; export const meta = { @@ -63,15 +62,3 @@ export default define(meta, paramDef, async () => { }, }; }); - -async function meilisearchStatus() { - if (meilisearch) { - return meilisearch.serverStats(); - } else { - return { - health: "unconfigured", - size: 0, - indexed_count: 0, - }; - } -} diff --git a/packages/backend/src/server/api/endpoints/users/search.ts b/packages/backend/src/server/api/endpoints/users/search.ts index 3aef6dc901..a15a0feb4b 100644 --- a/packages/backend/src/server/api/endpoints/users/search.ts +++ b/packages/backend/src/server/api/endpoints/users/search.ts @@ -78,8 +78,8 @@ export default define(meta, paramDef, async (ps, me) => { const nameQuery = Users.createQueryBuilder("user") .where( new Brackets((qb) => { - qb.where("user.name ILIKE :query", { - query: `%${sqlLikeEscape(ps.query)}%`, + qb.where("user.name &@~ :query", { + query: `${sqlLikeEscape(ps.query)}`, }); // Also search username if it qualifies as username @@ -115,8 +115,8 @@ export default define(meta, paramDef, async (ps, me) => { if (users.length < ps.limit) { const profQuery = UserProfiles.createQueryBuilder("prof") .select("prof.userId") - .where("prof.description ILIKE :query", { - query: `%${sqlLikeEscape(ps.query)}%`, + .where("prof.description &@~ :query", { + query: `${sqlLikeEscape(ps.query)}`, }); if (ps.origin === "local") { diff --git a/packages/backend/src/server/nodeinfo.ts b/packages/backend/src/server/nodeinfo.ts index f15eb8c6e2..73266f26a7 100644 --- a/packages/backend/src/server/nodeinfo.ts +++ b/packages/backend/src/server/nodeinfo.ts @@ -82,7 +82,6 @@ const nodeinfo2 = async () => { disableRecommendedTimeline: meta.disableRecommendedTimeline, disableGlobalTimeline: meta.disableGlobalTimeline, emailRequiredForSignup: meta.emailRequiredForSignup, - searchFilters: config.meilisearch ? true : false, postEditing: true, postImports: meta.experimentalFeatures?.postImports || false, enableHcaptcha: meta.enableHcaptcha, diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 59665e2219..d2ec137d38 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -1,6 +1,4 @@ import * as mfm from "mfm-js"; -import es from "@/db/elasticsearch.js"; -import sonic from "@/db/sonic.js"; import { publishMainStream, publishNotesStream, @@ -59,7 +57,6 @@ import type { UserProfile } from "@/models/entities/user-profile.js"; import { db } from "@/db/postgre.js"; import { getActiveWebhooks } from "@/misc/webhook-cache.js"; import { shouldSilenceInstance } from "@/misc/should-block-instance.js"; -import meilisearch from "@/db/meilisearch.js"; import { redisClient } from "@/db/redis.js"; import { Mutex } from "redis-semaphore"; import { langmap } from "@/misc/langmap.js"; @@ -166,7 +163,6 @@ export default async ( createdAt: User["createdAt"]; isBot: User["isBot"]; inbox?: User["inbox"]; - isIndexable?: User["isIndexable"]; }, data: Option, silent = false, @@ -654,11 +650,6 @@ export default async ( } }); } - - // Register to search database - if (user.isIndexable) { - await index(note, false); - } }); async function renderNoteOrRenoteActivity(data: Option, note: Note) { @@ -814,40 +805,6 @@ async function insertNote( } } -export async function index(note: Note, reindexing: boolean): Promise { - if (!note.text || note.visibility !== "public") return; - - if (config.elasticsearch && es) { - es.index({ - index: config.elasticsearch.index || "misskey_note", - id: note.id.toString(), - body: { - text: normalizeForSearch(note.text), - userId: note.userId, - userHost: note.userHost, - }, - }); - } - - if (sonic) { - await sonic.ingest.push( - sonic.collection, - sonic.bucket, - JSON.stringify({ - id: note.id, - userId: note.userId, - userHost: note.userHost, - channelId: note.channelId, - }), - note.text, - ); - } - - if (meilisearch && !reindexing) { - await meilisearch.ingestNote(note); - } -} - async function notifyToWatchersOfRenotee( renote: Note, user: { id: User["id"] }, diff --git a/packages/backend/src/services/note/delete.ts b/packages/backend/src/services/note/delete.ts index f22fc35eba..988e1c8c48 100644 --- a/packages/backend/src/services/note/delete.ts +++ b/packages/backend/src/services/note/delete.ts @@ -16,7 +16,6 @@ import { import { countSameRenotes } from "@/misc/count-same-renotes.js"; import { registerOrFetchInstanceDoc } from "@/services/register-or-fetch-instance-doc.js"; import { deliverToRelays } from "@/services/relay.js"; -import meilisearch from "@/db/meilisearch.js"; /** * 投稿を削除します。 @@ -117,10 +116,6 @@ export default async function ( userId: user.id, }); } - - if (meilisearch) { - await meilisearch.deleteNotes(note.id); - } } async function findCascadingNotes(note: Note) { diff --git a/packages/client/src/pages/admin/database.vue b/packages/client/src/pages/admin/database.vue index e2da648123..c746c7032e 100644 --- a/packages/client/src/pages/admin/database.vue +++ b/packages/client/src/pages/admin/database.vue @@ -7,9 +7,6 @@ :display-back-button="true" /> - {{ - i18n.ts.indexPosts - }} diff --git a/packages/client/src/pages/admin/index.vue b/packages/client/src/pages/admin/index.vue index ee5bbabb33..1dd8540414 100644 --- a/packages/client/src/pages/admin/index.vue +++ b/packages/client/src/pages/admin/index.vue @@ -78,7 +78,6 @@ import * as os from "@/os"; import { lookupUser } from "@/scripts/lookup-user"; import { lookupFile } from "@/scripts/lookup-file"; import { lookupInstance } from "@/scripts/lookup-instance"; -import { indexPosts } from "@/scripts/index-posts"; import { defaultStore } from "@/store"; import { useRouter } from "@/router"; import { @@ -156,16 +155,6 @@ const menuDef = computed(() => [ }, ] : []), - ...($i.isAdmin - ? [ - { - type: "button", - icon: `${icon("ph-list-magnifying-glass")}`, - text: i18n.ts.indexPosts, - action: indexPosts, - }, - ] - : []), ], }, { diff --git a/packages/client/src/pages/admin/overview.metrics.vue b/packages/client/src/pages/admin/overview.metrics.vue index c8d7efadfb..498659b194 100644 --- a/packages/client/src/pages/admin/overview.metrics.vue +++ b/packages/client/src/pages/admin/overview.metrics.vue @@ -29,24 +29,6 @@

Used: {{ bytes(diskUsed, 1) }}

- -
- -
-

MeiliSearch

-

- {{ i18n.ts._widgets.meiliStatus }}: {{ meiliAvailable }} -

-

- {{ i18n.ts._widgets.meiliSize }}: - {{ bytes(meiliTotalSize, 1) }} -

-

- {{ i18n.ts._widgets.meiliIndexCount }}: - {{ meiliIndexCount }} -

-
-
@@ -57,7 +39,6 @@ import XPie from "../../widgets/server-metric/pie.vue"; import bytes from "@/filters/bytes"; import { useStream } from "@/stream"; import * as os from "@/os"; -import { i18n } from "@/i18n"; import icon from "@/scripts/icon"; const stream = useStream(); @@ -72,11 +53,6 @@ const memTotal = ref(0); const memUsed = ref(0); const memFree = ref(0); -const meiliProgress = ref(0); -const meiliTotalSize = ref(0); -const meiliIndexCount = ref(0); -const meiliAvailable = ref("unavailable"); - const diskUsage = computed(() => meta.fs.used / meta.fs.total); const diskTotal = computed(() => meta.fs.total); const diskUsed = computed(() => meta.fs.used); @@ -89,11 +65,6 @@ function onStats(stats) { memTotal.value = stats.mem.total; memUsed.value = stats.mem.active; memFree.value = memTotal.value - memUsed.value; - - meiliTotalSize.value = stats.meilisearch.size; - meiliIndexCount.value = stats.meilisearch.indexed_count; - meiliAvailable.value = stats.meilisearch.health; - meiliProgress.value = meiliIndexCount.value / serverStats.notesCount; } const connection = stream.useChannel("serverStats"); diff --git a/packages/client/src/scripts/index-posts.ts b/packages/client/src/scripts/index-posts.ts deleted file mode 100644 index 94b545e90e..0000000000 --- a/packages/client/src/scripts/index-posts.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { i18n } from "@/i18n"; -import * as os from "@/os"; - -export async function indexPosts() { - const { canceled, result: index } = await os.inputText({ - title: i18n.ts.indexFrom, - text: i18n.ts.indexFromDescription, - }); - if (canceled) return; - - if (index == null || index === "") { - await os.api("admin/search/index-all"); - await os.alert({ - type: "info", - text: i18n.ts.indexNotice, - }); - } else { - await os.api("admin/search/index-all", { - cursor: index, - }); - await os.alert({ - type: "info", - text: i18n.ts.indexNotice, - }); - } -} diff --git a/packages/client/src/widgets/server-metric/index.vue b/packages/client/src/widgets/server-metric/index.vue index 9276f5a110..a9026efeac 100644 --- a/packages/client/src/widgets/server-metric/index.vue +++ b/packages/client/src/widgets/server-metric/index.vue @@ -46,13 +46,6 @@ :connection="connection" :meta="meta" /> - @@ -66,7 +59,6 @@ import XNet from "./net.vue"; import XCpu from "./cpu.vue"; import XMemory from "./mem.vue"; import XDisk from "./disk.vue"; -import XMeili from "./meilisearch.vue"; import MkContainer from "@/components/MkContainer.vue"; import type { GetFormResultType } from "@/scripts/form"; import * as os from "@/os"; diff --git a/packages/client/src/widgets/server-metric/meilisearch.vue b/packages/client/src/widgets/server-metric/meilisearch.vue deleted file mode 100644 index ea9d1120a7..0000000000 --- a/packages/client/src/widgets/server-metric/meilisearch.vue +++ /dev/null @@ -1,86 +0,0 @@ - - - - - diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ad105accc..3715d96ad7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -75,9 +75,6 @@ importers: '@discordapp/twemoji': specifier: ^15.0.2 version: 15.0.2 - '@elastic/elasticsearch': - specifier: 8.12.2 - version: 8.12.2 '@koa/cors': specifier: 5.0.0 version: 5.0.0 @@ -243,9 +240,6 @@ importers: megalodon: specifier: workspace:* version: link:../megalodon - meilisearch: - specifier: 0.37.0 - version: 0.37.0 mfm-js: specifier: 0.24.0 version: 0.24.0 @@ -336,9 +330,6 @@ importers: sharp: specifier: 0.33.2 version: 0.33.2 - sonic-channel: - specifier: ^1.3.1 - version: 1.3.1 stringz: specifier: 2.1.0 version: 2.1.0 @@ -1889,30 +1880,6 @@ packages: universalify: 0.1.2 dev: false - /@elastic/elasticsearch@8.12.2: - resolution: {integrity: sha512-04NvH3LIgcv1Uwguorfw2WwzC9Lhfsqs9f0L6uq6MrCw0lqe/HOQ6E8vJ6EkHAA15iEfbhtxOtenbZVVcE+mAQ==} - engines: {node: '>=18'} - dependencies: - '@elastic/transport': 8.4.1 - tslib: 2.6.2 - transitivePeerDependencies: - - supports-color - dev: false - - /@elastic/transport@8.4.1: - resolution: {integrity: sha512-/SXVuVnuU5b4dq8OFY4izG+dmGla185PcoqgK6+AJMpmOeY1QYVNbWtCwvSvoAANN5D/wV+EBU8+x7Vf9EphbA==} - engines: {node: '>=16'} - dependencies: - debug: 4.3.4(supports-color@8.1.1) - hpagent: 1.2.0 - ms: 2.1.3 - secure-json-parse: 2.7.0 - tslib: 2.6.2 - undici: 5.23.0 - transitivePeerDependencies: - - supports-color - dev: false - /@emnapi/runtime@0.45.0: resolution: {integrity: sha512-Txumi3td7J4A/xTTwlssKieHKTGl3j4A1tglBx72auZ49YK7ePY6XZricgIg9mnZT4xPfA+UPCUdnhRuEFDL+w==} requiresBuild: true @@ -7182,6 +7149,7 @@ packages: node-fetch: 2.6.12 transitivePeerDependencies: - encoding + dev: true /cross-spawn@5.1.0: resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==} @@ -12529,14 +12497,6 @@ packages: engines: {node: '>= 0.6'} dev: false - /meilisearch@0.37.0: - resolution: {integrity: sha512-LdbK6JmRghCawrmWKJSEQF0OiE82md+YqJGE/U2JcCD8ROwlhTx0KM6NX4rQt0u0VpV0QZVG9umYiu3CSSIJAQ==} - dependencies: - cross-fetch: 3.1.8 - transitivePeerDependencies: - - encoding - dev: false - /memoize@10.0.0: resolution: {integrity: sha512-H6cBLgsi6vMWOcCpvVCdFFnl3kerEXbrYh9q+lY6VXvQSmM6CkmV08VOwT+WE2tzIEqRPFfAq3fm4v/UIW6mSA==} engines: {node: '>=18'} @@ -15181,10 +15141,6 @@ packages: ajv-keywords: 3.5.2(ajv@6.12.6) dev: true - /secure-json-parse@2.7.0: - resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} - dev: false - /seedrandom@2.4.2: resolution: {integrity: sha512-uQ72txMoObtuJooiBLSVs5Yu2e9d/lHQz0boaqHjW8runXB9vR8nFtaZV54wYii613N0C8ZqTBLsfwDhAdpvqQ==} dev: false @@ -15462,11 +15418,6 @@ packages: smart-buffer: 4.2.0 dev: false - /sonic-channel@1.3.1: - resolution: {integrity: sha512-+K4IZVFE7Tf2DB4EFZ23xo7a/+gJaiOHhFzXVZpzkX6Rs/rvf4YbSxnEGdYw8mrTcjtpG+jLVQEhP8sNTtN5VA==} - engines: {node: '>= 6.0.0'} - dev: false - /sort-keys-length@1.0.1: resolution: {integrity: sha512-GRbEOUqCxemTAk/b32F2xa8wDTs+Z1QHOkbhJDQTvv/6G3ZkbJ+frYWsTcc7cBB3Fu4wy4XlLCuNtJuMn7Gsvw==} engines: {node: '>=0.10.0'} From b30e68c98ccda393b3e065ec4cda593f0b7e870a Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 1 Mar 2024 22:17:02 +0900 Subject: [PATCH 002/191] feat: post search filters Co-authored-by: sup39 --- docs/api-change.md | 7 +- docs/changelog.md | 2 +- docs/downgrade.sql | 5 + locales/en-US.yml | 8 + locales/ja-JP.yml | 8 + .../1708872574733-index-alt-text-and-cw.js | 17 ++ .../src/server/api/endpoints/notes/search.ts | 61 ++++- packages/client/package.json | 1 + packages/client/src/components/MkDialog.vue | 120 --------- .../client/src/components/MkSearchBox.vue | 248 ++++++++++++++++++ .../src/components/MkSimpleTextWindow.vue | 43 +++ packages/client/src/pages/search.vue | 44 +++- packages/client/src/router.ts | 6 + packages/client/src/scripts/search.ts | 128 +++++---- pnpm-lock.yaml | 7 + 15 files changed, 531 insertions(+), 174 deletions(-) create mode 100644 packages/backend/migration/1708872574733-index-alt-text-and-cw.js create mode 100644 packages/client/src/components/MkSearchBox.vue create mode 100644 packages/client/src/components/MkSimpleTextWindow.vue diff --git a/docs/api-change.md b/docs/api-change.md index cf9799f00b..c024b00d90 100644 --- a/docs/api-change.md +++ b/docs/api-change.md @@ -4,7 +4,12 @@ Breaking changes are indicated by the :warning: icon. ## Unreleased -- `admin/search/index-all` is removed since posts are now indexed automatically. +- :warning: `admin/search/index-all` is removed since posts are now indexed automatically. +- New optional parameters are added to `notes/search` endpoint: + - `sinceDate` + - `untilDate` + - `withFiles` + - `searchCwAndAlt` ## v20240301 diff --git a/docs/changelog.md b/docs/changelog.md index 049e5e0ed2..197db641b2 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -4,7 +4,7 @@ Critical security updates are indicated by the :warning: icon. ## Unreleased -- Introduce new full-text search engine +- Introduce new full-text search engine and post search filters ## v20240301 diff --git a/docs/downgrade.sql b/docs/downgrade.sql index b1fc95eab7..7f0921edb6 100644 --- a/docs/downgrade.sql +++ b/docs/downgrade.sql @@ -1,6 +1,7 @@ BEGIN; DELETE FROM "migrations" WHERE name IN ( + 'IndexAltTextAndCw1708872574733', 'Pgroonga1698420787202', 'ChangeDefaultConfigs1709251460718', 'AddReplyMuting1704851359889', @@ -13,6 +14,10 @@ DELETE FROM "migrations" WHERE name IN ( 'RemoveNativeUtilsMigration1705877093218' ); +-- index-alt-text-and-cw +DROP INDEX "IDX_f4f7b93d05958527300d79ac82"; +DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f"; + -- pgroonga DROP INDEX "IDX_f27f5d88941e57442be75ba9c8"; DROP INDEX "IDX_065d4d8f3b5adb4a08841eae3c"; diff --git a/locales/en-US.yml b/locales/en-US.yml index 476b340492..1c2d210512 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1181,6 +1181,14 @@ pullDownToReload: "Pull down to reload" releaseToReload: "Release to reload" reloading: "Reloading" enableTimelineStreaming: "Update timelines automatically" +searchWords: "Words to search / ID or URL to lookup" +searchWordsDescription: "To search for posts, enter the search term. Separate words with a space for an AND search, or 'OR' (without quotes) between words for an OR search.\nFor example, 'morning night' will find posts that contain both 'morning' and 'night', and 'morning OR night' will find posts that contain either 'morning' or 'night' (or both).\nYou can also combine AND/OR conditions like '(morning OR night) sleepy'.\n\nIf you want to go to a specific user page or post page, enter the ID or URL in this field and click the 'Lookup' button. Clicking 'Search' will search for posts that literally contain the ID/URL." +searchUsers: "Posted by (optional)" +searchUsersDescription: "To search for posts by a specific user/server, enter the ID (@user@example.com, or @user for a local user) or domain name (example.com).\n\nIf you enter 'me' (without quotes), all of your posts (including unlisted, followers-only, direct, and secret posts) will be searched.\n\nIf you enter 'local' (without quotes), the results will be filtered to include only posts from this server." +searchRange: "Posted within (optional)" +searchRangeDescription: "If you want to filter the time period, enter it in this format: 20220615-20231031\n\nIf you leave out the year (like 0105-0106 or 20231105-0110), it's interpreted as the current year.\n\nYou can also omit either the start or end date. For example, -0102 will filter the search results to show only posts made before 2 January this year, and 20231026- will filter the results to show only posts made after 26 October 2023." +searchPostsWithFiles: "Only posts with files" +searchCwAndAlt: "Include content warnings and file descriptions" _emojiModPerm: unauthorized: "None" diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 64626b5fe9..8ddf99aa21 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1001,6 +1001,14 @@ pullDownToReload: "下に引っ張って再読み込み" releaseToReload: "離して再読み込み" reloading: "読み込み中" enableTimelineStreaming: "タイムラインを自動で更新する" +searchWords: "検索語句・照会するIDやURL" +searchWordsDescription: "投稿を検索するには、ここに検索語句を入力してください。空白区切りでAND検索になり、ORを挟むとOR検索になります。\n例えば「朝 夜」と入力すると「朝」と「夜」が両方含まれた投稿を検索し、「朝 OR 夜」と入力すると「朝」または「夜」（または両方）が含まれた投稿を検索します。\n「(朝 OR 夜) 眠い」のように、AND検索とOR検索を同時に行うこともできます。\n\n特定のユーザーや投稿のページに飛びたい場合には、この欄にID (@user@example.com) や投稿のURLを入力し「照会」を押してください。「検索」を押すとそのIDやURLが文字通り含まれる投稿を検索します。" +searchUsers: "投稿元（オプション）" +searchUsersDescription: "投稿検索で投稿者を絞りたい場合、@user@example.com（ローカルユーザーなら @user）の形式で投稿者のIDを入力してください。ユーザーIDではなくドメイン名 (example.com) を指定すると、そのサーバーの投稿を検索します。\n\nme とだけ入力すると、自分の投稿を検索します。この検索結果には未収載・フォロワー限定・ダイレクト・秘密を含む全ての投稿が含まれます。\n\nlocal とだけ入力すると、ローカルサーバーの投稿を検索します。" +searchRange: "投稿期間（オプション）" +searchRangeDescription: "投稿検索で投稿期間を絞りたい場合、20220615-20231031 のような形式で投稿期間を入力してください。今年の日付を指定する場合には年の指定を省略できます（0105-0106 や 20231105-0110 のように）。\n\n開始日と終了日のどちらか一方は省略可能です。例えば -0102 とすると今年1月2日までの投稿のみを、20231026- とすると2023年10月26日以降の投稿のみを検索します。" +searchPostsWithFiles: "添付ファイルのある投稿のみ" +searchCwAndAlt: "閲覧注意の注釈と添付ファイルの代替テキストも検索する" _sensitiveMediaDetection: description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てられます。サーバーの負荷が少し増えます。" diff --git a/packages/backend/migration/1708872574733-index-alt-text-and-cw.js b/packages/backend/migration/1708872574733-index-alt-text-and-cw.js new file mode 100644 index 0000000000..dbe4ee3c13 --- /dev/null +++ b/packages/backend/migration/1708872574733-index-alt-text-and-cw.js @@ -0,0 +1,17 @@ +export class IndexAltTextAndCw1708872574733 { + name = "IndexAltTextAndCw1708872574733"; + + async up(queryRunner) { + await queryRunner.query( + `CREATE INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2)`, + ); + await queryRunner.query( + `CREATE INDEX "IDX_f4f7b93d05958527300d79ac82" ON "drive_file" USING "pgroonga" ("comment" pgroonga_varchar_full_text_search_ops_v2)`, + ); + } + + async down(queryRunner) { + await queryRunner.query(`DROP INDEX "IDX_f4f7b93d05958527300d79ac82"`); + await queryRunner.query(`DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f"`); + } +} diff --git a/packages/backend/src/server/api/endpoints/notes/search.ts b/packages/backend/src/server/api/endpoints/notes/search.ts index 6846a7dd14..0bc70d37f9 100644 --- a/packages/backend/src/server/api/endpoints/notes/search.ts +++ b/packages/backend/src/server/api/endpoints/notes/search.ts @@ -1,3 +1,4 @@ +import { Brackets } from "typeorm"; import { Notes } from "@/models/index.js"; import { Note } from "@/models/entities/note.js"; import define from "@/server/api/define.js"; @@ -34,6 +35,8 @@ export const paramDef = { query: { type: "string" }, sinceId: { type: "string", format: "misskey:id" }, untilId: { type: "string", format: "misskey:id" }, + sinceDate: { type: "number", nullable: true }, + untilDate: { type: "number", nullable: true }, limit: { type: "integer", minimum: 1, maximum: 100, default: 10 }, offset: { type: "integer", default: 0 }, host: { @@ -47,6 +50,8 @@ export const paramDef = { nullable: true, default: null, }, + withFiles: { type: "boolean", nullable: true }, + searchCwAndAlt: { type: "boolean", nullable: true }, channelId: { type: "string", format: "misskey:id", @@ -68,6 +73,8 @@ export default define(meta, paramDef, async (ps, me) => { Notes.createQueryBuilder("note"), ps.sinceId, ps.untilId, + ps.sinceDate ?? undefined, + ps.untilDate ?? undefined, ); if (ps.userId != null) { @@ -80,11 +87,57 @@ export default define(meta, paramDef, async (ps, me) => { }); } + if (ps.query != null) { + const q = sqlLikeEscape(ps.query); + + if (ps.searchCwAndAlt) { + query.andWhere( + new Brackets((qb) => { + qb.where("note.text &@~ :q", { q }) + .orWhere("note.cw &@~ :q", { q }) + .orWhere( + `EXISTS ( + SELECT FROM "drive_file" + WHERE + comment &@~ :q + AND + drive_file."id" = ANY(note."fileIds") + )`, + { q }, + ); + }), + ); + } else { + query.andWhere("note.text &@~ :q", { q }); + } + } + + query.innerJoinAndSelect("note.user", "user"); + + // "from: me": search all (public, home, followers, specified) my posts + // otherwise: search public indexable posts only + if (ps.userId == null || ps.userId !== me?.id) { + query + .andWhere("note.visibility = 'public'") + .andWhere("user.isIndexable = TRUE"); + } + + if (ps.userId != null) { + query.andWhere("note.userId = :userId", { userId: ps.userId }); + } + + if (ps.host === null) { + query.andWhere("note.userHost IS NULL"); + } + if (ps.host != null) { + query.andWhere("note.userHost = :userHost", { userHost: ps.host }); + } + + if (ps.withFiles === true) { + query.andWhere("note.fileIds != '{}'"); + } + query - .andWhere("note.text &@~ :q", { q: `${sqlLikeEscape(ps.query)}` }) - .andWhere("note.visibility = 'public'") - .innerJoinAndSelect("note.user", "user") - .andWhere("user.isIndexable = TRUE") .leftJoinAndSelect("user.avatar", "avatar") .leftJoinAndSelect("user.banner", "banner") .leftJoinAndSelect("note.reply", "reply") diff --git a/packages/client/package.json b/packages/client/package.json index 93d1e68a50..9ff60dda54 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -65,6 +65,7 @@ "libopenmpt-wasm": "github:TheEssem/libopenmpt-packaging#build", "matter-js": "0.19.0", "mfm-js": "0.24.0", + "moment": "2.30.1", "photoswipe": "5.4.3", "prettier": "3.2.5", "prismjs": "1.29.0", diff --git a/packages/client/src/components/MkDialog.vue b/packages/client/src/components/MkDialog.vue index f93e164997..3c9e0e3a8c 100644 --- a/packages/client/src/components/MkDialog.vue +++ b/packages/client/src/components/MkDialog.vue @@ -101,15 +101,6 @@ " /> - { - os.selectUser().then((user) => { - inputValue.value = appendFilter( - "from:@" + acct.toString(user), - ); - }); - }, - }, - { - type: "parent", - text: i18n.ts._filters.withFile, - icon: `${iconClass("ph-paperclip")}`, - children: [ - { - text: i18n.ts.image, - icon: `${iconClass("ph-image-square")}`, - action: () => { - inputValue.value = appendFilter("has:image"); - }, - }, - { - text: i18n.ts.video, - icon: `${iconClass("ph-video-camera")}`, - action: () => { - inputValue.value = appendFilter("has:video"); - }, - }, - { - text: i18n.ts.audio, - icon: `${iconClass("ph-music-note")}`, - action: () => { - inputValue.value = appendFilter("has:audio"); - }, - }, - { - text: i18n.ts.file, - icon: `${iconClass("ph-file")}`, - action: () => { - inputValue.value = appendFilter("has:file"); - }, - }, - ], - }, - { - icon: `${iconClass("ph-link")}`, - text: i18n.ts._filters.fromDomain, - action: () => { - inputValue.value = appendFilter("domain:"); - }, - }, - { - icon: `${iconClass("ph-calendar-blank")}`, - text: i18n.ts._filters.notesBefore, - action: () => { - os.inputDate({ - title: i18n.ts._filters.notesBefore, - }).then((res) => { - if (res.canceled) return; - inputValue.value = appendFilter( - "before:" + formatDateToYYYYMMDD(res.result), - ); - }); - }, - }, - { - icon: `${iconClass("ph-calendar-blank")}`, - text: i18n.ts._filters.notesAfter, - action: () => { - os.inputDate({ - title: i18n.ts._filters.notesAfter, - }).then((res) => { - if (res.canceled) return; - inputValue.value = appendFilter( - "after:" + formatDateToYYYYMMDD(res.result), - ); - }); - }, - }, - { - icon: `${iconClass("ph-eye")}`, - text: i18n.ts._filters.followingOnly, - action: () => { - inputValue.value = appendFilter("filter:following"); - }, - }, - { - icon: `${iconClass("ph-users-three")}`, - text: i18n.ts._filters.followersOnly, - action: () => { - inputValue.value = appendFilter("filter:followers"); - }, - }, - ], - ev.target, - { noReturnFocus: true }, - ); - inputEl.value?.focus(); - if (typeof inputValue.value === "string") { - inputEl.value?.selectRange( - inputValue.value.length, - inputValue.value.length, - ); // cursor at end - } -} - onMounted(() => { document.addEventListener("keydown", onKeydown); }); diff --git a/packages/client/src/components/MkSearchBox.vue b/packages/client/src/components/MkSearchBox.vue new file mode 100644 index 0000000000..0a1e7315af --- /dev/null +++ b/packages/client/src/components/MkSearchBox.vue @@ -0,0 +1,248 @@ + + + + + diff --git a/packages/client/src/components/MkSimpleTextWindow.vue b/packages/client/src/components/MkSimpleTextWindow.vue new file mode 100644 index 0000000000..e8766476e7 --- /dev/null +++ b/packages/client/src/components/MkSimpleTextWindow.vue @@ -0,0 +1,43 @@ + + + + + diff --git a/packages/client/src/pages/search.vue b/packages/client/src/pages/search.vue index 61f211bcf3..5967b4e545 100644 --- a/packages/client/src/pages/search.vue +++ b/packages/client/src/pages/search.vue @@ -43,6 +43,7 @@ import { computed, onMounted, ref, watch } from "vue"; import { Virtual } from "swiper/modules"; import { Swiper, SwiperSlide } from "swiper/vue"; +import moment from "moment"; import XNotes from "@/components/MkNotes.vue"; import XUserList from "@/components/MkUserList.vue"; import { i18n } from "@/i18n"; @@ -50,19 +51,37 @@ import { definePageMetadata } from "@/scripts/page-metadata"; import { defaultStore } from "@/store"; import { deviceKind } from "@/scripts/device-kind"; import icon from "@/scripts/icon"; +import { $i } from "@/reactiveAccount"; import "swiper/scss"; import "swiper/scss/virtual"; +import { api } from "@/os"; const props = defineProps<{ - query: string; + query?: string; + user?: string; + host?: string; + since?: string; + until?: string; channel?: string; + withFiles: "0" | "1"; + searchCwAndAlt: "0" | "1"; }>(); +const userId = props.user == null ? undefined : await getUserId(props.user); + const notesPagination = { endpoint: "notes/search" as const, limit: 10, params: computed(() => ({ - query: props.query, + query: props.query ?? undefined, + userId, + host: props.host == null ? undefined : getHost(props.host), + sinceDate: + props.since == null ? undefined : getUnixTime(props.since, false), + untilDate: + props.until == null ? undefined : getUnixTime(props.until, true), + withFiles: props.withFiles === "1", + searchCwAndAlt: props.searchCwAndAlt === "1", channelId: props.channel, })), }; @@ -76,6 +95,27 @@ const usersPagination = { })), }; +async function getUserId(user: string): Promise { + if (user === "me") return $i!.id; + + const split = (user.startsWith("@") ? user.slice(1) : user).split("@"); + const username = split[0]; + const host = split.length === 1 ? undefined : split[1]; + + return (await api("users/show", { username, host })).id; +} + +function getHost(host: string): string | null { + if (host === "local") return null; + return host; +} + +function getUnixTime(date: string, nextDay: boolean): number { + return moment(date, date.length === 4 ? "MMDD" : "YYYYMMDD") + .add(nextDay ? 1 : 0, "days") + .valueOf(); +} + const tabs = ["notes", "users"]; const tab = ref(tabs[0]); watch(tab, () => syncSlide(tabs.indexOf(tab.value))); diff --git a/packages/client/src/router.ts b/packages/client/src/router.ts index 3dae7e16bd..deca773d60 100644 --- a/packages/client/src/router.ts +++ b/packages/client/src/router.ts @@ -306,7 +306,13 @@ export const routes = [ component: page(() => import("./pages/search.vue")), query: { q: "query", + user: "user", + host: "host", + since: "since", + until: "until", + withFiles: "withFiles", channel: "channel", + detailed: "searchCwAndAlt", }, }, { diff --git a/packages/client/src/scripts/search.ts b/packages/client/src/scripts/search.ts index 7b75771358..552b87f7b8 100644 --- a/packages/client/src/scripts/search.ts +++ b/packages/client/src/scripts/search.ts @@ -1,65 +1,101 @@ -import * as os from "@/os"; import { i18n } from "@/i18n"; +import { api, popup, promiseDialog } from "@/os"; import { mainRouter } from "@/router"; -import { instance } from "@/instance"; +import MkSearchBox from "@/components/MkSearchBox.vue"; export async function search() { - const { canceled, result: query } = await os.inputText({ - type: instance.features.searchFilters ? "search" : "text", - title: i18n.ts.search, - placeholder: i18n.ts.searchPlaceholder, + const { canceled, result } = await new Promise< + | { canceled: true; result: undefined } + | { + canceled: false; + result: { + action: "lookup"; + query: string; + }; + } + | { + canceled: false; + result: { + action: "search"; + query?: string; + from?: string; + range?: string; + withFiles: boolean; + searchCwAndAlt: boolean; + }; + } + >((resolve, _) => { + popup( + MkSearchBox, + {}, + { + done: (result) => { + resolve(result ?? { canceled: true }); + }, + }, + "closed", + ); }); - if (canceled || query == null || query === "") return; - const q = query.trim(); + if (canceled || result == null || result.query === "") return; - if (q.startsWith("@") && !q.includes(" ")) { - mainRouter.push(`/${q}`); - return; - } + if (result.action === "lookup") { + if (result.query.startsWith("#")) { + mainRouter.push(`/tags/${encodeURIComponent(result.query.slice(1))}`); + return; + } + if (result.query.startsWith("@")) { + mainRouter.push(`/${result.query}`); + return; + } + if (result.query.startsWith("https://")) { + const promise = api("ap/show", { + uri: result.query, + }); - if (q.startsWith("#")) { - mainRouter.push(`/tags/${encodeURIComponent(q.slice(1))}`); - return; - } + promiseDialog(promise, null, null, i18n.ts.fetchingAsApObject); + const res = await promise; - // like 2018/03/12 - if (/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}/.test(q.replace(/-/g, "/"))) { - const date = new Date(q.replace(/-/g, "/")); + if (res.type === "User") { + mainRouter.push(`/@${res.object.username}@${res.object.host}`); + } else if (res.type === "Note") { + mainRouter.push(`/notes/${res.object.id}`); + } - // 日付しか指定されてない場合、例えば 2018/03/12 ならユーザーは - // 2018/03/12 のコンテンツを「含む」結果になることを期待するはずなので - // 23時間59分進める(そのままだと 2018/03/12 00:00:00 「まで」の - // 結果になってしまい、2018/03/12 のコンテンツは含まれない) - if (q.replace(/-/g, "/").match(/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}$/)) { - date.setHours(23, 59, 59, 999); + return; } - // TODO - // v.$root.$emit('warp', date); - os.alert({ - type: "waiting", - }); - return; + // fallback + mainRouter.push(`/search?q=${encodeURIComponent(result.query)}`); } - if (q.startsWith("https://")) { - const promise = os.api("ap/show", { - uri: q, - }); + if (result.action === "search") { + const params = new URLSearchParams(); - os.promiseDialog(promise, null, null, i18n.ts.fetchingAsApObject); - - const res = await promise; - - if (res.type === "User") { - mainRouter.push(`/@${res.object.username}@${res.object.host}`); - } else if (res.type === "Note") { - mainRouter.push(`/notes/${res.object.id}`); + if (result.query != null) { + params.append("q", result.query); } - return; - } + if (result.from != null) { + if (result.from === "me" || result.from.includes("@")) + params.append("user", result.from); + else params.append("host", result.from); + } - mainRouter.push(`/search?q=${encodeURIComponent(q)}`); + if (result.range != null) { + const split = result.range.split("-"); + if (split.length === 1) { + params.append("since", result.range); + params.append("until", result.range); + } else { + if (split[0] !== "") params.append("since", split[0]); + if (split[1] !== "") params.append("until", split[1]); + } + } + + params.append("detailed", result.searchCwAndAlt ? "1" : "0"); + params.append("withFiles", result.withFiles ? "1" : "0"); + + mainRouter.push(`/search?${params.toString()}`); + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3715d96ad7..8313fdaeb3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -731,6 +731,9 @@ importers: mfm-js: specifier: 0.24.0 version: 0.24.0 + moment: + specifier: 2.30.1 + version: 2.30.1 photoswipe: specifier: 5.4.3 version: 5.4.3 @@ -12809,6 +12812,10 @@ packages: resolution: {integrity: sha512-5LC9SOxjSc2HF6vO2CyuTDNivEdoz2IvyJJGj6X8DJ0eFyfszE0QiEd+iXmBvUP3WHxSjFH/vIsA0EN00cgr8w==} dev: false + /moment@2.30.1: + resolution: {integrity: sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==} + dev: true + /ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} dev: false From d70d0f4232a66dea5622ee6a72adbeb6a994a61a Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 1 Mar 2024 22:52:21 +0900 Subject: [PATCH 003/191] refactor (backend): separate muted words and muted patterns Co-authored-by: sup39 --- docs/changelog.md | 1 + docs/downgrade.sql | 5 + locales/en-US.yml | 1 + locales/ja-JP.yml | 3 +- .../src/model/entity/user_profile.rs | 2 + ...9-separate-hard-mute-words-and-patterns.js | 32 ++++++ .../backend/src/misc/check-hit-antenna.ts | 29 +++-- packages/backend/src/misc/check-word-mute.ts | 67 ++++++------ .../src/models/entities/user-profile.ts | 6 ++ .../backend/src/models/repositories/user.ts | 27 ++--- packages/backend/src/models/schema/user.ts | 10 ++ .../server/api/endpoints/admin/show-user.ts | 1 + .../src/server/api/endpoints/i/update.ts | 58 +++++++--- .../api/stream/channels/global-timeline.ts | 7 +- .../api/stream/channels/home-timeline.ts | 7 +- .../api/stream/channels/hybrid-timeline.ts | 7 +- .../api/stream/channels/local-timeline.ts | 7 +- .../stream/channels/recommended-timeline.ts | 7 +- packages/backend/src/services/note/create.ts | 37 ++++--- .../client/src/pages/settings/word-mute.vue | 101 ++++++++++++++++-- packages/firefish-js/src/entities.ts | 1 + 21 files changed, 319 insertions(+), 97 deletions(-) create mode 100644 packages/backend/migration/1706413792769-separate-hard-mute-words-and-patterns.js diff --git a/docs/changelog.md b/docs/changelog.md index 197db641b2..65548e3189 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,7 @@ Critical security updates are indicated by the :warning: icon. ## Unreleased - Introduce new full-text search engine and post search filters +- Refactoring ## v20240301 diff --git a/docs/downgrade.sql b/docs/downgrade.sql index 7f0921edb6..31fa160299 100644 --- a/docs/downgrade.sql +++ b/docs/downgrade.sql @@ -1,6 +1,7 @@ BEGIN; DELETE FROM "migrations" WHERE name IN ( + 'SeparateHardMuteWordsAndPatterns1706413792769', 'IndexAltTextAndCw1708872574733', 'Pgroonga1698420787202', 'ChangeDefaultConfigs1709251460718', @@ -14,6 +15,10 @@ DELETE FROM "migrations" WHERE name IN ( 'RemoveNativeUtilsMigration1705877093218' ); +-- separate-hard-mute-words-and-patterns +UPDATE "user_profile" SET "mutedWords" = "mutedWords" || array_to_json("mutedPatterns")::jsonb; +ALTER TABLE "user_profile" DROP "mutedPatterns"; + -- index-alt-text-and-cw DROP INDEX "IDX_f4f7b93d05958527300d79ac82"; DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f"; diff --git a/locales/en-US.yml b/locales/en-US.yml index 1c2d210512..0bca5e0c19 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1429,6 +1429,7 @@ _menuDisplay: hide: "Hide" _wordMute: muteWords: "Muted words" + mutePatterns: "Muted patterns" muteLangs: "Muted Languages" muteWordsDescription: "Separate with spaces for an AND condition or with line breaks for an OR condition." diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 8ddf99aa21..a79cda6706 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1220,7 +1220,8 @@ _menuDisplay: hide: "隠す" _wordMute: muteWords: "ミュートするワード" - muteLangs: "ミュートされた言語" + mutePatterns: "ミュートするパターン" + muteLangs: "ミュートする言語" muteWordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります。" muteWordsDescription2: "キーワードをスラッシュで囲むと正規表現になります。" muteLangsDescription: "OR 条件の場合はスペースまたは改行で区切ります。" diff --git a/packages/backend-rs/src/model/entity/user_profile.rs b/packages/backend-rs/src/model/entity/user_profile.rs index daccf57d1e..c05fd34ed3 100644 --- a/packages/backend-rs/src/model/entity/user_profile.rs +++ b/packages/backend-rs/src/model/entity/user_profile.rs @@ -71,6 +71,8 @@ pub struct Model { pub prevent_ai_learning: bool, #[sea_orm(column_name = "isIndexable")] pub is_indexable: bool, + #[sea_orm(column_name = "mutedPatterns")] + pub muted_patterns: Vec, } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] diff --git a/packages/backend/migration/1706413792769-separate-hard-mute-words-and-patterns.js b/packages/backend/migration/1706413792769-separate-hard-mute-words-and-patterns.js new file mode 100644 index 0000000000..ace5734018 --- /dev/null +++ b/packages/backend/migration/1706413792769-separate-hard-mute-words-and-patterns.js @@ -0,0 +1,32 @@ +export class SeparateHardMuteWordsAndPatterns1706413792769 { + name = "SeparateHardMuteWordsAndPatterns1706413792769"; + + async up(queryRunner) { + await queryRunner.query( + `ALTER TABLE "user_profile" ADD "mutedPatterns" text[] DEFAULT '{}'`, + ); + await queryRunner.query(` + UPDATE "user_profile" SET + "mutedPatterns" = ARRAY( + SELECT jsonb_array_elements_text(jsonb_path_query_array( + "mutedWords", + '$ ? (@.type() == "string")' + )) + ), + "mutedWords" = jsonb_path_query_array( + "mutedWords", + '$ ? (@.type() == "array")' + ) + `); + await queryRunner.query( + `ALTER TABLE "user_profile" ALTER "mutedPatterns" SET NOT NULL`, + ); + } + + async down(queryRunner) { + await queryRunner.query( + `UPDATE "user_profile" SET "mutedWords" = "mutedWords" || array_to_json("mutedPatterns")::jsonb`, + ); + await queryRunner.query(`ALTER TABLE "user_profile" DROP "mutedPatterns"`); + } +} diff --git a/packages/backend/src/misc/check-hit-antenna.ts b/packages/backend/src/misc/check-hit-antenna.ts index 1f4f2f2fae..81776ae55e 100644 --- a/packages/backend/src/misc/check-hit-antenna.ts +++ b/packages/backend/src/misc/check-hit-antenna.ts @@ -1,6 +1,7 @@ import type { Antenna } from "@/models/entities/antenna.js"; import type { Note } from "@/models/entities/note.js"; import type { User } from "@/models/entities/user.js"; +import type { UserProfile } from "@/models/entities/user-profile.js"; import { Blockings, UserProfiles } from "@/models/index.js"; import { getFullApAccount } from "@/misc/convert-host.js"; import * as Acct from "@/misc/acct.js"; @@ -9,7 +10,11 @@ import { Cache } from "@/misc/cache.js"; import { getWordHardMute } from "@/misc/check-word-mute.js"; const blockingCache = new Cache("blocking", 60 * 5); -const mutedWordsCache = new Cache("mutedWords", 60 * 5); +const hardMutesCache = new Cache<{ + userId: UserProfile["userId"]; + mutedWords: UserProfile["mutedWords"]; + mutedPatterns: UserProfile["mutedPatterns"]; +}>("hardMutes", 60 * 5); export async function checkHitAntenna( antenna: Antenna, @@ -89,12 +94,24 @@ export async function checkHitAntenna( ); if (blockings.includes(antenna.userId)) return false; - const mutedWords = await mutedWordsCache.fetch(antenna.userId, () => - UserProfiles.findOneBy({ userId: antenna.userId }).then( - (profile) => profile?.mutedWords, - ), + const mutes = await hardMutesCache.fetch(antenna.userId, () => + UserProfiles.findOneByOrFail({ + userId: antenna.userId, + }).then((profile) => { + return { + userId: antenna.userId, + mutedWords: profile.mutedWords, + mutedPatterns: profile.mutedPatterns, + }; + }), ); - if (await getWordHardMute(note, antenna.userId, mutedWords)) return false; + if ( + mutes.mutedWords != null && + mutes.mutedPatterns != null && + antenna.userId !== note.userId && + (await getWordHardMute(note, mutes.mutedWords, mutes.mutedPatterns)) + ) + return false; // TODO: eval expression diff --git a/packages/backend/src/misc/check-word-mute.ts b/packages/backend/src/misc/check-word-mute.ts index 5686aef2f7..f07f2a0fe5 100644 --- a/packages/backend/src/misc/check-word-mute.ts +++ b/packages/backend/src/misc/check-word-mute.ts @@ -6,11 +6,14 @@ type NoteLike = { text: Note["text"]; files?: Note["files"]; cw?: Note["cw"]; + reply?: NoteLike | null; + renote?: NoteLike | null; }; function checkWordMute( - note: NoteLike, - mutedWords: Array, + note: NoteLike | null | undefined, + mutedWords: string[][], + mutedPatterns: string[], ): boolean { if (note == null) return false; @@ -21,33 +24,33 @@ function checkWordMute( if (text === "") return false; - for (const mutePattern of mutedWords) { - if (Array.isArray(mutePattern)) { - // Clean up - const keywords = mutePattern.filter((keyword) => keyword !== ""); + for (const mutedWord of mutedWords) { + // Clean up + const keywords = mutedWord.filter((keyword) => keyword !== ""); - if ( - keywords.length > 0 && - keywords.every((keyword) => - text.toLowerCase().includes(keyword.toLowerCase()), - ) + if ( + keywords.length > 0 && + keywords.every((keyword) => + text.toLowerCase().includes(keyword.toLowerCase()), ) - return true; - } else { - // represents RegExp - const regexp = mutePattern.match(/^\/(.+)\/(.*)$/); + ) + return true; + } + for (const mutedPattern of mutedPatterns) { + // represents RegExp + const regexp = mutedPattern.match(/^\/(.+)\/(.*)$/); + + // This should never happen due to input sanitisation. + if (!regexp) { + console.warn(`Found invalid regex in word mutes: ${mutedPattern}`); + continue; + } + + try { + if (new RE2(regexp[1], regexp[2]).test(text)) return true; + } catch (err) { // This should never happen due to input sanitisation. - if (!regexp) { - console.warn(`Found invalid regex in word mutes: ${mutePattern}`); - continue; - } - - try { - if (new RE2(regexp[1], regexp[2]).test(text)) return true; - } catch (err) { - // This should never happen due to input sanitisation. - } } } @@ -55,17 +58,17 @@ function checkWordMute( } export async function getWordHardMute( - note: NoteLike, - meId: string | null | undefined, - mutedWords?: Array, + note: NoteLike | null, + mutedWords: string[][], + mutedPatterns: string[], ): Promise { - if (note.userId === meId || mutedWords == null) return false; + if (note == null || mutedWords == null || mutedPatterns == null) return false; if (mutedWords.length > 0) { return ( - checkWordMute(note, mutedWords) || - checkWordMute(note.reply, mutedWords) || - checkWordMute(note.renote, mutedWords) + checkWordMute(note, mutedWords, mutedPatterns) || + checkWordMute(note.reply, mutedWords, mutedPatterns) || + checkWordMute(note.renote, mutedWords, mutedPatterns) ); } diff --git a/packages/backend/src/models/entities/user-profile.ts b/packages/backend/src/models/entities/user-profile.ts index 0cdf4e8393..2a99d58226 100644 --- a/packages/backend/src/models/entities/user-profile.ts +++ b/packages/backend/src/models/entities/user-profile.ts @@ -217,6 +217,12 @@ export class UserProfile { }) public mutedWords: string[][]; + @Column("text", { + array: true, + nullable: false, + }) + public mutedPatterns: string[]; + @Column("jsonb", { default: [], comment: "List of instances muted by the user.", diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index 28abe7a9cc..59bba8b44d 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -544,13 +544,13 @@ export const UserRepository = db.getRepository(User).extend({ ? { avatarId: user.avatarId, bannerId: user.bannerId, - injectFeaturedNote: profile!.injectFeaturedNote, - receiveAnnouncementEmail: profile!.receiveAnnouncementEmail, - alwaysMarkNsfw: profile!.alwaysMarkNsfw, - carefulBot: profile!.carefulBot, - autoAcceptFollowed: profile!.autoAcceptFollowed, - noCrawle: profile!.noCrawle, - preventAiLearning: profile!.preventAiLearning, + injectFeaturedNote: profile?.injectFeaturedNote, + receiveAnnouncementEmail: profile?.receiveAnnouncementEmail, + alwaysMarkNsfw: profile?.alwaysMarkNsfw, + carefulBot: profile?.carefulBot, + autoAcceptFollowed: profile?.autoAcceptFollowed, + noCrawle: profile?.noCrawle, + preventAiLearning: profile?.preventAiLearning, isExplorable: user.isExplorable, isDeleted: user.isDeleted, hideOnlineStatus: user.hideOnlineStatus, @@ -571,17 +571,18 @@ export const UserRepository = db.getRepository(User).extend({ hasUnreadNotification: this.getHasUnreadNotification(user.id), hasPendingReceivedFollowRequest: this.getHasPendingReceivedFollowRequest(user.id), - mutedWords: profile!.mutedWords, - mutedInstances: profile!.mutedInstances, - mutingNotificationTypes: profile!.mutingNotificationTypes, - emailNotificationTypes: profile!.emailNotificationTypes, + mutedWords: profile?.mutedWords, + mutedPatterns: profile?.mutedPatterns, + mutedInstances: profile?.mutedInstances, + mutingNotificationTypes: profile?.mutingNotificationTypes, + emailNotificationTypes: profile?.emailNotificationTypes, } : {}), ...(opts.includeSecrets ? { - email: profile!.email, - emailVerified: profile!.emailVerified, + email: profile?.email, + emailVerified: profile?.emailVerified, securityKeysList: UserSecurityKeys.find({ where: { userId: user.id, diff --git a/packages/backend/src/models/schema/user.ts b/packages/backend/src/models/schema/user.ts index d6a5cd8b8d..bcdd718dd1 100644 --- a/packages/backend/src/models/schema/user.ts +++ b/packages/backend/src/models/schema/user.ts @@ -468,6 +468,16 @@ export const packedMeDetailedOnlySchema = { }, }, }, + mutedPatterns: { + type: "array", + nullable: false, + optional: false, + items: { + type: "string", + nullable: false, + optional: false, + }, + }, mutedInstances: { type: "array", nullable: true, diff --git a/packages/backend/src/server/api/endpoints/admin/show-user.ts b/packages/backend/src/server/api/endpoints/admin/show-user.ts index 164d6ae419..3ad255ddde 100644 --- a/packages/backend/src/server/api/endpoints/admin/show-user.ts +++ b/packages/backend/src/server/api/endpoints/admin/show-user.ts @@ -60,6 +60,7 @@ export default define(meta, paramDef, async (ps, me) => { injectFeaturedNote: profile.injectFeaturedNote, receiveAnnouncementEmail: profile.receiveAnnouncementEmail, mutedWords: profile.mutedWords, + mutedPatterns: profile.mutedPatterns, mutedInstances: profile.mutedInstances, mutingNotificationTypes: profile.mutingNotificationTypes, isModerator: user.isModerator, diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index d7974ce6cc..08bf885d49 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -120,6 +120,7 @@ export const paramDef = { ffVisibility: { type: "string", enum: ["public", "followers", "private"] }, pinnedPageId: { type: "string", format: "misskey:id", nullable: true }, mutedWords: { type: "array" }, + mutedPatterns: { type: "array", items: { type: "string" } }, mutedInstances: { type: "array", items: { @@ -159,23 +160,52 @@ export default define(meta, paramDef, async (ps, _user, token) => { profileUpdates.ffVisibility = ps.ffVisibility; if (ps.avatarId !== undefined) updates.avatarId = ps.avatarId; if (ps.bannerId !== undefined) updates.bannerId = ps.bannerId; + if (ps.mutedPatterns !== undefined) { + for (const item of ps.mutedPatterns) { + const regexp = item.match(/^\/(.+)\/(.*)$/); + if (!regexp) throw new ApiError(meta.errors.invalidRegexp); + + try { + new RegExp(regexp[1], regexp[2]); + } catch (err) { + throw new ApiError(meta.errors.invalidRegexp); + } + + profileUpdates.mutedPatterns = profileUpdates.mutedPatterns ?? []; + profileUpdates.mutedPatterns.push(item); + } + } if (ps.mutedWords !== undefined) { - // validate regular expression syntax - ps.mutedWords - .filter((x) => !Array.isArray(x)) - .forEach((x) => { - const regexp = x.match(/^\/(.+)\/(.*)$/); - if (!regexp) throw new ApiError(meta.errors.invalidRegexp); + // for backward compatibility + for (const item of ps.mutedWords) { + if (Array.isArray(item)) continue; - try { - new RE2(regexp[1], regexp[2]); - } catch (err) { - throw new ApiError(meta.errors.invalidRegexp); - } - }); + const regexp = item.match(/^\/(.+)\/(.*)$/); + if (!regexp) throw new ApiError(meta.errors.invalidRegexp); - profileUpdates.mutedWords = ps.mutedWords; - profileUpdates.enableWordMute = ps.mutedWords.length > 0; + try { + new RegExp(regexp[1], regexp[2]); + } catch (err) { + throw new ApiError(meta.errors.invalidRegexp); + } + + profileUpdates.mutedPatterns = profileUpdates.mutedPatterns ?? []; + profileUpdates.mutedPatterns.push(item); + } + + profileUpdates.mutedWords = ps.mutedWords.filter((item) => + Array.isArray(item), + ); + } + if ( + profileUpdates.mutedWords !== undefined || + profileUpdates.mutedPatterns !== undefined + ) { + profileUpdates.enableWordMute = + (profileUpdates.mutedWords != null && + profileUpdates.mutedWords.length > 0) || + (profileUpdates.mutedPatterns != null && + profileUpdates.mutedPatterns.length > 0); } if (ps.mutedInstances !== undefined) profileUpdates.mutedInstances = ps.mutedInstances; diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index 74345b9395..39d0719dd3 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -69,7 +69,12 @@ export default class extends Channel { // そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる if ( this.userProfile && - (await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords)) + this.user?.id !== note.userId && + (await getWordHardMute( + note, + this.userProfile.mutedWords, + this.userProfile.mutedPatterns, + )) ) return; diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts index f24218df92..8f23946259 100644 --- a/packages/backend/src/server/api/stream/channels/home-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts @@ -68,7 +68,12 @@ export default class extends Channel { // そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる if ( this.userProfile && - (await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords)) + this.user?.id !== note.userId && + (await getWordHardMute( + note, + this.userProfile.mutedWords, + this.userProfile.mutedPatterns, + )) ) return; diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index f87d7ab6d2..7f5c662b8c 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -85,7 +85,12 @@ export default class extends Channel { // そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる if ( this.userProfile && - (await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords)) + this.user?.id !== note.userId && + (await getWordHardMute( + note, + this.userProfile.mutedWords, + this.userProfile.mutedPatterns, + )) ) return; diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 2cba992b9e..88eabe991e 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -61,7 +61,12 @@ export default class extends Channel { // そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる if ( this.userProfile && - (await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords)) + this.user?.id !== note.userId && + (await getWordHardMute( + note, + this.userProfile.mutedWords, + this.userProfile.mutedPatterns, + )) ) return; diff --git a/packages/backend/src/server/api/stream/channels/recommended-timeline.ts b/packages/backend/src/server/api/stream/channels/recommended-timeline.ts index 12df39fca2..a9da732f89 100644 --- a/packages/backend/src/server/api/stream/channels/recommended-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/recommended-timeline.ts @@ -83,7 +83,12 @@ export default class extends Channel { // そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる if ( this.userProfile && - (await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords)) + this.user?.id !== note.userId && + (await getWordHardMute( + note, + this.userProfile.mutedWords, + this.userProfile.mutedPatterns, + )) ) return; diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index d2ec137d38..fc7f1265b3 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -65,9 +65,13 @@ import { inspect } from "node:util"; const logger = new Logger("create-note"); -const mutedWordsCache = new Cache< - { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] ->("mutedWords", 60 * 5); +const hardMutesCache = new Cache< + { + userId: UserProfile["userId"]; + mutedWords: UserProfile["mutedWords"]; + mutedPatterns: UserProfile["mutedPatterns"]; + }[] +>("hardMutes", 60 * 5); type NotificationType = "reply" | "renote" | "quote" | "mention"; @@ -357,27 +361,30 @@ export default async ( incNotesCountOfUser(user); // Word mute - mutedWordsCache + hardMutesCache .fetch(null, () => UserProfiles.find({ where: { enableWordMute: true, }, - select: ["userId", "mutedWords"], + select: ["userId", "mutedWords", "mutedPatterns"], }), ) .then((us) => { for (const u of us) { - getWordHardMute(data, u.userId, u.mutedWords).then((shouldMute) => { - if (shouldMute) { - MutedNotes.insert({ - id: genId(), - userId: u.userId, - noteId: note.id, - reason: "word", - }); - } - }); + if (u.userId === user.id) return; + getWordHardMute(note, u.mutedWords, u.mutedPatterns).then( + (shouldMute: boolean) => { + if (shouldMute) { + MutedNotes.insert({ + id: genId(), + userId: u.userId, + noteId: note.id, + reason: "word", + }); + } + }, + ); } }); diff --git a/packages/client/src/pages/settings/word-mute.vue b/packages/client/src/pages/settings/word-mute.vue index 68b6b09a45..8a6a0408ac 100644 --- a/packages/client/src/pages/settings/word-mute.vue +++ b/packages/client/src/pages/settings/word-mute.vue @@ -36,11 +36,15 @@ > {{ i18n.ts._wordMute.muteWords }} - + + + + {{ i18n.ts._wordMute.mutePatterns }} + { changed.value = true; }); +watch(hardMutedPatterns, () => { + changed.value = true; +}); + async function save() { - const parseMutes = (mutes, tab) => { + const parseSoftMutes = (mutes, tab) => { // split into lines, remove empty lines and unnecessary whitespace const lines = mutes .trim() @@ -151,11 +160,80 @@ async function save() { return lines; }; - let softMutes, softMLangs, hardMutes; + const parseMutedWords = (mutes) => { + // split into lines, remove empty lines and unnecessary whitespace + return mutes + .trim() + .split("\n") + .map((line) => line.trim()) + .filter((line) => line !== "") + .map((line) => line.split(" ")) + .filter((line) => line.length > 0); + }; + + const parseMutedPatterns = (mutes, tab) => { + // split into lines, remove empty lines and unnecessary whitespace + const lines = mutes + .trim() + .split("\n") + .map((line) => line.trim()) + .filter((line) => line !== ""); + + // check each line if it is a RegExp or not + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const regexp = line.match(/^\/(.+)\/(.*)$/); + if (regexp) { + // check that the RegExp is valid + try { + new RegExp(regexp[1], regexp[2]); + // note that regex lines will not be split by spaces! + } catch (err: any) { + // invalid syntax: do not save, do not reset changed flag + os.alert({ + type: "error", + title: i18n.ts.regexpError, + text: + i18n.t("regexpErrorDescription", { + tab, + line: i + 1, + }) + + "\n" + + err.toString(), + }); + // re-throw error so these invalid settings are not saved + throw err; + } + } else { + // invalid syntax: do not save, do not reset changed flag + os.alert({ + type: "error", + title: i18n.ts.regexpError, + text: i18n.t("regexpErrorDescription", { + tab, + line: i + 1, + }), + }); + // re-throw error so these invalid settings are not saved + throw new Error("Invalid regular expression"); + } + } + + return lines; + }; + + let softMutes, softMLangs, hardMWords, hardMPatterns; try { - softMutes = parseMutes(softMutedWords.value, i18n.ts._wordMute.soft); - softMLangs = parseMutes(softMutedLangs.value, i18n.ts._wordMute.lang); - hardMutes = parseMutes(hardMutedWords.value, i18n.ts._wordMute.hard); + softMutes = parseSoftMutes( + softMutedWords.value, + i18n.ts._wordMute.soft, + ); + softMLangs = parseMutedWords(softMutedLangs.value); + hardMWords = parseMutedWords(hardMutedWords.value); + hardMPatterns = parseMutedPatterns( + hardMutedPatterns.value, + i18n.ts._wordMute.hard, + ); } catch (err) { // already displayed error message in parseMutes return; @@ -164,7 +242,8 @@ async function save() { defaultStore.set("mutedWords", softMutes); defaultStore.set("mutedLangs", softMLangs); await os.api("i/update", { - mutedWords: hardMutes, + mutedWords: hardMWords, + mutedPatterns: hardMPatterns, }); changed.value = false; diff --git a/packages/firefish-js/src/entities.ts b/packages/firefish-js/src/entities.ts index 6d4340890f..ed028efa2d 100644 --- a/packages/firefish-js/src/entities.ts +++ b/packages/firefish-js/src/entities.ts @@ -107,6 +107,7 @@ export type MeDetailed = UserDetailed & { isDeleted: boolean; isExplorable: boolean; mutedWords: string[][]; + mutedPatterns: string[]; mutingNotificationTypes: string[]; noCrawle: boolean; preventAiLearning: boolean; From 7a78dceb52bc0aadb4f3e8ab9d9ee75e800d3b8d Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 1 Mar 2024 23:03:13 +0900 Subject: [PATCH 004/191] docs: minor updates in changelog, notice-for-admins --- docs/changelog.md | 1 + docs/notice-for-admins.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 65548e3189..99ece6a663 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -24,6 +24,7 @@ Critical security updates are indicated by the :warning: icon. - Set reaction history to public - Change default server settings (existing servers are not affected) - Disable new user registration +- Fix bugs ## v20240229 diff --git a/docs/notice-for-admins.md b/docs/notice-for-admins.md index 276ffea7b8..6fa5689642 100644 --- a/docs/notice-for-admins.md +++ b/docs/notice-for-admins.md @@ -1,6 +1,6 @@ # Unreleased -The full-text search engine in Firefish has been changed to [PGroonga](https://pgroonga.github.io/). This is no longer an optional feature, so please enable PGroonga on your system. If you are using Sonic, Meilisearch, or Elasticsearch, you can also uninstall it from your system and remove the settings from `.config/default.yml`. +The full-text search engine used in Firefish has been changed to [PGroonga](https://pgroonga.github.io/). This is no longer an optional feature, so please enable PGroonga on your system. If you are using Sonic, Meilisearch, or Elasticsearch, you can also uninstall it from your system and remove the settings from `.config/default.yml`. ## For systemd/pm2 users @@ -33,7 +33,7 @@ There are official installation instructions for many operating systems on Date: Fri, 1 Mar 2024 23:41:55 +0900 Subject: [PATCH 005/191] feat: show unlisted posts from following users in antennas --- docs/changelog.md | 1 + .../backend/src/misc/check-hit-antenna.ts | 31 ++++++++++++++----- .../server/api/endpoints/antennas/notes.ts | 3 +- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 99ece6a663..865ea86050 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -6,6 +6,7 @@ Critical security updates are indicated by the :warning: icon. - Introduce new full-text search engine and post search filters - Refactoring +- Show unlisted posts from following users in antennas (similar to [Fedibird](https://github.com/fedibird/mastodon/tree/fedibird) and [kmyblue](https://github.com/kmycode/mastodon), unlisted posts from people you don't follow won't be shown) ## v20240301 diff --git a/packages/backend/src/misc/check-hit-antenna.ts b/packages/backend/src/misc/check-hit-antenna.ts index 81776ae55e..b93cb459e8 100644 --- a/packages/backend/src/misc/check-hit-antenna.ts +++ b/packages/backend/src/misc/check-hit-antenna.ts @@ -2,12 +2,12 @@ import type { Antenna } from "@/models/entities/antenna.js"; import type { Note } from "@/models/entities/note.js"; import type { User } from "@/models/entities/user.js"; import type { UserProfile } from "@/models/entities/user-profile.js"; -import { Blockings, UserProfiles } from "@/models/index.js"; +import { Blockings, Followings, UserProfiles } from "@/models/index.js"; import { getFullApAccount } from "@/misc/convert-host.js"; import * as Acct from "@/misc/acct.js"; +import { getWordHardMute } from "@/misc/check-word-mute.js"; import type { Packed } from "@/misc/schema.js"; import { Cache } from "@/misc/cache.js"; -import { getWordHardMute } from "@/misc/check-word-mute.js"; const blockingCache = new Cache("blocking", 60 * 5); const hardMutesCache = new Cache<{ @@ -15,6 +15,7 @@ const hardMutesCache = new Cache<{ mutedWords: UserProfile["mutedWords"]; mutedPatterns: UserProfile["mutedPatterns"]; }>("hardMutes", 60 * 5); +const followingCache = new Cache("following", 60 * 5); export async function checkHitAntenna( antenna: Antenna, @@ -22,11 +23,10 @@ export async function checkHitAntenna( noteUser: { id: User["id"]; username: string; host: string | null }, ): Promise { if (note.visibility === "specified") return false; - if (note.visibility === "home") return false; - if (!antenna.withReplies && note.replyId != null) return false; if (antenna.withFile) { if (note.fileIds && note.fileIds.length === 0) return false; } + if (!antenna.withReplies && note.replyId != null) return false; if (antenna.src === "users") { const accts = antenna.users.map((x) => { @@ -53,14 +53,19 @@ export async function checkHitAntenna( .map((xs) => xs.filter((x) => x !== "")) .filter((xs) => xs.length > 0); + let text = `${note.text ?? ""} ${note.cw ?? ""}`; + if (note.files != null) + text += ` ${note.files.map((f) => f.comment ?? "").join(" ")}`; + text = text.trim(); + if (keywords.length > 0) { if (note.text == null) return false; const matched = keywords.some((and) => and.every((keyword) => antenna.caseSensitive - ? note.text!.includes(keyword) - : note.text!.toLowerCase().includes(keyword.toLowerCase()), + ? text.includes(keyword) + : text.toLowerCase().includes(keyword.toLowerCase()), ), ); @@ -78,8 +83,8 @@ export async function checkHitAntenna( const matched = excludeKeywords.some((and) => and.every((keyword) => antenna.caseSensitive - ? note.text!.includes(keyword) - : note.text!.toLowerCase().includes(keyword.toLowerCase()), + ? note.text?.includes(keyword) + : note.text?.toLowerCase().includes(keyword.toLowerCase()), ), ); @@ -94,6 +99,16 @@ export async function checkHitAntenna( ); if (blockings.includes(antenna.userId)) return false; + if (note.visibility === "followers" || note.visibility === "home") { + const following = await followingCache.fetch(antenna.userId, () => + Followings.find({ + where: { followerId: antenna.userId }, + select: ["followeeId"], + }).then((relations) => relations.map((relation) => relation.followeeId)), + ); + if (!following.includes(note.userId)) return false; + } + const mutes = await hardMutesCache.fetch(antenna.userId, () => UserProfiles.findOneByOrFail({ userId: antenna.userId, diff --git a/packages/backend/src/server/api/endpoints/antennas/notes.ts b/packages/backend/src/server/api/endpoints/antennas/notes.ts index 1512cb9281..cbe0318525 100644 --- a/packages/backend/src/server/api/endpoints/antennas/notes.ts +++ b/packages/backend/src/server/api/endpoints/antennas/notes.ts @@ -111,8 +111,7 @@ export default define(meta, paramDef, async (ps, user) => { .leftJoinAndSelect("replyUser.banner", "replyUserBanner") .leftJoinAndSelect("renote.user", "renoteUser") .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner") - .andWhere("note.visibility != 'home'"); + .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); generateVisibilityQuery(query, user); generateMutedUserQuery(query, user); From 9ce6a23266e6f8a5f2bdf6497fff74a92ea9a57d Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 1 Mar 2024 23:17:45 +0900 Subject: [PATCH 006/191] chore: remove unused dependencies --- package.json | 3 - packages/backend-rs/Cargo.lock | 277 +++++++++++++-------------------- packages/backend-rs/Cargo.toml | 2 - packages/backend/package.json | 2 - packages/client/package.json | 1 - pnpm-lock.yaml | 53 ------- 6 files changed, 110 insertions(+), 228 deletions(-) diff --git a/package.json b/package.json index a860ad932d..5150845ee3 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,6 @@ "js-yaml": "4.1.0", "gulp": "4.0.2", "gulp-cssnano": "2.1.3", - "gulp-rename": "2.0.0", "gulp-replace": "1.1.4", "gulp-terser": "2.1.0" }, @@ -49,8 +48,6 @@ "@biomejs/cli-darwin-x64": "^1.5.3", "@biomejs/cli-linux-arm64": "^1.5.3", "@biomejs/cli-linux-x64": "^1.5.3", - "@types/gulp": "4.0.17", - "@types/gulp-rename": "2.0.6", "@types/node": "20.11.21", "execa": "8.0.1", "pnpm": "8.15.4", diff --git a/packages/backend-rs/Cargo.lock b/packages/backend-rs/Cargo.lock index 38ca51f3da..e318891dcd 100644 --- a/packages/backend-rs/Cargo.lock +++ b/packages/backend-rs/Cargo.lock @@ -30,9 +30,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" +checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b" dependencies = [ "cfg-if", "getrandom", @@ -80,9 +80,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.11" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -128,9 +128,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "arrayvec" @@ -157,7 +157,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -168,7 +168,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -205,7 +205,6 @@ dependencies = [ "cfg-if", "chrono", "cuid2", - "derive_more", "jsonschema", "napi", "napi-build", @@ -220,7 +219,6 @@ dependencies = [ "serde_json", "thiserror", "tokio", - "utoipa", ] [[package]] @@ -338,15 +336,15 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "syn_derive", ] [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" [[package]] name = "bytecheck" @@ -390,12 +388,9 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" [[package]] name = "cfg-if" @@ -421,14 +416,14 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] name = "clap" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", "clap_derive", @@ -436,9 +431,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstream", "anstyle", @@ -455,7 +450,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -476,12 +471,6 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "convert_case" version = "0.6.0" @@ -558,12 +547,12 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e" +checksum = "ad291aa74992b9b7a7e88c38acbbf6ad7e107f1d90ee8775b7bc1fc3394f485c" dependencies = [ "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -616,19 +605,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive_more" -version = "0.99.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", -] - [[package]] name = "diff" version = "0.1.13" @@ -655,9 +631,9 @@ checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] name = "dyn-clone" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "either" @@ -924,7 +900,7 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "allocator-api2", ] @@ -948,9 +924,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -1078,13 +1054,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.3" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown 0.14.3", - "serde", ] [[package]] @@ -1095,7 +1070,7 @@ checksum = "0122b7114117e64a63ac49f752a5ca4624d534c7b1c7de796ac196381cd2d947" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1143,7 +1118,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a071f4f7efc9a9118dfb627a0a94ef247986e1ab8606a4c806ae2b3aa3b6978" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "anyhow", "base64", "bytecount", @@ -1236,9 +1211,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "md-5" @@ -1290,9 +1265,9 @@ dependencies = [ [[package]] name = "napi" -version = "2.15.1" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43792514b0c95c5beec42996da0c1b39265b02b75c97baa82d163d3ef55cbfa7" +checksum = "54a63d0570e4c3e0daf7a8d380563610e159f538e20448d6c911337246f40e84" dependencies = [ "bitflags 2.4.2", "ctor", @@ -1304,37 +1279,37 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.1.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4b4532cf86bfef556348ac65e561e3123879f0e7566cca6d43a6ff5326f13df" +checksum = "2f9130fccc5f763cf2069b34a089a18f0d0883c66aceb81f2fad541a3d823c43" [[package]] name = "napi-derive" -version = "2.15.1" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d56bb899c164ab1be5e542ae7db8b26750c864bf2eef07295f17754e6358777" +checksum = "05bb7c37e3c1dda9312fdbe4a9fc7507fca72288ba154ec093e2d49114e727ce" dependencies = [ "cfg-if", - "convert_case 0.6.0", + "convert_case", "napi-derive-backend", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] name = "napi-derive-backend" -version = "1.0.60" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cf2d74ac66fd1cccb646be75fdd1c1dce8acfe20a68f61566a31da0d3eb9786" +checksum = "f785a8b8d7b83e925f5aa6d2ae3c159d17fe137ac368dc185bef410e7acdaeb4" dependencies = [ - "convert_case 0.6.0", + "convert_case", "once_cell", "proc-macro2", "quote", "regex", "semver", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1527,7 +1502,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1576,7 +1551,7 @@ dependencies = [ "regex", "regex-syntax 0.7.5", "structmeta", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1635,9 +1610,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "powerfmt" @@ -1859,16 +1834,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin 0.9.8", "untrusted", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1942,15 +1918,6 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" -[[package]] -name = "rustc_version" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" -dependencies = [ - "semver", -] - [[package]] name = "rustix" version = "0.38.31" @@ -1996,9 +1963,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "schemars" @@ -2051,7 +2018,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2092,7 +2059,7 @@ dependencies = [ "proc-macro2", "quote", "sea-bae", - "syn 2.0.48", + "syn 2.0.52", "unicode-ident", ] @@ -2137,28 +2104,28 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "semver" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "serde" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2174,9 +2141,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.113" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -2269,12 +2236,12 @@ checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2332,7 +2299,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d84b0a3c3739e220d94b3239fd69fb1f74bc36e16643423bd99de3b43c21bfbd" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "atoi", "bigdecimal", "byteorder", @@ -2564,7 +2531,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2575,7 +2542,7 @@ checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2603,9 +2570,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2", "quote", @@ -2621,7 +2588,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2659,9 +2626,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", @@ -2686,7 +2653,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2762,7 +2729,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2833,7 +2800,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2871,9 +2838,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] @@ -2919,30 +2886,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" -[[package]] -name = "utoipa" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "272ebdfbc99111033031d2f10e018836056e4d2c8e2acda76450ec7974269fa7" -dependencies = [ - "indexmap", - "serde", - "serde_json", - "utoipa-gen", -] - -[[package]] -name = "utoipa-gen" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3c9f4d08338c1bfa70dde39412a040a884c6f318b3d09aaaf3437a1e52027fc" -dependencies = [ - "proc-macro-error", - "proc-macro2", - "quote", - "syn 2.0.48", -] - [[package]] name = "uuid" version = "1.7.0" @@ -3000,7 +2943,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "wasm-bindgen-shared", ] @@ -3034,7 +2977,7 @@ checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3073,7 +3016,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -3091,7 +3034,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -3111,17 +3054,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -3132,9 +3075,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -3144,9 +3087,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -3156,9 +3099,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -3168,9 +3111,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -3180,9 +3123,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -3192,9 +3135,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -3204,9 +3147,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winnow" @@ -3259,7 +3202,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] diff --git a/packages/backend-rs/Cargo.toml b/packages/backend-rs/Cargo.toml index 38d88cf3e7..1c80914c54 100644 --- a/packages/backend-rs/Cargo.toml +++ b/packages/backend-rs/Cargo.toml @@ -16,7 +16,6 @@ async-trait = "0.1.75" cfg-if = "1.0.0" chrono = "0.4.31" cuid2 = "0.1.2" -derive_more = "0.99.17" jsonschema = "0.17.1" once_cell = "1.19.0" parse-display = "0.8.2" @@ -27,7 +26,6 @@ serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" thiserror = "1.0.52" tokio = { version = "1.35.1", features = ["full"] } -utoipa = "4.1.0" # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix napi = { version = "2.14.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true } diff --git a/packages/backend/package.json b/packages/backend/package.json index ecb9f80da3..b51fecfa08 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -174,8 +174,6 @@ "@types/ws": "8.5.10", "cross-env": "7.0.3", "eslint": "^8.57.0", - "execa": "8.0.1", - "json5-loader": "4.0.1", "mocha": "10.3.0", "pug": "3.0.2", "strict-event-emitter-types": "2.0.0", diff --git a/packages/client/package.json b/packages/client/package.json index 9ff60dda54..3ca4a7893a 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -33,7 +33,6 @@ "@types/tinycolor2": "1.4.6", "@types/uuid": "9.0.8", "@vitejs/plugin-vue": "5.0.4", - "@vue/compiler-sfc": "3.4.21", "@vue/runtime-core": "3.4.21", "autobind-decorator": "2.4.0", "autosize": "6.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8313fdaeb3..d811635bed 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -14,9 +14,6 @@ importers: gulp-cssnano: specifier: 2.1.3 version: 2.1.3 - gulp-rename: - specifier: 2.0.0 - version: 2.0.0 gulp-replace: specifier: 1.1.4 version: 1.1.4 @@ -42,12 +39,6 @@ importers: '@biomejs/cli-linux-x64': specifier: ^1.5.3 version: 1.5.3 - '@types/gulp': - specifier: 4.0.17 - version: 4.0.17 - '@types/gulp-rename': - specifier: 2.0.6 - version: 2.0.6 '@types/node': specifier: 20.11.21 version: 20.11.21 @@ -521,12 +512,6 @@ importers: eslint: specifier: ^8.57.0 version: 8.57.0 - execa: - specifier: 8.0.1 - version: 8.0.1 - json5-loader: - specifier: 4.0.1 - version: 4.0.1(webpack@5.90.3) mocha: specifier: 10.3.0 version: 10.3.0 @@ -635,9 +620,6 @@ importers: '@vitejs/plugin-vue': specifier: 5.0.4 version: 5.0.4(vite@5.1.4)(vue@3.4.21) - '@vue/compiler-sfc': - specifier: 3.4.21 - version: 3.4.21 '@vue/runtime-core': specifier: 3.4.21 version: 3.4.21 @@ -5833,10 +5815,6 @@ packages: resolution: {integrity: sha512-V/Hy/X9Vt7f3BbPJEi8BdVFMByHi+jNXrYkW3huaybV/kQ0KJg0Y6PkEMbn+zeT+i+SiKZ/HMqJGIIt4LZDqNQ==} dev: false - /big.js@5.2.2: - resolution: {integrity: sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==} - dev: true - /bin-check@4.1.0: resolution: {integrity: sha512-b6weQyEUKsDGFlACWSIOfveEnImkJyK/FGW6FAG42loyoquvjdtOIqO6yBFzHyqyVVhNgNkQxxx09SFLK28YnA==} engines: {node: '>=4'} @@ -7752,11 +7730,6 @@ packages: resolution: {integrity: sha512-OuML9z640prB+0Rms1RgzJrBAqNC9mFdqT2HeFcXSGSGNEiZh9adsaPPxWbqRoMyJPzu+zNKPyGKv4/sPmJDyg==} dev: true - /emojis-list@3.0.0: - resolution: {integrity: sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==} - engines: {node: '>= 4'} - dev: true - /encode-utf8@1.0.3: resolution: {integrity: sha512-ucAnuBEhUK4boH2HjVYG5Q2mQyPorvv0u/ocS+zhdw0S8AlHYY+GOFhP1Gio5z4icpP2ivFSvhtFjQi8+T9ppw==} dev: false @@ -9944,11 +9917,6 @@ packages: vinyl-sourcemaps-apply: 0.2.1 dev: false - /gulp-rename@2.0.0: - resolution: {integrity: sha512-97Vba4KBzbYmR5VBs9mWmK+HwIf5mj+/zioxfZhOKeXtx5ZjBk57KFlePf5nxq9QsTtFl0ejnHE3zTC9MHXqyQ==} - engines: {node: '>=4'} - dev: false - /gulp-replace@1.1.4: resolution: {integrity: sha512-SVSF7ikuWKhpAW4l4wapAqPPSToJoiNKsbDoUnRrSgwZHH7lH8pbPeQj1aOVYQrbZKhfSVBxVW+Py7vtulRktw==} engines: {node: '>=10'} @@ -11669,18 +11637,6 @@ packages: resolution: {integrity: sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==} dev: false - /json5-loader@4.0.1(webpack@5.90.3): - resolution: {integrity: sha512-c9viNZlZTz0MTIcf/4qvek5Dz1/PU3DNCB4PwUhlEZIV3qb1bSD6vQQymlV17/Wm6ncra1aCvmIPsuRj+KfEEg==} - engines: {node: '>= 10.13.0'} - peerDependencies: - webpack: ^4.0.0 || ^5.0.0 - dependencies: - json5: 2.2.3 - loader-utils: 2.0.4 - schema-utils: 3.3.0 - webpack: 5.90.3(@swc/core@1.4.2) - dev: true - /json5@1.0.2: resolution: {integrity: sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==} hasBin: true @@ -12192,15 +12148,6 @@ packages: engines: {node: '>=6.11.5'} dev: true - /loader-utils@2.0.4: - resolution: {integrity: sha512-xXqpXoINfFhgua9xiqD8fPFHgkoq1mmmpE92WlDbm9rNRd/EbRb+Gqf908T2DMfuHjjJlksiK2RbHVOdD/MqSw==} - engines: {node: '>=8.9.0'} - dependencies: - big.js: 5.2.2 - emojis-list: 3.0.0 - json5: 2.2.3 - dev: true - /loadjs@4.2.0: resolution: {integrity: sha512-AgQGZisAlTPbTEzrHPb6q+NYBMD+DP9uvGSIjSUM5uG+0jG15cb8axWpxuOIqrmQjn6scaaH8JwloiP27b2KXA==} dev: true From ec18c532ca65c69332771bb40d1f88276acdad94 Mon Sep 17 00:00:00 2001 From: naskya Date: Sat, 2 Mar 2024 00:19:05 +0900 Subject: [PATCH 007/191] feat: ability to publish timelines on signed out page --- docs/api-change.md | 1 + docs/changelog.md | 1 + locales/en-US.yml | 2 + locales/ja-JP.yml | 2 + packages/backend/src/models/entities/meta.ts | 5 + .../src/server/api/endpoints/admin/meta.ts | 1 + .../server/api/endpoints/admin/update-meta.ts | 5 + .../backend/src/server/api/endpoints/meta.ts | 8 +- .../api/stream/channels/global-timeline.ts | 2 + .../api/stream/channels/local-timeline.ts | 2 + packages/client/src/components/MkTimeline.vue | 2 +- packages/client/src/pages/admin/settings.vue | 20 +++- packages/client/src/pages/timeline.vue | 93 +++++++++++-------- packages/client/src/router.ts | 6 ++ packages/client/src/store.ts | 8 +- 15 files changed, 115 insertions(+), 43 deletions(-) diff --git a/docs/api-change.md b/docs/api-change.md index c024b00d90..1fac31a7a9 100644 --- a/docs/api-change.md +++ b/docs/api-change.md @@ -10,6 +10,7 @@ Breaking changes are indicated by the :warning: icon. - `untilDate` - `withFiles` - `searchCwAndAlt` +- Added `enableGuestTimeline` field to the response of `meta` and `admin/meta`, and the request of `admin/update-meta` (optional). ## v20240301 diff --git a/docs/changelog.md b/docs/changelog.md index 865ea86050..16fe645f19 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -7,6 +7,7 @@ Critical security updates are indicated by the :warning: icon. - Introduce new full-text search engine and post search filters - Refactoring - Show unlisted posts from following users in antennas (similar to [Fedibird](https://github.com/fedibird/mastodon/tree/fedibird) and [kmyblue](https://github.com/kmycode/mastodon), unlisted posts from people you don't follow won't be shown) +- Add ability to publish the Local and Global timelines on `/timeline` page ## v20240301 diff --git a/locales/en-US.yml b/locales/en-US.yml index 0bca5e0c19..5d81f535b0 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1189,6 +1189,8 @@ searchRange: "Posted within (optional)" searchRangeDescription: "If you want to filter the time period, enter it in this format: 20220615-20231031\n\nIf you leave out the year (like 0105-0106 or 20231105-0110), it's interpreted as the current year.\n\nYou can also omit either the start or end date. For example, -0102 will filter the search results to show only posts made before 2 January this year, and 20231026- will filter the results to show only posts made after 26 October 2023." searchPostsWithFiles: "Only posts with files" searchCwAndAlt: "Include content warnings and file descriptions" +publishTimelines: "Publish timelines for visitors" +publishTimelinesDescription: "If enabled, the Local and Global timeline will be shown on {url} even when signed out." _emojiModPerm: unauthorized: "None" diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index a79cda6706..033efd642e 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -1009,6 +1009,8 @@ searchRange: "投稿期間（オプション）" searchRangeDescription: "投稿検索で投稿期間を絞りたい場合、20220615-20231031 のような形式で投稿期間を入力してください。今年の日付を指定する場合には年の指定を省略できます（0105-0106 や 20231105-0110 のように）。\n\n開始日と終了日のどちらか一方は省略可能です。例えば -0102 とすると今年1月2日までの投稿のみを、20231026- とすると2023年10月26日以降の投稿のみを検索します。" searchPostsWithFiles: "添付ファイルのある投稿のみ" searchCwAndAlt: "閲覧注意の注釈と添付ファイルの代替テキストも検索する" +publishTimelines: "非ログインユーザーにもタイムラインを公開する" +publishTimelinesDescription: "有効にすると、{url} でローカルタイムラインとグローバルタイムラインが公開されます。" _sensitiveMediaDetection: description: "機械学習を使って自動でセンシティブなメディアを検出し、モデレーションに役立てられます。サーバーの負荷が少し増えます。" diff --git a/packages/backend/src/models/entities/meta.ts b/packages/backend/src/models/entities/meta.ts index a14967185f..aebe97c437 100644 --- a/packages/backend/src/models/entities/meta.ts +++ b/packages/backend/src/models/entities/meta.ts @@ -61,6 +61,11 @@ export class Meta { }) public disableGlobalTimeline: boolean; + @Column("boolean", { + default: false, + }) + public enableGuestTimeline: boolean; + @Column("varchar", { length: 256, default: "⭐", diff --git a/packages/backend/src/server/api/endpoints/admin/meta.ts b/packages/backend/src/server/api/endpoints/admin/meta.ts index b8b645bc32..bb86042b0c 100644 --- a/packages/backend/src/server/api/endpoints/admin/meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/meta.ts @@ -479,6 +479,7 @@ export default define(meta, paramDef, async () => { disableLocalTimeline: instance.disableLocalTimeline, disableRecommendedTimeline: instance.disableRecommendedTimeline, disableGlobalTimeline: instance.disableGlobalTimeline, + enableGuestTimeline: instance.enableGuestTimeline, driveCapacityPerLocalUserMb: instance.localDriveCapacityMb, driveCapacityPerRemoteUserMb: instance.remoteDriveCapacityMb, emailRequiredForSignup: instance.emailRequiredForSignup, diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 765219fcdc..955a1d95d5 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -17,6 +17,7 @@ export const paramDef = { disableLocalTimeline: { type: "boolean", nullable: true }, disableRecommendedTimeline: { type: "boolean", nullable: true }, disableGlobalTimeline: { type: "boolean", nullable: true }, + enableGuestTimeline: { type: "boolean", nullable: true }, defaultReaction: { type: "string", nullable: true }, recommendedInstances: { type: "array", @@ -206,6 +207,10 @@ export default define(meta, paramDef, async (ps, me) => { set.disableGlobalTimeline = ps.disableGlobalTimeline; } + if (typeof ps.enableGuestTimeline === "boolean") { + set.enableGuestTimeline = ps.enableGuestTimeline; + } + if (typeof ps.defaultReaction === "string") { set.defaultReaction = ps.defaultReaction; } diff --git a/packages/backend/src/server/api/endpoints/meta.ts b/packages/backend/src/server/api/endpoints/meta.ts index 819fccc826..bdfcd30dd9 100644 --- a/packages/backend/src/server/api/endpoints/meta.ts +++ b/packages/backend/src/server/api/endpoints/meta.ts @@ -111,6 +111,11 @@ export const meta = { optional: false, nullable: false, }, + enableGuestTimeline: { + type: "boolean", + optional: false, + nullable: false, + }, driveCapacityPerLocalUserMb: { type: "number", optional: false, @@ -432,6 +437,7 @@ export default define(meta, paramDef, async (ps, me) => { disableLocalTimeline: instance.disableLocalTimeline, disableRecommendedTimeline: instance.disableRecommendedTimeline, disableGlobalTimeline: instance.disableGlobalTimeline, + enableGuestTimeline: instance.enableGuestTimeline, driveCapacityPerLocalUserMb: instance.localDriveCapacityMb, driveCapacityPerRemoteUserMb: instance.remoteDriveCapacityMb, emailRequiredForSignup: instance.emailRequiredForSignup, @@ -506,8 +512,8 @@ export default define(meta, paramDef, async (ps, me) => { localTimeLine: !instance.disableLocalTimeline, recommendedTimeline: !instance.disableRecommendedTimeline, globalTimeLine: !instance.disableGlobalTimeline, + guestTimeline: instance.enableGuestTimeline, emailRequiredForSignup: instance.emailRequiredForSignup, - searchFilters: config.meilisearch ? true : false, hcaptcha: instance.enableHcaptcha, recaptcha: instance.enableRecaptcha, objectStorage: instance.useObjectStorage, diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts index 39d0719dd3..79d2fe90ec 100644 --- a/packages/backend/src/server/api/stream/channels/global-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts @@ -23,6 +23,8 @@ export default class extends Channel { return; } + if (!meta.enableGuestTimeline && this.user == null) return; + this.withReplies = params != null ? !!params.withReplies : true; // Subscribe events diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 88eabe991e..1df87dbfc8 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -22,6 +22,8 @@ export default class extends Channel { return; } + if (!meta.enableGuestTimeline && this.user == null) return; + this.withReplies = params != null ? !!params.withReplies : true; // Subscribe events diff --git a/packages/client/src/components/MkTimeline.vue b/packages/client/src/components/MkTimeline.vue index 0c7bb68734..ade27ced52 100644 --- a/packages/client/src/components/MkTimeline.vue +++ b/packages/client/src/components/MkTimeline.vue @@ -1,6 +1,6 @@ diff --git a/packages/client/src/components/MkNoteSub.vue b/packages/client/src/components/MkNoteSub.vue index 194a72d4b1..2e5b594a62 100644 --- a/packages/client/src/components/MkNoteSub.vue +++ b/packages/client/src/components/MkNoteSub.vue @@ -51,6 +51,7 @@ :text="translation.text" :author="appearNote.user" :i="$i" + :lang="targetLang" :custom-emojis="appearNote.emojis" /> @@ -287,15 +288,12 @@ const enableEmojiReactions = defaultStore.state.enableEmojiReactions; const expandOnNoteClick = defaultStore.state.expandOnNoteClick; const lang = localStorage.getItem("lang"); const translateLang = localStorage.getItem("translateLang"); +const targetLang = (translateLang || lang || navigator.language)?.slice(0, 2); const isForeignLanguage: boolean = defaultStore.state.detectPostLanguage && appearNote.value.text != null && (() => { - const targetLang = (translateLang || lang || navigator.language)?.slice( - 0, - 2, - ); const postLang = detectLanguage(appearNote.value.text); return postLang !== "" && postLang !== targetLang; })(); diff --git a/packages/client/src/components/MkNotification.vue b/packages/client/src/components/MkNotification.vue index 4c31793a36..7ea46b5a57 100644 --- a/packages/client/src/components/MkNotification.vue +++ b/packages/client/src/components/MkNotification.vue @@ -118,6 +118,7 @@ :text="getNoteSummary(notification.note)" :plain="true" :nowrap="!full" + :lang="notification.note.lang" :custom-emojis="notification.note.emojis" /> @@ -131,6 +132,7 @@ :text="getNoteSummary(notification.note.renote)" :plain="true" :nowrap="!full" + :lang="notification.note.lang" :custom-emojis="notification.note.renote.emojis" /> @@ -144,6 +146,7 @@ :text="getNoteSummary(notification.note)" :plain="true" :nowrap="!full" + :lang="notification.note.lang" :custom-emojis="notification.note.emojis" /> @@ -157,6 +160,7 @@ :text="getNoteSummary(notification.note)" :plain="true" :nowrap="!full" + :lang="notification.note.lang" :custom-emojis="notification.note.emojis" /> @@ -170,6 +174,7 @@ :text="getNoteSummary(notification.note)" :plain="true" :nowrap="!full" + :lang="notification.note.lang" :custom-emojis="notification.note.emojis" /> @@ -183,6 +188,7 @@ :text="getNoteSummary(notification.note)" :plain="true" :nowrap="!full" + :lang="notification.note.lang" :custom-emojis="notification.note.emojis" /> @@ -196,6 +202,7 @@ :text="getNoteSummary(notification.note)" :plain="true" :nowrap="!full" + :lang="notification.note.lang" :custom-emojis="notification.note.emojis" /> diff --git a/packages/client/src/components/MkPoll.vue b/packages/client/src/components/MkPoll.vue index fb5445c6c7..4a3b79f471 100644 --- a/packages/client/src/components/MkPoll.vue +++ b/packages/client/src/components/MkPoll.vue @@ -22,6 +22,7 @@