From 1548250360d32ee8d66ec1d3f320a1847cf2b65a Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Thu, 6 Jul 2023 11:50:34 -0700 Subject: [PATCH 1/8] feat: :zap: cache server --- .config/example.yml | 14 ++++++++++++++ README.md | 9 ++++++++- packages/backend/src/config/types.ts | 10 ++++++++++ packages/backend/src/db/redis.ts | 20 ++++++++++++-------- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/.config/example.yml b/.config/example.yml index 51d380e7e7..ba74df8a59 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -67,6 +67,20 @@ redis: #db: 1 #user: default +# ┌─────────────────────────────┐ +#───┘ Cache server configuration └───────────────────────────────────── + +# A Redis-compatible server (DragonflyDB, Keydb, Redis) for caching +# If left blank, it will use the Redis server from above + +#cacheServer: + #host: localhost + #port: 6379 + #family: 0 # 0=Both, 4=IPv4, 6=IPv6 + #pass: example-pass + #prefix: example-prefix + #db: 1 + # Please configure either MeiliSearch *or* Sonic. # If both MeiliSearch and Sonic configurations are present, MeiliSearch will take precedence. diff --git a/README.md b/README.md index f66d14a324..0ad935667e 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,10 @@ If you have access to a server that supports one of the sources below, I recomme - 🦔 [Sonic](https://crates.io/crates/sonic-server) - [MeiliSearch](https://www.meilisearch.com/) - [ElasticSearch](https://www.elastic.co/elasticsearch/) - +- Caching server + - 🐲 At least [Dragonfly](https://www.dragonflydb.io/) v1.4.0 (recommended) + - 👻 [KeyDB](https://keydb.dev/) (untested) + - 🍱 Another [Redis](https://redis.io/) server, at least v6 ### 🏗️ Build dependencies - 🦀 At least [Rust](https://www.rust-lang.org/) v1.68.0 @@ -161,6 +164,10 @@ psql postgres -c "create database calckey with encoding = 'UTF8';" In Calckey's directory, fill out the `db` section of `.config/default.yml` with the correct information, where the `db` key is `calckey`. +## 💰 Caching server + +If you experience a lot of traffic, it's a good idea to set up another Redis-compatible caching server. If you don't set one one up, it'll falll back to the mandatory Redis server. + ## 🔎 Set up search ### 🦔 Sonic diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 84808413c0..43168662c4 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -26,6 +26,16 @@ export type Source = { user?: string; tls?: { [y: string]: string }; }; + cacheServer: { + host: string; + port: number; + family?: number; + pass?: string; + db?: number; + prefix?: string; + user?: string; + tls?: { [z: string]: string }; + }; elasticsearch: { host: string; port: number; diff --git a/packages/backend/src/db/redis.ts b/packages/backend/src/db/redis.ts index a1f3279f35..215effd8ea 100644 --- a/packages/backend/src/db/redis.ts +++ b/packages/backend/src/db/redis.ts @@ -2,15 +2,19 @@ import Redis from "ioredis"; import config from "@/config/index.js"; export function createConnection() { + let source = config.redis; + if (config.cacheServer) { + source = config.cacheServer; + } return new Redis({ - port: config.redis.port, - host: config.redis.host, - family: config.redis.family ?? 0, - password: config.redis.pass, - username: config.redis.user ?? "default", - keyPrefix: `${config.redis.prefix}:`, - db: config.redis.db || 0, - tls: config.redis.tls, + port: source.port, + host: source.host, + family: source.family ?? 0, + password: source.pass, + username: source.user ?? "default", + keyPrefix: `${source.prefix}:`, + db: source.db || 0, + tls: source.tls, }); } From 5aef784ecdb0b05379f1b3ae89977724c2b1c6ed Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Thu, 6 Jul 2023 11:52:48 -0700 Subject: [PATCH 2/8] docs: :memo: dragonfly flag --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0ad935667e..d66560d0ea 100644 --- a/README.md +++ b/README.md @@ -106,8 +106,8 @@ If you have access to a server that supports one of the sources below, I recomme - [ElasticSearch](https://www.elastic.co/elasticsearch/) - Caching server - 🐲 At least [Dragonfly](https://www.dragonflydb.io/) v1.4.0 (recommended) - - 👻 [KeyDB](https://keydb.dev/) (untested) - 🍱 Another [Redis](https://redis.io/) server, at least v6 + - 👻 [KeyDB](https://keydb.dev/) (untested) ### 🏗️ Build dependencies - 🦀 At least [Rust](https://www.rust-lang.org/) v1.68.0 @@ -166,7 +166,9 @@ In Calckey's directory, fill out the `db` section of `.config/default.yml` with ## 💰 Caching server -If you experience a lot of traffic, it's a good idea to set up another Redis-compatible caching server. If you don't set one one up, it'll falll back to the mandatory Redis server. +If you experience a lot of traffic, it's a good idea to set up another Redis-compatible caching server. If you don't set one one up, it'll fall back to the mandatory Redis server. + +For DragonflyDB, launch with the flag `--default_lua_flags='allow-undeclared-keys'`. ## 🔎 Set up search From 5fb2d9559814b9d4bee38c754164e15b271a7138 Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Thu, 6 Jul 2023 11:55:02 -0700 Subject: [PATCH 3/8] docs: :pencil2: dragonflydb typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d66560d0ea..9701bc86ef 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ If you have access to a server that supports one of the sources below, I recomme - [MeiliSearch](https://www.meilisearch.com/) - [ElasticSearch](https://www.elastic.co/elasticsearch/) - Caching server - - 🐲 At least [Dragonfly](https://www.dragonflydb.io/) v1.4.0 (recommended) + - 🐲 At least [DragonflyDB](https://www.dragonflydb.io/) v1.4.0 (recommended) - 🍱 Another [Redis](https://redis.io/) server, at least v6 - 👻 [KeyDB](https://keydb.dev/) (untested) ### 🏗️ Build dependencies From aa1482224c1ac04c912b519a8c03877ae74a7adb Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Thu, 6 Jul 2023 12:25:54 -0700 Subject: [PATCH 4/8] fix: use unique key for my page query --- packages/client/src/pages/pages.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/client/src/pages/pages.vue b/packages/client/src/pages/pages.vue index 75a32ff295..a3a08be7cf 100644 --- a/packages/client/src/pages/pages.vue +++ b/packages/client/src/pages/pages.vue @@ -66,10 +66,10 @@ :pagination="myPagesPagination" > From e61b229229987329c69e91b5ed9722c82277d6cf Mon Sep 17 00:00:00 2001 From: ThatOneCalculator Date: Thu, 6 Jul 2023 12:26:37 -0700 Subject: [PATCH 5/8] fix: use unique key for my galleries --- packages/client/src/pages/gallery/index.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/client/src/pages/gallery/index.vue b/packages/client/src/pages/gallery/index.vue index 66dc28d174..a4a09325c9 100644 --- a/packages/client/src/pages/gallery/index.vue +++ b/packages/client/src/pages/gallery/index.vue @@ -92,9 +92,9 @@ >
From c51e5074cf5449151554a60898b965096ec6c6e3 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 6 Jul 2023 17:06:31 -0400 Subject: [PATCH 6/8] fix: use host as prefix of cacheServer if undefined --- packages/backend/src/config/load.ts | 2 ++ packages/backend/src/config/types.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/config/load.ts b/packages/backend/src/config/load.ts index 9b8ee5edbb..fa98789554 100644 --- a/packages/backend/src/config/load.ts +++ b/packages/backend/src/config/load.ts @@ -55,6 +55,8 @@ export default function load() { mixin.clientEntry = clientManifest["src/init.ts"]; if (!config.redis.prefix) config.redis.prefix = mixin.host; + if (config.cacheServer && !config.cacheServer.prefix) + config.cacheServer.prefix = mixin.host; return Object.assign(config, mixin); } diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 43168662c4..7789c26e07 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -26,7 +26,7 @@ export type Source = { user?: string; tls?: { [y: string]: string }; }; - cacheServer: { + cacheServer?: { host: string; port: number; family?: number; From f5edaaf130eebe4322358f4c334a8e262c4328c6 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 6 Jul 2023 17:22:15 -0400 Subject: [PATCH 7/8] perf: use msgpackr to encode and decode --- packages/backend/package.json | 4 ++-- packages/backend/src/misc/cache.ts | 2 +- pnpm-lock.yaml | 11 +++-------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/backend/package.json b/packages/backend/package.json index 0600644899..ec6451f61c 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -28,13 +28,11 @@ "@bull-board/api": "5.2.0", "@bull-board/koa": "5.2.0", "@bull-board/ui": "5.2.0", - "megalodon": "workspace:*", "@discordapp/twemoji": "14.1.2", "@elastic/elasticsearch": "7.17.0", "@koa/cors": "3.4.3", "@koa/multer": "3.0.2", "@koa/router": "9.0.1", - "@msgpack/msgpack": "3.0.0-beta2", "@peertube/http-signature": "1.7.0", "@redocly/openapi-core": "1.0.0-beta.120", "@sinonjs/fake-timers": "9.1.2", @@ -87,9 +85,11 @@ "koa-send": "5.0.1", "koa-slow": "2.1.0", "koa-views": "7.0.2", + "megalodon": "workspace:*", "meilisearch": "0.33.0", "mfm-js": "0.23.3", "mime-types": "2.1.35", + "msgpackr": "1.9.5", "multer": "1.4.4-lts.1", "native-utils": "link:native-utils", "nested-property": "4.0.0", diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index fe68908e57..913258f05a 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -1,5 +1,5 @@ import { redisClient } from "@/db/redis.js"; -import { encode, decode } from "@msgpack/msgpack"; +import { encode, decode } from "msgpackr"; import { ChainableCommander } from "ioredis"; export class Cache { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3fc168cf1d..b9522c4346 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -105,9 +105,6 @@ importers: '@koa/router': specifier: 9.0.1 version: 9.0.1 - '@msgpack/msgpack': - specifier: 3.0.0-beta2 - version: 3.0.0-beta2 '@peertube/http-signature': specifier: 1.7.0 version: 1.7.0 @@ -276,6 +273,9 @@ importers: mime-types: specifier: 2.1.35 version: 2.1.35 + msgpackr: + specifier: 1.9.5 + version: 1.9.5 multer: specifier: 1.4.4-lts.1 version: 1.4.4-lts.1 @@ -2607,11 +2607,6 @@ packages: os-filter-obj: 2.0.0 dev: true - /@msgpack/msgpack@3.0.0-beta2: - resolution: {integrity: sha512-y+l1PNV0XDyY8sM3YtuMLK5vE3/hkfId+Do8pLo/OPxfxuFAUwcGz3oiiUuV46/aBpwTzZ+mRWVMtlSKbradhw==} - engines: {node: '>= 14'} - dev: false - /@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.2: resolution: {integrity: sha512-9bfjwDxIDWmmOKusUcqdS4Rw+SETlp9Dy39Xui9BEGEk19dDwH0jhipwFzEff/pFg95NKymc6TOTbRKcWeRqyQ==} cpu: [arm64] From 3705a1ecb17982068f6fdc8cefe37db09490bd56 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 6 Jul 2023 17:49:41 -0400 Subject: [PATCH 8/8] refactor: cache relays for a longer time --- packages/backend/src/services/relay.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/services/relay.ts b/packages/backend/src/services/relay.ts index 6f7829c218..e969d783a0 100644 --- a/packages/backend/src/services/relay.ts +++ b/packages/backend/src/services/relay.ts @@ -15,7 +15,7 @@ import { createSystemUser } from "./create-system-user.js"; const ACTOR_USERNAME = "relay.actor" as const; -const relaysCache = new Cache("relay", 60 * 10); +const relaysCache = new Cache("relay", 60 * 60); export async function getRelayActor(): Promise { const user = await Users.findOneBy({