From 063b2ff2662c549cfa658b5ed8db985aa555d23d Mon Sep 17 00:00:00 2001 From: Hazel K Date: Thu, 3 Oct 2024 19:53:57 -0400 Subject: [PATCH 1/4] increase character limits and share between text and CW --- .config/ci.yml | 2 +- .config/docker_example.yml | 2 +- .config/example.yml | 2 +- .../1727998351561-increase_character_limits.js | 18 ++++++++++++++++++ packages/backend/src/const.ts | 5 +++-- packages/backend/src/models/DriveFile.ts | 3 ++- packages/backend/src/models/Note.ts | 4 ++-- 7 files changed, 28 insertions(+), 8 deletions(-) create mode 100644 packages/backend/migration/1727998351561-increase_character_limits.js diff --git a/.config/ci.yml b/.config/ci.yml index 44092d3662..19ffe18d2c 100644 --- a/.config/ci.yml +++ b/.config/ci.yml @@ -167,7 +167,7 @@ id: 'aidx' # IP address family used for outgoing request (ipv4, ipv6 or dual) #outgoingAddressFamily: ipv4 -# Amount of characters that can be used when writing notes (maximum: 8192, minimum: 1) +# Amount of characters that can be used when writing notes (maximum: 100000, minimum: 1) maxNoteLength: 3000 # Proxy for HTTP/HTTPS diff --git a/.config/docker_example.yml b/.config/docker_example.yml index de95f1b21a..b697e7dbab 100644 --- a/.config/docker_example.yml +++ b/.config/docker_example.yml @@ -242,7 +242,7 @@ id: 'aidx' # IP address family used for outgoing request (ipv4, ipv6 or dual) #outgoingAddressFamily: ipv4 -# Amount of characters that can be used when writing notes (maximum: 8192, minimum: 1) +# Amount of characters that can be used when writing notes (maximum: 100000, minimum: 1) maxNoteLength: 3000 # Proxy for HTTP/HTTPS diff --git a/.config/example.yml b/.config/example.yml index 21e85b7b89..c87897d24c 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -251,7 +251,7 @@ id: 'aidx' # IP address family used for outgoing request (ipv4, ipv6 or dual) #outgoingAddressFamily: ipv4 -# Amount of characters that can be used when writing notes (maximum: 8192, minimum: 1) +# Amount of characters that can be used when writing notes (maximum: 100000, minimum: 1) maxNoteLength: 3000 # Proxy for HTTP/HTTPS diff --git a/packages/backend/migration/1727998351561-increase_character_limits.js b/packages/backend/migration/1727998351561-increase_character_limits.js new file mode 100644 index 0000000000..8754163846 --- /dev/null +++ b/packages/backend/migration/1727998351561-increase_character_limits.js @@ -0,0 +1,18 @@ +/* + * SPDX-FileCopyrightText: hazelnoot and other Sharkey contributors + * SPDX-License-Identifier: AGPL-3.0-only + */ + +export class IncreaseCharacterLimits1727998351561 { + name = 'IncreaseCharacterLimits1727998351561' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "drive_file" ALTER COLUMN "comment" TYPE varchar(100000)`); + await queryRunner.query(`ALTER TABLE "note" ALTER COLUMN "cw" TYPE text`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" ALTER COLUMN "cw" TYPE varchar(512)`); + await queryRunner.query(`ALTER TABLE "drive_file" ALTER COLUMN "comment" TYPE varchar(8192)`); + } +} diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts index 02c27779ca..29b796e161 100644 --- a/packages/backend/src/const.ts +++ b/packages/backend/src/const.ts @@ -13,15 +13,16 @@ export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days /** * Maximum note text length that can be stored in DB. + * Content Warnings are included in this limit. * Surrogate pairs count as one */ -export const DB_MAX_NOTE_TEXT_LENGTH = 8192; +export const DB_MAX_NOTE_TEXT_LENGTH = 100_000; /** * Maximum image description length that can be stored in DB. * Surrogate pairs count as one */ -export const DB_MAX_IMAGE_COMMENT_LENGTH = 8192; +export const DB_MAX_IMAGE_COMMENT_LENGTH = 100_000; //#endregion // ブラウザで直接表示することを許可するファイルの種類のリスト diff --git a/packages/backend/src/models/DriveFile.ts b/packages/backend/src/models/DriveFile.ts index dd810681c5..f536a7e4c6 100644 --- a/packages/backend/src/models/DriveFile.ts +++ b/packages/backend/src/models/DriveFile.ts @@ -7,6 +7,7 @@ import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typ import { id } from './util/id.js'; import { MiUser } from './User.js'; import { MiDriveFolder } from './DriveFolder.js'; +import {DB_MAX_IMAGE_COMMENT_LENGTH} from "@/const.js"; @Entity('drive_file') @Index(['userId', 'folderId', 'id']) @@ -61,7 +62,7 @@ export class MiDriveFile { public size: number; @Column('varchar', { - length: 8192, + length: DB_MAX_IMAGE_COMMENT_LENGTH, nullable: true, comment: 'The comment of the DriveFile.', }) diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index b11e2ec62b..408e023ff7 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -66,8 +66,8 @@ export class MiNote { }) public name: string | null; - @Column('varchar', { - length: 512, nullable: true, + @Column('text', { + nullable: true, }) public cw: string | null; From 3842a1ee8c32d136b97b2a0b2b7c0f8f1e733fe7 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Thu, 3 Oct 2024 20:05:15 -0400 Subject: [PATCH 2/4] fix length validation --- packages/backend/src/core/NoteCreateService.ts | 12 ++++++++++++ packages/backend/src/core/NoteEditService.ts | 12 ++++++++++++ .../backend/src/server/api/endpoints/notes/create.ts | 3 ++- .../backend/src/server/api/endpoints/notes/edit.ts | 4 +++- 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index c252336f99..beaa75e737 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -346,6 +346,18 @@ export class NoteCreateService implements OnApplicationShutdown { data.text = null; } + if (data.cw) { + if (data.cw.length > DB_MAX_NOTE_TEXT_LENGTH) { + data.cw = data.cw.slice(0, DB_MAX_NOTE_TEXT_LENGTH); + } + data.cw = data.cw.trim(); + if (data.cw === '') { + data.cw = null; + } + } else { + data.cw = null; + } + let tags = data.apHashtags; let emojis = data.apEmojis; let mentionedUsers = data.apMentions; diff --git a/packages/backend/src/core/NoteEditService.ts b/packages/backend/src/core/NoteEditService.ts index f42d14b466..aecd37aeb9 100644 --- a/packages/backend/src/core/NoteEditService.ts +++ b/packages/backend/src/core/NoteEditService.ts @@ -371,6 +371,18 @@ export class NoteEditService implements OnApplicationShutdown { data.text = null; } + if (data.cw) { + if (data.cw.length > DB_MAX_NOTE_TEXT_LENGTH) { + data.cw = data.cw.slice(0, DB_MAX_NOTE_TEXT_LENGTH); + } + data.cw = data.cw.trim(); + if (data.cw === '') { + data.cw = null; + } + } else { + data.cw = null; + } + let tags = data.apHashtags; let emojis = data.apEmojis; let mentionedUsers = data.apMentions; diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 626f03b758..a4c2e28129 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -252,7 +252,8 @@ export default class extends Endpoint { // eslint- private noteCreateService: NoteCreateService, ) { super(meta, paramDef, async (ps, me) => { - if (ps.text && (ps.text.length > this.config.maxNoteLength)) { + const contentLength = (ps.text?.length ?? 0) + (ps.cw?.length ?? 0); + if (contentLength > this.config.maxNoteLength) { throw new ApiError(meta.errors.maxLength); } diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index 835cbc14fa..b9be145caf 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -297,9 +297,11 @@ export default class extends Endpoint { // eslint- private noteEditService: NoteEditService, ) { super(meta, paramDef, async (ps, me) => { - if (ps.text && (ps.text.length > this.config.maxNoteLength)) { + const contentLength = (ps.text?.length ?? 0) + (ps.cw?.length ?? 0); + if (contentLength > this.config.maxNoteLength) { throw new ApiError(meta.errors.maxLength); } + let visibleUsers: MiUser[] = []; if (ps.visibleUserIds) { visibleUsers = await this.usersRepository.findBy({ From 8da3b064b75130d9efd4eea2ca322d19b2cb1ab3 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Fri, 4 Oct 2024 09:14:17 -0400 Subject: [PATCH 3/4] fix lint errors --- packages/backend/src/models/DriveFile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/models/DriveFile.ts b/packages/backend/src/models/DriveFile.ts index f536a7e4c6..7de4be4f96 100644 --- a/packages/backend/src/models/DriveFile.ts +++ b/packages/backend/src/models/DriveFile.ts @@ -4,10 +4,10 @@ */ import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm'; +import { DB_MAX_IMAGE_COMMENT_LENGTH } from '@/const.js'; import { id } from './util/id.js'; import { MiUser } from './User.js'; import { MiDriveFolder } from './DriveFolder.js'; -import {DB_MAX_IMAGE_COMMENT_LENGTH} from "@/const.js"; @Entity('drive_file') @Index(['userId', 'folderId', 'id']) From 95a0e814bc0c7678d7de64d6c6f7a106b6443199 Mon Sep 17 00:00:00 2001 From: Hazel K Date: Fri, 4 Oct 2024 09:15:12 -0400 Subject: [PATCH 4/4] remove numeric separators --- packages/backend/src/const.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/const.ts b/packages/backend/src/const.ts index 29b796e161..b8ad86497c 100644 --- a/packages/backend/src/const.ts +++ b/packages/backend/src/const.ts @@ -16,13 +16,13 @@ export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days * Content Warnings are included in this limit. * Surrogate pairs count as one */ -export const DB_MAX_NOTE_TEXT_LENGTH = 100_000; +export const DB_MAX_NOTE_TEXT_LENGTH = 100000; /** * Maximum image description length that can be stored in DB. * Surrogate pairs count as one */ -export const DB_MAX_IMAGE_COMMENT_LENGTH = 100_000; +export const DB_MAX_IMAGE_COMMENT_LENGTH = 100000; //#endregion // ブラウザで直接表示することを許可するファイルの種類のリスト