From c05723ca6ad4f17b823662e83ed8b442fe10626a Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Tue, 31 May 2022 17:44:22 +0900 Subject: [PATCH 1/8] Fix IP address rate limit (#8758) * Fix IP address rate limit * CHANGELOG * Tune getIpHash --- CHANGELOG.md | 2 +- packages/backend/src/misc/get-ip-hash.ts | 9 +++++++++ packages/backend/src/server/api/call.ts | 11 +++-------- packages/backend/src/server/api/private/signin.ts | 3 ++- 4 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 packages/backend/src/misc/get-ip-hash.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b086ddba9..cfcf52ce92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ You should also include the user name that made the change. Your own theme color may be unset if it was in an invalid format. Admins should check their instance settings if in doubt. - Perform port diagnosis at startup only when Listen fails @mei23 -- Rate limiting is now also usable for non-authenticated users. @Johann150 +- Rate limiting is now also usable for non-authenticated users. @Johann150 @mei23 Admins should make sure the reverse proxy sets the `X-Forwarded-For` header to the original address. ### Bugfixes diff --git a/packages/backend/src/misc/get-ip-hash.ts b/packages/backend/src/misc/get-ip-hash.ts new file mode 100644 index 0000000000..379325bb13 --- /dev/null +++ b/packages/backend/src/misc/get-ip-hash.ts @@ -0,0 +1,9 @@ +import IPCIDR from 'ip-cidr'; + +export function getIpHash(ip: string) { + // because a single person may control many IPv6 addresses, + // only a /64 subnet prefix of any IP will be taken into account. + // (this means for IPv4 the entire address is used) + const prefix = IPCIDR.createAddress(ip).mask(64); + return 'ip-' + BigInt('0b' + prefix).toString(36); +} diff --git a/packages/backend/src/server/api/call.ts b/packages/backend/src/server/api/call.ts index fbe25e1732..cd3e0abc06 100644 --- a/packages/backend/src/server/api/call.ts +++ b/packages/backend/src/server/api/call.ts @@ -6,7 +6,7 @@ import endpoints, { IEndpointMeta } from './endpoints.js'; import { ApiError } from './error.js'; import { apiLogger } from './logger.js'; import { AccessToken } from '@/models/entities/access-token.js'; -import IPCIDR from 'ip-cidr'; +import { getIpHash } from '@/misc/get-ip-hash.js'; const accessDenied = { message: 'Access denied.', @@ -33,18 +33,13 @@ export default async (endpoint: string, user: CacheableLocalUser | null | undefi throw new ApiError(accessDenied); } - if (ep.meta.requireCredential && ep.meta.limit && !isModerator) { + if (ep.meta.limit && !isModerator) { // koa will automatically load the `X-Forwarded-For` header if `proxy: true` is configured in the app. let limitActor: string; if (user) { limitActor = user.id; } else { - // because a single person may control many IPv6 addresses, - // only a /64 subnet prefix of any IP will be taken into account. - // (this means for IPv4 the entire address is used) - const ip = IPCIDR.createAddress(ctx.ip).mask(64); - - limitActor = 'ip-' + parseInt(ip, 2).toString(36); + limitActor = getIpHash(ctx!.ip); } const limit = Object.assign({}, ep.meta.limit); diff --git a/packages/backend/src/server/api/private/signin.ts b/packages/backend/src/server/api/private/signin.ts index b304550e29..79b31764fd 100644 --- a/packages/backend/src/server/api/private/signin.ts +++ b/packages/backend/src/server/api/private/signin.ts @@ -10,6 +10,7 @@ import { verifyLogin, hash } from '../2fa.js'; import { randomBytes } from 'node:crypto'; import { IsNull } from 'typeorm'; import { limiter } from '../limiter.js'; +import { getIpHash } from '@/misc/get-ip-hash.js'; export default async (ctx: Koa.Context) => { ctx.set('Access-Control-Allow-Origin', config.url); @@ -27,7 +28,7 @@ export default async (ctx: Koa.Context) => { try { // not more than 1 attempt per second and not more than 10 attempts per hour - await limiter({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, ctx.ip); + await limiter({ key: 'signin', duration: 60 * 60 * 1000, max: 10, minInterval: 1000 }, getIpHash(ctx.ip)); } catch (err) { ctx.status = 429; ctx.body = { From c56e45ecef571d8eece5fba9054906ab8077e417 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Tue, 31 May 2022 10:54:02 +0200 Subject: [PATCH 2/8] fix: always remove completed tasks (#8771) --- packages/backend/src/queue/index.ts | 2 ++ packages/backend/src/services/note/create.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index 67d5f5d248..c5fd7de1cb 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -305,11 +305,13 @@ export default function() { systemQueue.add('resyncCharts', { }, { repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, }); systemQueue.add('cleanCharts', { }, { repeat: { cron: '0 0 * * *' }, + removeOnComplete: true, }); systemQueue.add('checkExpiredMutings', { diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index ceb5e8cc71..e2bf9d5b59 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -312,7 +312,8 @@ export default async (user: { id: User['id']; username: User['username']; host: endedPollNotificationQueue.add({ noteId: note.id, }, { - delay + delay, + removeOnComplete: true, }); } From 95a3565d1c412983c0380a6cb3e4588c313d081a Mon Sep 17 00:00:00 2001 From: MeiMei <30769358+mei23@users.noreply.github.com> Date: Tue, 31 May 2022 17:55:07 +0900 Subject: [PATCH 3/8] Fix `Cannot find module` issue (#8770) * Add --force to yarn in the installation script * CHAGELOG --- CHANGELOG.md | 1 + scripts/install-packages.js | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cfcf52ce92..9687595010 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ You should also include the user name that made the change. - Server: use correct order of attachments on notes @Johann150 - Server: prevent crash when processing certain PNGs @syuilo - Server: Fix unable to generate video thumbnails @mei23 +- Server: Fix `Cannot find module` issue @mei23 ## 12.110.1 (2022/04/23) diff --git a/scripts/install-packages.js b/scripts/install-packages.js index bc8e016a3c..d1dea3ebe5 100644 --- a/scripts/install-packages.js +++ b/scripts/install-packages.js @@ -3,7 +3,7 @@ const execa = require('execa'); (async () => { console.log('installing dependencies of packages/backend ...'); - await execa('yarn', ['install'], { + await execa('yarn', ['--force', 'install'], { cwd: __dirname + '/../packages/backend', stdout: process.stdout, stderr: process.stderr, From d3e242a7f25e72bd65f27feebd878f5a45e7ae3b Mon Sep 17 00:00:00 2001 From: Andreas Nedbal Date: Tue, 31 May 2022 10:57:01 +0200 Subject: [PATCH 4/8] Extract commonly used test logic to commands (#8767) * meta(tests): enable workflows to run in branch * feat(tests): move commonly used logic to Cypress commands * chore(tests): replace more code with commands * meta(tests): disable workflows to run in branch --- cypress/integration/basic.js | 63 ++++++---------------------------- cypress/integration/widgets.js | 27 +++------------ cypress/support/commands.js | 30 ++++++++++++++++ 3 files changed, 44 insertions(+), 76 deletions(-) diff --git a/cypress/integration/basic.js b/cypress/integration/basic.js index eb15cfe223..eb5195c4b2 100644 --- a/cypress/integration/basic.js +++ b/cypress/integration/basic.js @@ -1,11 +1,6 @@ describe('Before setup instance', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); }); afterEach(() => { @@ -35,18 +30,10 @@ describe('Before setup instance', () => { describe('After setup instance', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); }); afterEach(() => { @@ -76,24 +63,13 @@ describe('After setup instance', () => { describe('After user signup', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); // ユーザー作成 - cy.request('POST', '/api/signup', { - username: 'alice', - password: 'alice1234', - }).its('body').as('alice'); + cy.registerUser('alice', 'alice1234'); }); afterEach(() => { @@ -138,34 +114,15 @@ describe('After user signup', () => { describe('After user singed in', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); + cy.resetState(); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); // ユーザー作成 - cy.request('POST', '/api/signup', { - username: 'alice', - password: 'alice1234', - }).its('body').as('alice'); + cy.registerUser('alice', 'alice1234'); - cy.visit('/'); - - cy.intercept('POST', '/api/signin').as('signin'); - - cy.get('[data-cy-signin]').click(); - cy.get('[data-cy-signin-username] input').type('alice'); - cy.get('[data-cy-signin-password] input').type('alice1234{enter}'); - - cy.wait('@signin').as('signedIn'); + cy.login('alice', 'alice1234'); }); afterEach(() => { diff --git a/cypress/integration/widgets.js b/cypress/integration/widgets.js index d63ff274bd..56ad95ee94 100644 --- a/cypress/integration/widgets.js +++ b/cypress/integration/widgets.js @@ -1,34 +1,15 @@ describe('After user signed in', () => { beforeEach(() => { - cy.window(win => { - win.indexedDB.deleteDatabase('keyval-store'); - }); + cy.resetState(); cy.viewport('macbook-16'); - cy.request('POST', '/api/reset-db').as('reset'); - cy.get('@reset').its('status').should('equal', 204); - cy.reload(true); // インスタンス初期セットアップ - cy.request('POST', '/api/admin/accounts/create', { - username: 'admin', - password: 'pass', - }).its('body').as('admin'); + cy.registerUser('admin', 'pass', true); // ユーザー作成 - cy.request('POST', '/api/signup', { - username: 'alice', - password: 'alice1234', - }).its('body').as('alice'); + cy.registerUser('alice', 'alice1234'); - cy.visit('/'); - - cy.intercept('POST', '/api/signin').as('signin'); - - cy.get('[data-cy-signin]').click(); - cy.get('[data-cy-signin-username] input').type('alice'); - cy.get('[data-cy-signin-password] input').type('alice1234{enter}'); - - cy.wait('@signin').as('signedIn'); + cy.login('alice', 'alice1234'); }); afterEach(() => { diff --git a/cypress/support/commands.js b/cypress/support/commands.js index 119ab03f7c..95bfcf6855 100644 --- a/cypress/support/commands.js +++ b/cypress/support/commands.js @@ -23,3 +23,33 @@ // // -- This will overwrite an existing command -- // Cypress.Commands.overwrite('visit', (originalFn, url, options) => { ... }) + +Cypress.Commands.add('resetState', () => { + cy.window(win => { + win.indexedDB.deleteDatabase('keyval-store'); + }); + cy.request('POST', '/api/reset-db').as('reset'); + cy.get('@reset').its('status').should('equal', 204); + cy.reload(true); +}); + +Cypress.Commands.add('registerUser', (username, password, isAdmin = false) => { + const route = isAdmin ? '/api/admin/accounts/create' : '/api/signup'; + + cy.request('POST', route, { + username: username, + password: password, + }).its('body').as(username); +}); + +Cypress.Commands.add('login', (username, password) => { + cy.visit('/'); + + cy.intercept('POST', '/api/signin').as('signin'); + + cy.get('[data-cy-signin]').click(); + cy.get('[data-cy-signin-username] input').type(username); + cy.get('[data-cy-signin-password] input').type(`${password}{enter}`); + + cy.wait('@signin').as('signedIn'); +}); From 025bf4a5e77f78928b21e77ceb33bc05a8ce642e Mon Sep 17 00:00:00 2001 From: Johann150 Date: Tue, 31 May 2022 11:57:55 +0200 Subject: [PATCH 5/8] fix(mfm): remove duplicate br tag/newline (#8616) --- packages/backend/src/mfm/from-html.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/backend/src/mfm/from-html.ts b/packages/backend/src/mfm/from-html.ts index 623cb0e71c..15110b6b70 100644 --- a/packages/backend/src/mfm/from-html.ts +++ b/packages/backend/src/mfm/from-html.ts @@ -6,6 +6,9 @@ const urlRegex = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+/; const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/; export function fromHtml(html: string, hashtagNames?: string[]): string { + // some AP servers like Pixelfed use br tags as well as newlines + html = html.replace(/\r?\n/gi, '\n'); + const dom = parse5.parseFragment(html); let text = ''; From 121a1784a2be9264740e3cc229ad0a282e6a6e20 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Tue, 31 May 2022 16:22:00 +0200 Subject: [PATCH 6/8] fix(lint): indentation --- packages/client/src/ui/deck.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/client/src/ui/deck.vue b/packages/client/src/ui/deck.vue index 1e0d9a1652..e538a93f06 100644 --- a/packages/client/src/ui/deck.vue +++ b/packages/client/src/ui/deck.vue @@ -17,7 +17,7 @@ :key="ids[0]" class="column" :column="columns.find(c => c.id === ids[0])" - :is-stacked="false" + :is-stacked="false" :style="columns.find(c => c.id === ids[0])!.flexible ? { flex: 1, minWidth: '350px' } : { width: columns.find(c => c.id === ids[0])!.width + 'px' }" @parent-focus="moveFocus(ids[0], $event)" /> From 118f3546616ab71e37e02eaf2330ccb195b57f04 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Wed, 1 Jun 2022 08:51:00 +0200 Subject: [PATCH 7/8] fix: server metrics widget --- packages/client/src/widgets/server-metric/net.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/client/src/widgets/server-metric/net.vue b/packages/client/src/widgets/server-metric/net.vue index 82b3a67d76..b698953f97 100644 --- a/packages/client/src/widgets/server-metric/net.vue +++ b/packages/client/src/widgets/server-metric/net.vue @@ -94,10 +94,10 @@ function onStats(connStats) { inPolygonPoints = `${viewBoxX - (stats.length - 1)},${viewBoxY} ${inPolylinePoints} ${viewBoxX},${viewBoxY}`; outPolygonPoints = `${viewBoxX - (stats.length - 1)},${viewBoxY} ${outPolylinePoints} ${viewBoxX},${viewBoxY}`; - inHeadX = inPolylinePoints[inPolylinePoints.length - 1][0]; - inHeadY = inPolylinePoints[inPolylinePoints.length - 1][1]; - outHeadX = outPolylinePoints[outPolylinePoints.length - 1][0]; - outHeadY = outPolylinePoints[outPolylinePoints.length - 1][1]; + inHeadX = inPolylinePointsStats[inPolylinePointsStats.length - 1][0]; + inHeadY = inPolylinePointsStats[inPolylinePointsStats.length - 1][1]; + outHeadX = outPolylinePointsStats[outPolylinePointsStats.length - 1][0]; + outHeadY = outPolylinePointsStats[outPolylinePointsStats.length - 1][1]; inRecent = connStats.net.rx; outRecent = connStats.net.tx; From 0263a783a6ef7cb8ebab8aa5e745ba2ed48528a1 Mon Sep 17 00:00:00 2001 From: Johann150 Date: Wed, 1 Jun 2022 09:34:40 +0200 Subject: [PATCH 8/8] fix(dev): no labels for l10n_develop --- .github/workflows/labeler.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 057208eda3..fa4a58c3a9 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,8 @@ name: "Pull Request Labeler" on: -- pull_request_target + pull_request_target: + branches-ignore: + - 'l10n_develop' jobs: triage: