HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: index all

Browse source
This commit is contained in:
Namekuji 2023-09-06 04:00:51 -04:00
parent 1402131b9e
commit 7f00bace29
No known key found for this signature in database
GPG key ID: 1D62332C07FBA532
1 changed files with 21 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
packages/backend/src/queue/processors/background/index-all-notes.ts
View file

@ -7,7 +7,7 @@ import { MoreThan } from "typeorm";
import { index } from "@/services/note/create.js";
import { Note } from "@/models/entities/note.js";
import meilisearch from "../../../db/meilisearch.js";
import { fetchPostCount, scyllaClient } from "@/db/scylla.js";
import { fetchPostCount, parseScyllaNote, scyllaClient } from "@/db/scylla.js";
const logger = queueLogger.createSubLogger("index-all-notes");
@ -21,6 +21,26 @@ export default async function indexAllNotes(
let indexedCount: number = (job.data.indexedCount as number) ?? 0;
let total: number = (job.data.total as number) ?? 0;
if (scyllaClient) {
total = await fetchPostCount(false);
scyllaClient.eachRow("SELECT * FROM note", [], (n, row) => {
if (n % 1000 === 0) {
job.progress(((n / total) * 100).toFixed(1));
logger.info(`Indexed notes ${n}/${total}`);
}
const note = parseScyllaNote(row);
if (meilisearch) {
meilisearch.ingestNote([note]).then(() => index(note, true));
} else {
index(note, true);
}
});
done();
logger.info("All notes have been indexed.");
return;
}
let running = true;
const take = 10000;
const batch = 100;