From 90c7ecf92fd9d96e9ddf88f662fc083908d25bb6 Mon Sep 17 00:00:00 2001 From: Lily Cohen Date: Wed, 7 Jun 2023 11:34:23 -0600 Subject: [PATCH 1/9] update to node 20 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d8671911e4..2227d57a33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ## Install dev and compilation dependencies, build files -FROM node:19-alpine as build +FROM node:20-alpine as build WORKDIR /calckey # Install compilation dependencies @@ -32,7 +32,7 @@ RUN pnpm i --prod --frozen-lockfile ## Runtime container -FROM node:19-alpine +FROM node:20-alpine WORKDIR /calckey # Install runtime dependencies From 90ad37c3a32293a1d089ebbe263b1a3d11f7cdd3 Mon Sep 17 00:00:00 2001 From: Lily Cohen Date: Wed, 7 Jun 2023 11:37:42 -0600 Subject: [PATCH 2/9] remove apk update as 'no-cache' already updates --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2227d57a33..29922af4f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ FROM node:20-alpine as build WORKDIR /calckey # Install compilation dependencies -RUN apk update RUN apk add --no-cache --no-progress git alpine-sdk python3 rust cargo vips # Copy only the dependency-related files first, to cache efficiently From 48f58feae634a60057e43537686c47391edbd288 Mon Sep 17 00:00:00 2001 From: Lily Cohen Date: Wed, 7 Jun 2023 11:43:32 -0600 Subject: [PATCH 3/9] precompile rust components --- .dockerignore | 3 +++ Dockerfile | 27 +++++++++++++++++++++++++-- docker-compose.yml | 2 -- package.json | 4 +++- packages/backend/package.json | 4 ++-- 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.dockerignore b/.dockerignore index 8aa17c3a89..4a75603f77 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,9 @@ node_modules **/node_modules report.*.json +# Rust +packages/backend/native-utils/target/* + # Cypress cypress/screenshots cypress/videos diff --git a/Dockerfile b/Dockerfile index 29922af4f4..cbdf7407e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,16 @@ WORKDIR /calckey # Install compilation dependencies RUN apk add --no-cache --no-progress git alpine-sdk python3 rust cargo vips +# Copy only the cargo dependency-related files first, to cache efficiently +COPY packages/backend/native-utils/Cargo.toml packages/backend/native-utils/Cargo.toml +COPY packages/backend/native-utils/Cargo.lock packages/backend/native-utils/Cargo.lock +COPY packages/backend/native-utils/migration/Cargo.toml packages/backend/native-utils/migration/Cargo.toml +COPY packages/backend/native-utils/src/*.rs packages/backend/native-utils/src/ + +# Install cargo dependencies +RUN cd packages/backend && \ + cargo fetch --locked --manifest-path ./native-utils/migration/Cargo.toml + # Copy only the dependency-related files first, to cache efficiently COPY package.json pnpm*.yaml ./ COPY packages/backend/package.json packages/backend/package.json @@ -22,13 +32,26 @@ RUN corepack prepare pnpm@latest --activate # Install dev mode dependencies for compilation RUN pnpm i --frozen-lockfile +# Copy in the rest of the native-utils rust files +COPY packages/backend/native-utils/.cargo packages/backend/native-utils/.cargo +COPY packages/backend/native-utils/src packages/backend/native-utils/src +COPY packages/backend/native-utils/migration packages/backend/native-utils/migration +COPY packages/backend/native-utils/tests packages/backend/native-utils/tests +COPY packages/backend/native-utils/*.rs packages/backend/native-utils/ + +# native-utils cargo build +RUN pnpm run build:cargo + # Copy in the rest of the files, to compile from TS to JS COPY . ./ -RUN pnpm run build +RUN pnpm run build:recursive +RUN pnpm run gulp # Trim down the dependencies to only the prod deps RUN pnpm i --prod --frozen-lockfile +# Clean up the cargo deps +RUN rm -rf /calckey/packages/backend/native-utils/target/release/deps ## Runtime container FROM node:20-alpine @@ -51,7 +74,7 @@ COPY --from=build /calckey/built /calckey/built COPY --from=build /calckey/packages/backend/built /calckey/packages/backend/built COPY --from=build /calckey/packages/backend/assets/instance.css /calckey/packages/backend/assets/instance.css COPY --from=build /calckey/packages/backend/native-utils/built /calckey/packages/backend/native-utils/built -COPY --from=build /calckey/packages/backend/native-utils/target /calckey/packages/backend/native-utils/target +COPY --from=build /calckey/packages/backend/native-utils/target/release /calckey/packages/backend/native-utils/target/release RUN corepack enable ENTRYPOINT [ "/sbin/tini", "--" ] diff --git a/docker-compose.yml b/docker-compose.yml index d6ad26a059..abb1882ea5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,8 +19,6 @@ services: environment: NODE_ENV: production volumes: - - ./.cargo-cache:/root/.cargo - - ./.cargo-target:/calckey/packages/backend/native-utils/target - ./files:/calckey/files - ./.config:/calckey/.config:ro diff --git a/package.json b/package.json index 42a18a33c1..a6de546fff 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,9 @@ "private": true, "scripts": { "rebuild": "pnpm run clean && pnpm -r run build && pnpm run gulp", - "build": "pnpm -r run build && pnpm run gulp", + "build": "pnpm -r run build:cargo && pnpm -r run build:recursive && pnpm run gulp", + "build:recursive": "pnpm -r run build", + "build:cargo": "pnpm --filter backend run build:cargo", "start": "pnpm --filter backend run start", "start:test": "pnpm --filter backend run start:test", "init": "pnpm run migrate", diff --git a/packages/backend/package.json b/packages/backend/package.json index 2a19b916cf..90b1e1e41b 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -8,10 +8,10 @@ "start:test": "NODE_ENV=test pnpm node ./built/index.js", "migrate": "pnpm run migrate:typeorm && pnpm run migrate:cargo", "migrate:typeorm": "typeorm migration:run -d ormconfig.js", - "migrate:cargo": "cargo run --manifest-path ./native-utils/migration/Cargo.toml -- up", + "migrate:cargo": "./native-utils/target/release/migration up", "revertmigration": "pnpm run revertmigration:cargo && pnpm run revertmigration:typeorm", "revertmigration:typeorm": "typeorm migration:revert -d ormconfig.js", - "revertmigration:cargo": "cargo run --manifest-path ./native-utils/migration/Cargo.toml -- down", + "revertmigration:cargo": "./native-utils/target/release/migration down", "check:connect": "node ./check_connect.js", "build": "pnpm swc src -d built -D", "watch": "pnpm swc src -d built -D -w", From e232332a469718b5f567fbf75224f987df2415e4 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Wed, 21 Jun 2023 21:25:25 -0400 Subject: [PATCH 4/9] fix build and clean scripts --- package.json | 4 +--- packages/backend/native-utils/package.json | 4 ++-- scripts/clean-all.js | 26 +++++++++++++--------- scripts/clean.js | 12 +++++----- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index a6de546fff..42a18a33c1 100644 --- a/package.json +++ b/package.json @@ -10,9 +10,7 @@ "private": true, "scripts": { "rebuild": "pnpm run clean && pnpm -r run build && pnpm run gulp", - "build": "pnpm -r run build:cargo && pnpm -r run build:recursive && pnpm run gulp", - "build:recursive": "pnpm -r run build", - "build:cargo": "pnpm --filter backend run build:cargo", + "build": "pnpm -r run build && pnpm run gulp", "start": "pnpm --filter backend run start", "start:test": "pnpm --filter backend run start:test", "init": "pnpm run migrate", diff --git a/packages/backend/native-utils/package.json b/packages/backend/native-utils/package.json index 2e6a721f4c..e326a2d3fa 100644 --- a/packages/backend/native-utils/package.json +++ b/packages/backend/native-utils/package.json @@ -34,8 +34,8 @@ }, "scripts": { "artifacts": "napi artifacts", - "build": "napi build --features napi --platform --release ./built/", - "build:debug": "napi build --platform", + "build": "napi build --features napi --platform --release ./built/ && cargo build --locked --release --manifest-path ./migration/Cargo.toml", + "build:debug": "napi build --platform ./built/ && cargo build --manifest-path ./migration/Cargo.toml", "prepublishOnly": "napi prepublish -t npm", "test": "pnpm run cargo:test && pnpm run build && ava", "universal": "napi universal", diff --git a/scripts/clean-all.js b/scripts/clean-all.js index c3d85bf3a5..c5f6829886 100644 --- a/scripts/clean-all.js +++ b/scripts/clean-all.js @@ -1,26 +1,30 @@ -const fs = require('fs'); +const fs = require('node:fs'); const execa = require('execa'); +const { join } = require('node:path'); (async () => { - fs.rmSync(__dirname + '/../packages/backend/built', { recursive: true, force: true }); - fs.rmSync(__dirname + '/../packages/backend/node_modules', { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/backend/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/backend/node_modules'), { recursive: true, force: true }); - fs.rmSync(__dirname + '/../packages/client/built', { recursive: true, force: true }); - fs.rmSync(__dirname + '/../packages/client/node_modules', { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/backend/native-utils/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/backend/native-utils/node_modules'), { recursive: true, force: true }); - fs.rmSync(__dirname + '/../packages/sw/built', { recursive: true, force: true }); - fs.rmSync(__dirname + '/../packages/sw/node_modules', { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/client/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/client/node_modules'), { recursive: true, force: true }); - fs.rmSync(__dirname + '/../built', { recursive: true, force: true }); - fs.rmSync(__dirname + '/../node_modules', { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/sw/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/sw/node_modules'), { recursive: true, force: true }); + + fs.rmSync(join(__dirname, '/../built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../node_modules'), { recursive: true, force: true }); execa('pnpm', ['store', 'prune'], { - cwd: __dirname + '/../', + cwd: join(__dirname, '/../'), stdio: 'inherit' }); execa('cargo', ['clean'], { - cwd: __dirname + '/../packages/backend/native-utils', + cwd: join(__dirname, '/../packages/backend/native-utils'), stdio: 'inherit' }); })(); diff --git a/scripts/clean.js b/scripts/clean.js index 70b9d882b5..cf8ee9bc56 100644 --- a/scripts/clean.js +++ b/scripts/clean.js @@ -1,8 +1,10 @@ -const fs = require('fs'); +const fs = require('node:fs'); +const { join } = require('node:path'); (async () => { - fs.rmSync(__dirname + '/../packages/backend/built', { recursive: true, force: true }); - fs.rmSync(__dirname + '/../packages/client/built', { recursive: true, force: true }); - fs.rmSync(__dirname + '/../packages/sw/built', { recursive: true, force: true }); - fs.rmSync(__dirname + '/../built', { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/backend/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/backend/native-utils/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/client/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../packages/sw/built'), { recursive: true, force: true }); + fs.rmSync(join(__dirname, '/../built'), { recursive: true, force: true }); })(); From 3780be57c052ca231cc876c9889cc69f56f1207d Mon Sep 17 00:00:00 2001 From: Namekuji Date: Wed, 21 Jun 2023 22:58:02 -0400 Subject: [PATCH 5/9] clean unnecessary artifacts --- Dockerfile | 41 ++++++---------------- packages/backend/native-utils/package.json | 6 ++-- packages/backend/package.json | 7 ++-- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index cbdf7407e0..c126855544 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ ## Install dev and compilation dependencies, build files -FROM node:20-alpine as build +FROM alpine:3.18 as build WORKDIR /calckey # Install compilation dependencies -RUN apk add --no-cache --no-progress git alpine-sdk python3 rust cargo vips +RUN apk add --no-cache --no-progress git alpine-sdk python3 nodejs-current npm rust cargo vips # Copy only the cargo dependency-related files first, to cache efficiently COPY packages/backend/native-utils/Cargo.toml packages/backend/native-utils/Cargo.toml @@ -25,40 +25,22 @@ COPY packages/backend/native-utils/package.json packages/backend/native-utils/pa COPY packages/backend/native-utils/npm/linux-x64-musl/package.json packages/backend/native-utils/npm/linux-x64-musl/package.json COPY packages/backend/native-utils/npm/linux-arm64-musl/package.json packages/backend/native-utils/npm/linux-arm64-musl/package.json -# Configure corepack and pnpm -RUN corepack enable -RUN corepack prepare pnpm@latest --activate +# Configure corepack and pnpm, and install dev mode dependencies for compilation +RUN corepack enable && corepack prepare pnpm@latest --activate && pnpm i --frozen-lockfile -# Install dev mode dependencies for compilation -RUN pnpm i --frozen-lockfile - -# Copy in the rest of the native-utils rust files -COPY packages/backend/native-utils/.cargo packages/backend/native-utils/.cargo -COPY packages/backend/native-utils/src packages/backend/native-utils/src -COPY packages/backend/native-utils/migration packages/backend/native-utils/migration -COPY packages/backend/native-utils/tests packages/backend/native-utils/tests -COPY packages/backend/native-utils/*.rs packages/backend/native-utils/ - -# native-utils cargo build -RUN pnpm run build:cargo - -# Copy in the rest of the files, to compile from TS to JS +# Copy in the rest of the files to compile COPY . ./ -RUN pnpm run build:recursive -RUN pnpm run gulp +RUN pnpm run build -# Trim down the dependencies to only the prod deps -RUN pnpm i --prod --frozen-lockfile - -# Clean up the cargo deps -RUN rm -rf /calckey/packages/backend/native-utils/target/release/deps +# Trim down the artifacts and dependencies to only the prod deps +RUN cargo clean --manifest-path /calckey/packages/backend/native-utils/Cargo.toml && pnpm i --prod --frozen-lockfile ## Runtime container -FROM node:20-alpine +FROM alpine:3.18 WORKDIR /calckey # Install runtime dependencies -RUN apk add --no-cache --no-progress tini ffmpeg vips-dev zip unzip rust cargo +RUN apk add --no-cache --no-progress tini ffmpeg vips-dev zip unzip nodejs-current COPY . ./ @@ -74,8 +56,7 @@ COPY --from=build /calckey/built /calckey/built COPY --from=build /calckey/packages/backend/built /calckey/packages/backend/built COPY --from=build /calckey/packages/backend/assets/instance.css /calckey/packages/backend/assets/instance.css COPY --from=build /calckey/packages/backend/native-utils/built /calckey/packages/backend/native-utils/built -COPY --from=build /calckey/packages/backend/native-utils/target/release /calckey/packages/backend/native-utils/target/release -RUN corepack enable +RUN corepack enable && corepack prepare pnpm@latest --activate ENTRYPOINT [ "/sbin/tini", "--" ] CMD [ "pnpm", "run", "migrateandstart" ] diff --git a/packages/backend/native-utils/package.json b/packages/backend/native-utils/package.json index e326a2d3fa..faf678e2d2 100644 --- a/packages/backend/native-utils/package.json +++ b/packages/backend/native-utils/package.json @@ -34,10 +34,12 @@ }, "scripts": { "artifacts": "napi artifacts", - "build": "napi build --features napi --platform --release ./built/ && cargo build --locked --release --manifest-path ./migration/Cargo.toml", + "build": "pnpm run build:napi && pnpm run build:migration", + "build:napi": "napi build --features napi --platform --release ./built/", + "build:migration": "cargo build --locked --release --manifest-path ./migration/Cargo.toml && cp -v ./target/release/migration ./built/migration", "build:debug": "napi build --platform ./built/ && cargo build --manifest-path ./migration/Cargo.toml", "prepublishOnly": "napi prepublish -t npm", - "test": "pnpm run cargo:test && pnpm run build && ava", + "test": "pnpm run cargo:test && pnpm run build:napi && ava", "universal": "napi universal", "version": "napi version", "format": "cargo fmt --all", diff --git a/packages/backend/package.json b/packages/backend/package.json index 90b1e1e41b..f7d19d85b2 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -8,10 +8,10 @@ "start:test": "NODE_ENV=test pnpm node ./built/index.js", "migrate": "pnpm run migrate:typeorm && pnpm run migrate:cargo", "migrate:typeorm": "typeorm migration:run -d ormconfig.js", - "migrate:cargo": "./native-utils/target/release/migration up", + "migrate:cargo": "./native-utils/built/migration up", "revertmigration": "pnpm run revertmigration:cargo && pnpm run revertmigration:typeorm", "revertmigration:typeorm": "typeorm migration:revert -d ormconfig.js", - "revertmigration:cargo": "./native-utils/target/release/migration down", + "revertmigration:cargo": "./native-utils/built/migration down", "check:connect": "node ./check_connect.js", "build": "pnpm swc src -d built -D", "watch": "pnpm swc src -d built -D -w", @@ -20,9 +20,6 @@ "test": "pnpm run mocha", "format": "pnpm rome format * --write" }, - "resolutions": { - "chokidar": "^3.3.1" - }, "optionalDependencies": { "@swc/core-android-arm64": "1.3.11", "@tensorflow/tfjs-node": "3.21.1" From 9b009918acbebab5b5fbf6f01dda9478714111e0 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 22 Jun 2023 02:00:28 -0400 Subject: [PATCH 6/9] add environment variable --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c126855544..08aedaf7c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN corepack enable && corepack prepare pnpm@latest --activate && pnpm i --froze # Copy in the rest of the files to compile COPY . ./ -RUN pnpm run build +RUN env NODE_ENV=production pnpm run build # Trim down the artifacts and dependencies to only the prod deps RUN cargo clean --manifest-path /calckey/packages/backend/native-utils/Cargo.toml && pnpm i --prod --frozen-lockfile @@ -58,5 +58,6 @@ COPY --from=build /calckey/packages/backend/assets/instance.css /calckey/package COPY --from=build /calckey/packages/backend/native-utils/built /calckey/packages/backend/native-utils/built RUN corepack enable && corepack prepare pnpm@latest --activate +ENV NODE_ENV=production ENTRYPOINT [ "/sbin/tini", "--" ] CMD [ "pnpm", "run", "migrateandstart" ] From 80f21df0e072d29bcc9fb12c1ff9a3ec9f7e1234 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 22 Jun 2023 11:56:33 -0400 Subject: [PATCH 7/9] mount files directory --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 08aedaf7c6..47452be3fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,5 +59,6 @@ COPY --from=build /calckey/packages/backend/native-utils/built /calckey/packages RUN corepack enable && corepack prepare pnpm@latest --activate ENV NODE_ENV=production +VOLUME [ "/calckey/files" ] ENTRYPOINT [ "/sbin/tini", "--" ] CMD [ "pnpm", "run", "migrateandstart" ] From 99e1f1ad84a908c15af4edf7300a68b2c5ed2aa1 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 22 Jun 2023 12:28:36 -0400 Subject: [PATCH 8/9] chore: format --- packages/backend/native-utils/package.json | 86 +++++++++++----------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/packages/backend/native-utils/package.json b/packages/backend/native-utils/package.json index faf678e2d2..432074f251 100644 --- a/packages/backend/native-utils/package.json +++ b/packages/backend/native-utils/package.json @@ -1,50 +1,50 @@ { - "name": "native-utils", - "version": "0.0.0", - "main": "built/index.js", - "types": "built/index.d.ts", - "napi": { - "name": "native-utils", - "triples": { - "additional": [ - "aarch64-apple-darwin", - "aarch64-linux-android", - "aarch64-unknown-linux-gnu", - "aarch64-unknown-linux-musl", - "aarch64-pc-windows-msvc", - "armv7-unknown-linux-gnueabihf", - "x86_64-unknown-linux-musl", - "x86_64-unknown-freebsd", - "i686-pc-windows-msvc", - "armv7-linux-androideabi", - "universal-apple-darwin" - ] - } - }, - "license": "MIT", - "devDependencies": { - "@napi-rs/cli": "2.16.1", - "ava": "5.1.1" - }, - "ava": { - "timeout": "3m" - }, - "engines": { - "node": ">= 10" - }, - "scripts": { - "artifacts": "napi artifacts", + "name": "native-utils", + "version": "0.0.0", + "main": "built/index.js", + "types": "built/index.d.ts", + "napi": { + "name": "native-utils", + "triples": { + "additional": [ + "aarch64-apple-darwin", + "aarch64-linux-android", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "aarch64-pc-windows-msvc", + "armv7-unknown-linux-gnueabihf", + "x86_64-unknown-linux-musl", + "x86_64-unknown-freebsd", + "i686-pc-windows-msvc", + "armv7-linux-androideabi", + "universal-apple-darwin" + ] + } + }, + "license": "MIT", + "devDependencies": { + "@napi-rs/cli": "2.16.1", + "ava": "5.1.1" + }, + "ava": { + "timeout": "3m" + }, + "engines": { + "node": ">= 10" + }, + "scripts": { + "artifacts": "napi artifacts", "build": "pnpm run build:napi && pnpm run build:migration", "build:napi": "napi build --features napi --platform --release ./built/", "build:migration": "cargo build --locked --release --manifest-path ./migration/Cargo.toml && cp -v ./target/release/migration ./built/migration", - "build:debug": "napi build --platform ./built/ && cargo build --manifest-path ./migration/Cargo.toml", - "prepublishOnly": "napi prepublish -t npm", - "test": "pnpm run cargo:test && pnpm run build:napi && ava", - "universal": "napi universal", - "version": "napi version", + "build:debug": "napi build --platform ./built/ && cargo build --manifest-path ./migration/Cargo.toml", + "prepublishOnly": "napi prepublish -t npm", + "test": "pnpm run cargo:test && pnpm run build:napi && ava", + "universal": "napi universal", + "version": "napi version", "format": "cargo fmt --all", "cargo:test": "pnpm run cargo:unit && pnpm run cargo:integration", - "cargo:unit": "cargo test unit_test && cargo test -F napi unit_test", - "cargo:integration": "cargo test -F noarray int_test -- --test-threads=1" - } + "cargo:unit": "cargo test unit_test && cargo test -F napi unit_test", + "cargo:integration": "cargo test -F noarray int_test -- --test-threads=1" + } } From f7631772a3bfffbc2d4102d67c4212b9f9c07f62 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 22 Jun 2023 13:16:59 -0400 Subject: [PATCH 9/9] cache native-utils --- .dockerignore | 4 +--- Dockerfile | 23 +++++++++++++------ .../backend/native-utils/migration/Cargo.toml | 6 ++--- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.dockerignore b/.dockerignore index 4a75603f77..90d15ddd90 100644 --- a/.dockerignore +++ b/.dockerignore @@ -28,9 +28,6 @@ coverage !/.config/example.yml !/.config/docker_example.env -#docker dev config -/dev/docker-compose.yml - # misskey built db @@ -50,3 +47,4 @@ packages/backend/assets/instance.css # dockerignore custom .git Dockerfile +docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 47452be3fa..e11cb2bf44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,12 +8,12 @@ RUN apk add --no-cache --no-progress git alpine-sdk python3 nodejs-current npm r # Copy only the cargo dependency-related files first, to cache efficiently COPY packages/backend/native-utils/Cargo.toml packages/backend/native-utils/Cargo.toml COPY packages/backend/native-utils/Cargo.lock packages/backend/native-utils/Cargo.lock +COPY packages/backend/native-utils/src/lib.rs packages/backend/native-utils/src/ COPY packages/backend/native-utils/migration/Cargo.toml packages/backend/native-utils/migration/Cargo.toml -COPY packages/backend/native-utils/src/*.rs packages/backend/native-utils/src/ +COPY packages/backend/native-utils/migration/src/lib.rs packages/backend/native-utils/migration/src/ # Install cargo dependencies -RUN cd packages/backend && \ - cargo fetch --locked --manifest-path ./native-utils/migration/Cargo.toml +RUN cargo fetch --locked --manifest-path /calckey/packages/backend/native-utils/Cargo.toml # Copy only the dependency-related files first, to cache efficiently COPY package.json pnpm*.yaml ./ @@ -28,12 +28,21 @@ COPY packages/backend/native-utils/npm/linux-arm64-musl/package.json packages/ba # Configure corepack and pnpm, and install dev mode dependencies for compilation RUN corepack enable && corepack prepare pnpm@latest --activate && pnpm i --frozen-lockfile +# Copy in the rest of the native-utils rust files +COPY packages/backend/native-utils/.cargo packages/backend/native-utils/.cargo +COPY packages/backend/native-utils/build.rs packages/backend/native-utils/ +COPY packages/backend/native-utils/src packages/backend/native-utils/src/ +COPY packages/backend/native-utils/migration/src packages/backend/native-utils/migration/src/ + +# Compile native-utils +RUN pnpm run --filter native-utils build + # Copy in the rest of the files to compile COPY . ./ -RUN env NODE_ENV=production pnpm run build +RUN env NODE_ENV=production sh -c "pnpm run --filter '!native-utils' build && pnpm run gulp" -# Trim down the artifacts and dependencies to only the prod deps -RUN cargo clean --manifest-path /calckey/packages/backend/native-utils/Cargo.toml && pnpm i --prod --frozen-lockfile +# Trim down the dependencies to only those for production +RUN pnpm i --prod --frozen-lockfile ## Runtime container FROM alpine:3.18 @@ -59,6 +68,6 @@ COPY --from=build /calckey/packages/backend/native-utils/built /calckey/packages RUN corepack enable && corepack prepare pnpm@latest --activate ENV NODE_ENV=production -VOLUME [ "/calckey/files" ] +VOLUME "/calckey/files" ENTRYPOINT [ "/sbin/tini", "--" ] CMD [ "pnpm", "run", "migrateandstart" ] diff --git a/packages/backend/native-utils/migration/Cargo.toml b/packages/backend/native-utils/migration/Cargo.toml index 4dee156ef3..7ed9fd5f03 100644 --- a/packages/backend/native-utils/migration/Cargo.toml +++ b/packages/backend/native-utils/migration/Cargo.toml @@ -10,14 +10,14 @@ path = "src/lib.rs" [features] default = [] -convert = ["dep:native-utils"] +convert = ["dep:native-utils", "dep:indicatif", "dep:futures"] [dependencies] serde_json = "1.0.96" native-utils = { path = "../", optional = true } -indicatif = { version = "0.17.4", features = ["tokio"] } +indicatif = { version = "0.17.4", features = ["tokio"], optional = true } tokio = { version = "1.28.2", features = ["full"] } -futures = "0.3.28" +futures = { version = "0.3.28", optional = true } serde_yaml = "0.9.21" serde = { version = "1.0.163", features = ["derive"] } urlencoding = "2.1.2"