From 9a63f855675391b961b93ba043b6033f0efcbd13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=80=81=E5=91=A8=E9=83=A8=E8=90=BD?= Date: Sun, 3 Mar 2024 14:02:27 +0800 Subject: [PATCH] feat(dev): add new docker-based dev env to replace broken nix dev env. --- .config/devenv.yml | 33 ++-- .dockerignore | 2 - .envrc | 4 - .gitignore | 4 - dev/docker-compose-onlydb.yml | 17 ++ dev/docker-compose.yml | 39 ++++- dev/docker-entrypoint.sh | 53 ++++++ docs/development.md | 112 ++----------- flake.lock | 294 ---------------------------------- flake.nix | 86 ---------- 10 files changed, 135 insertions(+), 509 deletions(-) delete mode 100644 .envrc create mode 100644 dev/docker-compose-onlydb.yml create mode 100644 dev/docker-entrypoint.sh delete mode 100644 flake.lock delete mode 100644 flake.nix diff --git a/.config/devenv.yml b/.config/devenv.yml index 65b819b569..ee8ab2b17b 100644 --- a/.config/devenv.yml +++ b/.config/devenv.yml @@ -2,31 +2,18 @@ url: http://localhost:3000 port: 3000 db: - host: 127.0.0.1 - port: 5432 + host: db + port: 25432 - db: firefish + db: firefish_db user: firefish - pass: firefish + pass: password redis: - host: localhost - port: 6379 + host: redis + port: 26379 family: 4 -#sonic: -# host: localhost -# port: 1491 -# auth: SecretPassword -# collection: notes -# bucket: default - -#elasticsearch: -# host: localhost -# port: 9200 -# ssl: false -# user: -# pass: id: 'aid' @@ -36,3 +23,11 @@ reservedUsernames: - administrator - me - system + +logLevel: [ + 'error', + 'success', + 'warning', + 'debug', + 'info' +] diff --git a/.dockerignore b/.dockerignore index 2efc98f8fc..f1c6bfd84a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -45,8 +45,6 @@ docker-compose.yml docker-compose.example.yml firefish.apache.conf firefish.nginx.conf -flake.lock -flake.nix title.svg /.gitlab /chart diff --git a/.envrc b/.envrc deleted file mode 100644 index 3ce7171a3c..0000000000 --- a/.envrc +++ /dev/null @@ -1,4 +0,0 @@ -if ! has nix_direnv_version || ! nix_direnv_version 2.3.0; then - source_url "https://raw.githubusercontent.com/nix-community/nix-direnv/2.3.0/direnvrc" "sha256-Dmd+j63L84wuzgyjITIfSxSD57Tx7v51DMxVZOsiUD8=" -fi -use flake . --impure diff --git a/.gitignore b/.gitignore index 2e08a1261e..746aff57c6 100644 --- a/.gitignore +++ b/.gitignore @@ -71,10 +71,6 @@ packages/megalodon/.idea .yarn yarn* -# Nix Development shell items -.devenv -.direnv - # Cargo cache for Docker /.cargo-cache /.cargo-target diff --git a/dev/docker-compose-onlydb.yml b/dev/docker-compose-onlydb.yml new file mode 100644 index 0000000000..d1037fd593 --- /dev/null +++ b/dev/docker-compose-onlydb.yml @@ -0,0 +1,17 @@ +version: "3" + +services: + redis: + image: docker.io/redis:7-alpine + ports: + - "26379:6379" + db: + image: docker.io/groonga/pgroonga:latest-alpine-16-slim + environment: + - "POSTGRES_PASSWORD=password" + - "POSTGRES_USER=firefish" + - "POSTGRES_DB=firefish_db" + ports: + - "25432:5432" + volumes: + - "./install.sql:/docker-entrypoint-initdb.d/install.sql:ro" diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index d1037fd593..b07352e60e 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -1,12 +1,44 @@ version: "3" services: + web: + image: docker.io/node:21-slim + container_name: firefish_web + restart: unless-stopped + depends_on: + - db + - redis + ports: + - "3000:3000" + networks: + - firefishnet + environment: + # Currently bug in the development env and it cannot run. + NODE_ENV: production + PGPASSWORD: password + URL: http://localhost:3000 + volumes: + - ./firefish:/firefish:rw + - ./docker-entrypoint.sh:/docker-entrypoint.sh:rw + entrypoint: /docker-entrypoint.sh + redis: - image: docker.io/redis:7-alpine + restart: unless-stopped + image: docker.io/redis:7.0-alpine + container_name: firefish_redis + networks: + - firefishnet ports: - "26379:6379" + volumes: + - ./redis:/data + db: + restart: unless-stopped image: docker.io/groonga/pgroonga:latest-alpine-16-slim + container_name: firefish_db + networks: + - firefishnet environment: - "POSTGRES_PASSWORD=password" - "POSTGRES_USER=firefish" @@ -14,4 +46,7 @@ services: ports: - "25432:5432" volumes: - - "./install.sql:/docker-entrypoint-initdb.d/install.sql:ro" + - ./db:/var/lib/postgresql/data + +networks: + firefishnet: diff --git a/dev/docker-entrypoint.sh b/dev/docker-entrypoint.sh new file mode 100644 index 0000000000..11f6f0b51a --- /dev/null +++ b/dev/docker-entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -xe +node -v + +# Check Environment Initialized Flag +if [ ! -f "/initialized_flag.txt" ];then + + # Install compilation dependencies + apt-get update + DEBIAN_FRONTEND='noninteractive' apt-get install -y --no-install-recommends curl build-essential ca-certificates python3 postgresql-client-15 + curl -vvv --proto '=https' --tlsv1.2 --show-error --fail https://sh.rustup.rs | sh -s -- -y + + # Add Cargo PATH + PATH="/root/.cargo/bin:${PATH}" + + if [ ! -f "/firefish/README.md" ];then + + # Download Firefish and decompress + curl -vvv -O --proto '=https' --tlsv1.2 --show-error --fail https://firefish.dev/firefish/firefish/-/archive/develop/firefish-develop.tar.bz2 + tar -xjvf firefish-develop.tar.bz2 --strip-components 1 -C /firefish + + # Configuring a new server + cd /firefish + cp .config/devenv.yml .config/default.yml + URL=${URL//\//\\\/} + sed -i "s/http:\/\/localhost:3000/${URL}/g" .config/default.yml + + fi + + # Configure postgres, add pgroonga search + psql -U firefish -p 25432 -h db -c "CREATE EXTENSION pgroonga IF NOT EXISTS;" + + # Configure pnpm, and install dev mode dependencies for compilation + cd /firefish + corepack enable + corepack prepare pnpm@latest --activate + pnpm install --prod false + +fi + +# Add Environment Initialized Flag +touch /initialized_flag.txt + +# Add Cargo PATH +PATH="/root/.cargo/bin:${PATH}" + +# Start a new server +cd /firefish +pnpm install --prod false +pnpm run build +pnpm run migrate +pnpm run start diff --git a/docs/development.md b/docs/development.md index 797c8273ee..f55a817f1e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,108 +1,24 @@ # Firefish Developer Docs -## Nix Dev Environment -The Firefish repo comes with a Nix-based shell environment to help make development as easy as possible! +## Docker-based Environment -Please note, however, that this environment will not work on Windows outside of a WSL2 environment. +The Firefish repo comes with a new Docker-based environment to help make development as easy as possible! ### Prerequisites -- Installed the [Nix Package Manager](https://nixos.org/download.html) (use the comman on their website) -- Installed [direnv](https://direnv.net/docs/installation.html) and added its hook to your shell. (package manager) +- Installed [Docker](https://docs.docker.com/get-docker/) (use the comman on their website) . +- If your Docker version is older, you may also need to manually install Docker Compose, and in the following instructions +`docker compose` should be replaced with `docker-compose` . +- It is necessary to confirm that port 3000, 25432 and 26379 are not used by other programs or services. -Once the repo is cloned to your computer, follow these next few steps inside the Firefish folder: +Once Docker is installed to your computer, follow these next few steps to running: -- Run `direnv allow`. This will build the environment and install all needed tools. -- Run `install-deps`, then `prepare-config`, to install the node dependencies and prepare the needed config files. -- In a second terminal, run `devenv up`. This will spawn a **Redis** server, a **Postgres** server, and the **Firefish** server in dev mode. -- Once you see the Firefish banner printed in your second terminal, run `migrate` in the first. -- Once migrations finish, open http://localhost:3000 in your web browser. +- Download `docker-compose.yml` and `docker-entrypoint.sh` in dev folder to the new folder on your computer. +- If you are not running on your PC, open `docker-compose.yml`, set `URL` and save. +- Run `chmod 777 docker-entrypoint.sh`, for initialization script can be execute. +- Run `docker compose up`, This will build the environment, dependencies and prepare the needed config files. +- Once you see the Firefish banner printed in your screen, means initialization finished. +- Open http://localhost:3000 or `URL` on `docker-compose.yml` in your web browser. - You should now see the admin user creation screen! -Note: When you want to restart a dev server, all you need to do is run `devenv up`, no other steps are necessary. - -# Possible Troubles with the dev enviroment -(this doesn't have to be done under normal conditions, this is for future reference) - -### direnv -If you have any trouble with `direnv allow` -Check that the contents of `.envrc` have the same version of nix-direnv that is specified here: -> nix-direnv under -> installation -> using direnv source url -> https://github.com/nix-community/nix-direnv#direnv-source_url - -there should be no errors during `direnv allow` - -### outdated nix packages -if `install-deps` or any subsequent command doesn't run due to versioning problems -`flake.nix` and `flake.lock` may be outdated - -delete `flake.lock`, or better, run `nix flake update --extra-experimental-features flakes --extra-experimental-features nix-command` -after that, run `direnv rebuild` - -if there are any errors, you might have to change `flake.nix` -(because the available options can change between versions - consider getting support in [the matrix channel](https://matrix.to/#/#firefish-community:nitro.chat)) - -### after changing a node version -in my case, i had to change the node version from 19, to 18 - -! before proceeding, make sure to delete all build artifacts! -remove `node_modules` and `built` folders, and maybe `.devenv` and `.direnv` as well -manually, or run `npm cache clean --force` and `pnpm cleanall` - -### Windows Subsystem for Linux -if `devenv up` terminates because of wrong folder permissions, - -create the file `/etc/wsl.conf` in your distro and add -```shell -[automount] -options = "metadata" -``` - -this allows `chmod` calls to actually have an effect. -the build scripts DO actually set the permissions, it just needs to work in wsl. - -### devenv up -devenv up may take a looong time. (some say this is fake news, maybe it was bad luck in my case) - -do not get spooked by this error: -``` -> firefish@14.0.0-dev32 start /mnt/.../firefish -> pnpm --filter backend run start - - -> backend@ start /mnt/.../firefish/packages/backend -> pnpm node ./built/index.js - -node:internal/modules/cjs/loader:1078 - throw err; - ^ - -Error: Cannot find module '/mnt/.../firefish/packages/backend/built/index.js' - at Module._resolveFilename (node:internal/modules/cjs/loader:1075:15) - at Module._load (node:internal/modules/cjs/loader:920:27) - at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:81:12) - at node:internal/main/run_main_module:23:47 { - code: 'MODULE_NOT_FOUND', - requireStack: [] -} - -Node.js v18.16.0 -undefined -/mnt/.../firefish/packages/backend: - ERR_PNPM_RECURSIVE_RUN_FIRST_FAIL  backend@ start: `pnpm node ./built/index.js` -Exit status 1 - ELIFECYCLE  Command failed with exit code 1. -``` - -the script is designed to constantly try to start the server, while the build is still running. -this just means that the build isn't finished yet. - -at some point you should see a banner that says "Firefish" in big letters - -then you're good to go and can run `migrate` (in another terminal)! - -if you don't see the banner, -and it's for some reason stuck on `Finished 'build' after 917 ms` for a view minutes, - -just leave devenv running and open another terminal in the folder -run `migrate` and then `pnpm --filter backend run start` by yourself -the server should start +Note: When you want to restart a dev server, all you need to do is press `Ctrl+C` and run `docker compose up`, no other steps are necessary. diff --git a/flake.lock b/flake.lock deleted file mode 100644 index f1ff690415..0000000000 --- a/flake.lock +++ /dev/null @@ -1,294 +0,0 @@ -{ - "nodes": { - "devenv": { - "inputs": { - "flake-compat": "flake-compat", - "nix": "nix", - "nixpkgs": "nixpkgs", - "pre-commit-hooks": "pre-commit-hooks" - }, - "locked": { - "lastModified": 1685521914, - "narHash": "sha256-0fdFP5IASLwJ0PSXrErW8PZon9TVYmi8VRF8OtjGkV4=", - "owner": "cachix", - "repo": "devenv", - "rev": "e206d8f2e3e8d6aa943656052f15bdfea8146b8d", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, - "fenix": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ], - "rust-analyzer-src": "rust-analyzer-src" - }, - "locked": { - "lastModified": 1685514167, - "narHash": "sha256-urRxF0ZGSNeZjM4kALNg3wTh7fBscbqQmS6S/HU7Wms=", - "owner": "nix-community", - "repo": "fenix", - "rev": "3abfea51663583186f687c49a157eab1639349ca", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "fenix", - "type": "github" - } - }, - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-parts": { - "inputs": { - "nixpkgs-lib": "nixpkgs-lib" - }, - "locked": { - "lastModified": 1685457039, - "narHash": "sha256-bEFtQm+YyLxQjKQAaBHJyPN1z2wbhBnr2g1NJWSYjwM=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "80717d11615b6f42d1ad2e18ead51193fc15de69", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" - } - }, - "flake-utils": { - "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "gitignore": { - "inputs": { - "nixpkgs": [ - "devenv", - "pre-commit-hooks", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1660459072, - "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=", - "owner": "hercules-ci", - "repo": "gitignore.nix", - "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73", - "type": "github" - }, - "original": { - "owner": "hercules-ci", - "repo": "gitignore.nix", - "type": "github" - } - }, - "lowdown-src": { - "flake": false, - "locked": { - "lastModified": 1633514407, - "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=", - "owner": "kristapsdz", - "repo": "lowdown", - "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8", - "type": "github" - }, - "original": { - "owner": "kristapsdz", - "repo": "lowdown", - "type": "github" - } - }, - "nix": { - "inputs": { - "lowdown-src": "lowdown-src", - "nixpkgs": [ - "devenv", - "nixpkgs" - ], - "nixpkgs-regression": "nixpkgs-regression" - }, - "locked": { - "lastModified": 1676545802, - "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=", - "owner": "domenkozar", - "repo": "nix", - "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f", - "type": "github" - }, - "original": { - "owner": "domenkozar", - "ref": "relaxed-flakes", - "repo": "nix", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1678875422, - "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-lib": { - "locked": { - "dir": "lib", - "lastModified": 1682879489, - "narHash": "sha256-sASwo8gBt7JDnOOstnps90K1wxmVfyhsTPPNTGBPjjg=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "da45bf6ec7bbcc5d1e14d3795c025199f28e0de0", - "type": "github" - }, - "original": { - "dir": "lib", - "owner": "NixOS", - "ref": "nixos-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs-regression": { - "locked": { - "lastModified": 1643052045, - "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - } - }, - "nixpkgs-stable": { - "locked": { - "lastModified": 1678872516, - "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-22.11", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1685399834, - "narHash": "sha256-Lt7//5snriXSdJo5hlVcDkpERL1piiih0UXIz1RUcC4=", - "owner": "nixos", - "repo": "nixpkgs", - "rev": "58c85835512b0db938600b6fe13cc3e3dc4b364e", - "type": "github" - }, - "original": { - "owner": "nixos", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [ - "devenv", - "flake-compat" - ], - "flake-utils": "flake-utils", - "gitignore": "gitignore", - "nixpkgs": [ - "devenv", - "nixpkgs" - ], - "nixpkgs-stable": "nixpkgs-stable" - }, - "locked": { - "lastModified": 1682596858, - "narHash": "sha256-Hf9XVpqaGqe/4oDGr30W8HlsWvJXtMsEPHDqHZA6dDg=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "fb58866e20af98779017134319b5663b8215d912", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, - "root": { - "inputs": { - "devenv": "devenv", - "fenix": "fenix", - "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs_2" - } - }, - "rust-analyzer-src": { - "flake": false, - "locked": { - "lastModified": 1685465261, - "narHash": "sha256-aJ2nUinUrNcFi+pb47bS5IIAeSiUEEPLJY8W4Q8Pcjk=", - "owner": "rust-lang", - "repo": "rust-analyzer", - "rev": "d2b3caa5b5694125fad04a9699e919444439f6a2", - "type": "github" - }, - "original": { - "owner": "rust-lang", - "ref": "nightly", - "repo": "rust-analyzer", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 60e2d54a25..0000000000 --- a/flake.nix +++ /dev/null @@ -1,86 +0,0 @@ -{ - description = "Firefish development flake"; - - inputs = { - nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable"; - # Flake Parts framework(https://flake.parts) - flake-parts.url = "github:hercules-ci/flake-parts"; - # Devenv for better devShells(https://devenv.sh) - devenv.url = "github:cachix/devenv"; - # Fenix for rust development - fenix.url = "github:nix-community/fenix"; - fenix.inputs.nixpkgs.follows = "nixpkgs"; - }; - outputs = inputs@{ flake-parts, ... }: - flake-parts.lib.mkFlake { inherit inputs; } { - imports = [ - inputs.devenv.flakeModule - ]; - - # Define the systems that this works on. Only tested with x66_64-linux, add more if you test and it works. - systems = [ - "x86_64-linux" - ]; - # Expose these attributes for every system defined above. - perSystem = { config, pkgs, ... }: { - # Devenv shells - devenv = { - shells = { - # The default shell, used by nix-direnv - default = { - name = "firefish-dev-shell"; - # Add additional packages to our environment - packages = [ - pkgs.nodePackages.pnpm - - pkgs.python3 - ]; - # No need to warn on a new version, we'll update as needed. - devenv.warnOnNewVersion = false; - # Enable typescript support - languages.typescript.enable = true; - # Enable javascript for NPM and PNPM - languages.javascript.enable = true; - languages.javascript.package = pkgs.nodejs_18; - # Enable stable Rust for the backend - languages.rust.enable = true; - languages.rust.version = "stable"; - processes = { - dev-server.exec = "pnpm run dev"; - }; - scripts = { - build.exec = "pnpm run build"; - clean.exec = "pnpm run clean"; - clear-state.exec = "rm -rf .devenv/state/redis .devenv/state/postgres"; - format.exec = "pnpm run format"; - install-deps.exec = "pnpm install"; - migrate.exec = "pnpm run migrate"; - prepare-config.exec = "cp .config/devenv.yml .config/default.yml"; - }; - services = { - postgres = { - enable = true; - package = pkgs.postgresql_12; - initialDatabases = [{ - name = "firefish"; - }]; - initialScript = '' - CREATE USER firefish WITH PASSWORD 'firefish'; - ALTER USER firefish WITH SUPERUSER; - GRANT ALL ON DATABASE firefish TO firefish; - ''; - listen_addresses = "127.0.0.1"; - port = 5432; - }; - redis = { - enable = true; - bind = "127.0.0.1"; - port = 6379; - }; - }; - }; - }; - }; - }; - }; -}