From 9ec9746233c1ef98f264f0c81a4c4d5c1b304e3e Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 13 Jul 2023 05:25:54 -0400 Subject: [PATCH] feat: scylladb schema migrator --- packages/backend/native-utils/Cargo.lock | 488 ++++++++++++++---- packages/backend/native-utils/Cargo.toml | 2 +- .../native-utils/scylla-migration/Cargo.toml | 15 + .../native-utils/scylla-migration/README.md | 56 ++ .../native-utils/scylla-migration/src/cli.rs | 109 ++++ .../scylla-migration/src/config.rs | 16 + .../scylla-migration/src/error.rs | 23 + .../native-utils/scylla-migration/src/lib.rs | 7 + .../native-utils/scylla-migration/src/main.rs | 8 + .../scylla-migration/src/migrator.rs | 210 ++++++++ packages/backend/package.json | 2 + 11 files changed, 848 insertions(+), 88 deletions(-) create mode 100644 packages/backend/native-utils/scylla-migration/Cargo.toml create mode 100644 packages/backend/native-utils/scylla-migration/README.md create mode 100644 packages/backend/native-utils/scylla-migration/src/cli.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/config.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/error.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/lib.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/main.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/migrator.rs diff --git a/packages/backend/native-utils/Cargo.lock b/packages/backend/native-utils/Cargo.lock index 1a20b792c2..27ba602448 100644 --- a/packages/backend/native-utils/Cargo.lock +++ b/packages/backend/native-utils/Cargo.lock @@ -8,6 +8,21 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +[[package]] +name = "addr2line" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.7.6" @@ -117,6 +132,12 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "arrayvec" version = "0.7.2" @@ -142,7 +163,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -153,7 +174,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -171,6 +192,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bae" version = "0.1.7" @@ -196,13 +232,24 @@ version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +[[package]] +name = "bigdecimal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256" +dependencies = [ + "num-bigint 0.3.3", + "num-integer", + "num-traits", +] + [[package]] name = "bigdecimal" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6773ddc0eafc0e509fb60e48dff7f450f8e674a0686ae8605e8d9901bd5eefa" dependencies = [ - "num-bigint", + "num-bigint 0.4.3", "num-integer", "num-traits", ] @@ -273,7 +320,7 @@ checksum = "0754613691538d51f329cce9af41d7b7ca150bc973056f1156611489475f54f7" dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", - "proc-macro-crate", + "proc-macro-crate 0.1.5", "proc-macro2", "syn 1.0.109", ] @@ -383,16 +430,16 @@ dependencies = [ "bitflags 1.3.2", "clap_derive 3.2.25", "clap_lex 0.2.4", - "indexmap", + "indexmap 1.9.3", "once_cell", "textwrap", ] [[package]] name = "clap" -version = "4.3.2" +version = "4.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "401a4694d2bf92537b6867d94de48c4842089645fdcdf6c71865b175d836e9c2" +checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" dependencies = [ "clap_builder", "clap_derive 4.3.2", @@ -401,13 +448,12 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.1" +version = "4.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72394f3339a76daf211e57d4bcb374410f3965dcc606dd0e03738c7888766980" +checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" dependencies = [ "anstream", "anstyle", - "bitflags 1.3.2", "clap_lex 0.5.0", "strsim", ] @@ -434,7 +480,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -530,16 +576,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "ctor" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" -dependencies = [ - "quote", - "syn 1.0.109", -] - [[package]] name = "ctor" version = "0.2.2" @@ -547,7 +583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1586fa608b1dab41f667475b4a41faec5ba680aee428bfa5de4ea520fdc6e901" dependencies = [ "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -562,6 +598,19 @@ dependencies = [ "sha3", ] +[[package]] +name = "dashmap" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" +dependencies = [ + "cfg-if", + "hashbrown 0.14.0", + "lock_api", + "once_cell", + "parking_lot_core 0.9.8", +] + [[package]] name = "derive_more" version = "0.99.17" @@ -645,6 +694,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -801,7 +856,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -857,6 +912,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "h2" version = "0.3.19" @@ -869,7 +930,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -894,6 +955,12 @@ dependencies = [ "ahash 0.8.3", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "hashlink" version = "0.8.2" @@ -942,6 +1009,12 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "histogram" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12cb882ccb290b8646e554b157ab0b71e64e8d5bef775cd66b6531e52d302669" + [[package]] name = "hkdf" version = "0.12.3" @@ -1011,7 +1084,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1062,6 +1135,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "indicatif" version = "0.17.5" @@ -1157,7 +1240,7 @@ dependencies = [ "anyhow", "base64 0.21.2", "bytecount", - "clap 4.3.2", + "clap 4.3.11", "fancy-regex", "fraction", "getrandom", @@ -1247,6 +1330,15 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de" +[[package]] +name = "lz4_flex" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a8cbbb2831780bc3b9c15a41f5b49222ef756b6730a95f3decfdd15903eb5a3" +dependencies = [ + "twox-hash", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1298,6 +1390,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "mio" version = "0.8.8" @@ -1316,7 +1417,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7f0a2e93526dd9c8c522d72a4d0c88678be8966fabe9fb8f2947fde6339b682" dependencies = [ "bitflags 2.3.1", - "ctor 0.2.2", + "ctor", "napi-derive", "napi-sys", "once_cell", @@ -1410,7 +1511,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" dependencies = [ - "num-bigint", + "num-bigint 0.4.3", "num-complex", "num-integer", "num-iter", @@ -1418,6 +1519,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -1472,7 +1584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg", - "num-bigint", + "num-bigint 0.4.3", "num-integer", "num-traits", ] @@ -1497,12 +1609,42 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" +dependencies = [ + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "number_prefix" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "object" +version = "0.30.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -1538,15 +1680,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "output_vt100" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" -dependencies = [ - "winapi", -] - [[package]] name = "parking_lot" version = "0.11.2" @@ -1597,9 +1730,9 @@ dependencies = [ [[package]] name = "parse-display" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f96cc033d72896bb9a2c239a14e1141c3e2eae6d649e7c10ef4e598d66bc86c" +checksum = "ddcac6cdc2aaa03a89780c6681ffb46ac5b4b7dbade8d2a20a6a501f6fd363a8" dependencies = [ "once_cell", "parse-display-derive", @@ -1608,9 +1741,9 @@ dependencies = [ [[package]] name = "parse-display-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5587062be441f3d868f7c4c9d13c67f286b03aa679d7f8176ef80bf2ee79e5d" +checksum = "6e503280f7d6dbf7566bc63d903c3b0f595fc11553a7e1f87f81adb8fcbca983" dependencies = [ "once_cell", "proc-macro2", @@ -1618,7 +1751,7 @@ dependencies = [ "regex", "regex-syntax 0.6.29", "structmeta", - "syn 1.0.109", + "syn 2.0.25", ] [[package]] @@ -1650,7 +1783,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -1685,13 +1818,11 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "pretty_assertions" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" dependencies = [ - "ctor 0.1.26", "diff", - "output_vt100", "yansi", ] @@ -1704,6 +1835,16 @@ dependencies = [ "toml", ] +[[package]] +name = "proc-macro-crate" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" +dependencies = [ + "once_cell", + "toml_edit", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1730,9 +1871,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.59" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aeca18b86b413c660b781aa319e4e2648a3e6f9eadc9b47e9038e6fe9f3451b" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] @@ -1785,9 +1926,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] @@ -1834,6 +1975,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core", +] + [[package]] name = "rand_xorshift" version = "0.3.0" @@ -2008,6 +2158,12 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2117,6 +2273,85 @@ dependencies = [ "untrusted", ] +[[package]] +name = "scylla" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b0771b8c0333f0dadd0d3f41a5ef2df5416c0b132a1e5297dfa0b1924406ef" +dependencies = [ + "arc-swap", + "async-trait", + "bigdecimal 0.2.2", + "byteorder", + "bytes", + "chrono", + "dashmap", + "futures", + "histogram", + "itertools", + "lz4_flex", + "num-bigint 0.3.3", + "num_enum", + "rand", + "rand_pcg", + "scylla-cql", + "scylla-macros", + "smallvec", + "snap", + "socket2 0.5.3", + "strum", + "strum_macros", + "thiserror", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "scylla-cql" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7ed843e6c6a19c7a719f02f4d9d264beb00f2867d994442fd8b7d66f5af2781" +dependencies = [ + "async-trait", + "bigdecimal 0.2.2", + "byteorder", + "bytes", + "chrono", + "lz4_flex", + "num-bigint 0.3.3", + "num_enum", + "scylla-macros", + "snap", + "thiserror", + "tokio", + "uuid", +] + +[[package]] +name = "scylla-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32d777dadbf7163d1524ea4f5a095146298d263a686febb96d022cf46d06df32" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "scylla-migration" +version = "0.1.0" +dependencies = [ + "chrono", + "clap 4.3.11", + "scylla", + "serde", + "serde_yaml", + "thiserror", + "tokio", +] + [[package]] name = "sea-orm" version = "0.11.3" @@ -2125,7 +2360,7 @@ checksum = "fade86e8d41fd1a4721f84cb834f4ca2783f973cc30e6212b7fafc134f169214" dependencies = [ "async-stream", "async-trait", - "bigdecimal", + "bigdecimal 0.3.1", "chrono", "futures", "log", @@ -2197,7 +2432,7 @@ version = "0.28.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbab99b8cd878ab7786157b7eb8df96333a6807cc6e45e8888c85b51534b401a" dependencies = [ - "bigdecimal", + "bigdecimal 0.3.1", "chrono", "rust_decimal", "sea-query-derive", @@ -2212,7 +2447,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cea85029985b40dfbf18318d85fe985c04db7c1b4e5e8e0a0a0cdff5f1e30f9" dependencies = [ - "bigdecimal", + "bigdecimal 0.3.1", "chrono", "rust_decimal", "sea-query", @@ -2294,22 +2529,22 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -2348,11 +2583,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.21" +version = "0.9.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" dependencies = [ - "indexmap", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -2430,6 +2665,12 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +[[package]] +name = "snap" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" + [[package]] name = "socket2" version = "0.4.9" @@ -2440,6 +2681,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -2485,7 +2736,7 @@ dependencies = [ "ahash 0.7.6", "atoi", "base64 0.13.1", - "bigdecimal", + "bigdecimal 0.3.1", "bitflags 1.3.2", "byteorder", "bytes", @@ -2505,14 +2756,14 @@ dependencies = [ "hex", "hkdf", "hmac", - "indexmap", + "indexmap 1.9.3", "itoa", "libc", "libsqlite3-sys", "log", "md-5", "memchr", - "num-bigint", + "num-bigint 0.4.3", "once_cell", "paste", "percent-encoding", @@ -2567,6 +2818,12 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stringprep" version = "0.1.2" @@ -2585,24 +2842,43 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "structmeta" -version = "0.1.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104842d6278bf64aa9d2f182ba4bde31e8aec7a131d29b7f444bb9b344a09e2a" +checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 1.0.109", + "syn 2.0.25", ] [[package]] name = "structmeta-derive" -version = "0.1.6" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24420be405b590e2d746d83b01f09af673270cf80e9b003a5fa7b651c58c7d93" +checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" dependencies = [ "proc-macro2", "quote", + "syn 2.0.25", +] + +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "rustversion", "syn 1.0.109", ] @@ -2625,9 +2901,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" dependencies = [ "proc-macro2", "quote", @@ -2661,22 +2937,22 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -2744,11 +3020,12 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", + "backtrace", "bytes", "libc", "mio", @@ -2756,7 +3033,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", "windows-sys 0.48.0", ] @@ -2769,7 +3046,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -2817,6 +3094,23 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" + +[[package]] +name = "toml_edit" +version = "0.19.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78" +dependencies = [ + "indexmap 2.0.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tower-service" version = "0.3.2" @@ -2844,7 +3138,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -2877,6 +3171,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typenum" version = "1.16.0" @@ -2969,7 +3273,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ae74ef183fae36d650f063ae7bde1cacbe1cd7e72b617cbe1e985551878b98" dependencies = [ - "indexmap", + "indexmap 1.9.3", "serde", "serde_json", "utoipa-gen", @@ -2984,7 +3288,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", ] [[package]] @@ -2993,6 +3297,7 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ + "getrandom", "serde", ] @@ -3060,7 +3365,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", "wasm-bindgen-shared", ] @@ -3094,7 +3399,7 @@ checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.25", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3307,6 +3612,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winnow" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a2094c43cc94775293eaa0e499fbc30048a6d824ac82c0351a8c0bf9112529" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/packages/backend/native-utils/Cargo.toml b/packages/backend/native-utils/Cargo.toml index f93180fe43..933186c1a8 100644 --- a/packages/backend/native-utils/Cargo.toml +++ b/packages/backend/native-utils/Cargo.toml @@ -4,7 +4,7 @@ name = "native-utils" version = "0.0.0" [workspace] -members = ["migration"] +members = ["migration", "scylla-migration"] [features] default = [] diff --git a/packages/backend/native-utils/scylla-migration/Cargo.toml b/packages/backend/native-utils/scylla-migration/Cargo.toml new file mode 100644 index 0000000000..d8a2f0db05 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "scylla-migration" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +chrono = "0.4.26" +clap = { version = "4.3.11", features = ["derive"] } +scylla = "0.8.2" +serde = { version = "1.0.171", features = ["derive"] } +serde_yaml = "0.9.22" +thiserror = "1.0.43" +tokio = { version = "1.29.1", features = ["full"] } diff --git a/packages/backend/native-utils/scylla-migration/README.md b/packages/backend/native-utils/scylla-migration/README.md new file mode 100644 index 0000000000..997d14034e --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/README.md @@ -0,0 +1,56 @@ +# ScyllaDB Migration Tool + +``` +$ cargo run -r -- help +ScyllaDB Migration Tool + +Usage: scylla-migration [OPTIONS] + +Commands: + generate Generate a new, empty migration + up Apply pending migrations + down Rollback applied migrations + help Print this message or the help of the given subcommand(s) + +Options: + -c, --config Path to 'default.yml' + -d, --migration-dir Directory to store migration files + -h, --help Print help + -V, --version Print version +``` + +## CLI Commands + +### Generate New Migration + +``` +cargo run -r -- -d ./cql generate +``` + +Edit `up.cql` and `down.cql` in `scylla-migration/cql/[timestamp]_`. + +Note: `down.cql` is for rollback. + +### Apply all pending migrations + +``` +cargo run -r -- -d ./cql -c ../../../.config/default.yml up +``` + +### Apply first 10 pending migrations + +``` +cargo run -r -- -d ./cql -c ../../../.config/default.yml up -n 10 +``` + +### Rollback last applied migrations + +``` +cargo run -r -- -d ./cql -c ../../../.config/default.yml down +``` + +### Rollback last 10 applied migrations + +``` +cargo run -r -- -d ./cql -c ../../../.config/default.yml down -n 10 +``` diff --git a/packages/backend/native-utils/scylla-migration/src/cli.rs b/packages/backend/native-utils/scylla-migration/src/cli.rs new file mode 100644 index 0000000000..12d4590a18 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/cli.rs @@ -0,0 +1,109 @@ +use clap::{Parser, Subcommand}; +use std::{fs, path::PathBuf}; + +use crate::config::Config; +use crate::error::Error; +use crate::migrator::Migrator; + +pub async fn run_cli() -> Result<(), Error> { + let cli = Cli::parse(); + + let migration_dir = cli + .migration_dir + .expect("Migration directory not specified"); + + match cli.subcommand { + MigrationCommand::Generate { migration_name } => { + Migrator::generate(migration_dir, &migration_name)?; + return Ok(()); + } + _ => {} + }; + + let yml = fs::File::open(cli.config.expect("Path to 'default.yml' not specified")) + .expect("Failed to open 'default.yml'"); + let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml"); + let config = config + .scylla + .expect("ScyllaDB config not found in 'default.yml'"); + + match cli.subcommand { + MigrationCommand::Up { num } => { + Migrator::new(migration_dir, &config).await?.up(num).await? + } + MigrationCommand::Down { num } => { + Migrator::new(migration_dir, &config) + .await? + .down(num) + .await? + } + _ => {} + }; + + Ok(()) +} + +#[derive(Parser)] +#[clap(version, about = "ScyllaDB Migration Tool")] +pub(crate) struct Cli { + #[clap( + value_parser, + global = true, + short = 'c', + long, + help = "Path to 'default.yml'" + )] + pub config: Option, + + #[clap( + value_parser, + global = true, + short = 'd', + long, + help = "Directory to store migration files" + )] + pub migration_dir: Option, + + #[clap(subcommand)] + pub subcommand: MigrationCommand, +} + +#[derive(Subcommand)] +pub(crate) enum MigrationCommand { + #[clap(about = "Generate a new, empty migration", display_order = 10)] + Generate { + #[clap( + value_parser, + required = true, + help = "Name of the new migration", + display_order = 11 + )] + migration_name: String, + }, + #[clap(about = "Apply pending migrations", display_order = 20)] + Up { + #[clap( + value_parser, + short, + long, + help = "Number of pending migrations to apply" + )] + num: Option, + }, + #[clap( + value_parser, + about = "Rollback applied migrations", + display_order = 30 + )] + Down { + #[clap( + value_parser, + short, + long, + default_value = "1", + help = "Number of applied migrations to be rolled back", + display_order = 31 + )] + num: u32, + }, +} diff --git a/packages/backend/native-utils/scylla-migration/src/config.rs b/packages/backend/native-utils/scylla-migration/src/config.rs new file mode 100644 index 0000000000..18f2a71ce8 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/config.rs @@ -0,0 +1,16 @@ +use serde::Deserialize; + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Config { + pub scylla: Option, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ScyllaConfig { + pub host: String, + pub port: u32, + pub keyspace: String, + pub replication_factor: i32, +} diff --git a/packages/backend/native-utils/scylla-migration/src/error.rs b/packages/backend/native-utils/scylla-migration/src/error.rs new file mode 100644 index 0000000000..11c0a4c0b1 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/error.rs @@ -0,0 +1,23 @@ +use scylla::{ + cql_to_rust::FromRowError, + transport::{ + errors::{NewSessionError, QueryError}, + query_result::SingleRowTypedError, + }, +}; +use std::io; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("Session error: {0}")] + Session(#[from] NewSessionError), + #[error("Query error: {0}")] + Query(#[from] QueryError), + #[error("Conversion error: {0}")] + Conversion(#[from] FromRowError), + #[error("Row error: {0}")] + Row(#[from] SingleRowTypedError), + #[error("File error: {0}")] + File(#[from] io::Error), +} diff --git a/packages/backend/native-utils/scylla-migration/src/lib.rs b/packages/backend/native-utils/scylla-migration/src/lib.rs new file mode 100644 index 0000000000..878200531e --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/lib.rs @@ -0,0 +1,7 @@ +pub use clap::Parser; + +pub mod cli; +pub mod config; +pub mod error; + +pub(crate) mod migrator; diff --git a/packages/backend/native-utils/scylla-migration/src/main.rs b/packages/backend/native-utils/scylla-migration/src/main.rs new file mode 100644 index 0000000000..257901d316 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/main.rs @@ -0,0 +1,8 @@ +use scylla_migration::{cli::run_cli, error::Error}; + +#[tokio::main] +async fn main() -> Result<(), Error> { + run_cli().await?; + + Ok(()) +} diff --git a/packages/backend/native-utils/scylla-migration/src/migrator.rs b/packages/backend/native-utils/scylla-migration/src/migrator.rs new file mode 100644 index 0000000000..b1da95be45 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/migrator.rs @@ -0,0 +1,210 @@ +use chrono::{Duration, TimeZone, Utc}; +use scylla::{ + frame::value::Timestamp, prepared_statement::PreparedStatement, FromRow, Session, + SessionBuilder, +}; +use std::{cmp, fs, io, path::PathBuf}; + +use crate::{config::ScyllaConfig, error::Error}; + +pub(crate) struct Migrator { + dir: PathBuf, + session: Session, + prep_up: PreparedStatement, + prep_down: PreparedStatement, +} + +#[derive(FromRow, Debug, Clone)] +struct Info { + version: String, + applied_at: Duration, +} + +const MIGRATION_TABLE: &str = "ck_migrations"; + +impl Migrator { + pub(crate) fn generate(dir: PathBuf, name: &str) -> Result<(), Error> { + let now = Utc::now().timestamp_millis(); + let new_dir = dir.join(format!("{}_{}", now, name)); + fs::create_dir_all(&new_dir)?; + fs::write(&new_dir.join("up.cql"), "")?; + fs::write(&new_dir.join("down.cql"), "")?; + + println!( + "Generated new migration in {}", + new_dir.canonicalize()?.display() + ); + + Ok(()) + } + + pub(crate) async fn new(dir: PathBuf, config: &ScyllaConfig) -> Result { + let session = SessionBuilder::new() + .known_node(format!("{}:{}", config.host, config.port)) + .build() + .await?; + + session + .query( + format!("CREATE KEYSPACE IF NOT EXISTS {keyspace} WITH replication = {{'class': 'NetworkTopologyStrategy', 'replication_factor' : {factor}}}", + keyspace = config.keyspace, + factor = config.replication_factor), &[] + ) + .await?; + session.use_keyspace(&config.keyspace, false).await?; + session + .query( + format!( + "CREATE TABLE IF NOT EXISTS {MIGRATION_TABLE} ( + version text, + applied_at timestamp, + PRIMARY KEY(version, applied_at))" + ), + &[], + ) + .await?; + + // Prepare for fast queries + let prep_up = session + .prepare(format!( + "INSERT INTO {MIGRATION_TABLE} (version, applied_at) VALUES (?, ?)" + )) + .await?; + let prep_down = session + .prepare(format!("DELETE FROM {MIGRATION_TABLE} WHERE version = ?")) + .await?; + + Ok(Self { + dir, + session, + prep_up, + prep_down, + }) + } + + async fn last(&self) -> Result, Error> { + let info = self + .session + .query(format!("SELECT * FROM {MIGRATION_TABLE}"), &[]) + .await? + .rows_typed_or_empty::() + .filter_map(|e| e.ok()); + + let mut info: Vec = info.collect(); + info.sort_unstable_by(|a, b| a.version.cmp(&b.version)); + + let last = info.last(); + if let Some(last) = last { + println!( + "Last version is {} applied at {}.", + last.version, + Utc.timestamp_millis_opt(last.applied_at.num_milliseconds()) + .latest() + .expect("Failed to get the latest timestamp") + ); + } else { + println!("No migration applied yet.") + } + + Ok(last.cloned()) + } + + fn entries_fs(&self) -> Result, io::Error> { + // Get names of subdirectories in the migration directory. + let mut entries: Vec = self + .dir + .read_dir()? + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_ok_and(|t| t.is_dir())) + .filter_map(|e| e.file_name().into_string().ok()) + .collect(); + + // Sort by the directory names. + entries.sort_unstable(); + + Ok(entries) + } + + fn cql_fs(&self, version: &str, filename: &str) -> Result, io::Error> { + let cql = fs::read_to_string(self.dir.join(version).join(filename))?; + let queries: Vec = cql + .split_terminator(';') + .filter_map(|q| { + let query = q.trim(); + (!query.is_empty()).then_some(query.to_string()) + }) + .collect(); + + Ok(queries) + } + + pub(crate) async fn up(&self, num: Option) -> Result<(), Error> { + let entries = self.entries_fs()?; + let length = entries.len(); + + let start_idx = match self.last().await? { + Some(last) => entries.partition_point(|e| e.clone() <= last.version), + None => 0, + }; + if start_idx == length { + println!("No pending migrations."); + return Ok(()); + } + let last_idx = match num { + None => length, + Some(n) => cmp::min(start_idx + n as usize, length), + }; + + println!( + "{} out of {} pending migration(s) will be applied.", + last_idx - start_idx, + length - start_idx + ); + + for version in entries[start_idx..last_idx].iter() { + println!("Applying {version}"); + + let now = Utc::now().timestamp_millis(); + let cql = self.cql_fs(version, "up.cql")?; + + for query in cql { + self.session.query(query, &[]).await?; + } + + // Save migration history + self.session + .execute( + &self.prep_up, + (version, Timestamp(Duration::milliseconds(now))), + ) + .await?; + } + + Ok(()) + } + + pub(crate) async fn down(&self, num: u32) -> Result<(), Error> { + if let Some(last) = self.last().await? { + let entries = self.entries_fs()?; + let last_idx = entries.partition_point(|e| e.clone() <= last.version); + let start_idx = cmp::max(last_idx as i64 - num as i64, 0) as usize; + + println!("{} migration(s) will be reverted.", last_idx - start_idx); + + for version in entries[start_idx..last_idx].iter().rev() { + println!("Reverting {version}."); + + let cql = self.cql_fs(version, "down.cql")?; + + for query in cql { + self.session.query(query, &[]).await?; + } + + // Remove version from migration history + self.session.execute(&self.prep_down, (version,)).await?; + } + } + + Ok(()) + } +} diff --git a/packages/backend/package.json b/packages/backend/package.json index 8564ca8327..cedd9bee71 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -9,9 +9,11 @@ "migrate": "pnpm run migrate:typeorm && pnpm run migrate:cargo", "migrate:typeorm": "typeorm migration:run -d ormconfig.js", "migrate:cargo": "./native-utils/built/migration up", + "migrate:scylla": "cargo run --release --locked --manifest-path ./native-utils/scylla-migration/Cargo.toml -- -d ./native-utils/scylla-migration/cql -c ../../.config/default.yml up", "revertmigration": "pnpm run revertmigration:cargo && pnpm run revertmigration:typeorm", "revertmigration:typeorm": "typeorm migration:revert -d ormconfig.js", "revertmigration:cargo": "./native-utils/built/migration down", + "revertmigration:scylla": "cargo run --release --locked --manifest-path ./native-utils/scylla-migration/Cargo.toml -- -d ./native-utils/scylla-migration/cql -c ../../.config/default.yml down", "check:connect": "node ./check_connect.js", "build": "pnpm swc src -d built -D", "watch": "pnpm swc src -d built -D -w",