diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 631c41c..9d5f894 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,10 +60,10 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - key: simd-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + key: simd-2-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} restore-keys: | - simd-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} - simd-${{ runner.os }}-cargo- + simd-2-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + simd-2-${{ runner.os }}-cargo- - name: Install nightly rust uses: actions-rs/toolchain@v1 with: @@ -78,6 +78,41 @@ jobs: which cargo-cache || cargo install cargo-cache cargo cache trim -l 1G + database_test: + runs-on: ubuntu-latest + env: + RUSTFLAGS: "-C target-cpu=skylake" + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: database-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + restore-keys: | + database-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + database-${{ runner.os }}-cargo- + - name: Install nightly rust + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: nightly-${{ env.RUST_TC_NIGHTLY_VER }} + override: true + - name: Check + run: cargo clippy --features database + - name: Build + run: cargo build --features database + - name: Run tests + run: cargo test --features database + - name: Trim cache + run: | + which cargo-cache || cargo install cargo-cache + cargo cache trim -l 1G + # make sure native-tls always builds openssl_build: runs-on: ubuntu-latest @@ -91,6 +126,10 @@ jobs: override: true - name: Check run: cargo clippy --no-default-features --features=native-tls + - name: Trim cache + run: | + which cargo-cache || cargo install cargo-cache + cargo cache trim -l 1G # cross compile from x86 mac to arm64, this is to make sure universal2 # release will build without error @@ -125,7 +164,7 @@ jobs: # set SDKROOT for C dependencies export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) cd roapi-http && \ - cargo build --bin roapi-http --target aarch64-apple-darwin + cargo build --bin roapi-http --features database --target aarch64-apple-darwin - name: Trim cache run: | which cargo-cache || cargo install cargo-cache diff --git a/Cargo.lock b/Cargo.lock index 4cdcd27..ed33837 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -64,6 +64,12 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "arrayvec" version = "0.7.2" @@ -72,9 +78,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "9.1.0" +version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9864ca2fdcd3d4883259495b4517879877c5991d9928cc9713794d8076d3e78b" +checksum = "db922bcac0aa9f3b6e191d73b0b5df925c2c10ab4893f7416359f289e5d23489" dependencies = [ "bitflags", "chrono", @@ -85,7 +91,7 @@ dependencies = [ "hex", "indexmap", "lazy_static", - "lexical-core", + "lexical-core 0.8.3", "multiversion", "num", "packed_simd_2", @@ -157,9 +163,9 @@ checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" [[package]] name = "async-trait" -version = "0.1.53" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" dependencies = [ "proc-macro2", "quote", @@ -246,12 +252,59 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bigdecimal" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256" +dependencies = [ + "num-bigint 0.3.3", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bindgen" +version = "0.58.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f8523b410d7187a43085e7e064416ea32ded16bd0a4e6fc025e21616d01258f" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "clap 2.34.0", + "env_logger 0.8.4", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.1.0", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitvec" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5237f00a8c86130a0cc317830e558b966dd7850d48a953d998c813f01a41b527" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake2" version = "0.10.4" @@ -268,7 +321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" dependencies = [ "arrayref", - "arrayvec", + "arrayvec 0.7.2", "cc", "cfg-if", "constant_time_eq", @@ -309,9 +362,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.3.4" +version = "3.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "f838e47a451d5a8fa552371f80024dd6ace9b7acdf25c4c3d0f9bc6816fb1c39" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -340,6 +393,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bufstream" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" + [[package]] name = "bumpalo" version = "3.9.1" @@ -373,6 +432,15 @@ dependencies = [ "jobserver", ] +[[package]] +name = "cexpr" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -394,18 +462,44 @@ dependencies = [ ] [[package]] -name = "clap" -version = "3.1.8" +name = "clang-sys" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c47df61d9e16dc010b55dba1952a57d8c215dbb533fd13cdd13369aac73b1c" +checksum = "4cc00842eed744b858222c4c9faf7243aafc6d33f92f96935263ef4d8a41ce21" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim 0.8.0", + "textwrap 0.11.0", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap" +version = "3.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c93436c21e4698bacadf42917db28b23017027a4deccb35dbe47a7e7840123" dependencies = [ "atty", "bitflags", "indexmap", "os_str_bytes", - "strsim", + "strsim 0.10.0", "termcolor", - "textwrap", + "textwrap 0.15.0", ] [[package]] @@ -434,8 +528,10 @@ version = "0.4.5" dependencies = [ "anyhow", "bytes", + "connectorx", "datafusion", "deltalake", + "dotenv", "futures", "graphql-parser", "hyper-rustls 0.23.0", @@ -466,10 +562,10 @@ name = "columnq-cli" version = "0.2.3" dependencies = [ "anyhow", - "clap", + "clap 3.1.6", "columnq", "dirs", - "env_logger", + "env_logger 0.9.0", "log", "pretty_assertions", "rustyline", @@ -511,6 +607,32 @@ dependencies = [ "cache-padded", ] +[[package]] +name = "connectorx" +version = "0.2.5" +source = "git+https://github.com/sfu-db/connector-x.git?rev=dba588ad26fb41a75d05cd54e08633b8063804c2#dba588ad26fb41a75d05cd54e08633b8063804c2" +dependencies = [ + "anyhow", + "arrow", + "chrono", + "fallible-streaming-iterator", + "fehler", + "itertools", + "log", + "num-traits", + "owning_ref", + "r2d2", + "r2d2_mysql", + "r2d2_sqlite", + "rayon", + "rusqlite", + "rust_decimal", + "serde_json", + "sqlparser 0.11.0", + "thiserror", + "urlencoding", +] + [[package]] name = "const_fn" version = "0.4.9" @@ -557,6 +679,51 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if", + "lazy_static", +] + [[package]] name = "crypto-common" version = "0.1.3" @@ -631,67 +798,97 @@ dependencies = [ [[package]] name = "datafusion" version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cf8e6735817bb021748d72cecc33e468d8775bf749470c52aa7f55ee5cdf9e" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f619d43494e182e260423f8333eed9ea789f0cb8#f619d43494e182e260423f8333eed9ea789f0cb8" dependencies = [ "ahash", "arrow", "async-trait", - "blake2", - "blake3", "chrono", "datafusion-common", + "datafusion-data-access", "datafusion-expr", + "datafusion-physical-expr", "futures", "hashbrown 0.12.0", "lazy_static", "log", - "md-5 0.10.1", "num_cpus", "ordered-float 2.10.0", - "parking_lot", + "parking_lot 0.12.0", "parquet", "paste", "pin-project-lite", "rand 0.8.5", - "regex", - "sha2 0.10.2", "smallvec", - "sqlparser 0.14.0", + "sqlparser 0.15.0", "tempfile", "tokio", "tokio-stream", - "unicode-segmentation", + "uuid", ] [[package]] name = "datafusion-common" version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40d2e2a1a6508f9e66c6deb84ab655be0e9db177f617ec1904de458439981b03" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f619d43494e182e260423f8333eed9ea789f0cb8#f619d43494e182e260423f8333eed9ea789f0cb8" dependencies = [ "arrow", "ordered-float 2.10.0", "parquet", - "sqlparser 0.14.0", + "sqlparser 0.15.0", +] + +[[package]] +name = "datafusion-data-access" +version = "1.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f619d43494e182e260423f8333eed9ea789f0cb8#f619d43494e182e260423f8333eed9ea789f0cb8" +dependencies = [ + "async-trait", + "chrono", + "futures", + "parking_lot 0.12.0", + "tempfile", + "tokio", ] [[package]] name = "datafusion-expr" version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1061dc27f2e4843ddb0f93a8e208d1984260c18c8bbf5e67598be9a111259fe2" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f619d43494e182e260423f8333eed9ea789f0cb8#f619d43494e182e260423f8333eed9ea789f0cb8" dependencies = [ "ahash", "arrow", "datafusion-common", - "sqlparser 0.14.0", + "sqlparser 0.15.0", +] + +[[package]] +name = "datafusion-physical-expr" +version = "7.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f619d43494e182e260423f8333eed9ea789f0cb8#f619d43494e182e260423f8333eed9ea789f0cb8" +dependencies = [ + "ahash", + "arrow", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-expr", + "hashbrown 0.12.0", + "lazy_static", + "md-5 0.10.1", + "ordered-float 2.10.0", + "paste", + "rand 0.8.5", + "regex", + "sha2 0.10.2", + "unicode-segmentation", ] [[package]] name = "deltalake" version = "0.4.1" -source = "git+https://github.com/delta-io/delta-rs.git?rev=f5e4b5f94393b517d3d88c200d7169cce11b304b#f5e4b5f94393b517d3d88c200d7169cce11b304b" +source = "git+https://github.com/delta-io/delta-rs.git?rev=0b99151d3a413bb3d90b318e7a825feb77e62902#0b99151d3a413bb3d90b318e7a825feb77e62902" dependencies = [ "arrow", "async-trait", @@ -724,6 +921,17 @@ dependencies = [ "uuid", ] +[[package]] +name = "derive_utils" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "532b4c15dccee12c7044f1fcad956e98410860b22231e44a3b827464797ca7bf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "diff" version = "0.1.12" @@ -797,6 +1005,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "dtoa" version = "0.4.8" @@ -806,7 +1020,7 @@ checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" [[package]] name = "dynamodb_lock" version = "0.4.1" -source = "git+https://github.com/delta-io/delta-rs.git?rev=f5e4b5f94393b517d3d88c200d7169cce11b304b#f5e4b5f94393b517d3d88c200d7169cce11b304b" +source = "git+https://github.com/delta-io/delta-rs.git?rev=0b99151d3a413bb3d90b318e7a825feb77e62902#0b99151d3a413bb3d90b318e7a825feb77e62902" dependencies = [ "async-trait", "log", @@ -839,6 +1053,19 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" +[[package]] +name = "env_logger" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19187fea3ac7e84da7dacf48de0c45d63c6a76f9490dae389aead16c243fce3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + [[package]] name = "env_logger" version = "0.9.0" @@ -889,6 +1116,18 @@ version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "1.7.0" @@ -900,9 +1139,9 @@ dependencies = [ [[package]] name = "fd-lock" -version = "3.0.5" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46e245f4c8ec30c6415c56cb132c07e69e74f1942f6b4a4061da748b49f486ca" +checksum = "02ecad9808e0596f8956d14f7fa868f996290bd01c8d7329d6e5bc2bb76adf8f" dependencies = [ "cfg-if", "rustix", @@ -910,10 +1149,30 @@ dependencies = [ ] [[package]] -name = "flatbuffers" -version = "2.0.0" +name = "fehler" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4c5738bcd7fad10315029c50026f83c9da5e4a21f8ed66826f43e0e2bde5f6" +checksum = "d5729fe49ba028cd550747b6e62cd3d841beccab5390aa398538c31a2d983635" +dependencies = [ + "fehler-macros", +] + +[[package]] +name = "fehler-macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccb5acb1045ebbfa222e2c50679e392a71dd77030b78fb0189f2d9c5974400f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "flatbuffers" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea97b4fe4b84e2f2765449bcea21cbdb3ee28cecb88afbf38a0c2e1639f5eb5" dependencies = [ "bitflags", "smallvec", @@ -929,6 +1188,7 @@ dependencies = [ "cfg-if", "crc32fast", "libc", + "libz-sys", "miniz_oxide", ] @@ -963,12 +1223,82 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "frunk" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cd67cf7d54b7e72d0ea76f3985c3747d74aee43e0218ad993b7903ba7a5395e" +dependencies = [ + "frunk_core", + "frunk_derives", + "frunk_proc_macros", +] + +[[package]] +name = "frunk_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1246cf43ec80bf8b2505b5c360b8fb999c97dabd17dbb604d85558d5cbc25482" + +[[package]] +name = "frunk_derives" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dbc4f084ec5a3f031d24ccedeb87ab2c3189a2f33b8d070889073837d5ea09e" +dependencies = [ + "frunk_proc_macro_helpers", + "quote", + "syn", +] + +[[package]] +name = "frunk_proc_macro_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99f11257f106c6753f5ffcb8e601fb39c390a088017aaa55b70c526bff15f63e" +dependencies = [ + "frunk_core", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "frunk_proc_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a078bd8459eccbb85e0b007b8f756585762a72a9efc53f359b371c3b6351dbcc" +dependencies = [ + "frunk_core", + "frunk_proc_macros_impl", + "proc-macro-hack", +] + +[[package]] +name = "frunk_proc_macros_impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ffba99f0fa4f57e42f57388fbb9a0ca863bc2b4261f3c5570fed579d5df6c32" +dependencies = [ + "frunk_core", + "frunk_proc_macro_helpers", + "proc-macro-hack", + "quote", + "syn", +] + [[package]] name = "fuchsia-cprng" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "funty" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1847abb9cb65d566acd5942e94aea9c8f547ad02c98e1649326fc0e8910b8b1e" + [[package]] name = "futures" version = "0.3.21" @@ -1085,9 +1415,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.6" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77" dependencies = [ "cfg-if", "libc", @@ -1103,6 +1433,12 @@ dependencies = [ "regex", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "graphql-parser" version = "0.4.0" @@ -1115,9 +1451,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.13" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" dependencies = [ "bytes", "fnv", @@ -1128,7 +1464,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util", + "tokio-util 0.6.9", "tracing", ] @@ -1143,6 +1479,9 @@ name = "hashbrown" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" @@ -1153,6 +1492,15 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown 0.11.2", +] + [[package]] name = "heck" version = "0.3.3" @@ -1197,6 +1545,15 @@ dependencies = [ "digest 0.9.0", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.3", +] + [[package]] name = "http" version = "0.2.6" @@ -1245,9 +1602,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.18" +version = "0.14.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd" dependencies = [ "bytes", "futures-channel", @@ -1323,9 +1680,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.8.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" dependencies = [ "autocfg", "hashbrown 0.11.2", @@ -1347,10 +1704,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" [[package]] -name = "io-lifetimes" -version = "0.6.1" +name = "io-enum" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9448015e586b611e5d322f6703812bbca2f1e709d5773ecd38ddb4e3bb649504" +checksum = "03e3306b0f260aad2872563eb0d5d1a59f2420fad270a661dce59a01e92d806b" +dependencies = [ + "autocfg", + "derive_utils", + "quote", + "syn", +] + +[[package]] +name = "io-lifetimes" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec58677acfea8a15352d42fc87d11d63596ade9239e0a7c9352914417515dbe6" [[package]] name = "ipnet" @@ -1403,6 +1772,35 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "lexical" +version = "5.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f404a90a744e32e8be729034fc33b90cf2a56418fbf594d69aa3c0214ad414e5" +dependencies = [ + "cfg-if", + "lexical-core 0.7.6", +] + +[[package]] +name = "lexical-core" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" +dependencies = [ + "arrayvec 0.5.2", + "bitflags", + "cfg-if", + "ryu", + "static_assertions", +] + [[package]] name = "lexical-core" version = "0.8.3" @@ -1473,12 +1871,44 @@ version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" +[[package]] +name = "libloading" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libm" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fc7aa29613bd6a620df431842069224d8bc9011086b1db4c0e0cd47fa03ec9a" +[[package]] +name = "libsqlite3-sys" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linked-hash-map" version = "0.5.4" @@ -1493,23 +1923,31 @@ checksum = "5284f00d480e1c39af34e72f8ad60b94f47007e3481cd3b731c1d67190ddc7b7" [[package]] name = "lock_api" -version = "0.4.7" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.16" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ "cfg-if", ] +[[package]] +name = "lru" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea2d928b485416e8908cff2d97d621db22b27f7b3b6729e438bcf42c671ba91" +dependencies = [ + "hashbrown 0.11.2", +] + [[package]] name = "lz4" version = "1.23.3" @@ -1649,10 +2087,82 @@ dependencies = [ ] [[package]] -name = "native-tls" -version = "0.2.10" +name = "mysql" +version = "21.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "06f5abe1c0f91831afd4d35298c08d958e80144869757b913891e5b0d00c2c96" +dependencies = [ + "bufstream", + "bytes", + "io-enum", + "libc", + "lru", + "mysql_common", + "named_pipe", + "native-tls", + "nix", + "once_cell", + "pem", + "percent-encoding", + "serde", + "serde_json", + "socket2", + "twox-hash", + "url", +] + +[[package]] +name = "mysql_common" +version = "0.27.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fa08ec695a40ed899b1239e81d0d74de5b40802d4fc8b513e2c541717c434e" +dependencies = [ + "base64", + "bigdecimal", + "bindgen", + "bitflags", + "bitvec", + "byteorder", + "bytes", + "cc", + "chrono", + "cmake", + "crc32fast", + "flate2", + "frunk", + "lazy_static", + "lexical", + "num-bigint 0.4.3", + "num-traits", + "rand 0.8.5", + "regex", + "rust_decimal", + "saturating", + "serde", + "serde_json", + "sha1", + "sha2 0.9.9", + "smallvec", + "subprocess", + "thiserror", + "time 0.2.27", + "uuid", +] + +[[package]] +name = "named_pipe" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad9c443cce91fc3e12f017290db75dde490d685cdaaf508d7159d7cf41f0eb2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" dependencies = [ "lazy_static", "libc", @@ -1688,6 +2198,16 @@ dependencies = [ "memoffset", ] +[[package]] +name = "nom" +version = "5.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb4262d26ed83a1c0a33a38fe2bb15797329c85770da05e6b828ddb782627af" +dependencies = [ + "memchr", + "version_check", +] + [[package]] name = "ntapi" version = "0.3.7" @@ -1703,7 +2223,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" dependencies = [ - "num-bigint", + "num-bigint 0.4.3", "num-complex", "num-integer", "num-iter", @@ -1711,6 +2231,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -1759,7 +2290,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a" dependencies = [ "autocfg", - "num-bigint", + "num-bigint 0.4.3", "num-integer", "num-traits", ] @@ -1783,15 +2314,6 @@ dependencies = [ "libc", ] -[[package]] -name = "num_threads" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba1801fb138d8e85e11d0fc70baf4fe1cdfffda7c6cd34a854905df588e5ed0" -dependencies = [ - "libc", -] - [[package]] name = "once_cell" version = "1.10.0" @@ -1883,6 +2405,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "owning_ref" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "packed_simd_2" version = "0.3.7" @@ -1899,6 +2430,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.5", +] + [[package]] name = "parking_lot" version = "0.12.0" @@ -1906,27 +2448,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.1", ] [[package]] name = "parking_lot_core" -version = "0.9.2" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995f667a6c822200b0433ac218e05582f0e2efa1b922a3fd2fbaadc5f87bab37" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys 0.34.0", + "windows-sys 0.32.0", ] [[package]] name = "parquet" -version = "9.1.0" +version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1697d963e6319c19099adcf131a5440685053d4902890f9e4bb272cbd0dc6532" +checksum = "1460c04ede29e7a2d825357cd764c403b5d28ebdabc129876552c8a86a4ee9a8" dependencies = [ "arrow", "base64", @@ -1936,7 +2492,7 @@ dependencies = [ "flate2", "lz4", "num", - "num-bigint", + "num-bigint 0.4.3", "parquet-format", "rand 0.8.5", "snap", @@ -1955,9 +2511,26 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.7" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" +checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "pem" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd56cbd21fea48d0c440b41cd69c589faacade08c992d9a54e471b79d0fd13eb" +dependencies = [ + "base64", + "once_cell", + "regex", +] [[package]] name = "percent-encoding" @@ -1965,6 +2538,24 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.0.10" @@ -1999,9 +2590,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe" [[package]] name = "polling" @@ -2016,6 +2607,49 @@ dependencies = [ "winapi", ] +[[package]] +name = "postgres" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb76d6535496f633fa799bb872ffb4790e9cbdedda9d35564ca0252f930c0dd5" +dependencies = [ + "bytes", + "fallible-iterator", + "futures", + "log", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "postgres-protocol" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79ec03bce71f18b4a27c4c64c6ba2ddf74686d69b91d8714fb32ead3adaed713" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "hmac 0.12.1", + "md-5 0.10.1", + "memchr", + "rand 0.8.5", + "sha2 0.10.2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d" +dependencies = [ + "bytes", + "fallible-iterator", + "postgres-protocol", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -2024,9 +2658,9 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "pretty_assertions" -version = "1.2.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" +checksum = "57c038cb5319b9c704bf9c227c261d275bfec0ad438118a2787ce47944fb228b" dependencies = [ "ansi_term", "ctor", @@ -2051,13 +2685,50 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.17" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" dependencies = [ "proc-macro2", ] +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log", + "parking_lot 0.11.2", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_mysql" +version = "21.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d05145690b395f5515feff202b8f4b9429c500f423ef7129175155c3c3a9e2" +dependencies = [ + "mysql", + "r2d2", +] + +[[package]] +name = "r2d2_sqlite" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d24607049214c5e42d3df53ac1d8a23c34cc6a5eefe3122acb2c72174719959" +dependencies = [ + "r2d2", + "rusqlite", +] + +[[package]] +name = "radium" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" + [[package]] name = "radix_trie" version = "0.2.1" @@ -2126,6 +2797,31 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -2137,18 +2833,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.13" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c" dependencies = [ "bitflags", ] [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "7776223e2696f1aa4c6b0170e83212f47296a00424305117d013dfe86fb0fe55" dependencies = [ "getrandom", "redox_syscall", @@ -2251,9 +2947,9 @@ dependencies = [ "async-process", "async-trait", "axum", - "clap", + "clap 3.1.6", "columnq", - "env_logger", + "env_logger 0.9.0", "hyper", "log", "pin-project", @@ -2479,6 +3175,42 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "rusqlite" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4b1eaf239b47034fb450ee9cdedd7d0226571689d8823030c4b6c2cb407152" +dependencies = [ + "bitflags", + "chrono", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "memchr", + "smallvec", +] + +[[package]] +name = "rust_decimal" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d37baa70cf8662d2ba1c1868c5983dda16ef32b105cce41fb5c47e72936a90b3" +dependencies = [ + "arrayvec 0.7.2", + "byteorder", + "bytes", + "num-traits", + "postgres", + "serde", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.2.3" @@ -2494,14 +3226,14 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.7", + "semver 1.0.6", ] [[package]] name = "rustix" -version = "0.34.1" +version = "0.33.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3cc851a13d30a34cb747ba2a0c5101a4b2e8b1677a29b213ee465365ea495e" +checksum = "ef7ec6a44fba95d21fa522760c03c16ca5ee95cebb6e4ef579cab3e6d7ba6c06" dependencies = [ "bitflags", "errno", @@ -2593,6 +3325,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + [[package]] name = "schannel" version = "0.1.19" @@ -2603,6 +3341,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot 0.11.2", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2669,9 +3416,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.7" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65bd28f48be7196d222d95b9243287f48d27aca604e08497513019ff0502cc4" +checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" [[package]] name = "semver-parser" @@ -2818,10 +3565,16 @@ dependencies = [ ] [[package]] -name = "slab" -version = "0.4.6" +name = "siphasher" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" [[package]] name = "smallvec" @@ -2871,9 +3624,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.14.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8f192f29f4aa49e57bebd0aa05858e0a1f32dd270af36efe49edb82cbfffab6" +checksum = "10e1ce16b71375ad72d28d111131069ce0d5f8603f4f86d8acd3456b41b57a51" dependencies = [ "log", ] @@ -2887,6 +3640,12 @@ dependencies = [ "log", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.17" @@ -2957,6 +3716,22 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d44a3643b4ff9caf57abcee9c2c621d6c03d9135e0d8b589bd9afb5992cb176a" +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + [[package]] name = "strsim" version = "0.10.0" @@ -2982,6 +3757,16 @@ dependencies = [ "syn", ] +[[package]] +name = "subprocess" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055cf3ebc2981ad8f0a5a17ef6652f652d87831f79fddcba2ac57bcb9a0aa407" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.4.1" @@ -2990,9 +3775,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.90" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704df27628939572cd88d33f171cd6f896f4eaca85252c6e0a72d8d8287ee86f" +checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" dependencies = [ "proc-macro2", "quote", @@ -3005,6 +3790,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempdir" version = "0.3.7" @@ -3038,6 +3829,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + [[package]] name = "textwrap" version = "0.15.0" @@ -3111,17 +3911,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "time" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" -dependencies = [ - "libc", - "num_threads", - "serde", -] - [[package]] name = "time-macros" version = "0.1.1" @@ -3172,7 +3961,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", - "parking_lot", + "parking_lot 0.12.0", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3201,6 +3990,29 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-postgres" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6c8b33df661b548dcd8f9bf87debb8c56c05657ed291122e1188698c2ece95" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator", + "futures", + "log", + "parking_lot 0.11.2", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "socket2", + "tokio", + "tokio-util 0.6.9", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -3236,16 +4048,30 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.1" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0edfdeb067411dba2044da6d1cb2df793dd35add7888d73c16e3381ded401764" +checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" dependencies = [ "bytes", "futures-core", "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -3259,7 +4085,7 @@ dependencies = [ "pin-project", "pin-project-lite", "tokio", - "tokio-util", + "tokio-util 0.7.0", "tower-layer", "tower-service", "tracing", @@ -3322,9 +4148,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.24" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90442985ee2f57c9e1b548ee72ae842f4a9a20e3f417cc38dbc5dc684d9bb4ee" +checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" dependencies = [ "lazy_static", ] @@ -3335,6 +4161,17 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "twox-hash" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" +dependencies = [ + "cfg-if", + "rand 0.8.5", + "static_assertions", +] + [[package]] name = "typenum" version = "1.15.0" @@ -3411,6 +4248,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68b90931029ab9b034b300b797048cf23723400aa757e8a2bfb9d748102f9821" + [[package]] name = "utf8parse" version = "0.2.0" @@ -3433,6 +4276,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + [[package]] name = "version_check" version = "0.9.4" @@ -3587,6 +4436,15 @@ dependencies = [ "cc", ] +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3633,15 +4491,15 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.34.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5acdd78cb4ba54c0045ac14f62d8f94a03d10047904ae2a40afa1e99d8f70825" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" dependencies = [ - "windows_aarch64_msvc 0.34.0", - "windows_i686_gnu 0.34.0", - "windows_i686_msvc 0.34.0", - "windows_x86_64_gnu 0.34.0", - "windows_x86_64_msvc 0.34.0", + "windows_aarch64_msvc 0.32.0", + "windows_i686_gnu 0.32.0", + "windows_i686_msvc 0.32.0", + "windows_x86_64_gnu 0.32.0", + "windows_x86_64_msvc 0.32.0", ] [[package]] @@ -3652,9 +4510,9 @@ checksum = "29277a4435d642f775f63c7d1faeb927adba532886ce0287bd985bffb16b6bca" [[package]] name = "windows_aarch64_msvc" -version = "0.34.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17cffbe740121affb56fad0fc0e421804adf0ae00891205213b5cecd30db881d" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" [[package]] name = "windows_i686_gnu" @@ -3664,9 +4522,9 @@ checksum = "1145e1989da93956c68d1864f32fb97c8f561a8f89a5125f6a2b7ea75524e4b8" [[package]] name = "windows_i686_gnu" -version = "0.34.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2564fde759adb79129d9b4f54be42b32c89970c18ebf93124ca8870a498688ed" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" [[package]] name = "windows_i686_msvc" @@ -3676,9 +4534,9 @@ checksum = "d4a09e3a0d4753b73019db171c1339cd4362c8c44baf1bcea336235e955954a6" [[package]] name = "windows_i686_msvc" -version = "0.34.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cd9d32ba70453522332c14d38814bceeb747d80b3958676007acadd7e166956" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" [[package]] name = "windows_x86_64_gnu" @@ -3688,9 +4546,9 @@ checksum = "8ca64fcb0220d58db4c119e050e7af03c69e6f4f415ef69ec1773d9aab422d5a" [[package]] name = "windows_x86_64_gnu" -version = "0.34.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfce6deae227ee8d356d19effc141a509cc503dfd1f850622ec4b0f84428e1f4" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" [[package]] name = "windows_x86_64_msvc" @@ -3700,9 +4558,9 @@ checksum = "08cabc9f0066848fef4bc6a1c1668e6efce38b661d2aeec75d18d8617eebb5f1" [[package]] name = "windows_x86_64_msvc" -version = "0.34.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19538ccc21819d01deaf88d6a17eae6596a12e9aafdbb97916fb49896d89de9" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" [[package]] name = "winreg" @@ -3713,6 +4571,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "wyz" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "129e027ad65ce1453680623c3fb5163cbf7107bfe1aa32257e7d0e63f9ced188" +dependencies = [ + "tap", +] + [[package]] name = "xml-rs" version = "0.8.4" @@ -3730,13 +4597,14 @@ dependencies = [ [[package]] name = "yup-oauth2" -version = "6.5.0" +version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfccb8835215946ebac32b89db2a4c7fbec7f289ea0e3ea53cfe2552983e257b" +checksum = "df0647712324d99fab8977022a015feeab311cecd8630ce2cd1e74643b651cf1" dependencies = [ "anyhow", "async-trait", "base64", + "chrono", "futures", "http", "hyper", @@ -3749,7 +4617,6 @@ dependencies = [ "seahash", "serde", "serde_json", - "time 0.3.9", "tokio", "url", ] @@ -3762,18 +4629,18 @@ checksum = "7eb5728b8afd3f280a869ce1d4c554ffaed35f45c231fc41bfbd0381bef50317" [[package]] name = "zstd" -version = "0.10.0+zstd.1.5.2" +version = "0.11.1+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b1365becbe415f3f0fcd024e2f7b45bacfb5bdd055f0dc113571394114e7bdd" +checksum = "77a16b8414fde0414e90c612eba70985577451c4c504b99885ebed24762cb81a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "4.1.4+zstd.1.5.2" +version = "5.0.1+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f7cd17c9af1a4d6c24beb1cc54b17e2ef7b593dc92f19e9d9acad8b182bbaee" +checksum = "7c12659121420dd6365c5c3de4901f97145b79651fb1d25814020ed2ed0585ae" dependencies = [ "libc", "zstd-sys", @@ -3781,9 +4648,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.6.3+zstd.1.5.2" +version = "2.0.1+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" +checksum = "9fd07cbbc53846d9145dbffdf6dd09a7a0aa52be46741825f5c97bdd4f73f12b" dependencies = [ "cc", "libc", diff --git a/columnq-cli/src/main.rs b/columnq-cli/src/main.rs index a5d1381..abde8cb 100644 --- a/columnq-cli/src/main.rs +++ b/columnq-cli/src/main.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use columnq::datafusion::arrow::util::pretty; use columnq::table::parse_table_uri_arg; -use columnq::{encoding, ColumnQ, ExecutionConfig}; +use columnq::{encoding, ColumnQ, SessionConfig}; #[cfg(snmalloc)] #[global_allocator] @@ -87,7 +87,7 @@ async fn console_loop(cq: &ColumnQ) -> anyhow::Result<()> { } async fn cmd_console(args: &clap::ArgMatches) -> anyhow::Result<()> { - let config = ExecutionConfig::default().with_information_schema(true); + let config = SessionConfig::default().with_information_schema(true); let mut cq = ColumnQ::new_with_config(config); if let Some(tables) = args.values_of("table") { @@ -107,7 +107,7 @@ fn bytes_to_stdout(bytes: &[u8]) -> anyhow::Result<()> { } async fn cmd_sql(args: &clap::ArgMatches) -> anyhow::Result<()> { - let config = ExecutionConfig::default().with_information_schema(true); + let config = SessionConfig::default().with_information_schema(true); let mut cq = ColumnQ::new_with_config(config); if let Some(tables) = args.values_of("table") { diff --git a/columnq/Cargo.toml b/columnq/Cargo.toml index 7bb9bc9..191d9ff 100644 --- a/columnq/Cargo.toml +++ b/columnq/Cargo.toml @@ -11,7 +11,7 @@ name = "columnq" path = "src/lib.rs" [dependencies] -datafusion = "7" +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f619d43494e182e260423f8333eed9ea789f0cb8", version = "7.0.0" } log = "0" regex = "1" @@ -34,7 +34,6 @@ reqwest = { version = "0.11", default-features = false, features = [ tokio = { version = "1", features = ["rt-multi-thread"] } futures = "0.3" - # S3 rusoto_core = { version = "0.47", default-features = false } rusoto_s3 = { version = "0.47", default-features = false } @@ -43,16 +42,25 @@ rusoto_sts = { version = "0.47", default-features = false } hyper-tls = { version = "0.5.0", default-features = false, optional = true } hyper-rustls = { version = "0.23.0", default-features = false, optional = true } +[dependencies.deltalake] +git = "https://github.com/delta-io/delta-rs.git" +rev = "0b99151d3a413bb3d90b318e7a825feb77e62902" +default-features = false +features = ["datafusion-ext"] -deltalake = { git = "https://github.com/delta-io/delta-rs.git", rev = "f5e4b5f94393b517d3d88c200d7169cce11b304b", default-features = false, features = [ - "datafusion-ext", -] } +[dependencies.connectorx] +git = "https://github.com/sfu-db/connector-x.git" +rev = "dba588ad26fb41a75d05cd54e08633b8063804c2" +version = "0.2.5" +features = ["default", "dst_arrow", "src_mysql", "src_sqlite"] +optional = true [dev-dependencies] anyhow = "1" serde_yaml = "0.8" tempdir = "0" pretty_assertions = "*" +dotenv = "*" [features] default = ["rustls"] @@ -82,3 +90,4 @@ native-tls = [ "yup-oauth2/hyper-tls", ] simd = ["datafusion/simd"] +database = ["connectorx"] diff --git a/columnq/src/columnq.rs b/columnq/src/columnq.rs index 509a628..8b68e3f 100644 --- a/columnq/src/columnq.rs +++ b/columnq/src/columnq.rs @@ -5,8 +5,8 @@ use std::sync::Arc; use datafusion::arrow; use datafusion::arrow::array::as_string_array; use datafusion::arrow::array::StringArray; -pub use datafusion::execution::context::ExecutionConfig; -use datafusion::execution::context::ExecutionContext; +pub use datafusion::execution::context::SessionConfig; +use datafusion::execution::context::SessionContext; use datafusion::physical_plan::collect; use crate::error::{ColumnQError, QueryError}; @@ -14,18 +14,18 @@ use crate::query; use crate::table::{self, KeyValueSource, TableSource}; pub struct ColumnQ { - dfctx: ExecutionContext, + dfctx: SessionContext, schema_map: HashMap, kv_catalog: HashMap>>, } impl ColumnQ { pub fn new() -> Self { - Self::new_with_config(ExecutionConfig::default()) + Self::new_with_config(SessionConfig::default()) } - pub fn new_with_config(config: ExecutionConfig) -> Self { - let dfctx = ExecutionContext::with_config(config); + pub fn new_with_config(config: SessionConfig) -> Self { + let dfctx = SessionContext::with_config(config); let schema_map = HashMap::::new(); Self { dfctx, @@ -59,7 +59,7 @@ impl ColumnQ { let filters = &[]; let exec_plan = table.scan(&projections, filters, None).await?; - let batches = collect(exec_plan, self.dfctx.runtime_env()).await?; + let batches = collect(exec_plan, self.dfctx.task_ctx()).await?; let mut kv = HashMap::new(); for batch in batches { let col_key = batch diff --git a/columnq/src/error.rs b/columnq/src/error.rs index 4abbd2f..d6236a8 100644 --- a/columnq/src/error.rs +++ b/columnq/src/error.rs @@ -65,6 +65,9 @@ pub enum ColumnQError { #[error("Generic error: {0}")] Generic(String), + + #[error("Database error: {0}")] + Database(String), } impl ColumnQError { diff --git a/columnq/src/query/graphql.rs b/columnq/src/query/graphql.rs index 42a6826..4853a70 100644 --- a/columnq/src/query/graphql.rs +++ b/columnq/src/query/graphql.rs @@ -181,9 +181,9 @@ fn to_datafusion_predicates<'a, 'b>( } pub fn query_to_df( - dfctx: &datafusion::execution::context::ExecutionContext, + dfctx: &datafusion::execution::context::SessionContext, q: &str, -) -> Result, QueryError> { +) -> Result, QueryError> { let doc = parse_query::<&str>(q)?; let def = match doc.definitions.len() { @@ -362,7 +362,7 @@ pub fn query_to_df( } pub async fn exec_query( - dfctx: &datafusion::execution::context::ExecutionContext, + dfctx: &datafusion::execution::context::SessionContext, q: &str, ) -> Result, QueryError> { query_to_df(dfctx, q)? @@ -374,7 +374,7 @@ pub async fn exec_query( #[cfg(test)] mod tests { use datafusion::arrow::array::*; - use datafusion::execution::context::ExecutionContext; + use datafusion::execution::context::SessionContext; use datafusion::logical_plan::{col, lit}; use super::*; @@ -382,7 +382,7 @@ mod tests { #[test] fn simple_query_planning() -> anyhow::Result<()> { - let mut dfctx = ExecutionContext::new(); + let mut dfctx = SessionContext::new(); register_table_properties(&mut dfctx)?; let df = query_to_df( @@ -414,7 +414,7 @@ mod tests { #[test] fn consistent_and_deterministics_logical_plan() -> anyhow::Result<()> { - let mut dfctx = ExecutionContext::new(); + let mut dfctx = SessionContext::new(); register_table_properties(&mut dfctx)?; let df = query_to_df( @@ -449,7 +449,7 @@ mod tests { #[tokio::test] async fn boolean_literal_as_predicate_operand() -> anyhow::Result<()> { - let mut dfctx = ExecutionContext::new(); + let mut dfctx = SessionContext::new(); register_table_properties(&mut dfctx)?; let batches = exec_query( diff --git a/columnq/src/query/rest.rs b/columnq/src/query/rest.rs index 6a339bc..195fef4 100644 --- a/columnq/src/query/rest.rs +++ b/columnq/src/query/rest.rs @@ -54,10 +54,10 @@ fn num_parse_err(e: std::num::ParseIntError) -> QueryError { } pub fn table_query_to_df( - dfctx: &datafusion::execution::context::ExecutionContext, + dfctx: &datafusion::execution::context::SessionContext, table_name: &str, params: &HashMap, -) -> Result, QueryError> { +) -> Result, QueryError> { lazy_static! { static ref RE_REST_FILTER: Regex = Regex::new(r"filter\[(?P.+)\](?P.+)?").unwrap(); @@ -167,7 +167,7 @@ pub fn table_query_to_df( } pub async fn query_table( - dfctx: &datafusion::execution::context::ExecutionContext, + dfctx: &datafusion::execution::context::SessionContext, table_name: &str, params: &HashMap, ) -> Result, QueryError> { @@ -180,14 +180,14 @@ mod tests { use super::*; use datafusion::arrow::array::*; - use datafusion::execution::context::ExecutionContext; + use datafusion::execution::context::SessionContext; use datafusion::prelude::*; use crate::test_util::*; #[tokio::test] async fn consistent_and_deterministics_logical_plan() -> anyhow::Result<()> { - let mut dfctx = ExecutionContext::new(); + let mut dfctx = SessionContext::new(); register_table_ubuntu_ami(&mut dfctx).await?; let mut params = HashMap::::new(); @@ -215,7 +215,7 @@ mod tests { #[tokio::test] async fn simple_filter() -> anyhow::Result<()> { - let mut dfctx = ExecutionContext::new(); + let mut dfctx = SessionContext::new(); register_table_ubuntu_ami(&mut dfctx).await?; let mut params = HashMap::::new(); diff --git a/columnq/src/query/sql.rs b/columnq/src/query/sql.rs index 4219951..e61e51d 100644 --- a/columnq/src/query/sql.rs +++ b/columnq/src/query/sql.rs @@ -5,15 +5,15 @@ use datafusion::arrow; use crate::error::QueryError; pub async fn exec_query( - dfctx: &datafusion::execution::context::ExecutionContext, + dfctx: &datafusion::execution::context::SessionContext, sql: &str, ) -> Result, QueryError> { let plan = dfctx .create_logical_plan(sql) .map_err(QueryError::plan_sql)?; - let df: Arc = Arc::new( - datafusion::execution::dataframe_impl::DataFrameImpl::new(dfctx.state.clone(), &plan), + let df: Arc = Arc::new( + datafusion::dataframe::DataFrame::new(dfctx.state.clone(), &plan), ); df.collect().await.map_err(QueryError::query_exec) @@ -22,14 +22,14 @@ pub async fn exec_query( #[cfg(test)] mod tests { use arrow::array::*; - use datafusion::execution::context::ExecutionContext; + use datafusion::execution::context::SessionContext; use super::*; use crate::test_util::*; #[tokio::test] async fn group_by_aggregation() -> anyhow::Result<()> { - let mut dfctx = ExecutionContext::new(); + let mut dfctx = SessionContext::new(); register_table_properties(&mut dfctx)?; let batches = exec_query( diff --git a/columnq/src/table/arrow_ipc_file.rs b/columnq/src/table/arrow_ipc_file.rs index 2c3ed30..03d9817 100644 --- a/columnq/src/table/arrow_ipc_file.rs +++ b/columnq/src/table/arrow_ipc_file.rs @@ -13,7 +13,7 @@ pub async fn to_mem_table( ) -> Result { debug!("loading arrow table data..."); let mut schema_and_partitions = partitions_from_table_source!(t, |mut r| { - let arrow_file_reader = arrow::ipc::reader::FileReader::try_new(&mut r)?; + let arrow_file_reader = arrow::ipc::reader::FileReader::try_new(&mut r, None)?; let schema = (*arrow_file_reader.schema()).clone(); arrow_file_reader diff --git a/columnq/src/table/arrow_ipc_stream.rs b/columnq/src/table/arrow_ipc_stream.rs index 70dc19c..bc68f3f 100644 --- a/columnq/src/table/arrow_ipc_stream.rs +++ b/columnq/src/table/arrow_ipc_stream.rs @@ -13,7 +13,7 @@ pub async fn to_mem_table( ) -> Result { debug!("loading arrow table data..."); let mut schema_and_partitions = partitions_from_table_source!(t, |mut r| { - let arrow_stream_reader = arrow::ipc::reader::StreamReader::try_new(&mut r)?; + let arrow_stream_reader = arrow::ipc::reader::StreamReader::try_new(&mut r, None)?; let schema = (*arrow_stream_reader.schema()).clone(); arrow_stream_reader diff --git a/columnq/src/table/database.rs b/columnq/src/table/database.rs new file mode 100644 index 0000000..e1ba4e5 --- /dev/null +++ b/columnq/src/table/database.rs @@ -0,0 +1,116 @@ +pub enum DatabaseLoader { + MySQL, + SQLite, + Postgres, +} + +#[cfg(feature = "database")] +mod imp { + use crate::error::ColumnQError; + use crate::table::TableSource; + use connectorx::prelude::*; + use connectorx::sources::mysql::BinaryProtocol; + use datafusion::arrow::record_batch::RecordBatch; + use log::debug; + + use super::DatabaseLoader; + + impl DatabaseLoader { + pub fn to_mem_table( + &self, + t: &TableSource, + ) -> Result { + debug!("loading database table data..."); + let queries = &[format!("SELECT * FROM {}", t.name)]; + let mut destination = ArrowDestination::new(); + match self { + DatabaseLoader::MySQL => { + let source = MySQLSource::::new(t.get_uri_str(), 2) + .map_err(|e| ColumnQError::Database(e.to_string()))?; + let dispatcher = + Dispatcher::< + MySQLSource, + ArrowDestination, + MySQLArrowTransport, + >::new(source, &mut destination, queries, None); + dispatcher + .run() + .map_err(|e| ColumnQError::Database(e.to_string()))?; + } + DatabaseLoader::SQLite => { + let uri = t.get_uri_str().replace("sqlite://", ""); + let source = SQLiteSource::new(&uri, 2) + .map_err(|e| ColumnQError::Database(e.to_string()))?; + let dispatcher = Dispatcher::< + SQLiteSource, + ArrowDestination, + SQLiteArrowTransport, + >::new( + source, &mut destination, queries, None + ); + dispatcher + .run() + .map_err(|e| ColumnQError::Database(e.to_string()))?; + } + DatabaseLoader::Postgres => { + // ToDo `Cannot start a runtime from within a runtime` error in `connector-x PostgresSource` + return Err(ColumnQError::Database( + "Postgres database features not be supported for now.".to_string(), + )); + } + }; + let schema_ref = destination.arrow_schema(); + let data: Vec = destination.arrow().unwrap(); + Ok(datafusion::datasource::MemTable::try_new( + schema_ref, + vec![data], + )?) + } + } +} + +#[cfg(not(feature = "database"))] +mod imp { + use crate::error::ColumnQError; + use crate::table::TableSource; + + use super::DatabaseLoader; + + impl DatabaseLoader { + pub fn to_mem_table( + &self, + _t: &TableSource, + ) -> Result { + Err(ColumnQError::Database( + "Enable 'database' feature flag to support this".to_string(), + )) + } + } +} + +pub use imp::*; + +#[cfg(feature = "database")] +#[cfg(test)] +mod tests { + use datafusion::datasource::TableProvider; + use dotenv::dotenv; + use std::env; + + use crate::table::TableSource; + + use super::*; + + #[tokio::test] + async fn load_mysql() -> anyhow::Result<()> { + dotenv().ok(); + if let Ok(name) = env::var("TABLE_NAME") { + let t = DatabaseLoader::MySQL + .to_mem_table(&TableSource::new(name, env::var("MYSQL_URL")?))?; + let stats = t.scan(&None, &[], None).await?.statistics(); + assert!(stats.num_rows.is_some()); + } + + Ok(()) + } +} diff --git a/columnq/src/table/mod.rs b/columnq/src/table/mod.rs index ba242dd..4df2eb2 100644 --- a/columnq/src/table/mod.rs +++ b/columnq/src/table/mod.rs @@ -16,6 +16,7 @@ use crate::error::ColumnQError; pub mod arrow_ipc_file; pub mod arrow_ipc_stream; pub mod csv; +pub mod database; pub mod delta; pub mod google_spreadsheets; pub mod json; @@ -108,12 +109,14 @@ impl TableOptionCsv { } #[inline] + #[must_use] pub fn with_delimiter(mut self, d: u8) -> Self { self.delimiter = d; self } #[inline] + #[must_use] pub fn with_has_header(mut self, has_header: bool) -> Self { self.has_header = has_header; self @@ -208,6 +211,9 @@ pub enum TableLoadOption { delta(TableOptionDelta), arrow {}, arrows {}, + mysql {}, + sqlite {}, + postgres {}, } impl TableLoadOption { @@ -250,6 +256,9 @@ impl TableLoadOption { Self::google_spreadsheet(_) | Self::delta { .. } => "", Self::arrow { .. } => "arrow", Self::arrows { .. } => "arrows", + Self::mysql { .. } => "mysql", + Self::sqlite { .. } => "sqlite", + Self::postgres { .. } => "postgres", } } } @@ -314,11 +323,13 @@ impl From for TableSource { impl TableSource { pub fn new(name: impl Into, source: impl Into) -> Self { + let io_source = source.into(); + let option = Self::parse_option(&io_source); Self { name: name.into(), - io_source: source.into(), + io_source, schema: None, - option: None, + option, batch_size: Self::default_batch_size(), } } @@ -332,11 +343,13 @@ impl TableSource { 8192 } + #[must_use] pub fn with_option(mut self, option: impl Into) -> Self { self.option = Some(option.into()); self } + #[must_use] pub fn with_schema(mut self, schema: impl Into) -> Self { self.schema = Some(schema.into()); self @@ -349,6 +362,22 @@ impl TableSource { } } + pub fn parse_option(source: &TableIoSource) -> Option { + match source { + TableIoSource::Uri(uri) => { + let uri = URIReference::try_from(uri.as_str()).ok()?; + let scheme = uri.scheme()?; + match scheme.as_str() { + "mysql" => Some(TableLoadOption::mysql {}), + "sqlite" => Some(TableLoadOption::sqlite {}), + "postgresql" => Some(TableLoadOption::postgres {}), + _ => None, + } + } + TableIoSource::Memory(_) => None, + } + } + pub fn parsed_uri(&self) -> Result { match &self.io_source { TableIoSource::Uri(uri) => URIReference::try_from(uri.as_str()).map_err(|_| { @@ -404,7 +433,7 @@ impl TableSource { (None, TableIoSource::Memory(_)) => { return Err(ColumnQError::Generic( "cannot detect table extension from memory IO source, please specify a format option".to_string() - )); + )); } }) } @@ -423,6 +452,15 @@ pub async fn load(t: &TableSource) -> Result, ColumnQErro TableLoadOption::delta { .. } => delta::to_datafusion_table(t).await?, TableLoadOption::arrow { .. } => Arc::new(arrow_ipc_file::to_mem_table(t).await?), TableLoadOption::arrows { .. } => Arc::new(arrow_ipc_stream::to_mem_table(t).await?), + TableLoadOption::mysql { .. } => { + Arc::new(database::DatabaseLoader::MySQL.to_mem_table(t)?) + } + TableLoadOption::sqlite { .. } => { + Arc::new(database::DatabaseLoader::SQLite.to_mem_table(t)?) + } + TableLoadOption::postgres { .. } => { + Arc::new(database::DatabaseLoader::Postgres.to_mem_table(t)?) + } }) } else { let t: Arc = match t.extension()? { @@ -538,11 +576,13 @@ impl KeyValueSource { } } + #[must_use] pub fn with_option(mut self, option: impl Into) -> Self { self.option = Some(option.into()); self } + #[must_use] pub fn with_schema(mut self, schema: impl Into) -> Self { self.schema = Some(schema.into()); self @@ -619,4 +659,15 @@ batch_size: 512 t ); } + + #[cfg(feature = "database")] + #[tokio::test] + async fn test_load_sqlite_table() -> anyhow::Result<()> { + let t = TableSource::new("uk_cities", "sqlite://../test_data/sqlite.db"); + let table = load(&t).await?; + let stats = table.scan(&None, &[], None).await?.statistics(); + assert_eq!(stats.num_rows, Some(37)); + + Ok(()) + } } diff --git a/columnq/src/table/parquet.rs b/columnq/src/table/parquet.rs index a328ab6..5710b34 100644 --- a/columnq/src/table/parquet.rs +++ b/columnq/src/table/parquet.rs @@ -7,9 +7,9 @@ use crate::table::{TableLoadOption, TableOptionParquet, TableSource}; use datafusion::arrow; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::datafusion_data_access::object_store::local::LocalFileSystem; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::{ListingOptions, ListingTable, ListingTableConfig}; -use datafusion::datasource::object_store::local::LocalFileSystem; use datafusion::datasource::TableProvider; use datafusion::parquet::arrow::{ArrowReader, ParquetFileArrowReader}; use datafusion::parquet::file::reader::SerializedFileReader; diff --git a/columnq/src/test_util.rs b/columnq/src/test_util.rs index 0015961..95fd259 100644 --- a/columnq/src/test_util.rs +++ b/columnq/src/test_util.rs @@ -6,7 +6,7 @@ use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; use datafusion::dataframe::DataFrame; use datafusion::datasource::MemTable; -use datafusion::execution::context::ExecutionContext; +use datafusion::execution::context::SessionContext; use crate::table; @@ -125,17 +125,17 @@ schema: Ok(table::load(&table_source).await?) } -pub fn register_table_properties(dfctx: &mut ExecutionContext) -> anyhow::Result<()> { +pub fn register_table_properties(dfctx: &mut SessionContext) -> anyhow::Result<()> { dfctx.register_table("properties", Arc::new(properties_table()?))?; Ok(()) } -pub async fn register_table_ubuntu_ami(dfctx: &mut ExecutionContext) -> anyhow::Result<()> { +pub async fn register_table_ubuntu_ami(dfctx: &mut SessionContext) -> anyhow::Result<()> { dfctx.register_table("ubuntu_ami", ubuntu_ami_table().await?)?; Ok(()) } -pub fn assert_eq_df(df1: Arc, df2: Arc) { +pub fn assert_eq_df(df1: Arc, df2: Arc) { assert_eq!( format!("{:?}", df1.to_logical_plan()), format!("{:?}", df2.to_logical_plan()) diff --git a/roapi-http/Cargo.toml b/roapi-http/Cargo.toml index 67459d1..32f1f04 100644 --- a/roapi-http/Cargo.toml +++ b/roapi-http/Cargo.toml @@ -46,6 +46,7 @@ native-tls-vendored = ["columnq/native-tls-vendored"] native-tls = ["columnq/native-tls"] simd = ["columnq/simd"] snmalloc = ["snmalloc-rs"] +database = ["columnq/database"] [dev-dependencies] reqwest = { version = "0.11", default-features = false, features = [ diff --git a/test_data/sqlite.db b/test_data/sqlite.db new file mode 100644 index 0000000..7434d77 Binary files /dev/null and b/test_data/sqlite.db differ