diff --git a/Cargo.lock b/Cargo.lock index 37a9517..acaa574 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,65 +85,110 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "26.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24e2bcd431a4aa0ff003fdd2dc21c78cfb42f31459c89d2312c2746fe17a5ac" +checksum = "1aea9fcb25bbb70f7f922f95b99ca29c1013dab47f6df61a6f24861842dd7f2e" dependencies = [ "ahash 0.8.3", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d967b42f7b12c91fd78acd396b20c2973b184c8866846674abbb00c963e93ab" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190f208ee7aa0f3596fa0098d42911dec5e123ca88c002a08b24877ad14c71e" +dependencies = [ + "ahash 0.8.3", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.13.2", + "num", + "packed_simd_2", +] + +[[package]] +name = "arrow-buffer" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d33c733c5b6c44a0fc526f29c09546e04eb56772a7a21e48e602f368be381f6" +dependencies = [ + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd349520b6a1ed4924ae2afc9d23330a3044319e4ec3d5b124c09e4d440ae87" +dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "bitflags", "chrono", "comfy-table", - "csv", - "flatbuffers", - "half", - "hashbrown 0.12.3", - "indexmap", - "lazy_static", "lexical-core", - "multiversion", "num", - "packed_simd_2", - "regex", - "regex-syntax", - "serde_json", ] [[package]] -name = "arrow-array" -version = "26.0.0" +name = "arrow-csv" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9044300874385f19e77cbf90911e239bd23630d8f23bb0f948f9067998a13b7" +checksum = "c80af3c3e290a2a7e1cc518f1471dff331878cb4af9a5b088bf030b89debf649" dependencies = [ - "ahash 0.8.3", + "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "chrono", - "half", - "hashbrown 0.12.3", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "26.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78476cbe9e3f808dcecab86afe42d573863c63e149c62e6e379ed2522743e626" -dependencies = [ - "half", - "num", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", ] [[package]] name = "arrow-data" -version = "26.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d916feee158c485dad4f701cba31bc9a90a8db87d9df8e2aa8adc0c20a2bbb9" +checksum = "b1c8361947aaa96d331da9df3f7a08bdd8ab805a449994c97f5c4d24c4b7e2cf" dependencies = [ "arrow-buffer", "arrow-schema", @@ -152,19 +197,84 @@ dependencies = [ ] [[package]] -name = "arrow-schema" -version = "26.0.0" +name = "arrow-ipc" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9406eb7834ca6bd8350d1baa515d18b9fcec487eddacfb62f5e19511f7bd37" +checksum = "9a46ee000b9fbd1e8db6e8b26acb8c760838512b39d8c9f9d73892cb55351d50" dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", +] + +[[package]] +name = "arrow-json" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bf2366607be867ced681ad7f272371a5cf1fc2941328eef7b4fee14565166fb" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "304069901c867200e21ec868ae7521165875470ef2f1f6d58f979a443d63997e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d57fe8ceef3392fdd493269d8a2d589de17bafce151aacbffbddac7a57f441a" +dependencies = [ + "ahash 0.8.3", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", + "hashbrown 0.13.2", +] + +[[package]] +name = "arrow-schema" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a16b88a93ac8350f0200b1cd336a1f887315925b8dd7aa145a37b8bdbd8497a4" +dependencies = [ + "bitflags 2.3.1", "serde", ] [[package]] name = "arrow-select" -version = "26.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6593a01586751c74498495d2f5a01fcd438102b52965c11dd98abf4ebcacef37" +checksum = "98e8a4d6ca37d5212439b24caad4d80743fcbb706706200dd174bb98e68fe9d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -173,6 +283,21 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-string" +version = "37.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb594efa397eb6a546f42b1f8df3d242ea84dbfda5232e06035dc2b2e2c8459" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "regex", + "regex-syntax", +] + [[package]] name = "ascii" version = "0.9.3" @@ -203,6 +328,9 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", + "xz2", + "zstd 0.11.2+zstd.1.5.2", + "zstd-safe 5.0.2+zstd.1.5.2", ] [[package]] @@ -222,7 +350,7 @@ dependencies = [ "slab", "socket2", "waker-fn", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -250,7 +378,7 @@ dependencies = [ "futures-lite", "libc", "signal-hook", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -267,7 +395,7 @@ checksum = "eff18d764974428cf3a9328e23fc5c986f5fbed46e6cd4cdf42544df5d297ec1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -478,7 +606,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.2", "lazy_static", "pin-project-lite", "tokio", @@ -587,7 +715,7 @@ checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43" dependencies = [ "async-trait", "axum-core", - "bitflags", + "bitflags 1.3.2", "bytes", "futures-util", "http", @@ -665,7 +793,7 @@ version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cexpr", "clang-sys", "lazy_static", @@ -684,6 +812,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" + [[package]] name = "bitvec" version = "1.0.1" @@ -771,7 +905,7 @@ dependencies = [ "borsh-schema-derive-internal", "proc-macro-crate", "proc-macro2", - "syn", + "syn 1.0.107", ] [[package]] @@ -782,7 +916,7 @@ checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -793,7 +927,7 @@ checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -859,7 +993,7 @@ checksum = "13e576ebe98e605500b3c8041bb888e966653577172df6dd97398714eb30b9bf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -870,9 +1004,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bytes-utils" @@ -960,6 +1094,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "chrono-tz" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9cc2b23599e6d7479755f3594285efb3f74a1bdca7a7374948bc831e23a552" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "clang-sys" version = "1.4.0" @@ -978,7 +1134,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_lex", "indexmap", "strsim", @@ -1048,7 +1204,7 @@ dependencies = [ "dotenvy", "futures", "graphql-parser", - "hyper-rustls", + "hyper-rustls 0.23.2", "hyper-tls", "lazy_static", "log", @@ -1062,7 +1218,7 @@ dependencies = [ "serde_derive", "serde_json", "serde_yaml", - "sqlparser 0.27.0", + "sqlparser 0.33.0", "tempfile", "thiserror", "tokio", @@ -1124,8 +1280,8 @@ dependencies = [ [[package]] name = "connectorx" -version = "0.3.2-alpha.3" -source = "git+https://github.com/sfu-db/connector-x.git?rev=962b396857979c813486ad842e91a0c21ce55f72#962b396857979c813486ad842e91a0c21ce55f72" +version = "0.3.2-alpha.5" +source = "git+https://github.com/roapi/connector-x.git?rev=dd58b6a90d28b1ee7e62da859a5ba1d2d6c0b179#dd58b6a90d28b1ee7e62da859a5ba1d2d6c0b179" dependencies = [ "anyhow", "arrow", @@ -1180,14 +1336,15 @@ checksum = "f3ad85c1f65dc7b37604eb0e89748faf0b9653065f2a8ef69f96a687ec1e9279" [[package]] name = "convergence" -version = "0.9.1" -source = "git+https://github.com/roapi/convergence.git?rev=1af3f7ea76b0586362b332f8a1c30053aca58c2e#1af3f7ea76b0586362b332f8a1c30053aca58c2e" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8f9d27087facca29d783c17a2079f6e124d2285ca1af4cba1b6da8738c3c93a" dependencies = [ "async-trait", "bytes", "chrono", "futures", - "sqlparser 0.27.0", + "sqlparser 0.33.0", "thiserror", "tokio", "tokio-util", @@ -1195,14 +1352,15 @@ dependencies = [ [[package]] name = "convergence-arrow" -version = "0.9.1" -source = "git+https://github.com/roapi/convergence.git?rev=1af3f7ea76b0586362b332f8a1c30053aca58c2e#1af3f7ea76b0586362b332f8a1c30053aca58c2e" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b551efb2c23303b6268dbbe5a725a3902b4af2729eeb13f004c4eb8dd5ae2a4" dependencies = [ "async-trait", "chrono", "convergence", "datafusion", - "sqlparser 0.27.0", + "sqlparser 0.33.0", "tokio", ] @@ -1355,6 +1513,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ct-logs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8" +dependencies = [ + "sct 0.6.1", +] + [[package]] name = "ctor" version = "0.1.26" @@ -1362,7 +1529,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1389,7 +1556,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 1.0.107", ] [[package]] @@ -1406,23 +1573,40 @@ checksum = "357f40d1f06a24b60ae1fe122542c1fb05d28d32acb2aed064e84bc2ad1e252e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", +] + +[[package]] +name = "dashmap" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +dependencies = [ + "cfg-if", + "hashbrown 0.12.3", + "lock_api", + "once_cell", + "parking_lot_core", ] [[package]] name = "datafusion" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a8411475928479fe57af18698626f0a44f3c29153e051dce45f7455c08a6d5" +checksum = "a8a7d4b334f4512ff2fdbce87f511f570ae895af1ac7c729e77c12583253b22a" dependencies = [ "ahash 0.8.3", "arrow", + "arrow-array", + "arrow-schema", "async-compression", "async-trait", "bytes", "bzip2", "chrono", + "dashmap", "datafusion-common", + "datafusion-execution", "datafusion-expr", "datafusion-optimizer", "datafusion-physical-expr", @@ -1431,61 +1615,80 @@ dependencies = [ "flate2", "futures", "glob", - "hashbrown 0.12.3", + "hashbrown 0.13.2", + "indexmap", "itertools", "lazy_static", "log", "num_cpus", "object_store", - "ordered-float 3.4.0", "parking_lot", "parquet", - "paste", "percent-encoding", "pin-project-lite", "rand", "smallvec", - "sqlparser 0.26.0", + "sqlparser 0.33.0", "tempfile", "tokio", "tokio-stream", "tokio-util", "url", "uuid", + "xz2", + "zstd 0.12.3+zstd.1.5.2", ] [[package]] name = "datafusion-common" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f1ffcbc1f040c9ab99f41db1c743d95aff267bb2e7286aaa010738b7402251" +checksum = "80abfcb1dbc6390f952f21de9069e6177ad6318fcae5fbceabb50666d96533dd" dependencies = [ "arrow", + "arrow-array", "chrono", + "num_cpus", "object_store", - "ordered-float 3.4.0", "parquet", - "sqlparser 0.26.0", + "sqlparser 0.33.0", +] + +[[package]] +name = "datafusion-execution" +version = "23.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2524f1b4b58319895b112809d2a59e54fa662d0e46330a455f22882c2cb7b9" +dependencies = [ + "dashmap", + "datafusion-common", + "datafusion-expr", + "hashbrown 0.13.2", + "log", + "object_store", + "parking_lot", + "rand", + "tempfile", + "url", ] [[package]] name = "datafusion-expr" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1883d9590d303ef38fa295567e7fdb9f8f5f511fcc167412d232844678cd295c" +checksum = "af8040b7a75b04685f4db0a1b11ffa93cd163c1bc13751df3f5cf76baabaf5a1" dependencies = [ "ahash 0.8.3", "arrow", "datafusion-common", - "log", - "sqlparser 0.26.0", + "sqlparser 0.33.0", ] [[package]] name = "datafusion-optimizer" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2127d46d566ab3463d70da9675fc07b9d634be8d17e80d0e1ce79600709fe651" +checksum = "74ceae25accc0f640a4238283f55f3a9fd181d55398703a4330fb2c46261e6a2" dependencies = [ "arrow", "async-trait", @@ -1493,18 +1696,21 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.12.3", + "hashbrown 0.13.2", + "itertools", "log", + "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d108b6fe8eeb317ecad1d74619e8758de49cccc8c771b56c97962fd52eaae23" +checksum = "df4cf228b312f2758cb78e93fe3d2dc602345028efdf7cfa5b338cb370d0a347" dependencies = [ "ahash 0.8.3", "arrow", + "arrow-array", "arrow-buffer", "arrow-schema", "blake2", @@ -1514,13 +1720,14 @@ dependencies = [ "datafusion-expr", "datafusion-row", "half", - "hashbrown 0.12.3", + "hashbrown 0.13.2", + "indexmap", "itertools", "lazy_static", + "libc", "md-5 0.10.5", - "num-traits", - "ordered-float 3.4.0", "paste", + "petgraph", "rand", "regex", "sha2 0.10.6", @@ -1530,24 +1737,24 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7956c5e25c01554bf36105db462382a4d9798b6331072a812408a84a5cf8cdfc" +checksum = "ed1bd1e6a9e2aaff2accdaa835d194db678f04d1527a5e6180af8fbcf0bc1cb6" dependencies = [ "arrow", + "chrono", "datafusion", "datafusion-common", "datafusion-expr", - "pbjson-build", + "object_store", "prost", - "prost-build", ] [[package]] name = "datafusion-row" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43537b6377d506e4788bf21e9ed943340e076b48ca4d077e6ea4405ca5e54a1c" +checksum = "b52b486fb3d81bb132e400304be01af5aba0ad6737e3518045bb98944991fe32" dependencies = [ "arrow", "datafusion-common", @@ -1557,20 +1764,22 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "14.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244d08d4710e1088d9c0949c9b5b8d68d9cf2cde7203134a4cc389e870fe2354" +checksum = "773e985c182e41cfd68f7a7b483ab6bfb68beaac241c348cd4b1bf9f9d61b762" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-expr", - "sqlparser 0.26.0", + "log", + "sqlparser 0.33.0", ] [[package]] name = "deltalake" -version = "0.5.0" -source = "git+https://github.com/delta-io/delta-rs.git?rev=d1e68cac9fc33b08dfb93260038f50c117c8534d#d1e68cac9fc33b08dfb93260038f50c117c8534d" +version = "0.10.0" +source = "git+https://github.com/delta-io/delta-rs.git?rev=72a9e5827e99c7d2a1cf05806ffce6f0a4449d47#72a9e5827e99c7d2a1cf05806ffce6f0a4449d47" dependencies = [ "arrow", "async-trait", @@ -1580,11 +1789,14 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-expr", + "datafusion-physical-expr", "datafusion-proto", + "datafusion-sql", "dynamodb_lock", - "errno", + "errno 0.3.1", "futures", "glibc_version", + "itertools", "lazy_static", "libc", "log", @@ -1602,6 +1814,7 @@ dependencies = [ "rusoto_sts", "serde", "serde_json", + "sqlparser 0.33.0", "thiserror", "tokio", "url", @@ -1616,7 +1829,7 @@ checksum = "7590f99468735a318c254ca9158d0c065aa9b5312896b5a043b5e39bc96f5fa2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1700,8 +1913,8 @@ checksum = "03d8c417d7a8cb362e0c37e5d815f5eb7c37f79ff93707329d5a194e42e54ca0" [[package]] name = "dynamodb_lock" -version = "0.4.1" -source = "git+https://github.com/delta-io/delta-rs.git?rev=d1e68cac9fc33b08dfb93260038f50c117c8534d#d1e68cac9fc33b08dfb93260038f50c117c8534d" +version = "0.4.3" +source = "git+https://github.com/delta-io/delta-rs.git?rev=72a9e5827e99c7d2a1cf05806ffce6f0a4449d47#72a9e5827e99c7d2a1cf05806ffce6f0a4449d47" dependencies = [ "async-trait", "log", @@ -1758,6 +1971,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "errno-dragonfly" version = "0.1.2" @@ -1813,7 +2037,7 @@ checksum = "28c0190ff0bd3b28bfdd4d0cf9f92faa12880fb0b8ae2054723dd6c76a4efd42" dependencies = [ "cfg-if", "rustix", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1833,7 +2057,7 @@ checksum = "ccb5acb1045ebbfa222e2c50679e392a71dd77030b78fb0189f2d9c5974400f9" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1844,12 +2068,12 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "22.9.29" +version = "23.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce016b9901aef3579617931fbb2df8fc9a9f7cb95a16eb8acc8148209bb9e70" +checksum = "77f5399c2c9c50ae9418e522842ad362f61ee48b346ac106807bd355a8a7c619" dependencies = [ - "bitflags", - "thiserror", + "bitflags 1.3.2", + "rustc_version", ] [[package]] @@ -1918,7 +2142,7 @@ checksum = "b83164912bb4c97cfe0772913c7af7387ee2e00cb6d4636fb65a35b3d0c8f173" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1930,7 +2154,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -1954,7 +2178,7 @@ dependencies = [ "frunk_proc_macro_helpers", "proc-macro-hack", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -2034,7 +2258,7 @@ checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -2091,7 +2315,7 @@ dependencies = [ [[package]] name = "glibc_version" version = "0.1.2" -source = "git+https://github.com/delta-io/delta-rs.git?rev=d1e68cac9fc33b08dfb93260038f50c117c8534d#d1e68cac9fc33b08dfb93260038f50c117c8534d" +source = "git+https://github.com/delta-io/delta-rs.git?rev=72a9e5827e99c7d2a1cf05806ffce6f0a4449d47#72a9e5827e99c7d2a1cf05806ffce6f0a4449d47" dependencies = [ "regex", ] @@ -2159,6 +2383,15 @@ dependencies = [ "ahash 0.7.6", ] +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +dependencies = [ + "ahash 0.8.3", +] + [[package]] name = "hashlink" version = "0.7.0" @@ -2287,6 +2520,23 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +dependencies = [ + "ct-logs", + "futures-util", + "hyper", + "log", + "rustls 0.19.1", + "rustls-native-certs 0.5.0", + "tokio", + "tokio-rustls 0.22.0", + "webpki 0.21.4", +] + [[package]] name = "hyper-rustls" version = "0.23.2" @@ -2296,10 +2546,10 @@ dependencies = [ "http", "hyper", "log", - "rustls", - "rustls-native-certs", + "rustls 0.20.8", + "rustls-native-certs 0.6.2", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -2382,7 +2632,7 @@ checksum = "e4b0d47a958cb166282b4dc4840a35783e861c2b39080af846e6481ebe145eee" dependencies = [ "derive_utils", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -2392,7 +2642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d6c6f8c91b4b9ed43484ad1a938e393caf35960fce7f82a040497207bd8e9e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2410,7 +2660,7 @@ dependencies = [ "hermit-abi 0.2.6", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2539,9 +2789,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.139" +version = "0.2.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "libloading" @@ -2656,6 +2906,17 @@ dependencies = [ "libc", ] +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "maplit" version = "1.0.2" @@ -2742,33 +3003,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" - -[[package]] -name = "multiversion" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "025c962a3dd3cc5e0e520aa9c612201d127dcdf28616974961a649dca64f5373" -dependencies = [ - "multiversion-macros", -] - -[[package]] -name = "multiversion-macros" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a3e2bde382ebf960c1f3e79689fa5941625fe9bf694a1cb64af3e85faff3af" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "windows-sys 0.42.0", ] [[package]] @@ -2806,7 +3041,7 @@ dependencies = [ "base64 0.13.1", "bigdecimal", "bindgen", - "bitflags", + "bitflags 1.3.2", "bitvec", "byteorder", "bytes", @@ -2877,7 +3112,7 @@ version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f3790c00a0150112de0f4cd161e3d7fc4b2d8a5542ffc35f099a2562aecb35c" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cc", "cfg-if", "libc", @@ -2992,9 +3227,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.5.4" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f344e51ec9584d2f51199c0c29c6f73dddd04ade986497875bf8fa2f178caf0" +checksum = "ec9cd6ca25e796a49fa242876d1c4de36a24a6da5258e9f0bc062dbf5e81c53b" dependencies = [ "async-trait", "aws-config", @@ -3007,7 +3242,7 @@ dependencies = [ "itertools", "parking_lot", "percent-encoding", - "quick-xml 0.27.1", + "quick-xml 0.28.2", "rand", "reqwest", "ring", @@ -3039,7 +3274,7 @@ version = "0.10.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b102428fd03bc5edf97f62620f7298614c45cedf287c271e7ed450bbaf83f2e1" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "foreign-types", "libc", @@ -3056,7 +3291,7 @@ checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -3090,18 +3325,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "1.1.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3305af35278dd29f46fcdd139e0b1fbfae2153f0e5928b39b035542dd31e37b7" -dependencies = [ - "num-traits", -] - -[[package]] -name = "ordered-float" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84eb1409416d254e4a9c8fa56cc24701755025b458f0fcd8e59e1f5f40c23bf" +checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" dependencies = [ "num-traits", ] @@ -3172,32 +3398,50 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] name = "parquet" -version = "26.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf8fa7ab6572791325a8595f55dc532dde88b996ae10a5ca8a2db746784ecc4" +checksum = "b5022d98333271f4ca3e87bab760498e61726bf5a6ca919123c80517e20ded29" dependencies = [ "ahash 0.8.3", - "arrow", - "base64 0.13.1", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.21.0", "brotli", "bytes", "chrono", "flate2", "futures", - "hashbrown 0.12.3", + "hashbrown 0.13.2", "lz4", "num", "num-bigint", + "object_store", + "paste", "seq-macro", "snap", "thrift", "tokio", - "zstd", + "twox-hash", + "zstd 0.12.3+zstd.1.5.2", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", ] [[package]] @@ -3206,18 +3450,6 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" -[[package]] -name = "pbjson-build" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbb7b706f2afc610f3853550cdbbf6372fd324824a087806bd4480ea4996e24" -dependencies = [ - "heck", - "itertools", - "prost", - "prost-types", -] - [[package]] name = "peeking_take_while" version = "0.1.2" @@ -3258,6 +3490,26 @@ dependencies = [ "phf_shared", ] +[[package]] +name = "phf_codegen" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +dependencies = [ + "phf_shared", + "rand", +] + [[package]] name = "phf_shared" version = "0.11.1" @@ -3284,7 +3536,7 @@ checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -3316,7 +3568,7 @@ dependencies = [ "libc", "log", "wepoll-ffi", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3397,9 +3649,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] @@ -3414,26 +3666,6 @@ dependencies = [ "prost-derive", ] -[[package]] -name = "prost-build" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" -dependencies = [ - "bytes", - "heck", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost", - "prost-types", - "regex", - "tempfile", - "which", -] - [[package]] name = "prost-derive" version = "0.11.6" @@ -3444,17 +3676,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e0526209433e96d83d750dd81a99118edbc55739e7e61a46764fd2ad537788" -dependencies = [ - "bytes", - "prost", + "syn 1.0.107", ] [[package]] @@ -3474,7 +3696,7 @@ checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -3489,9 +3711,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.27.1" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" dependencies = [ "memchr", "serde", @@ -3499,9 +3721,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -3611,7 +3833,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3681,7 +3903,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.2", "hyper-tls", "ipnet", "js-sys", @@ -3691,14 +3913,14 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.20.8", "rustls-pemfile 1.0.2", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util", "tower-service", "url", @@ -3747,7 +3969,7 @@ checksum = "6eaedadc88b53e36dd32d940ed21ae4d850d5916f2581526921f553a72ac34c4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -3783,9 +4005,9 @@ dependencies = [ [[package]] name = "rusoto_core" -version = "0.48.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" +checksum = "5b4f000e8934c1b4f70adde180056812e7ea6b1a247952db8ee98c94cd3116cc" dependencies = [ "async-trait", "base64 0.13.1", @@ -3794,8 +4016,7 @@ dependencies = [ "futures", "http", "hyper", - "hyper-rustls", - "hyper-tls", + "hyper-rustls 0.22.1", "lazy_static", "log", "rusoto_credential", @@ -3809,9 +4030,9 @@ dependencies = [ [[package]] name = "rusoto_credential" -version = "0.48.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" +checksum = "6a46b67db7bb66f5541e44db22b0a02fed59c9603e146db3a9e633272d3bac2f" dependencies = [ "async-trait", "chrono", @@ -3827,9 +4048,9 @@ dependencies = [ [[package]] name = "rusoto_dynamodb" -version = "0.48.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63ad8e126a46122a171587bbee590c5a51f311b65a5e83bb78a1f2adee720762" +checksum = "7935e1f9ca57c4ee92a4d823dcd698eb8c992f7e84ca21976ae72cd2b03016e7" dependencies = [ "async-trait", "bytes", @@ -3841,9 +4062,9 @@ dependencies = [ [[package]] name = "rusoto_signature" -version = "0.48.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" +checksum = "6264e93384b90a747758bcc82079711eacf2e755c3a8b5091687b5349d870bcc" dependencies = [ "base64 0.13.1", "bytes", @@ -3867,9 +4088,9 @@ dependencies = [ [[package]] name = "rusoto_sts" -version = "0.48.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1643f49aa67cb7cb895ebac5a2ff3f991c6dbdc58ad98b28158cd5706aecd1d" +checksum = "4e7edd42473ac006fd54105f619e480b0a94136e7f53cf3fb73541363678fd92" dependencies = [ "async-trait", "bytes", @@ -3886,7 +4107,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85127183a999f7db96d1a976a309eebbfb6ea3b0b400ddd8340190129de6eb7a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "chrono", "fallible-iterator", "fallible-streaming-iterator", @@ -3936,12 +4157,25 @@ version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fdebc4b395b7fbb9ab11e462e20ed9051e7b16e42d24042c776eca0ac81b03" dependencies = [ - "bitflags", - "errno", + "bitflags 1.3.2", + "errno 0.2.8", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.42.0", +] + +[[package]] +name = "rustls" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +dependencies = [ + "base64 0.13.1", + "log", + "ring", + "sct 0.6.1", + "webpki 0.21.4", ] [[package]] @@ -3952,8 +4186,20 @@ checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ "log", "ring", - "sct", - "webpki", + "sct 0.7.0", + "webpki 0.22.0", +] + +[[package]] +name = "rustls-native-certs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +dependencies = [ + "openssl-probe", + "rustls 0.19.1", + "schannel", + "security-framework", ] [[package]] @@ -3998,7 +4244,7 @@ version = "9.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db7826789c0e25614b03e5a54a0717a86f9ff6e6e5247f92b369472869320039" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "clipboard-win", "dirs-next", @@ -4043,7 +4289,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" dependencies = [ - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -4067,6 +4313,16 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +[[package]] +name = "sct" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "sct" version = "0.7.0" @@ -4089,7 +4345,7 @@ version = "2.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -4135,7 +4391,7 @@ checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -4282,7 +4538,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -4311,9 +4567,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -4336,20 +4592,23 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.26.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86be66ea0b2b22749cfa157d16e2e84bf793e626a3375f4d378dc289fa03affb" +checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a" dependencies = [ "log", + "sqlparser_derive", ] [[package]] -name = "sqlparser" -version = "0.27.0" +name = "sqlparser_derive" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba319938d4bfe250a769ac88278b629701024fe16f34257f9563bc628081970" +checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" dependencies = [ - "log", + "proc-macro2", + "quote", + "syn 1.0.107", ] [[package]] @@ -4402,7 +4661,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 1.0.107", ] [[package]] @@ -4432,6 +4691,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "0.1.1" @@ -4490,18 +4760,18 @@ checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] name = "thrift" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09678c4cdbb4eed72e18b7c2af1329c69825ed16fcbac62d083fc3e2b0590ff0" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float 1.1.1", + "ordered-float", ] [[package]] @@ -4569,14 +4839,13 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.24.2" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", @@ -4584,18 +4853,18 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.15", ] [[package]] @@ -4632,15 +4901,26 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-rustls" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +dependencies = [ + "rustls 0.19.1", + "tokio", + "webpki 0.21.4", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", - "webpki", + "webpki 0.22.0", ] [[package]] @@ -4733,7 +5013,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bytes", "futures-core", "futures-util", @@ -4779,7 +5059,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", ] [[package]] @@ -4989,7 +5269,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-shared", ] @@ -5023,7 +5303,7 @@ checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.107", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5057,6 +5337,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "webpki" version = "0.22.0" @@ -5073,7 +5363,7 @@ version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ - "webpki", + "webpki 0.22.0", ] [[package]] @@ -5085,17 +5375,6 @@ dependencies = [ "cc", ] -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - [[package]] name = "winapi" version = "0.3.9" @@ -5133,13 +5412,37 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -5148,30 +5451,60 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" @@ -5179,10 +5512,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.1" +name = "windows_x86_64_gnullvm" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" @@ -5223,6 +5568,15 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yaml-rust" version = "0.4.5" @@ -5250,12 +5604,12 @@ dependencies = [ "futures", "http", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.2", "hyper-tls", "itertools", "log", "percent-encoding", - "rustls", + "rustls 0.20.8", "rustls-pemfile 0.3.0", "seahash", "serde", @@ -5289,7 +5643,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.12.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +dependencies = [ + "zstd-safe 6.0.5+zstd.1.5.4", ] [[package]] @@ -5303,10 +5666,20 @@ dependencies = [ ] [[package]] -name = "zstd-sys" -version = "2.0.5+zstd.1.5.2" +name = "zstd-safe" +version = "6.0.5+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc50ffce891ad571e9f9afe5039c4837bede781ac4bb13052ed7ae695518596" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 757d986..18d8438 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,3 @@ split-debuginfo = "unpacked" [profile.release] lto = true codegen-units = 1 - -[patch.crates-io] -convergence = { git = "https://github.com/roapi/convergence.git", rev = "1af3f7ea76b0586362b332f8a1c30053aca58c2e" } -convergence-arrow = { git = "https://github.com/roapi/convergence.git", rev = "1af3f7ea76b0586362b332f8a1c30053aca58c2e" } diff --git a/columnq/Cargo.toml b/columnq/Cargo.toml index f0f9780..54260d0 100644 --- a/columnq/Cargo.toml +++ b/columnq/Cargo.toml @@ -13,10 +13,10 @@ path = "src/lib.rs" [dependencies] # pulling arrow-schema manually to enable the serde feature. # TODO: add serde feature in datafusion to avoid this workaround -arrow-schema = { version ="26", features = ["serde"] } +arrow-schema = { version ="37.0.0", features = ["serde"] } -datafusion = "14" -object_store = { version = "0.5.4", features = ["aws_profile", "gcp", "azure"] } +datafusion = "23" +object_store = { version = "0.5.6", features = ["aws_profile", "gcp", "azure"] } percent-encoding = "2.2.0" url = "2.2" @@ -24,7 +24,7 @@ log = "0" regex = "1" lazy_static = "1" graphql-parser = "0" -sqlparser = "0.27" # version need to be in sync with convergence +sqlparser = "0.33" # version need to be in sync with convergence yup-oauth2 = { version = "6.2", default-features = false, features = [ "service_account", ] } @@ -43,18 +43,18 @@ calamine = "0.19.1" tokio = { version = "1", features = ["rt-multi-thread"] } futures = "0.3" hyper-tls = { version = "0.5.0", default-features = false, optional = true } -hyper-rustls = { version = "0.23.0", default-features = false, optional = true } +hyper-rustls = { version = "0.23.2", default-features = false, optional = true } [dependencies.deltalake] git = "https://github.com/delta-io/delta-rs.git" -rev = "d1e68cac9fc33b08dfb93260038f50c117c8534d" +rev = "72a9e5827e99c7d2a1cf05806ffce6f0a4449d47" default-features = false features = ["datafusion-ext"] [dependencies.connectorx] -git = "https://github.com/sfu-db/connector-x.git" -rev = "962b396857979c813486ad842e91a0c21ce55f72" -version = "0.3.2-alpha.3" +git = "https://github.com/roapi/connector-x.git" +rev = "dd58b6a90d28b1ee7e62da859a5ba1d2d6c0b179" +version = "0.3.2-alpha.5" features = ["default", "dst_arrow"] optional = true @@ -72,7 +72,7 @@ default = ["rustls"] rustls = [ "hyper-rustls", "reqwest/rustls-tls", - "deltalake/s3-rustls", + "deltalake/s3", "yup-oauth2/hyper-rustls", ] native-tls-vendored = [ diff --git a/columnq/src/columnq.rs b/columnq/src/columnq.rs index 69192ac..aa3938e 100644 --- a/columnq/src/columnq.rs +++ b/columnq/src/columnq.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use datafusion::arrow; use datafusion::arrow::array::as_string_array; use datafusion::arrow::array::StringArray; -use datafusion::datasource::object_store::{ObjectStoreProvider, ObjectStoreRegistry}; +use datafusion::datasource::object_store::ObjectStoreRegistry; use datafusion::error::{DataFusionError, Result as DatafusionResult}; pub use datafusion::execution::context::SessionConfig; use datafusion::execution::context::SessionContext; @@ -20,11 +20,33 @@ use crate::table::{self, KeyValueSource, TableSource}; use object_store::aws::AmazonS3Builder; use object_store::azure::MicrosoftAzureBuilder; use object_store::gcp::GoogleCloudStorageBuilder; +use object_store::DynObjectStore; use url::Url; -pub struct ColumnQObjectStoreProvider {} -impl ObjectStoreProvider for ColumnQObjectStoreProvider { - fn get_by_url(&self, url: &Url) -> DatafusionResult> { +#[derive(Default)] +pub struct ColumnQObjectStoreRegistry {} + +impl std::fmt::Debug for ColumnQObjectStoreRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("ColumnQObjectStoreRegistry").finish() + } +} + +impl ColumnQObjectStoreRegistry { + pub fn get_by_url(&self, url: &Url) -> DatafusionResult> { + self.get_store(url) + } +} +impl ObjectStoreRegistry for ColumnQObjectStoreRegistry { + fn register_store( + &self, + _url: &Url, + _store: Arc, + ) -> Option> { + None + } + + fn get_store(&self, url: &Url) -> DatafusionResult> { match url.host_str() { None => Err(DataFusionError::Execution(format!( "Missing bucket name: {}", @@ -79,13 +101,15 @@ pub struct ColumnQ { impl ColumnQ { pub fn new() -> Self { - Self::new_with_config(SessionConfig::from_env().with_information_schema(true)) + Self::new_with_config( + SessionConfig::from_env() + .expect("Valid environment variables should be set to create SessionConfig") + .with_information_schema(true), + ) } pub fn new_with_config(config: SessionConfig) -> Self { - let object_store_provider = ColumnQObjectStoreProvider {}; - let object_store_registry = - ObjectStoreRegistry::new_with_provider(Some(Arc::new(object_store_provider))); + let object_store_registry = ColumnQObjectStoreRegistry::default(); let rn_config = RuntimeConfig::new().with_object_store_registry(Arc::new(object_store_registry)); let runtime_env = RuntimeEnv::new(rn_config).unwrap(); @@ -120,11 +144,11 @@ impl ColumnQ { return Err(ColumnQError::invalid_kv_key_type()); } let val_schema_idx = schema.index_of(&value)?; - let projections = Some(vec![key_schema_idx, val_schema_idx]); + let projections = vec![key_schema_idx, val_schema_idx]; let filters = &[]; let exec_plan = table - .scan(&self.dfctx.state(), &projections, filters, None) + .scan(&self.dfctx.state(), Some(&projections), filters, None) .await?; let batches = collect(exec_plan, self.dfctx.task_ctx()).await?; let mut kv = HashMap::new(); @@ -214,15 +238,14 @@ mod tests { use std::{env, str::FromStr}; use tempfile::Builder; - use datafusion::datasource::object_store::ObjectStoreProvider; use url::Url; - use super::ColumnQObjectStoreProvider; + use super::ColumnQObjectStoreRegistry; #[test] fn s3_object_store_type() { let host_url = "s3://bucket_name/path"; - let provider = ColumnQObjectStoreProvider {}; + let provider = ColumnQObjectStoreRegistry {}; let err = provider .get_by_url(&Url::from_str(host_url).unwrap()) @@ -242,7 +265,7 @@ mod tests { #[test] fn s3_object_store_type_no_bucket() { let host_url = "s3://"; - let provider = ColumnQObjectStoreProvider {}; + let provider = ColumnQObjectStoreRegistry {}; let err = provider .get_by_url(&Url::from_str(host_url).unwrap()) @@ -253,7 +276,7 @@ mod tests { #[tokio::test] async fn gcs_object_store_type() -> anyhow::Result<()> { let host_url = "gs://bucket_name/path"; - let provider = ColumnQObjectStoreProvider {}; + let provider = ColumnQObjectStoreRegistry {}; let tmp_dir = Builder::new().prefix("columnq.test.gcs").tempdir()?; let tmp_gcs_path = tmp_dir.path().join("service_account.json"); @@ -280,7 +303,7 @@ mod tests { #[test] fn azure_object_store_type() { let host_url = "az://bucket_name/path"; - let provider = ColumnQObjectStoreProvider {}; + let provider = ColumnQObjectStoreRegistry {}; // https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings env::set_var("AZURE_STORAGE_ACCOUNT_NAME", "devstoreaccount1"); env::set_var("AZURE_STORAGE_ACCOUNT_KEY", "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="); @@ -299,7 +322,7 @@ mod tests { #[test] fn unknown_object_store_type() { let unknown = "unknown://bucket_name/path"; - let provider = ColumnQObjectStoreProvider {}; + let provider = ColumnQObjectStoreRegistry {}; let err = provider .get_by_url(&Url::from_str(unknown).unwrap()) .unwrap_err(); diff --git a/columnq/src/encoding/json.rs b/columnq/src/encoding/json.rs index b388c37..ac5a846 100644 --- a/columnq/src/encoding/json.rs +++ b/columnq/src/encoding/json.rs @@ -42,11 +42,11 @@ mod tests { serde_json::json!([ { "d32": "1970-01-02", - "d64": "1970-01-01", + "d64": "1970-01-01T00:00:00.001", }, { "d32": "2021-04-12", - "d64": "2021-04-12", + "d64": "2021-04-12T04:04:28", }, ]) .to_string(), @@ -100,16 +100,16 @@ mod tests { std::str::from_utf8(&data).unwrap(), serde_json::json!([ { - "sec": "2021-04-12 04:04:28", - "msec": "2021-04-12 04:04:28", - "usec": "2021-04-12 04:04:28", - "nsec": "2021-04-12 04:04:28", + "sec": "2021-04-12T04:04:28", + "msec": "2021-04-12T04:04:28", + "usec": "2021-04-12T04:04:28", + "nsec": "2021-04-12T04:04:28", }, { - "sec": "2021-05-12 04:04:28", - "msec": "2021-05-12 04:04:28.001", - "usec": "2021-05-12 04:04:28.000002", - "nsec": "2021-05-12 04:04:28.000000003", + "sec": "2021-05-12T04:04:28", + "msec": "2021-05-12T04:04:28.001", + "usec": "2021-05-12T04:04:28.000002", + "nsec": "2021-05-12T04:04:28.000000003", } ]) .to_string(), diff --git a/columnq/src/io/object_store.rs b/columnq/src/io/object_store.rs index 382ccd1..f8e71f1 100644 --- a/columnq/src/io/object_store.rs +++ b/columnq/src/io/object_store.rs @@ -1,7 +1,6 @@ -use crate::columnq::ColumnQObjectStoreProvider; +use crate::columnq::ColumnQObjectStoreRegistry; use crate::error::ColumnQError; use crate::table::TableSource; -use datafusion::datasource::object_store::ObjectStoreProvider; use futures::TryStreamExt; use object_store::ObjectStore; use percent_encoding; @@ -27,7 +26,7 @@ where I: Iterator, F: FnMut(std::io::Cursor>) -> Result, { - let object_store_provider = ColumnQObjectStoreProvider {}; + let object_store_provider = ColumnQObjectStoreRegistry {}; let mut partitions = vec![]; for path_str in path_iter { @@ -49,7 +48,7 @@ pub async fn partitions_from_uri<'a, F, T>( where F: FnMut(std::io::Cursor>) -> Result, { - let object_store_provider = ColumnQObjectStoreProvider {}; + let object_store_provider = ColumnQObjectStoreRegistry {}; let url = &Url::from_str(t.get_uri_str()).unwrap(); let client = object_store_provider.get_by_url(url)?; let mut partitions = vec![]; diff --git a/columnq/src/query/graphql.rs b/columnq/src/query/graphql.rs index e43f993..d9f2b02 100644 --- a/columnq/src/query/graphql.rs +++ b/columnq/src/query/graphql.rs @@ -1,5 +1,4 @@ use std::convert::TryFrom; -use std::sync::Arc; use datafusion::arrow; use datafusion::logical_expr::Operator; @@ -40,9 +39,7 @@ fn invalid_query(message: String) -> QueryError { // convert order list from graphql argument to datafusion sort columns // // sort order matters, thus it's modeled as a list -fn to_datafusion_sort_columns<'b>( - sort_columns: &[Value<'b, &'b str>], -) -> Result, QueryError> { +fn to_datafusion_sort_columns(sort_columns: &[Value]) -> Result, QueryError> { sort_columns .iter() .map(|optval| match optval { @@ -82,7 +79,7 @@ fn to_datafusion_sort_columns<'b>( .collect() } -fn operand_to_datafusion_expr<'b>(operand: &Value<'b, &'b str>) -> Result { +fn operand_to_datafusion_expr(operand: &Value) -> Result { match operand { Value::Boolean(b) => Ok(Expr::Literal(ScalarValue::Boolean(Some(*b)))), Value::String(s) => Ok(Expr::Literal(ScalarValue::Utf8(Some(s.to_string())))), @@ -117,17 +114,14 @@ fn operand_to_datafusion_expr<'b>(operand: &Value<'b, &'b str>) -> Result( - col: &str, - filter: &Value<'b, &'b str>, -) -> Result, QueryError> { +fn to_datafusion_predicates(col: &str, filter: &Value) -> Result, QueryError> { match filter { Value::Object(obj) => obj .iter() .map(|(op, operand)| { let col_expr = Expr::Column(Column::from_name(col.to_string())); let right_expr = operand_to_datafusion_expr(operand)?; - match *op { + match op.as_str() { "eq" => Ok(binary_expr(col_expr, Operator::Eq, right_expr)), "lt" => Ok(binary_expr(col_expr, Operator::Lt, right_expr)), "lte" | "lteq" => Ok(binary_expr(col_expr, Operator::LtEq, right_expr)), @@ -153,11 +147,11 @@ fn to_datafusion_predicates<'b>( } } -pub fn query_to_df( +pub async fn query_to_df( dfctx: &datafusion::execution::context::SessionContext, q: &str, -) -> Result, QueryError> { - let doc = parse_query::<&str>(q)?; +) -> Result { + let doc = parse_query::(q)?; let def = match doc.definitions.len() { 1 => match &doc.definitions[0] { @@ -228,15 +222,16 @@ pub fn query_to_df( let field = field.ok_or_else(|| invalid_query("field not found in selection".to_string()))?; let mut df = dfctx - .table(field.name) - .map_err(|e| QueryError::invalid_table(e, field.name))?; + .table(field.name.as_str()) + .await + .map_err(|e| QueryError::invalid_table(e, field.name.as_str()))?; let mut filter = None; let mut sort = None; let mut limit = None; let mut page = None; for (key, value) in &field.arguments { - match *key { + match key.as_str() { "filter" => { filter = Some(value); } @@ -277,7 +272,7 @@ pub fn query_to_df( .items .iter() .map(|selection| match selection { - Selection::Field(f) => Ok(f.name), + Selection::Field(f) => Ok(f.name.as_str()), _ => Err(QueryError { error: "invalid graphql query".to_string(), message: "selection set in query should only contain Fields".to_string(), @@ -352,7 +347,8 @@ pub async fn exec_query( dfctx: &datafusion::execution::context::SessionContext, q: &str, ) -> Result, QueryError> { - query_to_df(dfctx, q)? + query_to_df(dfctx, q) + .await? .collect() .await .map_err(QueryError::query_exec) @@ -367,8 +363,8 @@ mod tests { use super::*; use crate::test_util::*; - #[test] - fn simple_query_planning() -> anyhow::Result<()> { + #[tokio::test] + async fn simple_query_planning() -> anyhow::Result<()> { let mut dfctx = SessionContext::new(); register_table_properties(&mut dfctx)?; @@ -386,21 +382,23 @@ mod tests { bath } }"#, - )?; + ) + .await?; let expected_df = dfctx - .table("properties")? + .table("properties") + .await? .filter(col("bath").gt_eq(lit(2i64)))? .filter(col("bed").gt(lit(3i64)))? .select(vec![col("address"), col("bed"), col("bath")])?; - assert_eq_df(df, expected_df); + assert_eq_df(df.into(), expected_df.into()); Ok(()) } - #[test] - fn consistent_and_deterministics_logical_plan() -> anyhow::Result<()> { + #[tokio::test] + async fn consistent_and_deterministics_logical_plan() -> anyhow::Result<()> { let mut dfctx = SessionContext::new(); register_table_properties(&mut dfctx)?; @@ -420,16 +418,18 @@ mod tests { bed } }"#, - )?; + ) + .await?; let expected_df = dfctx - .table("properties")? + .table("properties") + .await? .filter(col("bed").gt(lit(3i64)))? .select(vec![col("address"), col("bed")])? .sort(vec![column_sort_expr_asc("bed")])? .limit(0, Some(10))?; - assert_eq_df(df, expected_df); + assert_eq_df(df.into(), expected_df.into()); Ok(()) } @@ -460,18 +460,12 @@ mod tests { assert_eq!( batch.column(0).as_ref(), - Arc::new(StringArray::from(vec!["Kenmore, WA", "Fremont, WA",])).as_ref(), + &StringArray::from(vec!["Kenmore, WA", "Fremont, WA",]), ); - assert_eq!( - batch.column(1).as_ref(), - Arc::new(Int64Array::from(vec![4, 5])).as_ref(), - ); + assert_eq!(batch.column(1).as_ref(), &Int64Array::from(vec![4, 5]),); - assert_eq!( - batch.column(2).as_ref(), - Arc::new(Int64Array::from(vec![3, 3])).as_ref(), - ); + assert_eq!(batch.column(2).as_ref(), &Int64Array::from(vec![3, 3]),); Ok(()) } diff --git a/columnq/src/query/mod.rs b/columnq/src/query/mod.rs index d550c3f..c25d3e7 100644 --- a/columnq/src/query/mod.rs +++ b/columnq/src/query/mod.rs @@ -1,19 +1,20 @@ +use datafusion::logical_expr::expr::Sort; use datafusion::prelude::{Column, Expr}; pub fn column_sort_expr_desc(column: String) -> Expr { - Expr::Sort { + Expr::Sort(Sort { expr: Box::new(Expr::Column(Column::from_name(column))), asc: false, nulls_first: true, - } + }) } pub fn column_sort_expr_asc(column: impl Into) -> Expr { - Expr::Sort { + Expr::Sort(Sort { expr: Box::new(Expr::Column(Column::from_name(column))), asc: true, nulls_first: true, - } + }) } pub mod graphql; diff --git a/columnq/src/query/rest.rs b/columnq/src/query/rest.rs index bf23520..08aae69 100644 --- a/columnq/src/query/rest.rs +++ b/columnq/src/query/rest.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::sync::Arc; use datafusion::arrow::record_batch::RecordBatch; use datafusion::logical_expr::Operator; @@ -54,11 +53,11 @@ fn num_parse_err(e: std::num::ParseIntError) -> QueryError { } } -pub fn table_query_to_df( +pub async fn table_query_to_df( dfctx: &datafusion::execution::context::SessionContext, table_name: &str, params: &HashMap, -) -> Result, QueryError> { +) -> Result { lazy_static! { static ref RE_REST_FILTER: Regex = Regex::new(r"filter\[(?P.+)\](?P.+)?").unwrap(); @@ -66,6 +65,7 @@ pub fn table_query_to_df( let mut df = dfctx .table(table_name) + .await .map_err(|e| QueryError::invalid_table(e, table_name))?; // filter[col1]eq='foo' @@ -163,7 +163,7 @@ pub async fn query_table( table_name: &str, params: &HashMap, ) -> Result, QueryError> { - let df = table_query_to_df(dfctx, table_name, params)?; + let df = table_query_to_df(dfctx, table_name, params).await?; df.collect().await.map_err(QueryError::query_exec) } @@ -188,18 +188,20 @@ mod tests { params.insert("columns".to_string(), "ami_id,version".to_string()); params.insert("filter[arch]".to_string(), "'amd64'".to_string()); - let df = table_query_to_df(&dfctx, "ubuntu_ami", ¶ms)?; + let df = table_query_to_df(&dfctx, "ubuntu_ami", ¶ms).await?; assert_eq_df( - df, + df.into(), dfctx - .table("ubuntu_ami")? + .table("ubuntu_ami") + .await? .filter( col("arch").eq(Expr::Literal(ScalarValue::Utf8(Some("amd64".to_string())))), )? .select(vec![col("ami_id"), col("version")])? .sort(vec![column_sort_expr_asc("ami_id")])? - .limit(0, Some(10))?, + .limit(0, Some(10))? + .into(), ); Ok(()) @@ -221,7 +223,7 @@ mod tests { let batch = &batches[0]; assert_eq!( batch.column(0).as_ref(), - Arc::new(StringArray::from(vec!["ami-091a87cd1ff23d97c"])).as_ref(), + &StringArray::from(vec!["ami-091a87cd1ff23d97c"]), ); Ok(()) diff --git a/columnq/src/query/sql.rs b/columnq/src/query/sql.rs index 176bbde..66f28b0 100644 --- a/columnq/src/query/sql.rs +++ b/columnq/src/query/sql.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use datafusion::arrow; use crate::error::QueryError; @@ -9,12 +7,12 @@ pub async fn exec_query( sql: &str, ) -> Result, QueryError> { let plan = dfctx + .state() .create_logical_plan(sql) + .await .map_err(QueryError::plan_sql)?; - let df: Arc = Arc::new( - datafusion::dataframe::DataFrame::new(dfctx.state.clone(), &plan), - ); + let df = datafusion::dataframe::DataFrame::new(dfctx.state(), plan); df.collect().await.map_err(QueryError::query_exec) } @@ -47,15 +45,12 @@ mod tests { assert_eq!( batch.column(0).as_ref(), - Arc::new(StringArray::from(vec![ - "Carl", "Daniel", "Mike", "Roger", "Sam", - ])) - .as_ref(), + &StringArray::from(vec!["Carl", "Daniel", "Mike", "Roger", "Sam",]), ); assert_eq!( batch.column(1).as_ref(), - Arc::new(Int64Array::from(vec![3, 3, 4, 3, 2])).as_ref(), + &Int64Array::from(vec![3, 3, 4, 3, 2]), ); Ok(()) diff --git a/columnq/src/table/arrow_ipc_file.rs b/columnq/src/table/arrow_ipc_file.rs index 406e3ad..1e18b5f 100644 --- a/columnq/src/table/arrow_ipc_file.rs +++ b/columnq/src/table/arrow_ipc_file.rs @@ -79,7 +79,7 @@ mod tests { .await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(37 * 3)); Ok(()) @@ -92,7 +92,7 @@ mod tests { let t = to_mem_table(&TableSource::new("uk_cities".to_string(), test_path)).await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(37)); Ok(()) diff --git a/columnq/src/table/arrow_ipc_stream.rs b/columnq/src/table/arrow_ipc_stream.rs index 92d8308..767d43f 100644 --- a/columnq/src/table/arrow_ipc_stream.rs +++ b/columnq/src/table/arrow_ipc_stream.rs @@ -79,7 +79,7 @@ mod tests { .await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(37 * 3)); Ok(()) @@ -92,7 +92,7 @@ mod tests { let t = to_mem_table(&TableSource::new("uk_cities".to_string(), test_path)).await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(37)); Ok(()) diff --git a/columnq/src/table/csv.rs b/columnq/src/table/csv.rs index aff0307..eb4b107 100644 --- a/columnq/src/table/csv.rs +++ b/columnq/src/table/csv.rs @@ -132,7 +132,7 @@ mod tests { .await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(37 * 3)); Ok(()) @@ -155,7 +155,7 @@ c1,c2,c3 let t = to_mem_table(&source).await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(3)); Ok(()) diff --git a/columnq/src/table/delta.rs b/columnq/src/table/delta.rs index 94350bc..1dc89a0 100644 --- a/columnq/src/table/delta.rs +++ b/columnq/src/table/delta.rs @@ -140,7 +140,7 @@ mod tests { .await?; let ctx = SessionContext::new(); - validate_statistics(t.scan(&ctx.state(), &None, &[], None).await?.statistics()); + validate_statistics(t.scan(&ctx.state(), None, &[], None).await?.statistics()); match t.as_any().downcast_ref::() { Some(_) => Ok(()), diff --git a/columnq/src/table/google_spreadsheets.rs b/columnq/src/table/google_spreadsheets.rs index 4aab97a..cb495de 100644 --- a/columnq/src/table/google_spreadsheets.rs +++ b/columnq/src/table/google_spreadsheets.rs @@ -141,7 +141,7 @@ fn infer_schema(rows: &[Vec]) -> Schema { let dt = dt_iter.fold(dt_init, coerce_type); // normalize column name by replacing space with under score - Field::new(&col_name.replace(' ', "_"), dt, true) + Field::new(col_name.replace(' ', "_"), dt, true) }) .collect(); Schema::new(fields) diff --git a/columnq/src/table/json.rs b/columnq/src/table/json.rs index 6cc3886..3da436d 100644 --- a/columnq/src/table/json.rs +++ b/columnq/src/table/json.rs @@ -3,6 +3,7 @@ use std::sync::Arc; use datafusion::arrow; use datafusion::arrow::datatypes::Schema; +#[allow(deprecated)] use datafusion::arrow::json::reader::{Decoder, DecoderOptions}; use datafusion::arrow::record_batch::RecordBatch; use serde_json::value::Value; @@ -55,6 +56,8 @@ fn json_vec_to_partition( }; // decode to arrow record batch + #[allow(deprecated)] + // TODO: switch to RawDecoder let decoder = Decoder::new( Arc::new(schema.clone()), DecoderOptions::new().with_batch_size(batch_size), diff --git a/columnq/src/table/mod.rs b/columnq/src/table/mod.rs index 6d8ffcb..2bbbeb6 100644 --- a/columnq/src/table/mod.rs +++ b/columnq/src/table/mod.rs @@ -704,7 +704,7 @@ batch_size: 512 let ctx = datafusion::prelude::SessionContext::new(); let table = load(&t, &ctx).await?; let stats = table - .scan(&ctx.state(), &None, &[], None) + .scan(&ctx.state(), None, &[], None) .await? .statistics(); assert_eq!(stats.num_rows, Some(37)); @@ -726,7 +726,7 @@ uri: "sqlite://../test_data/sqlite/sample.{}" let ctx = datafusion::prelude::SessionContext::new(); let table = load(&t, &ctx).await?; let stats = table - .scan(&ctx.state(), &None, &[], None) + .scan(&ctx.state(), None, &[], None) .await? .statistics(); assert_eq!(stats.num_rows, Some(37)); diff --git a/columnq/src/table/ndjson.rs b/columnq/src/table/ndjson.rs index a20f8c9..a3a59eb 100644 --- a/columnq/src/table/ndjson.rs +++ b/columnq/src/table/ndjson.rs @@ -2,6 +2,7 @@ use std::io::{BufReader, Read}; use std::sync::Arc; use datafusion::arrow::datatypes::{Schema, SchemaRef}; +#[allow(deprecated)] use datafusion::arrow::json::reader::{infer_json_schema, Decoder, DecoderOptions, ValueIter}; use datafusion::arrow::record_batch::RecordBatch; @@ -18,6 +19,8 @@ fn decode_json_from_reader( schema_ref: SchemaRef, batch_size: usize, ) -> Result, ColumnQError> { + #[allow(deprecated)] + // TODO: switch to RawDecoder let decoder = Decoder::new( schema_ref, DecoderOptions::new().with_batch_size(batch_size), diff --git a/columnq/src/table/parquet.rs b/columnq/src/table/parquet.rs index 19e1ff9..29357dd 100644 --- a/columnq/src/table/parquet.rs +++ b/columnq/src/table/parquet.rs @@ -121,7 +121,7 @@ mod tests { .unwrap(); let stats = t - .scan(&ctx.state(), &None, &[], None) + .scan(&ctx.state(), None, &[], None) .await .unwrap() .statistics(); @@ -146,7 +146,7 @@ mod tests { .await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(500)); Ok(()) @@ -171,7 +171,7 @@ mod tests { .await?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert_eq!(stats.num_rows, Some(1500)); Ok(()) diff --git a/columnq/src/table/xlsx.rs b/columnq/src/table/xlsx.rs index 8fe8e9c..963fb20 100644 --- a/columnq/src/table/xlsx.rs +++ b/columnq/src/table/xlsx.rs @@ -64,7 +64,7 @@ fn infer_schema(r: &Range) -> Result { let mut dt_iter = set.iter().cloned(); let dt = dt_iter.next().unwrap_or(DataType::Utf8); - Field::new(&col_name.replace(' ', "_"), dt, true) + Field::new(col_name.replace(' ', "_"), dt, true) }) .collect(); Ok(Schema::new(fields)) @@ -182,7 +182,7 @@ sheet_name = "uk_cities_with_headers" let t = to_mem_table(&table_source).await.unwrap(); let ctx = SessionContext::new(); let stats = t - .scan(&ctx.state(), &None, &[], None) + .scan(&ctx.state(), None, &[], None) .await .unwrap() .statistics(); @@ -207,7 +207,7 @@ option: let t = to_mem_table(&table_source).await.unwrap(); let ctx = SessionContext::new(); let stats = t - .scan(&ctx.state(), &None, &[], None) + .scan(&ctx.state(), None, &[], None) .await .unwrap() .statistics(); diff --git a/columnq/src/test_util.rs b/columnq/src/test_util.rs index 78faa0e..34f0f3a 100644 --- a/columnq/src/test_util.rs +++ b/columnq/src/test_util.rs @@ -137,7 +137,7 @@ pub async fn register_table_ubuntu_ami(dfctx: &mut SessionContext) -> anyhow::Re pub fn assert_eq_df(df1: Arc, df2: Arc) { assert_eq!( - format!("{:?}", df1.to_logical_plan()), - format!("{:?}", df2.to_logical_plan()) + format!("{:?}", (*df1).clone().into_optimized_plan()), + format!("{:?}", (*df2).clone().into_optimized_plan()) ); } diff --git a/columnq/tests/table_mysql_test.rs b/columnq/tests/table_mysql_test.rs index 5433cea..5d54901 100644 --- a/columnq/tests/table_mysql_test.rs +++ b/columnq/tests/table_mysql_test.rs @@ -15,7 +15,7 @@ mod mysql { let t = DatabaseLoader::MySQL .to_mem_table(&TableSource::new(name, env::var("MYSQL_URL")?))?; let ctx = SessionContext::new(); - let stats = t.scan(&ctx.state(), &None, &[], None).await?.statistics(); + let stats = t.scan(&ctx.state(), None, &[], None).await?.statistics(); assert!(stats.num_rows.is_some()); } diff --git a/roapi/Cargo.toml b/roapi/Cargo.toml index 37f62ff..6867390 100644 --- a/roapi/Cargo.toml +++ b/roapi/Cargo.toml @@ -40,8 +40,8 @@ clap = { version = "3", features = ["color"] } thiserror = "1" anyhow = "1" -convergence = "0.9" -convergence-arrow = "0.9" +convergence = "0.11.0" +convergence-arrow = "0.11.0" [features] default = ["rustls", "snmalloc"] diff --git a/roapi/src/context.rs b/roapi/src/context.rs index 06e71dc..eac35a0 100644 --- a/roapi/src/context.rs +++ b/roapi/src/context.rs @@ -1,8 +1,7 @@ use std::collections::HashMap; -use std::sync::Arc; use async_trait::async_trait; -use columnq::datafusion::arrow; +use columnq::arrow::record_batch::RecordBatch; use columnq::datafusion::dataframe::DataFrame; use columnq::datafusion::error::DataFusionError; use columnq::encoding; @@ -61,25 +60,19 @@ pub trait RoapiContext: Send + Sync + 'static { async fn table_schema_json_bytes(&self, table_name: &str) -> Result, ApiErrResp>; - async fn query_graphql( - &self, - query: &str, - ) -> Result, QueryError>; + async fn query_graphql(&self, query: &str) -> Result, QueryError>; - async fn query_sql( - &self, - query: &str, - ) -> Result, QueryError>; + async fn query_sql(&self, query: &str) -> Result, QueryError>; async fn query_rest_table( &self, table_name: &str, params: &HashMap, - ) -> Result, QueryError>; + ) -> Result, QueryError>; async fn kv_get(&self, kv_name: &str, key: &str) -> Result, QueryError>; - async fn sql_to_df(&self, query: &str) -> Result, DataFusionError>; + async fn sql_to_df(&self, query: &str) -> Result; async fn get_response_format(&self) -> encoding::ContentType; } @@ -119,18 +112,12 @@ impl RoapiContext for RawRoapiContext { } #[inline] - async fn query_graphql( - &self, - query: &str, - ) -> Result, QueryError> { + async fn query_graphql(&self, query: &str) -> Result, QueryError> { self.cq.query_graphql(query).await } #[inline] - async fn query_sql( - &self, - query: &str, - ) -> Result, QueryError> { + async fn query_sql(&self, query: &str) -> Result, QueryError> { self.cq.query_sql(query).await } @@ -139,7 +126,7 @@ impl RoapiContext for RawRoapiContext { &self, table_name: &str, params: &HashMap, - ) -> Result, QueryError> { + ) -> Result, QueryError> { self.cq.query_rest_table(table_name, params).await } @@ -149,7 +136,7 @@ impl RoapiContext for RawRoapiContext { } #[inline] - async fn sql_to_df(&self, query: &str) -> Result, DataFusionError> { + async fn sql_to_df(&self, query: &str) -> Result { self.cq.dfctx.sql(query).await } @@ -195,19 +182,13 @@ impl RoapiContext for ConcurrentRoapiContext { } #[inline] - async fn query_graphql( - &self, - query: &str, - ) -> Result, QueryError> { + async fn query_graphql(&self, query: &str) -> Result, QueryError> { let ctx = self.read().await; ctx.cq.query_graphql(query).await } #[inline] - async fn query_sql( - &self, - query: &str, - ) -> Result, QueryError> { + async fn query_sql(&self, query: &str) -> Result, QueryError> { let ctx = self.read().await; ctx.cq.query_sql(query).await } @@ -217,7 +198,7 @@ impl RoapiContext for ConcurrentRoapiContext { &self, table_name: &str, params: &HashMap, - ) -> Result, QueryError> { + ) -> Result, QueryError> { let ctx = self.read().await; ctx.cq.query_rest_table(table_name, params).await } @@ -229,7 +210,7 @@ impl RoapiContext for ConcurrentRoapiContext { } #[inline] - async fn sql_to_df(&self, query: &str) -> Result, DataFusionError> { + async fn sql_to_df(&self, query: &str) -> Result { let ctx = self.read().await; ctx.cq.dfctx.sql(query).await } diff --git a/roapi/src/server/postgres.rs b/roapi/src/server/postgres.rs index 734fb37..844e7c3 100644 --- a/roapi/src/server/postgres.rs +++ b/roapi/src/server/postgres.rs @@ -20,18 +20,19 @@ use crate::context::RoapiContext; use crate::server::RunnableServer; fn df_err_to_sql(err: DataFusionError) -> ErrorResponse { - ErrorResponse::error(SqlState::DATA_EXCEPTION, err.to_string()) + ErrorResponse::error(SqlState::DataException, err.to_string()) } /// A portal built using a logical DataFusion query plan. pub struct DataFusionPortal { - df: Arc, + df: DataFrame, } #[async_trait] impl Portal for DataFusionPortal { async fn fetch(&mut self, batch: &mut DataRowBatch) -> Result<(), ErrorResponse> { - for arrow_batch in self.df.collect().await.map_err(df_err_to_sql)? { + let arrow_batches = self.df.clone().collect().await.map_err(df_err_to_sql)?; + for arrow_batch in arrow_batches { record_batch_to_rows(&arrow_batch, batch)?; } Ok(())