diff --git a/.github/workflows/roapi_release.yml b/.github/workflows/roapi_release.yml index 22d630f..4ba32a2 100644 --- a/.github/workflows/roapi_release.yml +++ b/.github/workflows/roapi_release.yml @@ -53,7 +53,7 @@ jobs: run: pip3 install 'maturin<0.12' - name: Build wheels - x86_64 run: | - maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database" + maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database-sqlite" pip install roapi --no-index --find-links dist --force-reinstall - name: Build wheels - universal2 env: @@ -62,7 +62,7 @@ jobs: run: | # set SDKROOT for C dependencies export SDKROOT=$(xcrun --sdk macosx --show-sdk-path) - maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database" + maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database-sqlite" pip install roapi --no-index --find-links dist --force-reinstall - name: Upload wheels uses: actions/upload-artifact@v2 @@ -88,7 +88,7 @@ jobs: strategy: matrix: platform: [ - { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd,database" }, + { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd,database-sqlite" }, # { python-architecture: "x86", target: "i686-pc-windows-msvc", features: "" }, ] steps: @@ -140,7 +140,7 @@ jobs: - manylinux: "2010" target: "x86_64-unknown-linux-musl" image_tag: "x86_64-musl" - features: "simd,rustls,database" + features: "simd,rustls,database-sqlite" name_suffix: "" upload: "true" # - manylinux: "2010" @@ -152,7 +152,7 @@ jobs: - manylinux: '2014' target: "aarch64-unknown-linux-musl" image_tag: "aarch64-musl" - features: "rustls,database" + features: "rustls,database-sqlite" name_suffix: "" upload: "true" # - manylinux: '2014' @@ -164,7 +164,7 @@ jobs: - manylinux: '2014' target: "armv7-unknown-linux-musleabihf" image_tag: "armv7-musleabihf" - features: "rustls,database" + features: "rustls,database-sqlite" name_suffix: "" upload: "true" # - manylinux: '2014' diff --git a/Cargo.lock b/Cargo.lock index db8d1e1..958cc0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -647,7 +647,7 @@ dependencies = [ [[package]] name = "connectorx" version = "0.3.1-alpha.8" -source = "git+https://github.com/roapi/connector-x.git?rev=cc6393c20a55e1e0ac664f18fabd18934a874e69#cc6393c20a55e1e0ac664f18fabd18934a874e69" +source = "git+https://github.com/roapi/connector-x.git?rev=47e40619740345209cbdb5e5b9ce4ad52e5768af#47e40619740345209cbdb5e5b9ce4ad52e5768af" dependencies = [ "anyhow", "arrow", diff --git a/columnq/Cargo.toml b/columnq/Cargo.toml index 8d52a74..387a7eb 100644 --- a/columnq/Cargo.toml +++ b/columnq/Cargo.toml @@ -50,9 +50,9 @@ features = ["datafusion-ext"] [dependencies.connectorx] git = "https://github.com/roapi/connector-x.git" -rev = "c9da0aae301beccc2e4f5db226f2a856cab82db5" +rev = "47e40619740345209cbdb5e5b9ce4ad52e5768af" version = "0.3.1-alpha.8" -features = ["default", "dst_arrow", "src_mysql", "src_sqlite"] +features = ["default", "dst_arrow"] optional = true [dev-dependencies] @@ -90,4 +90,9 @@ native-tls = [ "yup-oauth2/hyper-tls", ] simd = ["datafusion/simd"] -database = ["connectorx"] +database-sqlite = ["connectorx/src_sqlite"] +database-mysql = ["connectorx/src_mysql"] +database = [ + "database-sqlite", + "database-mysql", +] diff --git a/columnq/src/table/database.rs b/columnq/src/table/database.rs index c9a6c5a..feae6c4 100644 --- a/columnq/src/table/database.rs +++ b/columnq/src/table/database.rs @@ -4,11 +4,12 @@ pub enum DatabaseLoader { Postgres, } -#[cfg(feature = "database")] +#[cfg(any(feature = "database-sqlite", feature = "database-mysql"))] mod imp { use crate::error::ColumnQError; use crate::table::TableSource; use connectorx::prelude::*; + #[cfg(feature = "database-mysql")] use connectorx::sources::mysql::BinaryProtocol; use datafusion::arrow::record_batch::RecordBatch; use log::debug; @@ -25,32 +26,51 @@ mod imp { let mut destination = ArrowDestination::new(); match self { DatabaseLoader::MySQL => { - let source = MySQLSource::::new(t.get_uri_str(), 2) - .map_err(|e| ColumnQError::Database(e.to_string()))?; - let dispatcher = - Dispatcher::< + #[cfg(feature = "database-mysql")] + { + let source = MySQLSource::::new(t.get_uri_str(), 2) + .map_err(|e| ColumnQError::Database(e.to_string()))?; + let dispatcher = Dispatcher::< MySQLSource, ArrowDestination, MySQLArrowTransport, - >::new(source, &mut destination, queries, None); - dispatcher - .run() - .map_err(|e| ColumnQError::Database(e.to_string()))?; + >::new( + source, &mut destination, queries, None + ); + dispatcher + .run() + .map_err(|e| ColumnQError::Database(e.to_string()))?; + } + #[cfg(not(feature = "database-mysql"))] + { + return Err(ColumnQError::Database( + "MySQL database feature not enabled.".to_string(), + )); + } } DatabaseLoader::SQLite => { - let uri = t.get_uri_str().replace("sqlite://", ""); - let source = SQLiteSource::new(&uri, 2) - .map_err(|e| ColumnQError::Database(e.to_string()))?; - let dispatcher = Dispatcher::< - SQLiteSource, - ArrowDestination, - SQLiteArrowTransport, - >::new( - source, &mut destination, queries, None - ); - dispatcher - .run() - .map_err(|e| ColumnQError::Database(e.to_string()))?; + #[cfg(feature = "database-sqlite")] + { + let uri = t.get_uri_str().replace("sqlite://", ""); + let source = SQLiteSource::new(&uri, 2) + .map_err(|e| ColumnQError::Database(e.to_string()))?; + let dispatcher = Dispatcher::< + SQLiteSource, + ArrowDestination, + SQLiteArrowTransport, + >::new( + source, &mut destination, queries, None + ); + dispatcher + .run() + .map_err(|e| ColumnQError::Database(e.to_string()))?; + } + #[cfg(not(feature = "database-sqlite"))] + { + return Err(ColumnQError::Database( + "SQLite database feature not enabled.".to_string(), + )); + } } DatabaseLoader::Postgres => { // ToDo `Cannot start a runtime from within a runtime` error in `connector-x PostgresSource` @@ -69,7 +89,7 @@ mod imp { } } -#[cfg(not(feature = "database"))] +#[cfg(not(any(feature = "database-sqlite", feature = "database-mysql")))] mod imp { use crate::error::ColumnQError; use crate::table::TableSource; @@ -90,7 +110,7 @@ mod imp { pub use imp::*; -#[cfg(feature = "database")] +#[cfg(any(feature = "database-mysql"))] #[cfg(test)] mod tests { use datafusion::datasource::TableProvider; @@ -102,6 +122,7 @@ mod tests { use super::*; + #[cfg(feature = "database-mysql")] #[tokio::test] async fn load_mysql() -> anyhow::Result<()> { dotenv().ok(); diff --git a/columnq/src/table/mod.rs b/columnq/src/table/mod.rs index b3c0451..67ad073 100644 --- a/columnq/src/table/mod.rs +++ b/columnq/src/table/mod.rs @@ -671,7 +671,7 @@ batch_size: 512 ); } - #[cfg(feature = "database")] + #[cfg(feature = "database-sqlite")] #[tokio::test] async fn test_load_sqlite_table() -> anyhow::Result<()> { let t = TableSource::new("uk_cities", "sqlite://../test_data/sqlite.db"); diff --git a/roapi/Cargo.toml b/roapi/Cargo.toml index ef9806a..1c5e9d2 100644 --- a/roapi/Cargo.toml +++ b/roapi/Cargo.toml @@ -51,6 +51,8 @@ native-tls = ["columnq/native-tls"] simd = ["columnq/simd"] snmalloc = ["snmalloc-rs"] database = ["columnq/database"] +database-sqlite = ["columnq/database-sqlite"] +database-mysql = ["columnq/database-mysql"] [dev-dependencies] reqwest = { version = "0.11", default-features = false, features = [