split database feature flag into database-mysql and database-sqlite

2026-06-05 21:04:02 +08:00 · 2022-09-25 18:39:02 -07:00 · 2022-09-25 18:39:02 -07:00 · 4e812b9995
commit 4e812b9995
parent 37e5d5b03a
6 changed files with 63 additions and 35 deletions
--- a/.github/workflows/roapi_release.yml
+++ b/.github/workflows/roapi_release.yml
@ -53,7 +53,7 @@ jobs:
        run: pip3 install 'maturin<0.12'
      - name: Build wheels - x86_64
        run: |
-          maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database"
+          maturin build -m roapi/Cargo.toml -b bin --target x86_64-apple-darwin --release --out dist --cargo-extra-args="--features=simd,database-sqlite"
          pip install roapi --no-index --find-links dist --force-reinstall
      - name: Build wheels - universal2
        env:
@ -62,7 +62,7 @@ jobs:
        run: |
          # set SDKROOT for C dependencies
          export SDKROOT=$(xcrun --sdk macosx --show-sdk-path)
-          maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database"
+          maturin build -m roapi/Cargo.toml -b bin --release --universal2 --out dist --no-sdist --cargo-extra-args="--features=simd,database-sqlite"
          pip install roapi --no-index --find-links dist --force-reinstall
      - name: Upload wheels
        uses: actions/upload-artifact@v2
@ -88,7 +88,7 @@ jobs:
    strategy:
      matrix:
        platform: [
-          { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd,database" },
+          { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd,database-sqlite" },
          # { python-architecture: "x86", target: "i686-pc-windows-msvc", features: "" },
        ]
    steps:
@ -140,7 +140,7 @@ jobs:
          - manylinux: "2010"
            target: "x86_64-unknown-linux-musl"
            image_tag: "x86_64-musl"
-            features: "simd,rustls,database"
+            features: "simd,rustls,database-sqlite"
            name_suffix: ""
            upload: "true"
          # - manylinux: "2010"
@ -152,7 +152,7 @@ jobs:
          - manylinux: '2014'
            target: "aarch64-unknown-linux-musl"
            image_tag: "aarch64-musl"
-            features: "rustls,database"
+            features: "rustls,database-sqlite"
            name_suffix: ""
            upload: "true"
          # - manylinux: '2014'
@ -164,7 +164,7 @@ jobs:
          - manylinux: '2014'
            target: "armv7-unknown-linux-musleabihf"
            image_tag: "armv7-musleabihf"
-            features: "rustls,database"
+            features: "rustls,database-sqlite"
            name_suffix: ""
            upload: "true"
          # - manylinux: '2014'
--- a/Cargo.lock
+++ b/Cargo.lock
@ -647,7 +647,7 @@ dependencies = [
 [[package]]
 name = "connectorx"
 version = "0.3.1-alpha.8"
-source = "git+https://github.com/roapi/connector-x.git?rev=cc6393c20a55e1e0ac664f18fabd18934a874e69#cc6393c20a55e1e0ac664f18fabd18934a874e69"
+source = "git+https://github.com/roapi/connector-x.git?rev=47e40619740345209cbdb5e5b9ce4ad52e5768af#47e40619740345209cbdb5e5b9ce4ad52e5768af"
 dependencies = [
 "anyhow",
 "arrow",
--- a/columnq/Cargo.toml
+++ b/columnq/Cargo.toml
@ -50,9 +50,9 @@ features = ["datafusion-ext"]

 [dependencies.connectorx]
 git = "https://github.com/roapi/connector-x.git"
-rev = "c9da0aae301beccc2e4f5db226f2a856cab82db5"
+rev = "47e40619740345209cbdb5e5b9ce4ad52e5768af"
 version = "0.3.1-alpha.8"
-features = ["default", "dst_arrow", "src_mysql", "src_sqlite"]
+features = ["default", "dst_arrow"]
 optional = true

 [dev-dependencies]
@ -90,4 +90,9 @@ native-tls = [
    "yup-oauth2/hyper-tls",
 ]
 simd = ["datafusion/simd"]
-database = ["connectorx"]
+database-sqlite = ["connectorx/src_sqlite"]
+database-mysql = ["connectorx/src_mysql"]
+database = [
+    "database-sqlite",
+    "database-mysql",
+]
--- a/columnq/src/table/database.rs
+++ b/columnq/src/table/database.rs
@ -4,11 +4,12 @@ pub enum DatabaseLoader {
    Postgres,
 }

-#[cfg(feature = "database")]
+#[cfg(any(feature = "database-sqlite", feature = "database-mysql"))]
 mod imp {
    use crate::error::ColumnQError;
    use crate::table::TableSource;
    use connectorx::prelude::*;
+    #[cfg(feature = "database-mysql")]
    use connectorx::sources::mysql::BinaryProtocol;
    use datafusion::arrow::record_batch::RecordBatch;
    use log::debug;
@ -25,32 +26,51 @@ mod imp {
            let mut destination = ArrowDestination::new();
            match self {
                DatabaseLoader::MySQL => {
-                    let source = MySQLSource::<BinaryProtocol>::new(t.get_uri_str(), 2)
-                        .map_err(|e| ColumnQError::Database(e.to_string()))?;
-                    let dispatcher =
-                        Dispatcher::<
+                    #[cfg(feature = "database-mysql")]
+                    {
+                        let source = MySQLSource::<BinaryProtocol>::new(t.get_uri_str(), 2)
+                            .map_err(|e| ColumnQError::Database(e.to_string()))?;
+                        let dispatcher = Dispatcher::<
                            MySQLSource<BinaryProtocol>,
                            ArrowDestination,
                            MySQLArrowTransport<BinaryProtocol>,
-                        >::new(source, &mut destination, queries, None);
-                    dispatcher
-                        .run()
-                        .map_err(|e| ColumnQError::Database(e.to_string()))?;
+                        >::new(
+                            source, &mut destination, queries, None
+                        );
+                        dispatcher
+                            .run()
+                            .map_err(|e| ColumnQError::Database(e.to_string()))?;
+                    }
+                    #[cfg(not(feature = "database-mysql"))]
+                    {
+                        return Err(ColumnQError::Database(
+                            "MySQL database feature not enabled.".to_string(),
+                        ));
+                    }
                }
                DatabaseLoader::SQLite => {
-                    let uri = t.get_uri_str().replace("sqlite://", "");
-                    let source = SQLiteSource::new(&uri, 2)
-                        .map_err(|e| ColumnQError::Database(e.to_string()))?;
-                    let dispatcher = Dispatcher::<
-                        SQLiteSource,
-                        ArrowDestination,
-                        SQLiteArrowTransport,
-                    >::new(
-                        source, &mut destination, queries, None
-                    );
-                    dispatcher
-                        .run()
-                        .map_err(|e| ColumnQError::Database(e.to_string()))?;
+                    #[cfg(feature = "database-sqlite")]
+                    {
+                        let uri = t.get_uri_str().replace("sqlite://", "");
+                        let source = SQLiteSource::new(&uri, 2)
+                            .map_err(|e| ColumnQError::Database(e.to_string()))?;
+                        let dispatcher = Dispatcher::<
+                            SQLiteSource,
+                            ArrowDestination,
+                            SQLiteArrowTransport,
+                        >::new(
+                            source, &mut destination, queries, None
+                        );
+                        dispatcher
+                            .run()
+                            .map_err(|e| ColumnQError::Database(e.to_string()))?;
+                    }
+                    #[cfg(not(feature = "database-sqlite"))]
+                    {
+                        return Err(ColumnQError::Database(
+                            "SQLite database feature not enabled.".to_string(),
+                        ));
+                    }
                }
                DatabaseLoader::Postgres => {
                    // ToDo `Cannot start a runtime from within a runtime` error in `connector-x PostgresSource`
@ -69,7 +89,7 @@ mod imp {
    }
 }

-#[cfg(not(feature = "database"))]
+#[cfg(not(any(feature = "database-sqlite", feature = "database-mysql")))]
 mod imp {
    use crate::error::ColumnQError;
    use crate::table::TableSource;
@ -90,7 +110,7 @@ mod imp {

 pub use imp::*;

-#[cfg(feature = "database")]
+#[cfg(any(feature = "database-mysql"))]
 #[cfg(test)]
 mod tests {
    use datafusion::datasource::TableProvider;
@ -102,6 +122,7 @@ mod tests {

    use super::*;

+    #[cfg(feature = "database-mysql")]
    #[tokio::test]
    async fn load_mysql() -> anyhow::Result<()> {
        dotenv().ok();
--- a/columnq/src/table/mod.rs
+++ b/columnq/src/table/mod.rs
@ -671,7 +671,7 @@ batch_size: 512
        );
    }

-    #[cfg(feature = "database")]
+    #[cfg(feature = "database-sqlite")]
    #[tokio::test]
    async fn test_load_sqlite_table() -> anyhow::Result<()> {
        let t = TableSource::new("uk_cities", "sqlite://../test_data/sqlite.db");
--- a/roapi/Cargo.toml
+++ b/roapi/Cargo.toml
@ -51,6 +51,8 @@ native-tls = ["columnq/native-tls"]
 simd = ["columnq/simd"]
 snmalloc = ["snmalloc-rs"]
 database = ["columnq/database"]
+database-sqlite = ["columnq/database-sqlite"]
+database-mysql = ["columnq/database-mysql"]

 [dev-dependencies]
 reqwest = { version = "0.11", default-features = false, features = [