add native-tls feature to support linking with system provided openssl (#93)

other ci related changes:
* temporarily disable openssl build in release jobs
* upload windows build to github release

.github/workflows/build.yml

@@ -75,6 +75,36 @@ jobs:
         which cargo-cache || cargo install cargo-cache
         cargo cache trim -l 1G
 
+  # make sure native-tls always builds
+  openssl_build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.cargo/bin/
+          ~/.cargo/registry/index/
+          ~/.cargo/registry/cache/
+          ~/.cargo/git/db/
+          target/
+        key: openssl-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
+        restore-keys: |
+          openssl-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }}
+          openssl-${{ runner.os }}-cargo-
+    - name: Install minimal stable with clippy and rustfmt
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: default
+        toolchain: stable
+        override: true
+    - name: Check
+      run: cargo clippy --no-default-features --features=native-tls
+    - name: Trim cache
+      run: |
+        which cargo-cache || cargo install cargo-cache
+        cargo cache trim -l 1G
+
   # cross compile from x86 mac to arm64, this is to make sure universal2
   # release will build without error
   mac_cross_build:

.github/workflows/columnq_cli_release.yml

@@ -70,11 +70,13 @@ jobs:
           name: wheels
           path: dist
       - name: Archive binary
+        if: "startsWith(github.ref, 'refs/tags/')"
         run: tar czvf target/release/columnq-cli.tar.gz -C target/release columnq
       - name: Upload binary to GitHub Release
         uses: svenstaro/upload-release-action@v2
         if: "startsWith(github.ref, 'refs/tags/')"
         with:
+          overwrite: true
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           asset_name: columnq-cli-apple-darwin.tar.gz
           file: target/release/columnq-cli.tar.gz
@@ -85,10 +87,10 @@ jobs:
     runs-on: windows-latest
     strategy:
       matrix:
-        platform: [
-          { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd" },
-          # { python-architecture: "x86", target: "i686-pc-windows-msvc", features: "" },
-        ]
+        platform:
+          - python-architecture: "x64"
+            target: "x86_64-pc-windows-msvc"
+            features: "simd"
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
@@ -114,6 +116,18 @@ jobs:
         with:
           name: wheels
           path: dist
+      - name: Archive binary
+        if: "startsWith(github.ref, 'refs/tags/')"
+        run: tar czvf target/release/columnq-cli.tar.gz -C target/${{ matrix.platform.target }}/release columnq.exe
+      - name: Upload binary to GitHub Release
+        uses: svenstaro/upload-release-action@v2
+        if: "startsWith(github.ref, 'refs/tags/')"
+        with:
+          overwrite: true
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          asset_name: columnq-cli-windows.tar.gz
+          file: target/release/columnq-cli.tar.gz
+          tag: ${{ github.ref }}
 
   linux:
     needs: validate-release-tag
@@ -121,15 +135,43 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [
-          { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,rustls", upload: "true" },
-          { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,native-tls-vendored", upload: "false" },
-          # { manylinux: '2010', target: "i686-unknown-linux-musl", image_tag: "i686-musl", features: "" },
-          { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "rustls", upload: "true" },
-          { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "native-tls-vendored", upload: "false" },
-          { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "rustls", upload: "true" },
-          { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "native-tls-vendored", upload: "false" },
-        ]
+        platform:
+          - manylinux: "2010"
+            target: "x86_64-unknown-linux-musl"
+            image_tag: "x86_64-musl"
+            features: "simd,rustls"
+            name_suffix: ""
+            upload: "true"
+          # - manylinux: "2010"
+          #   target: "x86_64-unknown-linux-musl"
+          #   image_tag: "x86_64-musl"
+          #   features: "simd,native-tls"
+          #   name_suffix: "-openssl"
+          #   upload: "false"
+          - manylinux: '2014'
+            target: "aarch64-unknown-linux-musl"
+            image_tag: "aarch64-musl"
+            features: "rustls"
+            name_suffix: ""
+            upload: "true"
+          # - manylinux: '2014'
+          #   target: "aarch64-unknown-linux-musl"
+          #   image_tag: "aarch64-musl"
+          #   features: "native-tls"
+          #   name_suffix: "-openssl"
+          #   upload: "false"
+          - manylinux: '2014'
+            target: "armv7-unknown-linux-musleabihf"
+            image_tag: "armv7-musleabihf"
+            features: "rustls"
+            name_suffix: ""
+            upload: "true"
+          # - manylinux: '2014'
+          #   target: "armv7-unknown-linux-musleabihf"
+          #   image_tag: "armv7-musleabihf"
+          #   features: "native-tls"
+          #   name_suffix: "-openssl"
+          #   upload: "false"
     container:
       image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }}
       env:
@@ -144,6 +186,11 @@ jobs:
             --toolchain nightly-${{ env.RUST_TC_NIGHTLY_VER }} \
             ${{ matrix.platform.target }}
           rustup default nightly-${{ env.RUST_TC_NIGHTLY_VER }}
+      - name: Install build deps
+        if: "contains(matrix.platform.name_suffix, 'openssl')"
+        run: |
+          apt-get update
+          apt-get install -y libssl-dev
       - name: Build Wheels
         run: |
           sudo python3 -m pip install 'maturin<0.12'
@@ -158,14 +205,15 @@ jobs:
           name: wheels
           path: dist
       - name: Archive binary
-        if: "matrix.platform.upload == 'true'"
+        if: "startsWith(github.ref, 'refs/tags/')"
         run: tar czvf target/release/columnq-cli.tar.gz -C target/${{ matrix.platform.target }}/release columnq
       - name: Upload binary to GitHub Release
+        if: "startsWith(github.ref, 'refs/tags/')"
         uses: svenstaro/upload-release-action@v2
-        if: "startsWith(github.ref, 'refs/tags/') && matrix.platform.upload == 'true'"
         with:
+          overwrite: true
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          asset_name: columnq-cli-${{ matrix.platform.target }}.tar.gz
+          asset_name: columnq-cli-${{ matrix.platform.target }}${{ matrix.platform.name_suffix }}.tar.gz
           file: target/release/columnq-cli.tar.gz
           tag: ${{ github.ref }}
 

.github/workflows/roapi_http_release.yml

@@ -71,11 +71,13 @@ jobs:
           name: wheels
           path: dist
       - name: Archive binary
+        if: "startsWith(github.ref, 'refs/tags/')"
         run: tar czvf target/release/roapi-http.tar.gz -C target/release roapi-http
       - name: Upload binary to GitHub Release
         uses: svenstaro/upload-release-action@v2
         if: "startsWith(github.ref, 'refs/tags/')"
         with:
+          overwrite: true
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           asset_name: roapi-http-apple-darwin.tar.gz
           file: target/release/roapi-http.tar.gz
@@ -115,6 +117,19 @@ jobs:
         with:
           name: wheels
           path: dist
+      - name: Archive binary
+        if: "startsWith(github.ref, 'refs/tags/')"
+        run: |
+            tar czvf target/release/roapi-http.tar.gz -C target/${{ matrix.platform.target }}/release roapi-http.exe
+      - name: Upload binary to GitHub Release
+        uses: svenstaro/upload-release-action@v2
+        if: "startsWith(github.ref, 'refs/tags/')"
+        with:
+          overwrite: true
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          asset_name: roapi-http-windows.tar.gz
+          file: target/release/roapi-http.tar.gz
+          tag: ${{ github.ref }}
 
   linux:
     needs: validate-release-tag
@@ -122,15 +137,43 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [
-          { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,rustls", upload: "true" },
-          { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,native-tls-vendored", upload: "false" },
-          # { manylinux: '2010', target: "i686-unknown-linux-musl", image_tag: "i686-musl", features: "" },
-          { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "rustls", upload: "true"},
-          { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "native-tls-vendored", upload: "false" },
-          { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "rustls", upload: "true" },
-          { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "native-tls-vendored", upload: "false" },
-        ]
+        platform:
+          - manylinux: "2010"
+            target: "x86_64-unknown-linux-musl"
+            image_tag: "x86_64-musl"
+            features: "simd,rustls"
+            name_suffix: ""
+            upload: "true"
+          # - manylinux: "2010"
+          #   target: "x86_64-unknown-linux-musl"
+          #   image_tag: "x86_64-musl"
+          #   features: "simd,native-tls"
+          #   name_suffix: "-openssl"
+          #   upload: "false"
+          - manylinux: '2014'
+            target: "aarch64-unknown-linux-musl"
+            image_tag: "aarch64-musl"
+            features: "rustls"
+            name_suffix: ""
+            upload: "true"
+          # - manylinux: '2014'
+          #   target: "aarch64-unknown-linux-musl"
+          #   image_tag: "aarch64-musl"
+          #   features: "native-tls"
+          #   name_suffix: "-openssl"
+          #   upload: "false"
+          - manylinux: '2014'
+            target: "armv7-unknown-linux-musleabihf"
+            image_tag: "armv7-musleabihf"
+            features: "rustls"
+            name_suffix: ""
+            upload: "true"
+          # - manylinux: '2014'
+          #   target: "armv7-unknown-linux-musleabihf"
+          #   image_tag: "armv7-musleabihf"
+          #   features: "native-tls"
+          #   name_suffix: "-openssl"
+          #   upload: "false"
     container:
       image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }}
       env:
@@ -145,6 +188,11 @@ jobs:
             --toolchain nightly-${{ env.RUST_TC_NIGHTLY_VER }} \
             ${{ matrix.platform.target }}
           rustup default nightly-${{ env.RUST_TC_NIGHTLY_VER }}
+      - name: Install build deps
+        if: "contains(matrix.platform.name_suffix, 'openssl')"
+        run: |
+          apt-get update
+          apt-get install -y libssl-dev
       - name: Build Wheels
         run: |
           sudo python3 -m pip install 'maturin<0.12'
@@ -159,14 +207,15 @@ jobs:
           name: wheels
           path: dist
       - name: Archive binary
-        if: "matrix.platform.upload == 'true'"
+        if: "startsWith(github.ref, 'refs/tags/')"
         run: tar czvf target/release/roapi-http.tar.gz -C target/${{ matrix.platform.target }}/release roapi-http
       - name: Upload binary to GitHub Release
         uses: svenstaro/upload-release-action@v2
-        if: "startsWith(github.ref, 'refs/tags/') && matrix.platform.upload == 'true'"
+        if: "startsWith(github.ref, 'refs/tags/')"
         with:
+          overwrite: true
           repo_token: ${{ secrets.GITHUB_TOKEN }}
-          asset_name: roapi-http-${{ matrix.platform.target }}.tar.gz
+          asset_name: roapi-http-${{ matrix.platform.target }}${{ matrix.platform.name_suffix }}.tar.gz
           file: target/release/roapi-http.tar.gz
           tag: ${{ github.ref }}
 

columnq-cli/Cargo.toml

@@ -29,6 +29,7 @@ dirs = { version = "3" }
 default = ["rustls", "snmalloc"]
 rustls = ["columnq/rustls"]
 native-tls-vendored = ["columnq/native-tls-vendored"]
+native-tls = ["columnq/native-tls"]
 simd = ["columnq/simd"]
 snmalloc = ["snmalloc-rs"]
 

columnq/Cargo.toml

@@ -36,7 +36,7 @@ rusoto_core = { version = "0.47", default-features = false }
 rusoto_s3 = { version = "0.47", default-features = false }
 rusoto_credential = { version = "0.47" }
 rusoto_sts = { version = "0.47", default-features = false }
-hyper-tls = { version = "0.5.0", default-features = false, features = ["vendored"], optional = true }
+hyper-tls = { version = "0.5.0", default-features = false, optional = true }
 hyper-rustls = { version = "0.22.1", default-features = false, optional = true }
 
 
@@ -52,4 +52,5 @@ pretty_assertions = "*"
 default = ["rustls"]
 rustls = ["reqwest/rustls-tls", "rusoto_core/rustls", "rusoto_core/rustls", "rusoto_sts/rustls", "deltalake/s3-rustls", "yup-oauth2/hyper-rustls"]
 native-tls-vendored = ["reqwest/native-tls-vendored", "hyper-tls/vendored", "rusoto_core/native-tls", "rusoto_s3/native-tls", "rusoto_sts/native-tls", "deltalake/s3", "yup-oauth2/hyper-tls"]
+native-tls = ["reqwest/native-tls", "rusoto_core/native-tls", "rusoto_s3/native-tls", "rusoto_sts/native-tls", "deltalake/s3", "yup-oauth2/hyper-tls"]
 simd = ["datafusion/simd"]

roapi-http/Cargo.toml

@@ -40,6 +40,7 @@ anyhow = "1"
 default = ["rustls", "snmalloc"]
 rustls = ["columnq/rustls"]
 native-tls-vendored = ["columnq/native-tls-vendored"]
+native-tls = ["columnq/native-tls"]
 simd = ["columnq/simd"]
 snmalloc = ["snmalloc-rs"]