From 6ce319de5f3a94addc00d370e75ab8902a871953 Mon Sep 17 00:00:00 2001 From: QP Hou Date: Thu, 14 Oct 2021 09:05:54 -0700 Subject: [PATCH] add native-tls feature to support linking with system provided openssl (#93) other ci related changes: * temporarily disable openssl build in release jobs * upload windows build to github release --- .github/workflows/build.yml | 30 +++++++++ .github/workflows/columnq_cli_release.yml | 80 ++++++++++++++++++----- .github/workflows/roapi_http_release.yml | 73 +++++++++++++++++---- columnq-cli/Cargo.toml | 1 + columnq/Cargo.toml | 3 +- roapi-http/Cargo.toml | 1 + 6 files changed, 159 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d54e47a..286b7b8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -75,6 +75,36 @@ jobs: which cargo-cache || cargo install cargo-cache cargo cache trim -l 1G + # make sure native-tls always builds + openssl_build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: openssl-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + restore-keys: | + openssl-${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock', '**/Cargo.toml') }} + openssl-${{ runner.os }}-cargo- + - name: Install minimal stable with clippy and rustfmt + uses: actions-rs/toolchain@v1 + with: + profile: default + toolchain: stable + override: true + - name: Check + run: cargo clippy --no-default-features --features=native-tls + - name: Trim cache + run: | + which cargo-cache || cargo install cargo-cache + cargo cache trim -l 1G + # cross compile from x86 mac to arm64, this is to make sure universal2 # release will build without error mac_cross_build: diff --git a/.github/workflows/columnq_cli_release.yml b/.github/workflows/columnq_cli_release.yml index 04f9c43..36d3b15 100644 --- a/.github/workflows/columnq_cli_release.yml +++ b/.github/workflows/columnq_cli_release.yml @@ -70,11 +70,13 @@ jobs: name: wheels path: dist - name: Archive binary + if: "startsWith(github.ref, 'refs/tags/')" run: tar czvf target/release/columnq-cli.tar.gz -C target/release columnq - name: Upload binary to GitHub Release uses: svenstaro/upload-release-action@v2 if: "startsWith(github.ref, 'refs/tags/')" with: + overwrite: true repo_token: ${{ secrets.GITHUB_TOKEN }} asset_name: columnq-cli-apple-darwin.tar.gz file: target/release/columnq-cli.tar.gz @@ -85,10 +87,10 @@ jobs: runs-on: windows-latest strategy: matrix: - platform: [ - { python-architecture: "x64", target: "x86_64-pc-windows-msvc", features: "simd" }, - # { python-architecture: "x86", target: "i686-pc-windows-msvc", features: "" }, - ] + platform: + - python-architecture: "x64" + target: "x86_64-pc-windows-msvc" + features: "simd" steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -114,6 +116,18 @@ jobs: with: name: wheels path: dist + - name: Archive binary + if: "startsWith(github.ref, 'refs/tags/')" + run: tar czvf target/release/columnq-cli.tar.gz -C target/${{ matrix.platform.target }}/release columnq.exe + - name: Upload binary to GitHub Release + uses: svenstaro/upload-release-action@v2 + if: "startsWith(github.ref, 'refs/tags/')" + with: + overwrite: true + repo_token: ${{ secrets.GITHUB_TOKEN }} + asset_name: columnq-cli-windows.tar.gz + file: target/release/columnq-cli.tar.gz + tag: ${{ github.ref }} linux: needs: validate-release-tag @@ -121,15 +135,43 @@ jobs: strategy: fail-fast: false matrix: - platform: [ - { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,rustls", upload: "true" }, - { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,native-tls-vendored", upload: "false" }, - # { manylinux: '2010', target: "i686-unknown-linux-musl", image_tag: "i686-musl", features: "" }, - { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "rustls", upload: "true" }, - { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "native-tls-vendored", upload: "false" }, - { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "rustls", upload: "true" }, - { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "native-tls-vendored", upload: "false" }, - ] + platform: + - manylinux: "2010" + target: "x86_64-unknown-linux-musl" + image_tag: "x86_64-musl" + features: "simd,rustls" + name_suffix: "" + upload: "true" + # - manylinux: "2010" + # target: "x86_64-unknown-linux-musl" + # image_tag: "x86_64-musl" + # features: "simd,native-tls" + # name_suffix: "-openssl" + # upload: "false" + - manylinux: '2014' + target: "aarch64-unknown-linux-musl" + image_tag: "aarch64-musl" + features: "rustls" + name_suffix: "" + upload: "true" + # - manylinux: '2014' + # target: "aarch64-unknown-linux-musl" + # image_tag: "aarch64-musl" + # features: "native-tls" + # name_suffix: "-openssl" + # upload: "false" + - manylinux: '2014' + target: "armv7-unknown-linux-musleabihf" + image_tag: "armv7-musleabihf" + features: "rustls" + name_suffix: "" + upload: "true" + # - manylinux: '2014' + # target: "armv7-unknown-linux-musleabihf" + # image_tag: "armv7-musleabihf" + # features: "native-tls" + # name_suffix: "-openssl" + # upload: "false" container: image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }} env: @@ -144,6 +186,11 @@ jobs: --toolchain nightly-${{ env.RUST_TC_NIGHTLY_VER }} \ ${{ matrix.platform.target }} rustup default nightly-${{ env.RUST_TC_NIGHTLY_VER }} + - name: Install build deps + if: "contains(matrix.platform.name_suffix, 'openssl')" + run: | + apt-get update + apt-get install -y libssl-dev - name: Build Wheels run: | sudo python3 -m pip install 'maturin<0.12' @@ -158,14 +205,15 @@ jobs: name: wheels path: dist - name: Archive binary - if: "matrix.platform.upload == 'true'" + if: "startsWith(github.ref, 'refs/tags/')" run: tar czvf target/release/columnq-cli.tar.gz -C target/${{ matrix.platform.target }}/release columnq - name: Upload binary to GitHub Release + if: "startsWith(github.ref, 'refs/tags/')" uses: svenstaro/upload-release-action@v2 - if: "startsWith(github.ref, 'refs/tags/') && matrix.platform.upload == 'true'" with: + overwrite: true repo_token: ${{ secrets.GITHUB_TOKEN }} - asset_name: columnq-cli-${{ matrix.platform.target }}.tar.gz + asset_name: columnq-cli-${{ matrix.platform.target }}${{ matrix.platform.name_suffix }}.tar.gz file: target/release/columnq-cli.tar.gz tag: ${{ github.ref }} diff --git a/.github/workflows/roapi_http_release.yml b/.github/workflows/roapi_http_release.yml index ab94c0d..ac4a8df 100644 --- a/.github/workflows/roapi_http_release.yml +++ b/.github/workflows/roapi_http_release.yml @@ -71,11 +71,13 @@ jobs: name: wheels path: dist - name: Archive binary + if: "startsWith(github.ref, 'refs/tags/')" run: tar czvf target/release/roapi-http.tar.gz -C target/release roapi-http - name: Upload binary to GitHub Release uses: svenstaro/upload-release-action@v2 if: "startsWith(github.ref, 'refs/tags/')" with: + overwrite: true repo_token: ${{ secrets.GITHUB_TOKEN }} asset_name: roapi-http-apple-darwin.tar.gz file: target/release/roapi-http.tar.gz @@ -115,6 +117,19 @@ jobs: with: name: wheels path: dist + - name: Archive binary + if: "startsWith(github.ref, 'refs/tags/')" + run: | + tar czvf target/release/roapi-http.tar.gz -C target/${{ matrix.platform.target }}/release roapi-http.exe + - name: Upload binary to GitHub Release + uses: svenstaro/upload-release-action@v2 + if: "startsWith(github.ref, 'refs/tags/')" + with: + overwrite: true + repo_token: ${{ secrets.GITHUB_TOKEN }} + asset_name: roapi-http-windows.tar.gz + file: target/release/roapi-http.tar.gz + tag: ${{ github.ref }} linux: needs: validate-release-tag @@ -122,15 +137,43 @@ jobs: strategy: fail-fast: false matrix: - platform: [ - { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,rustls", upload: "true" }, - { manylinux: '2010', target: "x86_64-unknown-linux-musl", image_tag: "x86_64-musl", features: "simd,native-tls-vendored", upload: "false" }, - # { manylinux: '2010', target: "i686-unknown-linux-musl", image_tag: "i686-musl", features: "" }, - { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "rustls", upload: "true"}, - { manylinux: '2014', target: "aarch64-unknown-linux-musl", image_tag: "aarch64-musl", features: "native-tls-vendored", upload: "false" }, - { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "rustls", upload: "true" }, - { manylinux: '2014', target: "armv7-unknown-linux-musleabihf", image_tag: "armv7-musleabihf", features: "native-tls-vendored", upload: "false" }, - ] + platform: + - manylinux: "2010" + target: "x86_64-unknown-linux-musl" + image_tag: "x86_64-musl" + features: "simd,rustls" + name_suffix: "" + upload: "true" + # - manylinux: "2010" + # target: "x86_64-unknown-linux-musl" + # image_tag: "x86_64-musl" + # features: "simd,native-tls" + # name_suffix: "-openssl" + # upload: "false" + - manylinux: '2014' + target: "aarch64-unknown-linux-musl" + image_tag: "aarch64-musl" + features: "rustls" + name_suffix: "" + upload: "true" + # - manylinux: '2014' + # target: "aarch64-unknown-linux-musl" + # image_tag: "aarch64-musl" + # features: "native-tls" + # name_suffix: "-openssl" + # upload: "false" + - manylinux: '2014' + target: "armv7-unknown-linux-musleabihf" + image_tag: "armv7-musleabihf" + features: "rustls" + name_suffix: "" + upload: "true" + # - manylinux: '2014' + # target: "armv7-unknown-linux-musleabihf" + # image_tag: "armv7-musleabihf" + # features: "native-tls" + # name_suffix: "-openssl" + # upload: "false" container: image: docker://messense/rust-musl-cross:${{ matrix.platform.image_tag }} env: @@ -145,6 +188,11 @@ jobs: --toolchain nightly-${{ env.RUST_TC_NIGHTLY_VER }} \ ${{ matrix.platform.target }} rustup default nightly-${{ env.RUST_TC_NIGHTLY_VER }} + - name: Install build deps + if: "contains(matrix.platform.name_suffix, 'openssl')" + run: | + apt-get update + apt-get install -y libssl-dev - name: Build Wheels run: | sudo python3 -m pip install 'maturin<0.12' @@ -159,14 +207,15 @@ jobs: name: wheels path: dist - name: Archive binary - if: "matrix.platform.upload == 'true'" + if: "startsWith(github.ref, 'refs/tags/')" run: tar czvf target/release/roapi-http.tar.gz -C target/${{ matrix.platform.target }}/release roapi-http - name: Upload binary to GitHub Release uses: svenstaro/upload-release-action@v2 - if: "startsWith(github.ref, 'refs/tags/') && matrix.platform.upload == 'true'" + if: "startsWith(github.ref, 'refs/tags/')" with: + overwrite: true repo_token: ${{ secrets.GITHUB_TOKEN }} - asset_name: roapi-http-${{ matrix.platform.target }}.tar.gz + asset_name: roapi-http-${{ matrix.platform.target }}${{ matrix.platform.name_suffix }}.tar.gz file: target/release/roapi-http.tar.gz tag: ${{ github.ref }} diff --git a/columnq-cli/Cargo.toml b/columnq-cli/Cargo.toml index b295982..4dc7ef6 100644 --- a/columnq-cli/Cargo.toml +++ b/columnq-cli/Cargo.toml @@ -29,6 +29,7 @@ dirs = { version = "3" } default = ["rustls", "snmalloc"] rustls = ["columnq/rustls"] native-tls-vendored = ["columnq/native-tls-vendored"] +native-tls = ["columnq/native-tls"] simd = ["columnq/simd"] snmalloc = ["snmalloc-rs"] diff --git a/columnq/Cargo.toml b/columnq/Cargo.toml index ad9584e..7304130 100644 --- a/columnq/Cargo.toml +++ b/columnq/Cargo.toml @@ -36,7 +36,7 @@ rusoto_core = { version = "0.47", default-features = false } rusoto_s3 = { version = "0.47", default-features = false } rusoto_credential = { version = "0.47" } rusoto_sts = { version = "0.47", default-features = false } -hyper-tls = { version = "0.5.0", default-features = false, features = ["vendored"], optional = true } +hyper-tls = { version = "0.5.0", default-features = false, optional = true } hyper-rustls = { version = "0.22.1", default-features = false, optional = true } @@ -52,4 +52,5 @@ pretty_assertions = "*" default = ["rustls"] rustls = ["reqwest/rustls-tls", "rusoto_core/rustls", "rusoto_core/rustls", "rusoto_sts/rustls", "deltalake/s3-rustls", "yup-oauth2/hyper-rustls"] native-tls-vendored = ["reqwest/native-tls-vendored", "hyper-tls/vendored", "rusoto_core/native-tls", "rusoto_s3/native-tls", "rusoto_sts/native-tls", "deltalake/s3", "yup-oauth2/hyper-tls"] +native-tls = ["reqwest/native-tls", "rusoto_core/native-tls", "rusoto_s3/native-tls", "rusoto_sts/native-tls", "deltalake/s3", "yup-oauth2/hyper-tls"] simd = ["datafusion/simd"] diff --git a/roapi-http/Cargo.toml b/roapi-http/Cargo.toml index 995393b..84e031e 100644 --- a/roapi-http/Cargo.toml +++ b/roapi-http/Cargo.toml @@ -40,6 +40,7 @@ anyhow = "1" default = ["rustls", "snmalloc"] rustls = ["columnq/rustls"] native-tls-vendored = ["columnq/native-tls-vendored"] +native-tls = ["columnq/native-tls"] simd = ["columnq/simd"] snmalloc = ["snmalloc-rs"]