diff --git a/Cargo.lock b/Cargo.lock index 018ed85..37a9517 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1066,6 +1066,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "toml 0.7.4", "uriparse", "url", "yup-oauth2", @@ -3385,7 +3386,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" dependencies = [ - "toml", + "toml 0.5.11", ] [[package]] @@ -3774,6 +3775,7 @@ dependencies = [ "thiserror", "tokio", "tokio-postgres", + "toml 0.7.4", "tower-http", "tower-layer", "tracing", @@ -4147,6 +4149,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -4666,6 +4677,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d964908cec0d030b812013af25a0e57fddfadb1e066ecc6681d86253129d4f" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tower" version = "0.4.13" @@ -5139,6 +5184,15 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "winnow" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/README.md b/README.md index b3fc8bb..f804fda 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,7 @@ curl 'localhost:8080/api/schema' ### Config file -You can also configure multiple table sources using YAML config, which supports more +You can also configure multiple table sources using YAML or Toml config, which supports more advanced format specific table options: ```yaml @@ -167,7 +167,7 @@ tables: To run serve tables using config file: ```bash -roapi -c ./roapi.yml +roapi -c ./roapi.yml # or .toml ``` See [config @@ -180,7 +180,7 @@ source](https://roapi.github.io/docs/config/dataset-formats/gsheet.html). By default, ROAPI encodes responses in JSON format, but you can request different encodings by specifying the `ACCEPT` header: -``` +```bash curl -X POST \ -H 'ACCEPT: application/vnd.apache.arrow.stream' \ -d "SELECT launch_library_id FROM spacex_launches WHERE launch_library_id IS NOT NULL" \ @@ -259,10 +259,10 @@ Starlink-21 (v1.0)% ### Query through Postgres wire protocol -ROAPI can present itself as a Postgre server so users can use Postgres clients +ROAPI can present itself as a Postgres server so users can use Postgres clients to issue SQL queries. -``` +```bash $ psql -h 127.0.0.1 psql (12.10 (Ubuntu 12.10-0ubuntu0.20.04.1), server 13) WARNING: psql major version 12, server major version 13. @@ -270,7 +270,7 @@ WARNING: psql major version 12, server major version 13. Type "help" for help. houqp=> select count(*) from uk_cities; - COUNT(UInt8(1)) + COUNT(UInt8(1)) ----------------- 37 (1 row) @@ -329,7 +329,7 @@ Misc: ## Development -The core of ROAPI, including query frontends and data layer, lives in the +The core of ROAPI, including query front-ends and data layer, lives in the self-contained [columnq](https://github.com/roapi/roapi/tree/main/columnq) crate. It takes queries and outputs Arrow record batches. Data sources will also be loaded and stored in memory as Arrow record batches. diff --git a/columnq/Cargo.toml b/columnq/Cargo.toml index e092df6..f0f9780 100644 --- a/columnq/Cargo.toml +++ b/columnq/Cargo.toml @@ -61,6 +61,7 @@ optional = true [dev-dependencies] anyhow = "1" serde_yaml = "0.8" +toml = "0.7" tempfile = "3.3.0" pretty_assertions = "*" dotenvy = "*" diff --git a/columnq/src/table/xlsx.rs b/columnq/src/table/xlsx.rs index e5ed4b7..8fe8e9c 100644 --- a/columnq/src/table/xlsx.rs +++ b/columnq/src/table/xlsx.rs @@ -165,7 +165,32 @@ mod tests { } #[tokio::test] - async fn load_xlsx_with_config() { + async fn load_xlsx_with_toml_config() { + let mut table_source: TableSource = toml::from_str( + r#" +name = "test" +uri = "test_data/uk_cities_with_headers.xlsx" +[option] +format = "xlsx" +sheet_name = "uk_cities_with_headers" +"#, + ) + .unwrap(); + // patch uri path with the correct test data path + table_source.io_source = TableIoSource::Uri(test_data_path("uk_cities_with_headers.xlsx")); + + let t = to_mem_table(&table_source).await.unwrap(); + let ctx = SessionContext::new(); + let stats = t + .scan(&ctx.state(), &None, &[], None) + .await + .unwrap() + .statistics(); + assert_eq!(stats.num_rows, Some(37)); + } + + #[tokio::test] + async fn load_xlsx_with_yaml_config() { let mut table_source: TableSource = serde_yaml::from_str( r#" name: "test" diff --git a/roapi/Cargo.toml b/roapi/Cargo.toml index 4c4ef52..37f62ff 100644 --- a/roapi/Cargo.toml +++ b/roapi/Cargo.toml @@ -35,6 +35,7 @@ serde = { version = "1", features = ["rc"] } serde_json = "1" serde_derive = "1" serde_yaml = "0.8" +toml = "0.7" clap = { version = "3", features = ["color"] } thiserror = "1" anyhow = "1" diff --git a/roapi/src/config.rs b/roapi/src/config.rs index 915c09b..18b5f73 100644 --- a/roapi/src/config.rs +++ b/roapi/src/config.rs @@ -121,8 +121,13 @@ pub fn get_configuration() -> Result { Some(config_path) => { let config_content = fs::read_to_string(config_path) .with_context(|| format!("Failed to read config file: {config_path}"))?; - - serde_yaml::from_str(&config_content).context("Failed to parse YAML config")? + if config_path.ends_with(".yaml") || config_path.ends_with(".yml") { + serde_yaml::from_str(&config_content).context("Failed to parse YAML config")? + } else if config_path.ends_with(".toml") { + toml::from_str(&config_content).context("Failed to parse TOML config")? + } else { + bail!("Unsupported config file format: {}", config_path); + } } };