diff --git a/Cargo.lock b/Cargo.lock index 4b99308..53bee7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ "tokio", "tokio-util", "tower", - "tower-http 0.1.2", + "tower-http", "tower-layer", "tower-service", ] @@ -421,7 +421,7 @@ dependencies = [ [[package]] name = "columnq" -version = "0.4.3" +version = "0.4.5" dependencies = [ "anyhow", "bytes", @@ -454,7 +454,7 @@ dependencies = [ [[package]] name = "columnq-cli" -version = "0.2.1" +version = "0.2.3" dependencies = [ "anyhow", "clap", @@ -1166,12 +1166,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" - [[package]] name = "httparse" version = "1.5.1" @@ -2126,7 +2120,7 @@ dependencies = [ [[package]] name = "roapi-http" -version = "0.5.2" +version = "0.5.4" dependencies = [ "anyhow", "async-process", @@ -2145,7 +2139,7 @@ dependencies = [ "snmalloc-rs", "thiserror", "tokio", - "tower-http 0.2.0", + "tower-http", "tower-layer", "tracing", ] @@ -3107,24 +3101,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "tower-http" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39ee603d6e665ecc7e0f8d479eedb4626bd4726f0ee6119cee5b3a6bf184cac0" -dependencies = [ - "bitflags", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "pin-project-lite", - "tower-layer", - "tower-service", -] - [[package]] name = "tower-layer" version = "0.3.1" diff --git a/columnq-cli/Cargo.toml b/columnq-cli/Cargo.toml index a9601d4..c2ca5e5 100644 --- a/columnq-cli/Cargo.toml +++ b/columnq-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "columnq-cli" -version = "0.2.2" +version = "0.2.3" homepage = "https://github.com/roapi/roapi" license = "MIT" readme = "README.md" diff --git a/columnq/Cargo.toml b/columnq/Cargo.toml index d437afe..dded9d5 100644 --- a/columnq/Cargo.toml +++ b/columnq/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "columnq" -version = "0.4.4" +version = "0.4.5" homepage = "https://github.com/roapi/roapi" license = "MIT" authors = ["QP Hou "] diff --git a/columnq/src/table/google_spreadsheets.rs b/columnq/src/table/google_spreadsheets.rs index 45aa92c..1c4e66f 100644 --- a/columnq/src/table/google_spreadsheets.rs +++ b/columnq/src/table/google_spreadsheets.rs @@ -141,7 +141,7 @@ fn infer_schema(rows: &[Vec]) -> Schema { let dt = dt_iter.fold(dt_init, coerce_type); // normalize column name by replacing space with under score - Field::new(&col_name.replace(" ", "_"), dt, false) + Field::new(&col_name.replace(" ", "_"), dt, true) }) .collect(); Schema::new(fields) @@ -165,36 +165,43 @@ fn sheet_values_to_record_batch(values: &[Vec]) -> Result Arc::new( rows_iter - .map(|row| Some(parse_boolean(&row[i]))) + .map(|row| row.get(i).map(|v| parse_boolean(v))) .collect::(), ) as ArrayRef, DataType::Int64 => Arc::new( rows_iter .map(|row| { - Ok(Some(row[i].parse::().map_err(|_| { - ColumnQError::GoogleSpreadsheets(format!( - "Expect int64 value, got: {}", - row[i] - )) - })?)) + row.get(i) + .map(|v| { + v.parse::().map_err(|_| { + ColumnQError::GoogleSpreadsheets(format!( + "Expect int64 value, got: {}", + row[i] + )) + }) + }) + .transpose() }) .collect::, ColumnQError>>()?, ) as ArrayRef, DataType::Float64 => Arc::new( rows_iter .map(|row| { - Ok(Some(row[i].parse::().map_err(|_| { - ColumnQError::GoogleSpreadsheets(format!( - "Expect float64 value, got: {}", - row[i] - )) - })?)) + row.get(i) + .map(|v| { + v.parse::().map_err(|_| { + ColumnQError::GoogleSpreadsheets(format!( + "Expect float64 value, got: {}", + row[i] + )) + }) + }) + .transpose() }) .collect::, ColumnQError>>()?, ) as ArrayRef, - _ => Arc::new(rows_iter.map(|row| Some(&row[i])).collect::()) - as ArrayRef, + _ => Arc::new(rows_iter.map(|row| row.get(i)).collect::()) as ArrayRef, }) }) .collect::, ColumnQError>>()?; @@ -424,15 +431,15 @@ mod tests { assert_eq!( schema, Schema::new(vec![ - Field::new("Address", DataType::Utf8, false), - Field::new("Image", DataType::Utf8, false), - Field::new("Landlord", DataType::Utf8, false), - Field::new("Bed", DataType::Int64, false), - Field::new("Bath", DataType::Int64, false), - Field::new("Occupied", DataType::Boolean, false), - Field::new("Monthly_Rent", DataType::Utf8, false), - Field::new("Lease_Expiration_Date", DataType::Utf8, false), - Field::new("Days_Until_Expiration", DataType::Utf8, false), + Field::new("Address", DataType::Utf8, true), + Field::new("Image", DataType::Utf8, true), + Field::new("Landlord", DataType::Utf8, true), + Field::new("Bed", DataType::Int64, true), + Field::new("Bath", DataType::Int64, true), + Field::new("Occupied", DataType::Boolean, true), + Field::new("Monthly_Rent", DataType::Utf8, true), + Field::new("Lease_Expiration_Date", DataType::Utf8, true), + Field::new("Days_Until_Expiration", DataType::Utf8, true), ]) ); } @@ -458,4 +465,68 @@ mod tests { Ok(()) } + + #[test] + fn unaligned_sheetvalue_to_record_batch() -> anyhow::Result<()> { + // empty cells at the end of a row will not be returned from the server + let sheet = SpreadsheetValues { + range: "Properties!A1:AB1000".to_string(), + major_dimension: "ROWS".to_string(), + values: vec![ + row(&[ + "Address", + "Image", + "Landlord", + "Bed", + "Bath", + "Occupied", + "Monthly Rent", + "Lease Expiration Date", + "Days Until Expiration", + ]), + row(&[ + "Bothell, WA", + "https://a.com/1.jpeg", + "Roger", + "3", + "2", + "FALSE", + "$2,000", + "10/23/2020", + "Expired", + ]), + row(&[ + "Shoreline, WA", + "https://a.com/3.jpeg", + "Roger", + "1", + "1", + "TRUE", + "$1,200", + ]), + ], + }; + + let batch = sheet_values_to_record_batch(&sheet.values)?; + + assert_eq!(batch.num_columns(), 9); + assert_eq!( + batch.column(3).as_ref(), + Arc::new(Int64Array::from(vec![3, 1])).as_ref(), + ); + assert_eq!( + batch.column(5).as_ref(), + Arc::new(BooleanArray::from(vec![false, true])).as_ref(), + ); + assert_eq!( + batch.column(2).as_ref(), + Arc::new(StringArray::from(vec!["Roger", "Roger"])).as_ref(), + ); + assert_eq!( + batch.column(8).as_ref(), + Arc::new(StringArray::from(vec![Some("Expired"), None])).as_ref(), + ); + + Ok(()) + } } diff --git a/roapi-http/Cargo.toml b/roapi-http/Cargo.toml index d160b01..c5fbd45 100644 --- a/roapi-http/Cargo.toml +++ b/roapi-http/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "roapi-http" -version = "0.5.3" +version = "0.5.4" authors = ["QP Hou "] homepage = "https://github.com/roapi/roapi" license = "MIT"