Enable the definition of --from to include a local relative path
This requires some shenanigans to ensure that carto generates a file:// URL. I'm sure there are more tests which should be added here
This commit is contained in:
parent
7a0076c7a8
commit
4d2676eabc
|
@ -14,3 +14,5 @@ Cargo.lock
|
||||||
# MSVC Windows builds of rustc generate these, which store debugging information
|
# MSVC Windows builds of rustc generate these, which store debugging information
|
||||||
*.pdb
|
*.pdb
|
||||||
|
|
||||||
|
# Example output for local testing
|
||||||
|
output/
|
||||||
|
|
|
@ -7,7 +7,7 @@ edition = "2021"
|
||||||
anyhow = "*"
|
anyhow = "*"
|
||||||
async-std = { version = "1", features = ["attributes", "tokio1"] }
|
async-std = { version = "1", features = ["attributes", "tokio1"] }
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
deltalake = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["s3"]}
|
deltalake = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["s3", "parquet"]}
|
||||||
dotenv = "*"
|
dotenv = "*"
|
||||||
glob = "0.3"
|
glob = "0.3"
|
||||||
gumdrop = "0.8"
|
gumdrop = "0.8"
|
||||||
|
|
|
@ -6,12 +6,17 @@ fields:
|
||||||
- name: timestamp
|
- name: timestamp
|
||||||
type: timestamp
|
type: timestamp
|
||||||
nullable: false
|
nullable: false
|
||||||
|
# Make this optional
|
||||||
|
metadata:
|
||||||
- name: temp
|
- name: temp
|
||||||
type: integer
|
type: integer
|
||||||
nullable: false
|
nullable: false
|
||||||
|
metadata:
|
||||||
- name: lat
|
- name: lat
|
||||||
type: double
|
type: double
|
||||||
nullable: false
|
nullable: false
|
||||||
|
metadata:
|
||||||
- name: long
|
- name: long
|
||||||
type: double
|
type: double
|
||||||
nullable: false
|
nullable: false
|
||||||
|
metadata:
|
||||||
|
|
50
src/lib.rs
50
src/lib.rs
|
@ -52,13 +52,34 @@ pub fn table_path_to_uri(base_uri: &str, table_path: &PathBuf) -> Result<Url, an
|
||||||
"Converting table to URI: base: {} path: {:?}",
|
"Converting table to URI: base: {} path: {:?}",
|
||||||
base_uri, table_path
|
base_uri, table_path
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The Url.join method treats the trailing slash as something important. In the case where it
|
||||||
|
* doesn't exist, this should be added to ensure the join() works
|
||||||
|
*/
|
||||||
|
let base_uri = match base_uri.chars().last() {
|
||||||
|
Some(last) => {
|
||||||
|
if last != '/' {
|
||||||
|
format!("{}/", base_uri)
|
||||||
|
} else {
|
||||||
|
base_uri.to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
None => base_uri.to_string(),
|
||||||
|
};
|
||||||
|
|
||||||
match table_path.as_path().to_str() {
|
match table_path.as_path().to_str() {
|
||||||
Some(table_path_str) => match Url::parse(base_uri) {
|
Some(table_path_str) => match Url::parse(&base_uri) {
|
||||||
Ok(url) => Ok(url.join(table_path_str)?),
|
Ok(url) => Ok(url.join(table_path_str)?),
|
||||||
Err(url::ParseError::RelativeUrlWithoutBase) => filepath_to_url(&table_path),
|
Err(url::ParseError::RelativeUrlWithoutBase) => {
|
||||||
|
// If we don't have a base we need to convert this to a base and try again
|
||||||
|
let base_uri = filepath_to_url(&PathBuf::from(base_uri))?;
|
||||||
|
println!("Foo: {} {:?}", base_uri, table_path);
|
||||||
|
table_path_to_uri(&base_uri.into_string(), &table_path)
|
||||||
|
}
|
||||||
Err(other) => Err(other.into()),
|
Err(other) => Err(other.into()),
|
||||||
},
|
},
|
||||||
None => Err(anyhow!("Failed to convert a PathBuf")),
|
None => Err(anyhow!("Failed to convert to a PathBuf")),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -132,7 +153,26 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_path_to_url_relative() {
|
fn test_table_path_to_uri_with_local_base() {
|
||||||
|
let _ = pretty_env_logger::try_init();
|
||||||
|
let result = table_path_to_uri("output", &PathBuf::from("relative"));
|
||||||
|
assert!(result.is_ok());
|
||||||
|
let result = result.unwrap();
|
||||||
|
|
||||||
|
let pattern = "output/relative";
|
||||||
|
// Make sure that the output matches the pattern we expect
|
||||||
|
// Doing this with a matches() since the root the result will be dependent on the test
|
||||||
|
// runner's location
|
||||||
|
assert_eq!(
|
||||||
|
1,
|
||||||
|
result.path().matches(pattern).collect::<Vec<&str>>().len(),
|
||||||
|
"result was: {}",
|
||||||
|
result
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_path_to_uri_relative() {
|
||||||
let simple = filepath_to_url(&PathBuf::from("relative"));
|
let simple = filepath_to_url(&PathBuf::from("relative"));
|
||||||
assert!(simple.is_ok());
|
assert!(simple.is_ok());
|
||||||
let simple = simple.unwrap();
|
let simple = simple.unwrap();
|
||||||
|
@ -140,7 +180,7 @@ mod tests {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_path_to_url_relative_slash() {
|
fn test_path_to_uri_relative_slash() {
|
||||||
let simple = filepath_to_url(&PathBuf::from("./relative"));
|
let simple = filepath_to_url(&PathBuf::from("./relative"));
|
||||||
assert!(simple.is_ok());
|
assert!(simple.is_ok());
|
||||||
let simple = simple.unwrap();
|
let simple = simple.unwrap();
|
||||||
|
|
|
@ -20,7 +20,7 @@ struct CLIOptions {
|
||||||
async fn main() -> Result<(), anyhow::Error> {
|
async fn main() -> Result<(), anyhow::Error> {
|
||||||
dotenv().ok();
|
dotenv().ok();
|
||||||
match std::env::var("RUST_LOG") {
|
match std::env::var("RUST_LOG") {
|
||||||
Ok(_) => {},
|
Ok(_) => {}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
// Default to at least warn level logging
|
// Default to at least warn level logging
|
||||||
std::env::set_var("RUST_LOG", "warn");
|
std::env::set_var("RUST_LOG", "warn");
|
||||||
|
@ -57,7 +57,7 @@ async fn main() -> Result<(), anyhow::Error> {
|
||||||
error!("Failed to open table: {:?}", e);
|
error!("Failed to open table: {:?}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Failed to use local path to compute table path: {:?}", e);
|
error!("Failed to use local path to compute table path: {:?}", e);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue