Enable the definition of --from to include a local relative path

This requires some shenanigans to ensure that carto generates a file:// URL. I'm
sure there are more tests which should be added here

.gitignore

@@ -14,3 +14,5 @@ Cargo.lock
 # MSVC Windows builds of rustc generate these, which store debugging information
 *.pdb
 
+# Example output for local testing
+output/

Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 anyhow = "*"
 async-std = { version = "1", features = ["attributes", "tokio1"] }
 chrono = "0.4"
-deltalake = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["s3"]}
+deltalake = { git = "https://github.com/delta-io/delta-rs", branch = "main", features = ["s3", "parquet"]}
 dotenv = "*"
 glob = "0.3"
 gumdrop = "0.8"

examples/simple.yml

@@ -6,12 +6,17 @@ fields:
   - name: timestamp
     type: timestamp
     nullable: false
+    # Make this optional
+    metadata:
   - name: temp
     type: integer
     nullable: false
+    metadata:
   - name: lat
     type: double
     nullable: false
+    metadata:
   - name: long
     type: double
     nullable: false
+    metadata:

src/lib.rs

@@ -52,13 +52,34 @@ pub fn table_path_to_uri(base_uri: &str, table_path: &PathBuf) -> Result<Url, an
         "Converting table to URI: base: {} path: {:?}",
         base_uri, table_path
     );
+
+    /*
+     * The Url.join method treats the trailing slash as something important. In the case where it
+     * doesn't exist, this should be added to ensure the join() works
+     */
+    let base_uri = match base_uri.chars().last() {
+        Some(last) => {
+            if last != '/' {
+                format!("{}/", base_uri)
+            } else {
+                base_uri.to_string()
+            }
+        }
+        None => base_uri.to_string(),
+    };
+
     match table_path.as_path().to_str() {
-        Some(table_path_str) => match Url::parse(base_uri) {
+        Some(table_path_str) => match Url::parse(&base_uri) {
             Ok(url) => Ok(url.join(table_path_str)?),
-            Err(url::ParseError::RelativeUrlWithoutBase) => filepath_to_url(&table_path),
+            Err(url::ParseError::RelativeUrlWithoutBase) => {
+                // If we don't have a base we need to convert this to a base and try again
+                let base_uri = filepath_to_url(&PathBuf::from(base_uri))?;
+                println!("Foo: {} {:?}", base_uri, table_path);
+                table_path_to_uri(&base_uri.into_string(), &table_path)
+            }
             Err(other) => Err(other.into()),
         },
-        None => Err(anyhow!("Failed to convert a PathBuf")),
+        None => Err(anyhow!("Failed to convert to a PathBuf")),
     }
 }
 
@@ -132,7 +153,26 @@ mod tests {
     }
 
     #[test]
-    fn test_path_to_url_relative() {
+    fn test_table_path_to_uri_with_local_base() {
+        let _ = pretty_env_logger::try_init();
+        let result = table_path_to_uri("output", &PathBuf::from("relative"));
+        assert!(result.is_ok());
+        let result = result.unwrap();
+
+        let pattern = "output/relative";
+        // Make sure that the output matches the pattern we expect
+        // Doing this with a matches() since the root the result will be dependent on the test
+        // runner's location
+        assert_eq!(
+            1,
+            result.path().matches(pattern).collect::<Vec<&str>>().len(),
+            "result was: {}",
+            result
+        );
+    }
+
+    #[test]
+    fn test_path_to_uri_relative() {
         let simple = filepath_to_url(&PathBuf::from("relative"));
         assert!(simple.is_ok());
         let simple = simple.unwrap();
@@ -140,7 +180,7 @@ mod tests {
     }
 
     #[test]
-    fn test_path_to_url_relative_slash() {
+    fn test_path_to_uri_relative_slash() {
         let simple = filepath_to_url(&PathBuf::from("./relative"));
         assert!(simple.is_ok());
         let simple = simple.unwrap();

src/main.rs

@@ -20,7 +20,7 @@ struct CLIOptions {
 async fn main() -> Result<(), anyhow::Error> {
     dotenv().ok();
     match std::env::var("RUST_LOG") {
-        Ok(_) => {},
+        Ok(_) => {}
         Err(_) => {
             // Default to at least warn level logging
             std::env::set_var("RUST_LOG", "warn");
@@ -57,7 +57,7 @@ async fn main() -> Result<(), anyhow::Error> {
                                 error!("Failed to open table: {:?}", e);
                             }
                         }
-                    },
+                    }
                     Err(e) => {
                         error!("Failed to use local path to compute table path: {:?}", e);
                     }