Checkpointing some work before switching from one machine to another

In the process of moving some command line options around

README.adoc

@@ -11,6 +11,25 @@ been created, it not be modified when the `.yml` has been modified.
 ====
 
 
+== Running
+
+.Command line arguments
+[source,bash]
+----
+Optional arguments:
+  -h, --help       Print this very helpful message
+  -f, --from FROM  Path which contains yaml table definitions
+  -t, --to TO      Destination path to create/migrate the tables
+---
+
+.Creating local Delta Tables
+[source,bash]
+----
+carto -f schemas -t output-tables
+----
+
+
+
 == Development
 
 Carto is built in Rust, so assuming you have a modern link:https://rustup.sh[Rust] toolchain installed, you should be able to execute:

examples/simple.yml

@@ -0,0 +1,17 @@
+#
+# This is a simple table that will keep track of weather data
+---
+type: struct
+fields:
+  - name: timestamp
+    type: timestamp
+    nullable: false
+  - name: temp
+    type: integer
+    nullable: false
+  - name: lat
+    type: double
+    nullable: false
+  - name: long
+    type: double
+    nullable: false

src/lib.rs

@@ -65,8 +65,8 @@ pub fn table_path_to_uri(base_uri: &str, table_path: &PathBuf) -> Result<Url, an
 /*
  * Convert the path of a yaml file to the path of the table in S3
  */
-pub fn path_to_table(path: &Path) -> Result<PathBuf, anyhow::Error> {
-    let path = path.strip_prefix("namespaces")?;
+pub fn localpath_to_table(root: &str, local_path: &Path) -> Result<PathBuf, anyhow::Error> {
+    let path = local_path.strip_prefix(root)?;
 
     if let Some(table_prefix) = path.parent() {
         match path.file_stem() {
@@ -107,7 +107,7 @@ mod tests {
     fn test_path_to_table() {
         let path = Path::new("namespaces/home/pge/electric_bronze.yml");
 
-        let table_path = path_to_table(&path).expect("Failed to pull the path");
+        let table_path = localpath_to_table("namespaces", &path).expect("Failed to pull the path");
         assert_eq!(table_path, PathBuf::from("home/pge/electric_bronze"));
     }
 

src/main.rs

@@ -19,36 +19,51 @@ struct CLIOptions {
 #[async_std::main]
 async fn main() -> Result<(), anyhow::Error> {
     dotenv().ok();
+    match std::env::var("RUST_LOG") {
+        Ok(_) => {},
+        Err(_) => {
+            // Default to at least warn level logging
+            std::env::set_var("RUST_LOG", "warn");
+        }
+    }
     pretty_env_logger::init();
 
     let opts = CLIOptions::parse_args_default_or_exit();
     debug!("Starting with options: {:?}", opts);
 
     use glob::glob;
+    let pattern = format!("{}/**/*.yml", opts.from);
+    debug!("Using glob pattern: {}", pattern);
 
-    for entry in glob(&format!("{}/**/*.yml", opts.from)).expect("Failed to read glob pattern") {
+    for entry in glob(&pattern).expect("Failed to read glob pattern") {
         match entry {
             Ok(path) => {
-                if let Ok(table_path) = carto::path_to_table(&path) {
-                    let url = carto::table_path_to_uri(&opts.to, &table_path)
-                        .expect("Failed to create the table URI");
-                    debug!("Attempting to open table at: {}", url);
+                debug!("Considering: {:?}", path);
+                match carto::localpath_to_table(&opts.from, &path) {
+                    Ok(table_path) => {
+                        let url = carto::table_path_to_uri(&opts.to, &table_path)
+                            .expect("Failed to create the table URI");
+                        debug!("Attempting to open table at: {}", url);
 
-                    match deltalake::open_table(url.clone()).await {
-                        Ok(table) => {
-                            debug!("table: {:?}", table);
-                        }
-                        Err(deltalake::DeltaTableError::NotATable(st)) => {
-                            warn!("Table does not exist {}", st);
-                            carto::create_table(url, path).await?;
-                        }
-                        Err(e) => {
-                            error!("Failed to open table: {:?}", e);
+                        match deltalake::open_table(url.clone()).await {
+                            Ok(table) => {
+                                debug!("table: {:?}", table);
+                            }
+                            Err(deltalake::DeltaTableError::NotATable(st)) => {
+                                warn!("Table does not exist {}", st);
+                                carto::create_table(url, path).await?;
+                            }
+                            Err(e) => {
+                                error!("Failed to open table: {:?}", e);
+                            }
                         }
+                    },
+                    Err(e) => {
+                        error!("Failed to use local path to compute table path: {:?}", e);
                     }
                 }
             }
-            Err(e) => println!("{:?}", e),
+            Err(e) => error!("{:?}", e),
         }
     }
     Ok(())