Refactor the ServerConfig to allow for loading configuration from a directory

This makes it possible to load fragments of configuration into one large
configuration struct (ServerConfig). This will be really useful for
incrementally dropping configuration with a configuration management tool or
just git

Cargo.toml

@@ -12,15 +12,20 @@ name = "synchronik-agent"
 path = "src/agent/main.rs"
 
 [dependencies]
+anyhow = "*"
 async-std = { version = "1", features = ["attributes", "tokio1"] }
 chrono = "0.4"
 dotenv = "~0.15"
 driftwood = "0"
+# Library for handling filesystem globs
+glob = "0.3"
 # Command line parsing
 gumdrop = "0.8"
 handlebars = { version = "4", features = ["dir_source"] }
 html-escape = "0.2"
 log = "~0.4.8"
+# Used for filesystem notifications to reload data live
+notify = "5"
 # Needed for GitHub API calls
 octocrab = "0.18"
 os_pipe = "1"

examples/server.yml

@@ -1,3 +1,6 @@
+#
+# Example configuration of the Synchronik server. This file is also read by
+# some configuration parsing unit tests
 ---
 agents:
   'Local':

src/server/config.rs

@@ -1,5 +1,7 @@
 use std::collections::HashMap;
+use std::path::PathBuf;
 
+use log::*;
 use serde::{Deserialize, Serialize};
 use url::Url;
 
@@ -94,8 +96,73 @@ impl ServerConfig {
     pub fn has_project(&self, name: &str) -> bool {
         self.projects.contains_key(name)
     }
+
+    /*
+     * Load the ServerConfig from the given file.
+     */
+    fn from_filepath(path: &PathBuf) -> anyhow::Result<Self> {
+        let config_file = std::fs::File::open(path).expect("Failed to open config file");
+        serde_yaml::from_reader(config_file).map_err(anyhow::Error::from)
+    }
+
+    /*
+     * Load the ServerConfig from an amalgamation of yaml in the given directory
+     */
+    fn from_dirpath(path: &PathBuf) -> anyhow::Result<Self> {
+        use glob::glob;
+        use std::fs::File;
+
+        let pattern = format!("{}/**/*.yml", path.as_path().to_string_lossy());
+        debug!("Loading config from directory with pattern: {}", pattern);
+
+        let mut values: Vec<serde_yaml::Value> = vec![];
+
+        for entry in glob(&pattern).expect("Failed to read glob pattern") {
+            match entry {
+                Ok(path) => {
+                    if let Ok(file) = File::open(path) {
+                        if let Ok(value) = serde_yaml::from_reader(file) {
+                            values.push(value);
+                        }
+                    }
+                }
+                Err(e) => error!("Failed to read entry: {:?}", e),
+            }
+        }
+
+        /*
+         * At this point we should have enough partials to do a coercion to the ServerConfig
+         * structure
+         */
+        let mut v = serde_yaml::Value::Null;
+        for m in values.drain(0..) {
+            merge_yaml(&mut v, m);
+        }
+        serde_yaml::from_value(v).map_err(anyhow::Error::from)
+    }
+
+    /*
+     * Take the given path and do the necessary deserialization whether a file or a directory
+     */
+    pub fn from_path(path: &PathBuf) -> anyhow::Result<Self> {
+        if !path.exists() {
+            error!("The provided configuration path does not exist: {:?}", path);
+            return Err(std::io::Error::from(std::io::ErrorKind::NotFound).into());
+        }
+
+        match path.is_file() {
+            true => Self::from_filepath(&path),
+            false => Self::from_dirpath(&path),
+        }
+    }
 }
 
+/*
+ * Default trait implementation for ServerConfig, will result in an empty set of agents and
+ * projects
+ *
+ * Not really useful for anything other than tests
+ */
 impl Default for ServerConfig {
     fn default() -> Self {
         Self {
@@ -104,3 +171,85 @@ impl Default for ServerConfig {
         }
     }
 }
+
+/*
+ * Merge two Valus from <https://stackoverflow.com/a/67743348>
+ */
+fn merge_yaml(a: &mut serde_yaml::Value, b: serde_yaml::Value) {
+    match (a, b) {
+        (a @ &mut serde_yaml::Value::Mapping(_), serde_yaml::Value::Mapping(b)) => {
+            let a = a.as_mapping_mut().unwrap();
+            for (k, v) in b {
+                if v.is_sequence() && a.contains_key(&k) && a[&k].is_sequence() {
+                    let mut _b = a.get(&k).unwrap().as_sequence().unwrap().to_owned();
+                    _b.append(&mut v.as_sequence().unwrap().to_owned());
+                    a[&k] = serde_yaml::Value::from(_b);
+                    continue;
+                }
+                if !a.contains_key(&k) {
+                    a.insert(k.to_owned(), v.to_owned());
+                } else {
+                    merge_yaml(&mut a[&k], v);
+                }
+            }
+        }
+        (a, b) => *a = b,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::path::PathBuf;
+
+    #[test]
+    fn test_serverconfig_from_filepath() {
+        let path = PathBuf::from("./examples/server.yml");
+        let config = ServerConfig::from_path(&path);
+        match config {
+            Ok(config) => {
+                assert_eq!(
+                    config.agents.len(),
+                    1,
+                    "Unexpected number of agents: {:?}",
+                    config.agents
+                );
+            }
+            Err(e) => {
+                assert!(false, "Failed to process ServerConfig: {:?}", e);
+            }
+        }
+    }
+
+    #[test]
+    fn test_serverconfig_non0xistent() {
+        let path = PathBuf::from("./non-existing/path/withstuff");
+        let config = ServerConfig::from_path(&path);
+        assert!(config.is_err());
+    }
+
+    #[test]
+    fn test_serverconfig_from_filedir() {
+        let path = PathBuf::from("./examples/synchronik.d");
+        let config = ServerConfig::from_path(&path);
+        match config {
+            Ok(config) => {
+                assert_eq!(
+                    config.agents.len(),
+                    1,
+                    "Unexpected number of agents: {:?}",
+                    config.agents
+                );
+                assert_eq!(
+                    config.projects.len(),
+                    2,
+                    "Unexpected number of projects: {:?}",
+                    config.projects
+                );
+            }
+            Err(e) => {
+                assert!(false, "Failed to process ServerConfig: {:?}", e);
+            }
+        }
+    }
+}

src/server/main.rs

@@ -82,10 +82,7 @@ async fn main() -> Result<(), tide::Error> {
     debug!("Starting with options: {:?}", opts);
 
     let config = match opts.config {
-        Some(path) => {
-            let config_file = std::fs::File::open(path).expect("Failed to open config file");
-            serde_yaml::from_reader(config_file).expect("Failed to read config file")
-        }
+        Some(path) => ServerConfig::from_path(&path)?,
         None => ServerConfig::default(),
     };
     debug!("Starting with config: {:?}", config);

src/server/routes.rs

@@ -7,8 +7,8 @@
 use log::*;
 use tide::{Body, Request};
 
-use crate::AppState;
 use crate::models::Project;
+use crate::AppState;
 
 /**
  *  GET /