Example what the different directories are about in the main readme

This should make the compile cycles a little bit more sensible when just working
on the CLI components of the agent, for example.

Cargo.lock

@@ -1552,6 +1552,7 @@ version = "0.1.0"
 dependencies = [
  "flate2",
  "gumdrop",
+ "otto-models",
  "serde 1.0.117",
  "serde_yaml",
  "tar",
@@ -1564,9 +1565,7 @@ dependencies = [
  "async-std",
  "log",
  "os_pipe",
- "osp",
  "otto-models",
- "pretty_env_logger 0.4.0",
  "serde 1.0.117",
  "serde_json",
  "serde_yaml",
@@ -1576,6 +1575,19 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "otto-agent-cli"
+version = "0.1.0"
+dependencies = [
+ "async-std",
+ "log",
+ "otto-agent",
+ "pretty_env_logger 0.4.0",
+ "serde 1.0.117",
+ "serde_json",
+ "serde_yaml",
+]
+
 [[package]]
 name = "otto-auctioneer"
 version = "0.1.0"

Cargo.toml

@@ -1,9 +1,11 @@
 [workspace]
 
 members = [
-    "agent",
-    "models",
-    "osp",
+    "cli/agent",
+    "cli/osp",
+
+    "crates/agent",
+    "crates/models",
 
     "services/auctioneer",
     "services/eventbus",

README.adoc

@@ -38,6 +38,36 @@ The service-to-service communication is all done with JSON over HTTP.
 In order to contribute to the project, please join us on IRC or open pull
 requests and issues.
 
+== Subdirectories
+
+Each subdirectory should have its own README with a little more information,
+but at a glance:
+
+=== `cli/`
+
+This directory contains all the command-line interfaces for Otto.
+
+=== `crates/`
+
+This directory contains the various pieces of shared code
+
+=== `rfcs/`
+
+RFCs (Request for Comment) are design documents for different patterns or
+subcomponents within Otto.
+
+=== `services/`
+
+Projects in this directory are Otto's mesh of services, which speak HTTP to
+provide different aspects of functionality for the Otto project.
+
+=== `stdlib/`
+
+The Otto step "Standard Library." In essence, these are all the step libraries
+that are assumed to be installed by default with Otto.
+
+
+
 == Problems to Solve
 
 Below is an incomplete listing of the problems Otto aims to solve for:

cli/README.adoc

@@ -0,0 +1,5 @@
+= Command Line Interfaces
+
+This directory contains the various command-line interfaces used for different
+aspects of Otto.
+

cli/agent/Cargo.toml

@@ -0,0 +1,19 @@
+[package]
+name = "otto-agent-cli"
+version = "0.1.0"
+authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
+edition = "2018"
+
+[[bin]]
+name = "otto-agent"
+path = "src/main.rs"
+
+[dependencies]
+async-std = { version = "~1.7", features = ["attributes"]}
+log = "~0.4.8"
+otto-agent= { path = "../../crates/agent" }
+pretty_env_logger = "~0.4.0"
+serde_json = "~1.0.59"
+# Needed for reading manifest yamls
+serde_yaml = "~0.8.13"
+serde = {version = "~1.0.117", features = ["rc", "derive"]}

cli/osp/Cargo.toml

@@ -4,10 +4,6 @@ version = "0.1.0"
 authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
 edition = "2018"
 
-[lib]
-name = "osp"
-path = "src/lib.rs"
-
 [[bin]]
 name = "osp"
 path = "src/main.rs"
@@ -15,6 +11,7 @@ path = "src/main.rs"
 [dependencies]
 flate2 = "~1.0.18"
 gumdrop = "~0.8.0"
+otto-models = { path = "../../crates/models" }
 serde_yaml = "~0.8.13"
 serde = {version = "~1.0.117", features = ["rc", "derive"]}
 tar = "~0.4.30"

cli/osp/src/main.rs

@@ -0,0 +1,72 @@
+use otto_models::osp::Manifest;
+use std::fs::File;
+use std::path::Path;
+
+/**
+ * Create an artifact from the given manifest
+ */
+fn create_artifact(manifest: &Manifest, dir: &Path, output: &Path) -> Result<(), std::io::Error> {
+    use flate2::write::GzEncoder;
+    use flate2::Compression;
+
+    let tar_gz = File::create(output)?;
+    let enc = GzEncoder::new(tar_gz, Compression::default());
+    let mut tar = tar::Builder::new(enc);
+    let mut file = File::open(dir.join(Path::new("manifest.yml")))?;
+    tar.append_file(format!("{}/manifest.yml", manifest.symbol), &mut file)?;
+
+    for include in manifest.includes.iter() {
+        let mut f = File::open(match include.name.starts_with("./") {
+            true => {
+                // Relative to dir
+                dir.join(&include.name)
+            }
+            false => {
+                // Relative to $PWD
+                Path::new(&include.name).to_path_buf()
+            }
+        })?;
+
+        let archive_path = format!(
+            "{}/{}",
+            manifest.symbol,
+            match include.flatten {
+                true => {
+                    let p = Path::new(&include.name);
+                    p.file_name().unwrap().to_str().unwrap()
+                }
+                false => &include.name,
+            }
+        );
+        tar.append_file(&archive_path, &mut f)
+            .expect(&format!("Failed to append file: {}", &archive_path));
+    }
+    Ok(())
+}
+
+fn main() -> std::io::Result<()> {
+    // TODO use gumdrop for real argument parsing
+    let args: Vec<String> = std::env::args().collect();
+
+    if args.len() != 2 {
+        panic!("osp can only accept a single argument: the directory containing a manifest.yml");
+    }
+
+    let dir = Path::new(&args[1]);
+    if !dir.is_dir() {
+        panic!("The argument must be a directory");
+    }
+    let manifest = dir.join(Path::new("manifest.yml"));
+    let manifest = serde_yaml::from_reader::<File, Manifest>(File::open(manifest)?)
+        .expect("Failed to parse manifest.yml");
+
+    let step_name = dir
+        .file_name()
+        .expect("Failed to unwrap the directory filename")
+        .to_str()
+        .unwrap();
+    println!("default out: {:#?}", step_name);
+
+    create_artifact(&manifest, &dir, Path::new(&format!("{}.tar.gz", step_name)))?;
+    Ok(())
+}

crates/README.adoc

@@ -0,0 +1,5 @@
+= Crates
+
+This directory contains the various internally re-usable crates for Otto. These
+are intended to be for sharing code or object definitions across multiple
+services or between command lines and services.

crates/agent/Cargo.toml

@@ -4,21 +4,11 @@ version = "0.1.0"
 authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
 edition = "2018"
 
-[lib]
-name = "otto_agent"
-path = "src/lib.rs"
-
-[[bin]]
-name = "otto-agent"
-path = "src/main.rs"
-
 [dependencies]
 async-std = { version = "~1.7", features = ["attributes"]}
 log = "~0.4.8"
-otto-models = { path = "../models" }
-osp = { path = "../osp" }
 os_pipe = "~0.9.2"
-pretty_env_logger = "~0.4.0"
+otto-models = { path = "../../crates/models" }
 serde_json = "~1.0.59"
 # Needed for reading manifest yamls
 serde_yaml = "~0.8.13"

crates/agent/src/lib.rs

@@ -288,7 +288,7 @@ mod tests {
             parameters: StepParameters::Positional(vec![params]),
         };
         let manifests =
-            load_manifests_for("../stdlib", &vec![step]).expect("Failed to look into stdlib?");
+            load_manifests_for("../../stdlib", &vec![step]).expect("Failed to look into stdlib?");
         assert!(manifests.len() > 0);
     }
 
@@ -297,7 +297,7 @@ mod tests {
         use serde_json::Value;
         let arg = Value::String("ps".to_string());
         let parameters = StepParameters::Positional(vec![arg.clone()]);
-        let manifests = load_manifests_for_symbols("../stdlib", vec!["sh".to_string()])
+        let manifests = load_manifests_for_symbols("../../stdlib", vec!["sh".to_string()])
             .expect("Failed to look into stdlib?");
         let loaded = manifests.get("sh").expect("Must have a `sh` manifest");
 
@@ -315,7 +315,7 @@ mod tests {
         use serde_json::Value;
         let arg = Value::String("hi".to_string());
         let parameters = StepParameters::Positional(vec![arg.clone(), arg.clone()]);
-        let manifests = load_manifests_for_symbols("../stdlib", vec!["echo".to_string()])
+        let manifests = load_manifests_for_symbols("../../stdlib", vec!["echo".to_string()])
             .expect("Failed to look into stdlib?");
         let loaded = manifests.get("echo").expect("Must have a `echo` manifest");
 

crates/models/.gitignore

@@ -0,0 +1 @@
+target/

crates/models/src/lib.rs

@@ -3,6 +3,8 @@ use serde_json::Value;
 use std::collections::HashMap;
 use uuid::Uuid;
 
+pub mod osp;
+
 /**
  * A Pipeline contains the total configuration and steps for a single pipeline run
  */

crates/models/src/osp.rs

@@ -0,0 +1,53 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Manifest {
+    pub symbol: String,
+    #[serde(default = "default_false")]
+    pub cache: bool,
+    pub description: String,
+    pub includes: Vec<Include>,
+    pub entrypoint: Entrypoint,
+    pub parameters: Vec<Parameter>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Include {
+    pub name: String,
+    #[serde(default = "default_false")]
+    pub flatten: bool,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Entrypoint {
+    pub path: std::path::PathBuf,
+    #[serde(default = "default_false")]
+    pub multiarch: bool,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Parameter {
+    pub name: String,
+    pub required: bool,
+    #[serde(rename = "type")]
+    pub p_type: ParameterType,
+    pub description: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum ParameterType {
+    #[serde(rename = "string")]
+    StringParameter,
+    #[serde(rename = "boolean")]
+    BoolParameter,
+    #[serde(rename = "block")]
+    BlockParameter,
+}
+
+/** Simple function for serde defaults */
+fn default_false() -> bool {
+    false
+}
+
+#[cfg(test)]
+mod tests {}

osp/src/lib.rs

@@ -1,98 +0,0 @@
-use flate2::write::GzEncoder;
-use flate2::Compression;
-use serde::{Deserialize, Serialize};
-use std::fs::File;
-use std::path::{Path, PathBuf};
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct Manifest {
-    pub symbol: String,
-    #[serde(default = "default_false")]
-    pub cache: bool,
-    pub description: String,
-    pub includes: Vec<Include>,
-    pub entrypoint: Entrypoint,
-    pub parameters: Vec<Parameter>,
-}
-
-impl Manifest {
-    /**
-     * Create an artifact from the given manifest
-     */
-    pub fn create_artifact(&self, dir: &Path, output: &Path) -> Result<(), std::io::Error> {
-        let tar_gz = File::create(output)?;
-        let enc = GzEncoder::new(tar_gz, Compression::default());
-        let mut tar = tar::Builder::new(enc);
-        let mut manifest = File::open(dir.join(Path::new("manifest.yml")))?;
-        tar.append_file(format!("{}/manifest.yml", self.symbol), &mut manifest)?;
-
-        for include in self.includes.iter() {
-            let mut f = File::open(match include.name.starts_with("./") {
-                true => {
-                    // Relative to dir
-                    dir.join(&include.name)
-                }
-                false => {
-                    // Relative to $PWD
-                    Path::new(&include.name).to_path_buf()
-                }
-            })?;
-
-            let archive_path = format!(
-                "{}/{}",
-                self.symbol,
-                match include.flatten {
-                    true => {
-                        let p = Path::new(&include.name);
-                        p.file_name().unwrap().to_str().unwrap()
-                    }
-                    false => &include.name,
-                }
-            );
-            tar.append_file(&archive_path, &mut f)
-                .expect(&format!("Failed to append file: {}", &archive_path));
-        }
-        Ok(())
-    }
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct Include {
-    name: String,
-    #[serde(default = "default_false")]
-    flatten: bool,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct Entrypoint {
-    pub path: PathBuf,
-    #[serde(default = "default_false")]
-    multiarch: bool,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub struct Parameter {
-    pub name: String,
-    pub required: bool,
-    #[serde(rename = "type")]
-    pub p_type: ParameterType,
-    pub description: String,
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-pub enum ParameterType {
-    #[serde(rename = "string")]
-    StringParameter,
-    #[serde(rename = "boolean")]
-    BoolParameter,
-    #[serde(rename = "block")]
-    BlockParameter,
-}
-
-/** Simple function for serde defaults */
-fn default_false() -> bool {
-    false
-}
-
-#[cfg(test)]
-mod tests {}

osp/src/main.rs

@@ -1,30 +0,0 @@
-use osp::Manifest;
-use std::fs::File;
-use std::path::Path;
-
-fn main() -> std::io::Result<()> {
-    // TODO use gumdrop for real argument parsing
-    let args: Vec<String> = std::env::args().collect();
-
-    if args.len() != 2 {
-        panic!("osp can only accept a single argument: the directory containing a manifest.yml");
-    }
-
-    let dir = Path::new(&args[1]);
-    if !dir.is_dir() {
-        panic!("The argument must be a directory");
-    }
-    let manifest = dir.join(Path::new("manifest.yml"));
-    let manifest = serde_yaml::from_reader::<File, Manifest>(File::open(manifest)?)
-        .expect("Failed to parse manifest.yml");
-
-    let step_name = dir
-        .file_name()
-        .expect("Failed to unwrap the directory filename")
-        .to_str()
-        .unwrap();
-    println!("default out: {:#?}", step_name);
-
-    manifest.create_artifact(&dir, Path::new(&format!("{}.tar.gz", step_name)))?;
-    Ok(())
-}

services/local-orchestrator/Cargo.toml

@@ -7,8 +7,8 @@ edition = "2018"
 [dependencies]
 async-std = { version = "~1.7", features = ["attributes"]}
 log = "~0.4.11"
-otto-agent = { path = "../../agent" }
-otto-models = { path = "../../models" }
+otto-agent = { path = "../../crates/agent" }
+otto-models = { path = "../../crates/models" }
 pretty_env_logger = "~0.4.0"
 serde = {version = "~1.0.117", features = ["rc", "derive"]}
 serde_json = "~1.0.59"

services/parser/Cargo.toml

@@ -15,7 +15,7 @@ path = "src/main.rs"
 [dependencies]
 async-std = { version = "1.6.5", features = ["attributes"]}
 log = "~0.4.11"
-otto-models = { path = "../../models" }
+otto-models = { path = "../../crates/models" }
 pest = "~2.1.3"
 pest_derive = "~2.1.0"
 pretty_env_logger = "~0.4.0"

stdlib/archive/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 async-std = { version = "~1.7", features = ["attributes"] }
 flate2 = "~1.0.18"
 glob = "~0.3.0"
-otto-agent = { path = "../../agent" }
+otto-agent = { path = "../../crates/agent" }
 serde = {version = "~1.0.117", features = ["derive"]}
 # Not using the curl-client default feature to ensure that builds won't require
 # libcurl for now

stdlib/dir/Cargo.toml

@@ -7,5 +7,5 @@ edition = "2018"
 [dependencies]
 serde_yaml = "~0.8.13"
 serde = {version = "~1.0.117", features = ["derive"]}
-otto-agent = { path = "../../agent" }
-otto-models = { path = "../../models" }
+otto-agent = { path = "../../crates/agent" }
+otto-models = { path = "../../crates/models" }

stdlib/error/Cargo.toml

@@ -5,5 +5,5 @@ authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
 edition = "2018"
 
 [dependencies]
-otto-agent = { path = "../../agent" }
+otto-agent = { path = "../../crates/agent" }
 serde = {version = "~1.0.117", features = ["derive"]}

stdlib/git/Cargo.toml

@@ -6,6 +6,6 @@ edition = "2018"
 
 [dependencies]
 git2 = "~0.13.12"
-otto-agent = { path = "../../agent" }
+otto-agent = { path = "../../crates/agent" }
 serde = {version = "~1.0.117", features = ["derive"]}
 url = "~2.2.0"

stdlib/sh/Cargo.toml

@@ -5,6 +5,6 @@ authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
 edition = "2018"
 
 [dependencies]
-otto-agent = { path = "../../agent" }
+otto-agent = { path = "../../crates/agent" }
 serde = {version = "~1.0.117", features = ["derive"]}
 tempfile = "~3.1.0"

stdlib/unarchive/Cargo.toml

@@ -6,6 +6,6 @@ edition = "2018"
 
 [dependencies]
 async-std = { version = "~1.7", features = ["attributes"] }
-otto-agent = { path = "../../agent" }
+otto-agent = { path = "../../crates/agent" }
 serde = {version = "~1.0.117", features = ["derive"]}
 surf = { version = "~2.1.0", features = ["h1-client"]}