From ee2510ed6f94436e7b7861b82dbb0a1197ca4a0e Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Thu, 26 Nov 2020 10:40:37 -0800 Subject: [PATCH] Refactor the source tree to separate local executables (CLIs) and models a bit more This should make the compile cycles a little bit more sensible when just working on the CLI components of the agent, for example. --- Cargo.lock | 16 ++++- Cargo.toml | 8 ++- {agent => cli/agent}/.gitignore | 0 cli/agent/Cargo.toml | 19 +++++ {agent => cli/agent}/src/main.rs | 0 {models => cli/osp}/.gitignore | 0 {osp => cli/osp}/Cargo.toml | 5 +- {osp => cli/osp}/README.adoc | 0 cli/osp/src/main.rs | 72 +++++++++++++++++++ {osp => crates/agent}/.gitignore | 0 {agent => crates/agent}/Cargo.toml | 12 +--- {agent => crates/agent}/src/control.rs | 0 {agent => crates/agent}/src/lib.rs | 6 +- {agent => crates/agent}/src/step.rs | 0 crates/models/.gitignore | 1 + {models => crates/models}/Cargo.toml | 0 {models => crates/models}/src/lib.rs | 2 + crates/models/src/osp.rs | 53 ++++++++++++++ osp/src/lib.rs | 98 -------------------------- osp/src/main.rs | 30 -------- services/local-orchestrator/Cargo.toml | 4 +- services/parser/Cargo.toml | 2 +- stdlib/archive/Cargo.toml | 2 +- stdlib/dir/Cargo.toml | 4 +- stdlib/error/Cargo.toml | 2 +- stdlib/git/Cargo.toml | 2 +- stdlib/sh/Cargo.toml | 2 +- stdlib/unarchive/Cargo.toml | 2 +- 28 files changed, 181 insertions(+), 161 deletions(-) rename {agent => cli/agent}/.gitignore (100%) create mode 100644 cli/agent/Cargo.toml rename {agent => cli/agent}/src/main.rs (100%) rename {models => cli/osp}/.gitignore (100%) rename {osp => cli/osp}/Cargo.toml (86%) rename {osp => cli/osp}/README.adoc (100%) create mode 100644 cli/osp/src/main.rs rename {osp => crates/agent}/.gitignore (100%) rename {agent => crates/agent}/Cargo.toml (71%) rename {agent => crates/agent}/src/control.rs (100%) rename {agent => crates/agent}/src/lib.rs (97%) rename {agent => crates/agent}/src/step.rs (100%) create mode 100644 crates/models/.gitignore rename {models => crates/models}/Cargo.toml (100%) rename {models => crates/models}/src/lib.rs (99%) create mode 100644 crates/models/src/osp.rs delete mode 100644 osp/src/lib.rs delete mode 100644 osp/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index f7bb194..d528a66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1552,6 +1552,7 @@ version = "0.1.0" dependencies = [ "flate2", "gumdrop", + "otto-models", "serde 1.0.117", "serde_yaml", "tar", @@ -1564,9 +1565,7 @@ dependencies = [ "async-std", "log", "os_pipe", - "osp", "otto-models", - "pretty_env_logger 0.4.0", "serde 1.0.117", "serde_json", "serde_yaml", @@ -1576,6 +1575,19 @@ dependencies = [ "uuid", ] +[[package]] +name = "otto-agent-cli" +version = "0.1.0" +dependencies = [ + "async-std", + "log", + "otto-agent", + "pretty_env_logger 0.4.0", + "serde 1.0.117", + "serde_json", + "serde_yaml", +] + [[package]] name = "otto-auctioneer" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index fdc0262..356b844 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,11 @@ [workspace] members = [ - "agent", - "models", - "osp", + "cli/agent", + "cli/osp", + + "crates/agent", + "crates/models", "services/auctioneer", "services/eventbus", diff --git a/agent/.gitignore b/cli/agent/.gitignore similarity index 100% rename from agent/.gitignore rename to cli/agent/.gitignore diff --git a/cli/agent/Cargo.toml b/cli/agent/Cargo.toml new file mode 100644 index 0000000..c5dca81 --- /dev/null +++ b/cli/agent/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "otto-agent-cli" +version = "0.1.0" +authors = ["R. Tyler Croy "] +edition = "2018" + +[[bin]] +name = "otto-agent" +path = "src/main.rs" + +[dependencies] +async-std = { version = "~1.7", features = ["attributes"]} +log = "~0.4.8" +otto-agent= { path = "../../crates/agent" } +pretty_env_logger = "~0.4.0" +serde_json = "~1.0.59" +# Needed for reading manifest yamls +serde_yaml = "~0.8.13" +serde = {version = "~1.0.117", features = ["rc", "derive"]} diff --git a/agent/src/main.rs b/cli/agent/src/main.rs similarity index 100% rename from agent/src/main.rs rename to cli/agent/src/main.rs diff --git a/models/.gitignore b/cli/osp/.gitignore similarity index 100% rename from models/.gitignore rename to cli/osp/.gitignore diff --git a/osp/Cargo.toml b/cli/osp/Cargo.toml similarity index 86% rename from osp/Cargo.toml rename to cli/osp/Cargo.toml index 8d4506c..94505b0 100644 --- a/osp/Cargo.toml +++ b/cli/osp/Cargo.toml @@ -4,10 +4,6 @@ version = "0.1.0" authors = ["R. Tyler Croy "] edition = "2018" -[lib] -name = "osp" -path = "src/lib.rs" - [[bin]] name = "osp" path = "src/main.rs" @@ -15,6 +11,7 @@ path = "src/main.rs" [dependencies] flate2 = "~1.0.18" gumdrop = "~0.8.0" +otto-models = { path = "../../crates/models" } serde_yaml = "~0.8.13" serde = {version = "~1.0.117", features = ["rc", "derive"]} tar = "~0.4.30" diff --git a/osp/README.adoc b/cli/osp/README.adoc similarity index 100% rename from osp/README.adoc rename to cli/osp/README.adoc diff --git a/cli/osp/src/main.rs b/cli/osp/src/main.rs new file mode 100644 index 0000000..0d3d827 --- /dev/null +++ b/cli/osp/src/main.rs @@ -0,0 +1,72 @@ +use otto_models::osp::Manifest; +use std::fs::File; +use std::path::Path; + +/** + * Create an artifact from the given manifest + */ +fn create_artifact(manifest: &Manifest, dir: &Path, output: &Path) -> Result<(), std::io::Error> { + use flate2::write::GzEncoder; + use flate2::Compression; + + let tar_gz = File::create(output)?; + let enc = GzEncoder::new(tar_gz, Compression::default()); + let mut tar = tar::Builder::new(enc); + let mut file = File::open(dir.join(Path::new("manifest.yml")))?; + tar.append_file(format!("{}/manifest.yml", manifest.symbol), &mut file)?; + + for include in manifest.includes.iter() { + let mut f = File::open(match include.name.starts_with("./") { + true => { + // Relative to dir + dir.join(&include.name) + } + false => { + // Relative to $PWD + Path::new(&include.name).to_path_buf() + } + })?; + + let archive_path = format!( + "{}/{}", + manifest.symbol, + match include.flatten { + true => { + let p = Path::new(&include.name); + p.file_name().unwrap().to_str().unwrap() + } + false => &include.name, + } + ); + tar.append_file(&archive_path, &mut f) + .expect(&format!("Failed to append file: {}", &archive_path)); + } + Ok(()) +} + +fn main() -> std::io::Result<()> { + // TODO use gumdrop for real argument parsing + let args: Vec = std::env::args().collect(); + + if args.len() != 2 { + panic!("osp can only accept a single argument: the directory containing a manifest.yml"); + } + + let dir = Path::new(&args[1]); + if !dir.is_dir() { + panic!("The argument must be a directory"); + } + let manifest = dir.join(Path::new("manifest.yml")); + let manifest = serde_yaml::from_reader::(File::open(manifest)?) + .expect("Failed to parse manifest.yml"); + + let step_name = dir + .file_name() + .expect("Failed to unwrap the directory filename") + .to_str() + .unwrap(); + println!("default out: {:#?}", step_name); + + create_artifact(&manifest, &dir, Path::new(&format!("{}.tar.gz", step_name)))?; + Ok(()) +} diff --git a/osp/.gitignore b/crates/agent/.gitignore similarity index 100% rename from osp/.gitignore rename to crates/agent/.gitignore diff --git a/agent/Cargo.toml b/crates/agent/Cargo.toml similarity index 71% rename from agent/Cargo.toml rename to crates/agent/Cargo.toml index 0f2c2c4..1d00b99 100644 --- a/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -4,21 +4,11 @@ version = "0.1.0" authors = ["R. Tyler Croy "] edition = "2018" -[lib] -name = "otto_agent" -path = "src/lib.rs" - -[[bin]] -name = "otto-agent" -path = "src/main.rs" - [dependencies] async-std = { version = "~1.7", features = ["attributes"]} log = "~0.4.8" -otto-models = { path = "../models" } -osp = { path = "../osp" } os_pipe = "~0.9.2" -pretty_env_logger = "~0.4.0" +otto-models = { path = "../../crates/models" } serde_json = "~1.0.59" # Needed for reading manifest yamls serde_yaml = "~0.8.13" diff --git a/agent/src/control.rs b/crates/agent/src/control.rs similarity index 100% rename from agent/src/control.rs rename to crates/agent/src/control.rs diff --git a/agent/src/lib.rs b/crates/agent/src/lib.rs similarity index 97% rename from agent/src/lib.rs rename to crates/agent/src/lib.rs index 4dce406..a7689db 100644 --- a/agent/src/lib.rs +++ b/crates/agent/src/lib.rs @@ -288,7 +288,7 @@ mod tests { parameters: StepParameters::Positional(vec![params]), }; let manifests = - load_manifests_for("../stdlib", &vec![step]).expect("Failed to look into stdlib?"); + load_manifests_for("../../stdlib", &vec![step]).expect("Failed to look into stdlib?"); assert!(manifests.len() > 0); } @@ -297,7 +297,7 @@ mod tests { use serde_json::Value; let arg = Value::String("ps".to_string()); let parameters = StepParameters::Positional(vec![arg.clone()]); - let manifests = load_manifests_for_symbols("../stdlib", vec!["sh".to_string()]) + let manifests = load_manifests_for_symbols("../../stdlib", vec!["sh".to_string()]) .expect("Failed to look into stdlib?"); let loaded = manifests.get("sh").expect("Must have a `sh` manifest"); @@ -315,7 +315,7 @@ mod tests { use serde_json::Value; let arg = Value::String("hi".to_string()); let parameters = StepParameters::Positional(vec![arg.clone(), arg.clone()]); - let manifests = load_manifests_for_symbols("../stdlib", vec!["echo".to_string()]) + let manifests = load_manifests_for_symbols("../../stdlib", vec!["echo".to_string()]) .expect("Failed to look into stdlib?"); let loaded = manifests.get("echo").expect("Must have a `echo` manifest"); diff --git a/agent/src/step.rs b/crates/agent/src/step.rs similarity index 100% rename from agent/src/step.rs rename to crates/agent/src/step.rs diff --git a/crates/models/.gitignore b/crates/models/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/crates/models/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/models/Cargo.toml b/crates/models/Cargo.toml similarity index 100% rename from models/Cargo.toml rename to crates/models/Cargo.toml diff --git a/models/src/lib.rs b/crates/models/src/lib.rs similarity index 99% rename from models/src/lib.rs rename to crates/models/src/lib.rs index 12c7140..0ce0ef5 100644 --- a/models/src/lib.rs +++ b/crates/models/src/lib.rs @@ -3,6 +3,8 @@ use serde_json::Value; use std::collections::HashMap; use uuid::Uuid; +pub mod osp; + /** * A Pipeline contains the total configuration and steps for a single pipeline run */ diff --git a/crates/models/src/osp.rs b/crates/models/src/osp.rs new file mode 100644 index 0000000..a68d4db --- /dev/null +++ b/crates/models/src/osp.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Manifest { + pub symbol: String, + #[serde(default = "default_false")] + pub cache: bool, + pub description: String, + pub includes: Vec, + pub entrypoint: Entrypoint, + pub parameters: Vec, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Include { + pub name: String, + #[serde(default = "default_false")] + pub flatten: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Entrypoint { + pub path: std::path::PathBuf, + #[serde(default = "default_false")] + pub multiarch: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Parameter { + pub name: String, + pub required: bool, + #[serde(rename = "type")] + pub p_type: ParameterType, + pub description: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum ParameterType { + #[serde(rename = "string")] + StringParameter, + #[serde(rename = "boolean")] + BoolParameter, + #[serde(rename = "block")] + BlockParameter, +} + +/** Simple function for serde defaults */ +fn default_false() -> bool { + false +} + +#[cfg(test)] +mod tests {} diff --git a/osp/src/lib.rs b/osp/src/lib.rs deleted file mode 100644 index 76ee7e3..0000000 --- a/osp/src/lib.rs +++ /dev/null @@ -1,98 +0,0 @@ -use flate2::write::GzEncoder; -use flate2::Compression; -use serde::{Deserialize, Serialize}; -use std::fs::File; -use std::path::{Path, PathBuf}; - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Manifest { - pub symbol: String, - #[serde(default = "default_false")] - pub cache: bool, - pub description: String, - pub includes: Vec, - pub entrypoint: Entrypoint, - pub parameters: Vec, -} - -impl Manifest { - /** - * Create an artifact from the given manifest - */ - pub fn create_artifact(&self, dir: &Path, output: &Path) -> Result<(), std::io::Error> { - let tar_gz = File::create(output)?; - let enc = GzEncoder::new(tar_gz, Compression::default()); - let mut tar = tar::Builder::new(enc); - let mut manifest = File::open(dir.join(Path::new("manifest.yml")))?; - tar.append_file(format!("{}/manifest.yml", self.symbol), &mut manifest)?; - - for include in self.includes.iter() { - let mut f = File::open(match include.name.starts_with("./") { - true => { - // Relative to dir - dir.join(&include.name) - } - false => { - // Relative to $PWD - Path::new(&include.name).to_path_buf() - } - })?; - - let archive_path = format!( - "{}/{}", - self.symbol, - match include.flatten { - true => { - let p = Path::new(&include.name); - p.file_name().unwrap().to_str().unwrap() - } - false => &include.name, - } - ); - tar.append_file(&archive_path, &mut f) - .expect(&format!("Failed to append file: {}", &archive_path)); - } - Ok(()) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Include { - name: String, - #[serde(default = "default_false")] - flatten: bool, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Entrypoint { - pub path: PathBuf, - #[serde(default = "default_false")] - multiarch: bool, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Parameter { - pub name: String, - pub required: bool, - #[serde(rename = "type")] - pub p_type: ParameterType, - pub description: String, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub enum ParameterType { - #[serde(rename = "string")] - StringParameter, - #[serde(rename = "boolean")] - BoolParameter, - #[serde(rename = "block")] - BlockParameter, -} - -/** Simple function for serde defaults */ -fn default_false() -> bool { - false -} - -#[cfg(test)] -mod tests {} diff --git a/osp/src/main.rs b/osp/src/main.rs deleted file mode 100644 index e12198e..0000000 --- a/osp/src/main.rs +++ /dev/null @@ -1,30 +0,0 @@ -use osp::Manifest; -use std::fs::File; -use std::path::Path; - -fn main() -> std::io::Result<()> { - // TODO use gumdrop for real argument parsing - let args: Vec = std::env::args().collect(); - - if args.len() != 2 { - panic!("osp can only accept a single argument: the directory containing a manifest.yml"); - } - - let dir = Path::new(&args[1]); - if !dir.is_dir() { - panic!("The argument must be a directory"); - } - let manifest = dir.join(Path::new("manifest.yml")); - let manifest = serde_yaml::from_reader::(File::open(manifest)?) - .expect("Failed to parse manifest.yml"); - - let step_name = dir - .file_name() - .expect("Failed to unwrap the directory filename") - .to_str() - .unwrap(); - println!("default out: {:#?}", step_name); - - manifest.create_artifact(&dir, Path::new(&format!("{}.tar.gz", step_name)))?; - Ok(()) -} diff --git a/services/local-orchestrator/Cargo.toml b/services/local-orchestrator/Cargo.toml index 05cb3f5..52cea21 100644 --- a/services/local-orchestrator/Cargo.toml +++ b/services/local-orchestrator/Cargo.toml @@ -7,8 +7,8 @@ edition = "2018" [dependencies] async-std = { version = "~1.7", features = ["attributes"]} log = "~0.4.11" -otto-agent = { path = "../../agent" } -otto-models = { path = "../../models" } +otto-agent = { path = "../../crates/agent" } +otto-models = { path = "../../crates/models" } pretty_env_logger = "~0.4.0" serde = {version = "~1.0.117", features = ["rc", "derive"]} serde_json = "~1.0.59" diff --git a/services/parser/Cargo.toml b/services/parser/Cargo.toml index 6357572..05ba6fc 100644 --- a/services/parser/Cargo.toml +++ b/services/parser/Cargo.toml @@ -15,7 +15,7 @@ path = "src/main.rs" [dependencies] async-std = { version = "1.6.5", features = ["attributes"]} log = "~0.4.11" -otto-models = { path = "../../models" } +otto-models = { path = "../../crates/models" } pest = "~2.1.3" pest_derive = "~2.1.0" pretty_env_logger = "~0.4.0" diff --git a/stdlib/archive/Cargo.toml b/stdlib/archive/Cargo.toml index 63dd9f7..a52b7e8 100644 --- a/stdlib/archive/Cargo.toml +++ b/stdlib/archive/Cargo.toml @@ -8,7 +8,7 @@ edition = "2018" async-std = { version = "~1.7", features = ["attributes"] } flate2 = "~1.0.18" glob = "~0.3.0" -otto-agent = { path = "../../agent" } +otto-agent = { path = "../../crates/agent" } serde = {version = "~1.0.117", features = ["derive"]} # Not using the curl-client default feature to ensure that builds won't require # libcurl for now diff --git a/stdlib/dir/Cargo.toml b/stdlib/dir/Cargo.toml index 7de2ef7..1880584 100644 --- a/stdlib/dir/Cargo.toml +++ b/stdlib/dir/Cargo.toml @@ -7,5 +7,5 @@ edition = "2018" [dependencies] serde_yaml = "~0.8.13" serde = {version = "~1.0.117", features = ["derive"]} -otto-agent = { path = "../../agent" } -otto-models = { path = "../../models" } +otto-agent = { path = "../../crates/agent" } +otto-models = { path = "../../crates/models" } diff --git a/stdlib/error/Cargo.toml b/stdlib/error/Cargo.toml index dbfef86..8eb7ff1 100644 --- a/stdlib/error/Cargo.toml +++ b/stdlib/error/Cargo.toml @@ -5,5 +5,5 @@ authors = ["R. Tyler Croy "] edition = "2018" [dependencies] -otto-agent = { path = "../../agent" } +otto-agent = { path = "../../crates/agent" } serde = {version = "~1.0.117", features = ["derive"]} diff --git a/stdlib/git/Cargo.toml b/stdlib/git/Cargo.toml index dfb7b19..73ab74e 100644 --- a/stdlib/git/Cargo.toml +++ b/stdlib/git/Cargo.toml @@ -6,6 +6,6 @@ edition = "2018" [dependencies] git2 = "~0.13.12" -otto-agent = { path = "../../agent" } +otto-agent = { path = "../../crates/agent" } serde = {version = "~1.0.117", features = ["derive"]} url = "~2.2.0" diff --git a/stdlib/sh/Cargo.toml b/stdlib/sh/Cargo.toml index db23f68..d70afa4 100644 --- a/stdlib/sh/Cargo.toml +++ b/stdlib/sh/Cargo.toml @@ -5,6 +5,6 @@ authors = ["R. Tyler Croy "] edition = "2018" [dependencies] -otto-agent = { path = "../../agent" } +otto-agent = { path = "../../crates/agent" } serde = {version = "~1.0.117", features = ["derive"]} tempfile = "~3.1.0" diff --git a/stdlib/unarchive/Cargo.toml b/stdlib/unarchive/Cargo.toml index 87fd760..cfc919e 100644 --- a/stdlib/unarchive/Cargo.toml +++ b/stdlib/unarchive/Cargo.toml @@ -6,6 +6,6 @@ edition = "2018" [dependencies] async-std = { version = "~1.7", features = ["attributes"] } -otto-agent = { path = "../../agent" } +otto-agent = { path = "../../crates/agent" } serde = {version = "~1.0.117", features = ["derive"]} surf = { version = "~2.1.0", features = ["h1-client"]}