diff --git a/Cargo.lock b/Cargo.lock index f5e3148..3b8a951 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,6 +75,17 @@ name = "anyhow" version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "archive-step" +version = "0.1.0" +dependencies = [ + "flate2 1.0.18 (registry+https://github.com/rust-lang/crates.io-index)", + "glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "otto-agent 0.1.0", + "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", + "tar 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "arrayref" version = "0.3.6" @@ -773,6 +784,11 @@ dependencies = [ "polyval 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "gloo-timers" version = "0.2.1" @@ -1593,7 +1609,6 @@ version = "0.1.0" dependencies = [ "otto-agent 0.1.0", "serde 1.0.117 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_yaml 0.8.13 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2206,6 +2221,7 @@ dependencies = [ "checksum generic-array 0.14.4 (registry+https://github.com/rust-lang/crates.io-index)" = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" "checksum getrandom 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" "checksum ghash 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d6e27f0689a6e15944bdce7e45425efb87eaa8ab0c6e87f11d0987a9133e2531" +"checksum glob 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" "checksum gloo-timers 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" "checksum gumdrop 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46571f5d540478cf70d2a42dd0d6d8e9f4b9cc7531544b93311e657b86568a0b" "checksum gumdrop_derive 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "915ef07c710d84733522461de2a734d4d62a3fd39a4d4f404c2f385ef8618d05" diff --git a/Cargo.toml b/Cargo.toml index 5664018..0e60e97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "osp", + "stdlib/archive", "stdlib/dir", "stdlib/error", "stdlib/sh", diff --git a/stdlib/archive/.gitignore b/stdlib/archive/.gitignore new file mode 100644 index 0000000..2f7896d --- /dev/null +++ b/stdlib/archive/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/stdlib/archive/Cargo.toml b/stdlib/archive/Cargo.toml new file mode 100644 index 0000000..5228d31 --- /dev/null +++ b/stdlib/archive/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "archive-step" +version = "0.1.0" +authors = ["R. Tyler Croy "] +edition = "2018" + +[dependencies] +flate2 = "~1.0.18" +glob = "~0.3.0" +otto-agent = { path = "../../agent" } +serde = {version = "~1.0.117", features = ["derive"]} +tar = "~0.4.30" diff --git a/stdlib/archive/README.adoc b/stdlib/archive/README.adoc new file mode 100644 index 0000000..36e1893 --- /dev/null +++ b/stdlib/archive/README.adoc @@ -0,0 +1,6 @@ += archive + +The `archive` step will store files or file patterns associated with the +running pipeline in the Otto object store. + +This can be helpful for persisting logs, or built artifacts. diff --git a/stdlib/archive/manifest.yml b/stdlib/archive/manifest.yml new file mode 100644 index 0000000..075e482 --- /dev/null +++ b/stdlib/archive/manifest.yml @@ -0,0 +1,27 @@ +--- +symbol: archive +description: | + The `archive` step will archive a globbed pattern of artifacts + +includes: + - name: target/release/archive-step + flatten: true + +entrypoint: + path: archive-step + multiarch: false + +parameters: + - name: artifacts + required: true + type: string + description: | + A simple string or glob pattern identifying the artifacts to archive + + Glob patterns will result in a tarball storing the matching files. + + - name: followSymlinks + description: | + Whether the archive step should follow symbolic links in the archive proces + type: boolean + required: false diff --git a/stdlib/archive/src/main.rs b/stdlib/archive/src/main.rs new file mode 100644 index 0000000..34e55ae --- /dev/null +++ b/stdlib/archive/src/main.rs @@ -0,0 +1,101 @@ +/* + * The archive step will store artifacts on the server associated with the + * running pipeline + */ + +use glob::glob; +use serde::Deserialize; +use std::path::PathBuf; +use ottoagent::step::*; + +#[derive(Clone, Debug, Deserialize)] +struct Parameters { + artifacts: String, + #[serde(rename = "followSymlinks")] + follow_symlinks: Option, +} + +/** + * Will return a vec of PathBufs that the specified pattern matches + * + * If the pattern is an invalid glob, there will be a panic and the step will exit + */ +fn artifact_matches(pattern: &str) -> Vec { + glob(pattern).expect("Failed to read glob pattern") + .filter_map(std::result::Result::ok) + .map( |p| p ).collect() +} + +/** + * This function will create a tarball based on the given paths + */ +fn create_tarball(paths: Vec) -> std::io::Result<()> { + Ok(()) +} + +/** + * Actually archive the named file into the object store + * + * The path should be the path to a single file, or a generated tarball + */ +fn archive(path: &PathBuf) -> std::io::Result<()> { + Ok(()) +} + +fn main() -> std::io::Result<()> { + let args = std::env::args().collect(); + let invoke: Invocation = invocation_from_args(&args).unwrap(); + + let artifacts = artifact_matches(&invoke.parameters.artifacts); + + match artifacts.len() { + 0 => { + panic!("The `archive` step was given a pattern ({}) which doesn't match any files", invoke.parameters.artifacts); + }, + 1 => { + // no tarball, unless it's a directory + let file = &artifacts[0]; + if file.is_dir() { + create_tarball(artifacts); + } + else { + archive(file); + } + }, + _ => { + match create_tarball(artifacts) { + Err(e) => { + // TODO handle + }, + Ok(file) => { + //archive(file); + } + } + }, + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn artifact_matches_empty_pattern() { + let paths = artifact_matches("/this/will/never/exist"); + assert_eq!(paths.len(), 0); + } + + #[test] + fn artifact_matches_single_file() { + let paths = artifact_matches("/dev/null"); + assert_eq!(paths.len(), 1); + } + + #[test] + fn artifact_matches_wildcard() { + let paths = artifact_matches("*"); + assert!(paths.len() > 1); + } +} diff --git a/stdlib/sh/Cargo.toml b/stdlib/sh/Cargo.toml index 5153e34..db23f68 100644 --- a/stdlib/sh/Cargo.toml +++ b/stdlib/sh/Cargo.toml @@ -6,6 +6,5 @@ edition = "2018" [dependencies] otto-agent = { path = "../../agent" } -serde_yaml = "~0.8.13" serde = {version = "~1.0.117", features = ["derive"]} tempfile = "~3.1.0"