Merge pull request #29 from rtyler/step-libraries

Mainlining the work in progress on step libraries

.gitignore

@@ -5,3 +5,4 @@ node_modules/
 .cargo/
 build/
 .otto-ebc-history
+*.tar.gz

Cargo.toml

@@ -2,8 +2,25 @@
 
 members = [
     "auctioneer",
+
+    "agents/primitive",
+
     "eventbus",
     "eventbus-cli",
-    "eventbus-inmemory",
+
+    # Disabled, not compiling at the moment
+    # "eventbus-inmemory",
+
     "processors/travis-ci",
+
+    "osp",
+
+    "stdlib/sh",
 ]
+
+[profile.release]
+panic = "abort"
+lto = true
+codegen-units = 1
+incremental = false
+opt-level = "z"

Makefile

@@ -3,44 +3,33 @@
 # and helps organize the various tasks for preparation, compilation, and
 # testing.
 #
-# Execute `make` to get help ffor the various targets
+# Execute `make` to get help for the various targets
 ################################################################################
 
-# Set the PATH so we can automatically include our node binstubs
-export PATH:=./node_modules/.bin:${PATH}
-SUB_DIRS=grammar
-
 ################################################################################
 ## Phony targets
+
+release:
+	cargo build --release
+	# Strip all the executables for size, does impact debug symbols
+	find target/release -type f -executable -exec strip {} \;
+
+steps: release
+	for dir in $$(find stdlib -maxdepth 1 -type d | tail -n +2); do \
+		echo ">> Packaging $$dir"; \
+		./target/release/osp $$dir; \
+	done;
+
 # Cute hack thanks to:
 # https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 help: ## Display this help text
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-build: depends ## Build all components
-	$(foreach dir, $(SUB_DIRS), $(MAKE) -C $(dir) $@)
-	tsc
-
-lint: depends
-	tslint -c tslint.json -t stylish 'lib/**/*.ts' 'services/**/*.ts'
-
-check: depends lint build ## Run validation tests
-	jest
-	dredd
-
-swagger: depends ## Generate the swagger stubs based on apispecs
-
-depends: prereqs ## Download all dependencies
-
-prereqs: scripts/prereqs.sh ## Check that this system has the necessary tools to build otto
-	@sh scripts/prereqs.sh
-
 clean: ## Clean all temporary/working files
-	$(foreach dir, $(SUB_DIRS), $(MAKE) -C $(dir) $@)
 
 diagram: system.png system.dot ## Generate the diagrams describing otto
 	dot -Tpng -o system.png system.dot
 
 ################################################################################
 
-.PHONY: all build check clean depends lint swagger
+.PHONY: clean diagram help steps release

agents/primitive/.gitignore

@@ -0,0 +1 @@
+target/

agents/primitive/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "primitive-agent"
+version = "0.1.0"
+authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
+edition = "2018"
+
+[dependencies]
+serde_yaml = "~0.8.13"
+serde = {version = "~1.0.117", features = ["rc", "derive"]}
+osp = { path = "../../osp" }
+tempfile = "~3.1.0"

agents/primitive/src/main.rs

@@ -0,0 +1,98 @@
+
+
+use serde::Deserialize;
+use serde_yaml::Value;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::{stdout, stderr, Write};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+use tempfile::NamedTempFile;
+
+#[derive(Clone, Debug, Deserialize)]
+struct Pipeline {
+    steps: Vec<Step>,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+struct Step {
+    symbol: String,
+    parameters: Value,
+}
+
+fn run(steps_dir: &str, steps: &Vec<Step>) -> std::io::Result<()> {
+    let dir = Path::new(steps_dir);
+
+    if ! dir.is_dir() {
+        panic!("STEPS_DIR must be a directory! {:?}", dir);
+    }
+
+    let mut manifests: HashMap<String, osp::Manifest> = HashMap::new();
+    let mut m_paths: HashMap<String, PathBuf> = HashMap::new();
+
+    for step in steps.iter() {
+        let manifest_file = dir.join(&step.symbol).join("manifest.yml");
+
+        if manifest_file.is_file() {
+            println!("{} exists", step.symbol);
+
+            let file = File::open(manifest_file)?;
+            // TODO: This is dumb and inefficient
+            m_paths.insert(step.symbol.clone(), dir.join(&step.symbol).to_path_buf());
+            manifests.insert(step.symbol.clone(),
+                serde_yaml::from_reader::<File, osp::Manifest>(file).expect("Failed to parse manifest")
+            );
+        }
+        else {
+            println!("{}/manifest.yml does not exist, step cannot execute", step.symbol);
+            println!("NORMALLY THIS WOULD ERROR BEFORE ANYTHING EXECUTES");
+        }
+    }
+    println!("---");
+
+    // Now that things are valid and collected, let's executed
+    for step in steps.iter() {
+        if let Some(runner) = manifests.get(&step.symbol) {
+            let m_path = m_paths.get(&step.symbol).expect("Failed to grab the step library path");
+            let entrypoint = m_path.join(&runner.entrypoint.path);
+            println!("entry: {:?}", entrypoint);
+
+            let mut file = NamedTempFile::new()?;
+            let mut step_args = HashMap::new();
+            step_args.insert("parameters", &step.parameters);
+
+            serde_yaml::to_writer(&mut file, &step_args)
+                .expect("Failed to write temporary file for script");
+
+            let output = Command::new(entrypoint)
+                .arg(file.path())
+                .output()
+                .expect("Failed to invoke the script");
+            stdout().write_all(&output.stdout).unwrap();
+            stderr().write_all(&output.stderr).unwrap();
+        }
+    }
+
+    Ok(())
+}
+
+fn main() -> std::io::Result<()>{
+    let args: Vec<String> = std::env::args().collect();
+    let steps_dir = std::env::var("STEPS_DIR").expect("STEPS_DIR must be defined");
+
+    if args.len() != 2 {
+        panic!("The sh step can only accept a single argument: the parameters file path");
+    }
+
+    let file = File::open(&args[1])?;
+
+    match serde_yaml::from_reader::<File, Pipeline>(file) {
+        Err(e) => {
+            panic!("Failed to parse parameters file: {:#?}", e);
+        }
+        Ok(invoke) => {
+            run(&steps_dir, &invoke.steps);
+        },
+    };
+    Ok(())
+}

osp/.gitignore

@@ -0,0 +1 @@
+target/

osp/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "osp"
+version = "0.1.0"
+authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
+edition = "2018"
+
+[lib]
+name = "osp"
+path = "src/lib.rs"
+
+[[bin]]
+name = "osp"
+path = "src/main.rs"
+
+[dependencies]
+flate2 = "~1.0.18"
+gumdrop = "~0.8.0"
+serde_yaml = "~0.8.13"
+serde = {version = "~1.0.117", features = ["rc", "derive"]}
+tar = "~0.4.30"

osp/README.adoc

@@ -0,0 +1,5 @@
+= Otto Step Packager
+
+This directory contains the step library packaging tool `osp` which will read a
+`manifest.yml` and then package up an artifact suitable for consumption by Otto
+agents.

osp/src/lib.rs

@@ -0,0 +1,90 @@
+use flate2::Compression;
+use flate2::write::GzEncoder;
+use serde::{Deserialize, Serialize};
+use std::fs::File;
+use std::path::{Path, PathBuf};
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Manifest {
+    pub symbol: String,
+    pub description: String,
+    pub includes: Vec<Include>,
+    pub entrypoint: Entrypoint,
+    pub parameters: Vec<Parameter>,
+}
+
+impl Manifest {
+    /**
+     * Create an artifact from the given manifest
+     */
+    pub fn create_artifact(&self, dir: &Path, output: &Path) -> Result<(), std::io::Error> {
+        let tar_gz = File::create(output)?;
+        let enc = GzEncoder::new(tar_gz, Compression::default());
+        let mut tar = tar::Builder::new(enc);
+        let mut manifest = File::open(dir.join(Path::new("manifest.yml")))?;
+        tar.append_file(format!("{}/manifest.yml", self.symbol), &mut manifest)?;
+
+        for include in self.includes.iter() {
+            let mut f = File::open(match include.name.starts_with("./") {
+                true => {
+                    // Relative to dir
+                    dir.join(&include.name)
+                },
+                false => {
+                    // Relative to $PWD
+                    Path::new(&include.name).to_path_buf()
+                },
+            })?;
+
+            let archive_path = format!("{}/{}",
+                                    self.symbol,
+                                    match include.flatten {
+                                        true => {
+                                            let p = Path::new(&include.name);
+                                            p.file_name().unwrap().to_str().unwrap()
+                                        },
+                                        false => &include.name,
+                                    });
+            tar.append_file(archive_path, &mut f);
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Include {
+    name: String,
+    #[serde(default = "default_false")]
+    flatten: bool,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Entrypoint {
+    pub path: PathBuf,
+    #[serde(default = "default_false")]
+    multiarch: bool,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Parameter {
+    name: String,
+    required: bool,
+    #[serde(rename = "type")]
+    p_type: ParameterType,
+    description: String,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum ParameterType {
+    #[serde(rename = "string")]
+    StringParameter,
+    #[serde(rename = "boolean")]
+    BoolParameter,
+}
+
+/** Simple function for serde defaults */
+fn default_false() -> bool { false }
+
+#[cfg(test)]
+mod tests {
+}

osp/src/main.rs

@@ -0,0 +1,26 @@
+
+use osp::Manifest;
+use std::fs::File;
+use std::path::Path;
+
+fn main() -> std::io::Result<()> {
+    // TODO use gumdrop for real argument parsing
+    let args: Vec<String> = std::env::args().collect();
+
+    if args.len() != 2 {
+        panic!("osp can only accept a single argument: the directory containing a manifest.yml");
+    }
+
+    let dir = Path::new(&args[1]);
+    if ! dir.is_dir() {
+        panic!("The argument must be a directory");
+    }
+    let manifest = dir.join(Path::new("manifest.yml"));
+    let manifest = serde_yaml::from_reader::<File, Manifest>(File::open(manifest)?).expect("Failed to parse manifest.yml");
+
+    let step_name = dir.file_name().expect("Failed to unwrap the directory filename").to_str().unwrap();
+    println!("default out: {:#?}", step_name);
+
+    manifest.create_artifact(&dir, Path::new(&format!("{}.tar.gz", step_name)))?;
+    Ok(())
+}

rfc/0011-step-library-format.adoc

@@ -0,0 +1,259 @@
+= RFC-0011: Step Library Packaging Format
+:toc: preamble
+:toclevels: 3
+ifdef::env-github[]
+:tip-caption: :bulb:
+:note-caption: :information_source:
+:important-caption: :heavy_exclamation_mark:
+:caution-caption: :fire:
+:warning-caption: :warning:
+endif::[]
+
+.**RFC Template**
+
+.Metadata
+[cols="1h,1"]
+|===
+| RFC
+| 0011
+
+| Title
+| Step Library Packaging Format
+
+| Sponsor
+| link:https://github.com/rtyler[R Tyler Croy]
+
+| Status
+| Not Submitted :information_source:
+
+| Type
+| Standards
+
+| Created
+| 2020-10-17
+
+|===
+
+== Abstract
+
+In order to provide for a simple and extensible way to implement steps, the
+step library packaging format allows for native tools and scripts to be
+distributed and loaded by agents.
+
+
+== Specification
+
+Each step effectively has an `entrypoint` which is a binary executable. The
+agents in Otto will execute this file with an <<invocation-file>> containing
+all the necessary configuration and parameters.
+
+[[manifest-file]]
+== Manifest file
+
+The `manifest.yml` file is the step library package description. It contains
+all the information on how the step can be invokoed, but also details about how
+the build tooling should package the artifact for use within Otto.
+
+.Example manifest file
+[source,yaml]
+----
+# This manifest captures the basic functionality of the Jenkins Pipeline `sh`
+# step
+---
+# The symbol defines how this step should present in the pipeline
+symbol: sh
+# Description is help text
+description: |
+  The `sh` step executes a shell script within the given execution context
+
+# List all the files/globs to include in the packaged artifact
+includes:
+  # Paths are treated as relative from wherever osp is invoked from
+  - name: target/release/sh-step
+    # Steps the entire prefix of the file name, placing the file in the root of
+    # the artifact
+    flatten: true
+  # A name starting with ./ is treated to be relative to the manifest.yml
+  - name: ./README.adoc
+
+# The entrypoint tells the Otto agent which actual binary to use when
+# executing.
+entrypoint:
+  path: sh-step
+  # Multiarch tells the agent that this should be executed on all platforms. In
+  # which case case it may be "blindly" invoked.
+  #
+  # Non-multiarch steps will be attempt to be invoked with
+  # `${entrypoint.path}-${arch}-${vendor}-${system}-${abi}` similar to how
+  # Rust manages target triples: https://doc.rust-lang.org/nightly/rustc/platform-support.html
+  multiarch: false
+
+# The configuration helps the step self-express the configuration variables it
+# requires from Otto in order to execute properly
+#
+# The names of these variables are to be considered globally flat by default,
+# allowing for multiple steps to share the same configuration values. Should a
+# step wish to _not_ share its configuration values, it should namespace them
+# in the key name with the convention of `{step}.{key}` (e.g.
+# `sh.default_shell`)
+#
+# The configuration variables are also a means of requesting credentials from
+# Otto.
+configuration:
+  default_shell:
+    description: |
+      The default shell to use for the invocation of `sh` steps
+    required: false
+    default: '/bin/sh'
+
+# The parameters array allows for keyword invocation and positional invocation
+# of the step
+parameters:
+  - name: script
+    required: true
+    type: string
+    description: |
+      Runs a Bourne shell script, typically on a Unix node. Multiple lines are accepted.
+
+      An interpreter selector may be used, for example: `#!/usr/bin/perl`
+
+      Otherwise the system default shell will be run, using the `-xe` flags (you can specify `set +e` and/or `set +x` to disable those).
+
+  - name: encoding
+    description: |
+      Encoding of the stdout/stderr output, not typically needed as the system will
+      default to whatever `LC_TYPE` is defined.
+    type: string
+    required: false
+
+  - name: label
+    description: |
+      A label to identify the shell step in a GUI.
+    type: string
+    required: false
+
+  - name: returnStatus
+    description: Compatibility support only, doesn't do anything
+    type: boolean
+    required: false
+
+  - name: returnStdout
+    description: Compatibility support only, doesn't do anything
+    type: boolean
+    required: false
+----
+
+
+[[invocation-file]]
+=== Invocation file
+
+The invocation file is a YAML file generated at runtime and made available to
+the step binary on the agent. The invocation file should carry all parameters,
+environment variables, and internal configuration necessary for the step binary
+to execute correctly.
+
+.Example invocation file passed to entrypoint
+[source,yaml]
+----
+---
+env:
+  SOME_VAR: 'value'
+parameters:
+  script: 'ls -lah'
+----
+
+== Motivation
+
+[TIP]
+====
+Explain why the existing code base or process is inadequate to address the problem that the RFC solves.
+This section may also contain any historal context such as how things were done before this proposal.
+
+* Do not discuss design choices or alternative designs that were rejected, those belong in the Reasoning section.
+====
+
+== Reasoning
+
+[TIP]
+====
+Explain why particular design decisions were made.
+Describe alternate designs that were considered and related work, e.g. how the feature is supported in other systems.
+Provide evidence of consensus within the community and discuss important objections or concerns raised during discussion.
+
+* Use sub-headings to organize this section for ease of readability.
+* Do not talk about history or why this needs to be done, that is part of Motivation section.
+====
+
+== Backwards Compatibility
+
+[TIP]
+====
+Describe any incompatibilities and their severity.
+Describe how the RFC proposes to deal with these incompatibilities.
+
+If there are no backwards compatibility concerns, this section may simply say:
+There are no backwards compatibility concerns related to this proposal.
+====
+
+== Security
+
+[TIP]
+====
+Describe the security impact of this proposal.
+Outline what was done to identify and evaluate security issues,
+discuss of potential security issues and how they are mitigated or prevented,
+and how the RFC interacts with existing permissions, authentication, authorization, etc.
+
+If this proposal will have no impact on security, this section may simply say:
+There are no security risks related to this proposal.
+====
+
+
+== Testing
+
+[TIP]
+====
+If the RFC involves any kind of behavioral change to code give a summary of how
+its correctness (and, if applicable, compatibility, security, etc.) can be
+tested.
+
+In the preferred case that automated tests can be developed to cover all
+significant changes, simply give a short summary of the nature of these tests.
+
+If some or all of changes will require human interaction to verify, explain why
+automated tests are considered impractical.  Then summarize what kinds of test
+cases might be required: user scenarios with action steps and expected
+outcomes.  Might behavior vary by platform (operating system, servlet
+container, web browser, etc.)?  Are there foreseeable interactions between
+different permissible versions of components?
+Are any special tools, proprietary software, or online service accounts
+required to exercise a related code path (Active Directory server, GitHub
+login, etc.)?  When will testing take place relative to merging code changes,
+and might retesting be required if other changes are made to this area in the
+future?
+
+If this proposal requires no testing, this section may simply say:
+There are no testing issues related to this proposal.
+====
+
+== Prototype Implementation
+
+[TIP]
+====
+Link to any open source reference implementation of code changes for this proposal.
+The implementation need not be completed before the RFC is accepted
+but must be completed before the RFC is given "final" status.
+
+RFCs which will not include code changes may omit this section.
+====
+
+== References
+
+[TIP]
+====
+Provide links to any related documents.  This will include links to discussions
+on the mailing list, pull requests, and meeting notes.
+====
+
+
+

stdlib/echo/README.adoc

@@ -0,0 +1,11 @@
+= echo step
+
+The `echo` step is a simple step that just echoes a string into the log.
+
+
+[NOTE]
+====
+This is currently _not_ cross-platform to explicitly provide a step to test
+multiarch step libraries
+====
+

stdlib/echo/echo-step

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+
+cat $1 | gawk 'match($0, /\s+message:\s+["](.*)?["]/, a) {print a[1]}'

stdlib/echo/manifest.yml

@@ -0,0 +1,19 @@
+---
+symbol: echo
+description: |
+  The `echo` step is a simple step that just echoes a string into the log.
+
+includes:
+  - name: ./echo-step
+  - name: ./README.adoc
+
+entrypoint:
+  path: echo-step
+  multiarch: true
+
+parameters:
+  - name: message
+    required: true
+    type: string
+    description: |
+      The message to echo

stdlib/sh/.gitignore

@@ -0,0 +1 @@
+target/

stdlib/sh/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "sh-step"
+version = "0.1.0"
+authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
+edition = "2018"
+
+[dependencies]
+serde_yaml = "~0.8.13"
+serde = {version = "~1.0.117", features = ["derive"]}
+tempfile = "~3.1.0"

stdlib/sh/README.adoc

@@ -0,0 +1,13 @@
+= sh step
+
+The `sh` step is a fundamental building block and can effectively execute
+arbitrary scripts passed in by the user.
+
+
+.Example invocation file passed to entrypoint
+[source,yaml]
+----
+---
+parameters:
+  script: 'ls -lah'
+----

stdlib/sh/manifest.yml

@@ -0,0 +1,64 @@
+# This manifest captures the basic functionality of the Jenkins Pipeline `sh`
+# step
+---
+# The symbol defines how this step should present in the pipeline
+symbol: sh
+# Description is help text
+description: |
+  The `sh` step executes a shell script within the given execution context
+
+# List all the files/globs to include in the packaged artifact
+includes:
+  # Paths are treated as relative from wherever osp is invoked from
+  - name: target/release/sh-step
+    # Steps the entire prefix of the file name, placing the file in the root of
+    # the artifact
+    flatten: true
+  # A name starting with ./ is treated to be relative to the manifest.yml
+  - name: ./README.adoc
+
+# The entrypoint tells the Otto agent which actual binary to use when
+# executing.
+entrypoint:
+  path: sh-step
+  # Multiarch tells the agent that this should be executed on all platforms. In
+  # which case case it may be "blindly" invoked.
+  #
+  # Non-multiarch steps will be attempt to be invoked with
+  # `${entrypoint.path}-${arch}-${vendor}-${system}-${abi}` similar to how
+  # Rust manages target triples: https://doc.rust-lang.org/nightly/rustc/platform-support.html
+  multiarch: false
+
+parameters:
+  - name: script
+    required: true
+    type: string
+    description: |
+      Runs a Bourne shell script, typically on a Unix node. Multiple lines are accepted.
+
+      An interpreter selector may be used, for example: `#!/usr/bin/perl`
+
+      Otherwise the system default shell will be run, using the `-xe` flags (you can specify `set +e` and/or `set +x` to disable those).
+
+  - name: encoding
+    description: |
+      Encoding of the stdout/stderr output, not typically needed as the system will
+      default to whatever `LC_TYPE` is defined.
+    type: string
+    required: false
+
+  - name: label
+    description: |
+      A label to identify the shell step in a GUI.
+    type: string
+    required: false
+
+  - name: returnStatus
+    description: Compatibility support only, doesn't do anything
+    type: boolean
+    required: false
+
+  - name: returnStdout
+    description: Compatibility support only, doesn't do anything
+    type: boolean
+    required: false

stdlib/sh/src/main.rs

@@ -0,0 +1,62 @@
+/*
+ * A very simple step which just invokes a shell script with some flags
+ */
+
+use serde::Deserialize;
+use std::fs::File;
+use std::io::{stderr, stdout, Write};
+use std::process::Command;
+use tempfile::NamedTempFile;
+
+#[derive(Clone, Debug, Deserialize)]
+struct Invocation {
+    parameters: Parameters,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+struct Parameters {
+    script: String,
+    encoding: Option<String>,
+    label: Option<String>,
+    #[serde(rename = "returnStatus")]
+    return_status: Option<bool>,
+    #[serde(rename = "returnStdout")]
+    return_stdout: Option<bool>,
+}
+
+fn main() -> std::io::Result<()> {
+    let args: Vec<String> = std::env::args().collect();
+
+    if args.len() != 2 {
+        panic!("The sh step can only accept a single argument: the parameters file path");
+    }
+
+    let file = File::open(&args[1])?;
+
+    match serde_yaml::from_reader::<File, Invocation>(file) {
+        Err(e) => {
+            panic!("Failed to parse parameters file: {:#?}", e);
+        }
+        Ok(invoke) => {
+            // Create a file inside of `std::env::temp_dir()`.
+            let mut file = NamedTempFile::new()?;
+            writeln!(file, "{}", invoke.parameters.script)
+                .expect("Failed to write temporary file for script");
+
+            let output = Command::new("/bin/sh")
+                .arg(file.path())
+                .output()
+                .expect("Failed to invoke the script");
+
+            stdout().write_all(&output.stdout).unwrap();
+            stderr().write_all(&output.stderr).unwrap();
+
+            std::process::exit(
+                output
+                    .status
+                    .code()
+                    .expect("Failed to get status code of script"),
+            );
+        }
+    }
+}