Start working on providing steps with per-step cache that they can utilize

This is going to first be used with the `git` step to ensure that it can cache
clones between pipeline runs without hitting the network

See #40

agent/src/lib.rs

@@ -136,11 +136,23 @@ pub fn run(
 
             let mut file = NamedTempFile::new()?;
 
+            let cache = match runner.manifest.cache {
+                true => {
+                    if let Ok(dir) = std::env::var("CACHES_DIR") {
+                        Some(PathBuf::from(dir))
+                    } else {
+                        None
+                    }
+                }
+                false => None,
+            };
+
             // TODO: This is going to be wrong on nested steps
             let sock = control::agent_socket();
             let configuration = step::Configuration {
                 pipeline: pipeline,
                 uuid: step.uuid,
+                cache: cache,
                 ipc: sock,
                 endpoints: endpoints.clone(),
             };

agent/src/main.rs

@@ -6,6 +6,7 @@
 use async_std::sync::channel;
 use serde::Deserialize;
 use std::fs::File;
+use std::path::Path;
 use uuid::Uuid;
 
 use otto_agent::*;
@@ -27,6 +28,35 @@ struct Invocation {
     steps: Vec<otto_models::Step>,
 }
 
+/**
+ * Ensure the directory exists by making it or panicking
+ */
+fn mkdir_if_not_exists(path: &Path) -> std::io::Result<()> {
+    use std::io::{Error, ErrorKind};
+
+    if path.exists() {
+        if path.is_dir() {
+            return Ok(());
+        }
+        return Err(Error::new(
+            ErrorKind::AlreadyExists,
+            format!("{:?} exists and is not a directory", path),
+        ));
+    } else {
+        std::fs::create_dir(path)?;
+    }
+    Ok(())
+}
+
+/**
+ * Set common environment variables for all subprocesses to inherit
+ */
+fn set_common_env_vars() {
+    use std::env::set_var;
+    set_var("OTTO", "true");
+    set_var("CI", "true");
+}
+
 #[async_std::main]
 async fn main() -> std::io::Result<()> {
     pretty_env_logger::init();
@@ -34,10 +64,19 @@ async fn main() -> std::io::Result<()> {
     let steps_dir = std::env::var("STEPS_DIR").expect("STEPS_DIR must be defined");
 
     if args.len() != 2 {
-        panic!("The sh step can only accept a single argument: the parameters file path");
+        panic!("The agent can only accept a single argument: the invocation file path");
     }
 
     let file = File::open(&args[1])?;
+
+    let work_dir = Path::new("agent-work");
+    let cache_dir = work_dir.join("caches");
+    mkdir_if_not_exists(&work_dir)?;
+    mkdir_if_not_exists(&cache_dir)?;
+
+    std::env::set_var("CACHES_DIR", cache_dir);
+    std::env::set_current_dir(work_dir)?;
+
     let (sender, receiver) = channel(MAX_CONTROL_MSGS);
 
     match serde_json::from_reader::<File, Invocation>(file) {
@@ -50,6 +89,16 @@ async fn main() -> std::io::Result<()> {
                 control::run(sender).await.expect("Failed to bind control?");
             });
 
+            /*
+             * Enter into the pipeline specific work directory
+             */
+            let pipeline_dir = invoke.pipeline.to_hyphenated().to_string();
+            let pipeline_dir = Path::new(&pipeline_dir);
+            mkdir_if_not_exists(&pipeline_dir)?;
+            std::env::set_current_dir(pipeline_dir)?;
+
+            set_common_env_vars();
+
             run(&steps_dir, &invoke.steps, invoke.pipeline, Some(receiver))
                 .expect("Failed to run pipeline");
         }

agent/src/step.rs

@@ -6,6 +6,7 @@
 use log::*;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::path::PathBuf;
 use url::Url;
 use uuid::Uuid;
 
@@ -35,7 +36,8 @@ pub struct Configuration {
     pub pipeline: Uuid,
     /// The uuid of this specific step
     pub uuid: Uuid,
-    pub ipc: std::path::PathBuf,
+    pub cache: Option<PathBuf>,
+    pub ipc: PathBuf,
     pub endpoints: HashMap<String, Endpoint>,
 }
 

osp/src/lib.rs

@@ -7,6 +7,8 @@ use std::path::{Path, PathBuf};
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct Manifest {
     pub symbol: String,
+    #[serde(default = "default_false")]
+    pub cache: bool,
     pub description: String,
     pub includes: Vec<Include>,
     pub entrypoint: Entrypoint,

rfc/0011-step-library-format.adoc

@@ -66,6 +66,14 @@ symbol: sh
 description: |
   The `sh` step executes a shell script within the given execution context
 
+# Instructs the agent to create a local cache for this step, this will default
+# to false as _most_ steps do not need their own caching directory.
+#
+# Agents are expected to construct the caches in their
+# workdir/caches/<step-symbol> and to pass that into the step via the "cache"
+# invocation file parameter
+cache: true
+
 # List all the files/globs to include in the packaged artifact
 includes:
   # Paths are treated as relative from wherever osp is invoked from

stdlib/git/manifest.yml

@@ -11,6 +11,8 @@ entrypoint:
   path: git-step
   multiarch: false
 
+cache: true
+
 parameters:
   - name: url
     required: true
@@ -23,3 +25,10 @@ parameters:
     type: string
     description: |
       A git branch to clone instead of the default.
+
+  - name: into
+    required: false
+    type: string
+    description: |
+      Path into which the clone should be performed, can be used as `.` to
+      clone into the current working directory

stdlib/git/src/main.rs

@@ -10,6 +10,7 @@ use url::Url;
 struct Parameters {
     url: Url,
     branch: Option<String>,
+    into: Option<String>,
 }
 
 /**
@@ -32,21 +33,25 @@ fn main() -> std::io::Result<()> {
     let invoke: Invocation<Parameters> =
         invocation_from_args(&args).expect("Failed to deserialize the invocation for the step");
 
-    if let Some(path) = repo_from_url(&invoke.parameters.url) {
-        println!("Clone!");
-        let mut builder = git2::build::RepoBuilder::new();
-
-        if let Some(branch) = &invoke.parameters.branch {
-            builder.branch(&branch);
+    let clone_path = match invoke.parameters.into {
+        Some(into) => into,
+        None => {
+            repo_from_url(&invoke.parameters.url).expect("Failed to determine local path to clone")
         }
-        let _repo = match builder.clone(&invoke.parameters.url.into_string(), Path::new(&path)) {
-            Ok(repo) => repo,
-            Err(e) => panic!("failed to clone: {}", e),
-        };
-    } else {
-        println!("Failed to determine the right local path to clone the repository into");
-        std::process::exit(1);
+    };
+
+    println!("Clone!");
+    let mut builder = git2::build::RepoBuilder::new();
+
+    if let Some(branch) = &invoke.parameters.branch {
+        builder.branch(&branch);
     }
+
+    let _repo = match builder.clone(&invoke.parameters.url.into_string(), Path::new(&clone_path)) {
+        Ok(repo) => repo,
+        Err(e) => panic!("failed to clone: {}", e),
+    };
+
     Ok(())
 }
 

stdlib/git/tests/simple.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-INVOCATION_FILE=tmp_gittest_invocation_file.json
+INVOCATION_FILE=$PWD/tmp_gittest_invocation_file.json
 
 oneTimeTearDown() {
     rm -f $INVOCATION_FILE
@@ -46,10 +46,62 @@ test_clone_ref_tag() {
 EOF
 
     output=$(git-step $INVOCATION_FILE)
+    echo $output
     assertTrue "step should be able to clone the given url: ${output}" $?
     assertTrue "step should have cloned the repo" "test -d otto-test-repository"
     assertTrue "step should have cloned the repo to the branch" "test -f otto-test-repository/this-is-a-branch"
     rm -rf otto-test-repository
 }
 
+test_clone_into() {
+    cat > $INVOCATION_FILE<<EOF
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+        }
+    },
+    "parameters" : {
+        "url" : "https://git.brokenco.de/rtyler/otto-test-repository",
+        "into" : "."
+    }
+}
+EOF
+
+    mkdir work-dir
+    pushd work-dir
+        output=$(git-step $INVOCATION_FILE)
+        assertTrue "step should be able to clone the given url: ${output}" $?
+        assertTrue "step should have cloned the repo into $PWD" "test -f README.adoc"
+    popd
+}
+test_clone_with_cache() {
+    cache_dir="$PWD/caches"
+
+    cat > $INVOCATION_FILE<<EOF
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "cache" : "${cache_dir}",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+        }
+    },
+    "parameters" : {
+        "url" : "https://git.brokenco.de/rtyler/otto-test-repository"
+    }
+}
+EOF
+
+    mkdir work-dir
+    pushd work-dir
+        output=$(git-step $INVOCATION_FILE)
+        assertTrue "step should be able to clone the given url: ${output}" $?
+    popd
+
+}
+
 . $(dirname $0)/../../../contrib/shunit2/shunit2