Start working on providing steps with per-step cache that they can utilize

This is going to first be used with the `git` step to ensure that it can cache
clones between pipeline runs without hitting the network

See #40

agent/src/lib.rs

@@ -136,11 +136,23 @@ pub fn run(
 
             let mut file = NamedTempFile::new()?;
 
+            let cache = match runner.manifest.cache {
+                true => {
+                    if let Ok(dir) = std::env::var("CACHES_DIR") {
+                        Some(PathBuf::from(dir))
+                    } else {
+                        None
+                    }
+                }
+                false => None,
+            };
+
             // TODO: This is going to be wrong on nested steps
             let sock = control::agent_socket();
             let configuration = step::Configuration {
                 pipeline: pipeline,
                 uuid: step.uuid,
+                cache: cache,
                 ipc: sock,
                 endpoints: endpoints.clone(),
             };

agent/src/main.rs

@@ -36,16 +36,27 @@ fn mkdir_if_not_exists(path: &Path) -> std::io::Result<()> {
 
     if path.exists() {
         if path.is_dir() {
-            return Ok(())
+            return Ok(());
         }
-        return Err(Error::new(ErrorKind::AlreadyExists, format!("{:?} exists and is not a directory", path)));
-    }
-    else {
+        return Err(Error::new(
+            ErrorKind::AlreadyExists,
+            format!("{:?} exists and is not a directory", path),
+        ));
+    } else {
         std::fs::create_dir(path)?;
     }
     Ok(())
 }
 
+/**
+ * Set common environment variables for all subprocesses to inherit
+ */
+fn set_common_env_vars() {
+    use std::env::set_var;
+    set_var("OTTO", "true");
+    set_var("CI", "true");
+}
+
 #[async_std::main]
 async fn main() -> std::io::Result<()> {
     pretty_env_logger::init();
@@ -63,6 +74,7 @@ async fn main() -> std::io::Result<()> {
     mkdir_if_not_exists(&work_dir)?;
     mkdir_if_not_exists(&cache_dir)?;
 
+    std::env::set_var("CACHES_DIR", cache_dir);
     std::env::set_current_dir(work_dir)?;
 
     let (sender, receiver) = channel(MAX_CONTROL_MSGS);
@@ -85,6 +97,8 @@ async fn main() -> std::io::Result<()> {
             mkdir_if_not_exists(&pipeline_dir)?;
             std::env::set_current_dir(pipeline_dir)?;
 
+            set_common_env_vars();
+
             run(&steps_dir, &invoke.steps, invoke.pipeline, Some(receiver))
                 .expect("Failed to run pipeline");
         }

agent/src/step.rs

@@ -6,6 +6,7 @@
 use log::*;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
+use std::path::PathBuf;
 use url::Url;
 use uuid::Uuid;
 
@@ -35,7 +36,8 @@ pub struct Configuration {
     pub pipeline: Uuid,
     /// The uuid of this specific step
     pub uuid: Uuid,
-    pub ipc: std::path::PathBuf,
+    pub cache: Option<PathBuf>,
+    pub ipc: PathBuf,
     pub endpoints: HashMap<String, Endpoint>,
 }
 

osp/src/lib.rs

@@ -7,6 +7,8 @@ use std::path::{Path, PathBuf};
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct Manifest {
     pub symbol: String,
+    #[serde(default = "default_false")]
+    pub cache: bool,
     pub description: String,
     pub includes: Vec<Include>,
     pub entrypoint: Entrypoint,

rfc/0011-step-library-format.adoc

@@ -66,6 +66,14 @@ symbol: sh
 description: |
   The `sh` step executes a shell script within the given execution context
 
+# Instructs the agent to create a local cache for this step, this will default
+# to false as _most_ steps do not need their own caching directory.
+#
+# Agents are expected to construct the caches in their
+# workdir/caches/<step-symbol> and to pass that into the step via the "cache"
+# invocation file parameter
+cache: true
+
 # List all the files/globs to include in the packaged artifact
 includes:
   # Paths are treated as relative from wherever osp is invoked from

stdlib/git/manifest.yml

@@ -11,6 +11,8 @@ entrypoint:
   path: git-step
   multiarch: false
 
+cache: true
+
 parameters:
   - name: url
     required: true

stdlib/git/src/main.rs

@@ -35,7 +35,9 @@ fn main() -> std::io::Result<()> {
 
     let clone_path = match invoke.parameters.into {
         Some(into) => into,
-        None => repo_from_url(&invoke.parameters.url).expect("Failed to determine local path to clone"),
+        None => {
+            repo_from_url(&invoke.parameters.url).expect("Failed to determine local path to clone")
+        }
     };
 
     println!("Clone!");

stdlib/git/tests/simple.sh

@@ -46,6 +46,7 @@ test_clone_ref_tag() {
 EOF
 
     output=$(git-step $INVOCATION_FILE)
+    echo $output
     assertTrue "step should be able to clone the given url: ${output}" $?
     assertTrue "step should have cloned the repo" "test -d otto-test-repository"
     assertTrue "step should have cloned the repo to the branch" "test -f otto-test-repository/this-is-a-branch"
@@ -76,5 +77,31 @@ EOF
         assertTrue "step should have cloned the repo into $PWD" "test -f README.adoc"
     popd
 }
+test_clone_with_cache() {
+    cache_dir="$PWD/caches"
+
+    cat > $INVOCATION_FILE<<EOF
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "cache" : "${cache_dir}",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+        }
+    },
+    "parameters" : {
+        "url" : "https://git.brokenco.de/rtyler/otto-test-repository"
+    }
+}
+EOF
+
+    mkdir work-dir
+    pushd work-dir
+        output=$(git-step $INVOCATION_FILE)
+        assertTrue "step should be able to clone the given url: ${output}" $?
+    popd
+
+}
 
 . $(dirname $0)/../../../contrib/shunit2/shunit2