Start adding some docs for writing steps, only cache covered now

I first set this up without the "--mirror" like functionality, but unfortunately
I couldn't get my fetch updates to work properly. A future optimization would be
to remove the mirror-refspecs and instead update the refsspecs whenever a new
branch is fetched.

Fixes #40

Cargo.lock

@@ -170,7 +170,7 @@ dependencies = [
  "httparse",
  "lazy_static",
  "log",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
 ]
 
 [[package]]
@@ -246,7 +246,7 @@ dependencies = [
  "http-types",
  "log",
  "memchr",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
 ]
 
 [[package]]
@@ -260,7 +260,7 @@ dependencies = [
  "async-io",
  "async-mutex",
  "blocking",
- "crossbeam-utils 0.8.0",
+ "crossbeam-utils 0.8.1",
  "futures-channel",
  "futures-core",
  "futures-io",
@@ -271,7 +271,7 @@ dependencies = [
  "memchr",
  "num_cpus",
  "once_cell",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
  "pin-utils",
  "slab",
  "wasm-bindgen-futures",
@@ -619,13 +619,12 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec91540d98355f690a86367e566ecad2e9e579f230230eb7c21398372be73ea5"
+checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d"
 dependencies = [
  "autocfg",
  "cfg-if 1.0.0",
- "const_fn",
  "lazy_static",
 ]
 
@@ -913,7 +912,7 @@ dependencies = [
  "futures-io",
  "memchr",
  "parking",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
  "waker-fn",
 ]
 
@@ -1008,6 +1007,7 @@ dependencies = [
  "git2",
  "otto-agent",
  "serde 1.0.117",
+ "sha2",
  "url",
 ]
 
@@ -1143,7 +1143,7 @@ dependencies = [
  "cookie",
  "futures-lite",
  "infer",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
  "rand",
  "serde 1.0.117",
  "serde_json",
@@ -1200,7 +1200,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "80aafab09693e9fa74b76ef207c55dc1cba5d9d5dc6dcc1b6a96d008a98000e9"
 dependencies = [
  "bytes",
- "crossbeam-utils 0.8.0",
+ "crossbeam-utils 0.8.1",
  "curl",
  "curl-sys",
  "flume",
@@ -1290,9 +1290,9 @@ dependencies = [
 
 [[package]]
 name = "libnghttp2-sys"
-version = "0.1.4+1.41.0"
+version = "0.1.5+1.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03624ec6df166e79e139a2310ca213283d6b3c30810c54844f307086d4488df1"
+checksum = "9657455ff47889b70ffd37c3e118e8cdd23fd1f9f3293a285f141070621c4c79"
 dependencies = [
  "cc",
  "libc",
@@ -1300,9 +1300,9 @@ dependencies = [
 
 [[package]]
 name = "libssh2-sys"
-version = "0.2.19"
+version = "0.2.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca46220853ba1c512fc82826d0834d87b06bcd3c2a42241b7de72f3d2fe17056"
+checksum = "df40b13fe7ea1be9b9dffa365a51273816c345fc1811478b57ed7d964fbfc4ce"
 dependencies = [
  "cc",
  "libc",
@@ -1763,6 +1763,12 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b"
 
+[[package]]
+name = "pin-project-lite"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c"
+
 [[package]]
 name = "pin-utils"
 version = "0.1.0"
@@ -1790,11 +1796,11 @@ dependencies = [
 
 [[package]]
 name = "polyval"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5884790f1ce3553ad55fec37b5aaac5882e0e845a2612df744d6c85c9bf046c"
+checksum = "b3fd900a291ceb8b99799cc8cd3d1d3403a51721e015bc533528b2ceafcc443c"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "universal-hash",
 ]
 
@@ -2287,7 +2293,7 @@ dependencies = [
  "log",
  "mime_guess",
  "once_cell",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
  "serde 1.0.117",
  "serde_json",
  "web-sys",
@@ -2295,9 +2301,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.50"
+version = "1.0.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "443b4178719c5a851e1bde36ce12da21d74a0e60b4d982ec3385a933c812f0f6"
+checksum = "3b4f34193997d92804d359ed09953e25d5138df6bcc055a71bf68ee89fdf9223"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2394,7 +2400,7 @@ dependencies = [
  "http-types",
  "kv-log-macro",
  "log",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
  "route-recognizer",
  "serde 1.0.117",
  "serde_json",
@@ -2417,7 +2423,7 @@ dependencies = [
  "http-types",
  "kv-log-macro",
  "log",
- "pin-project-lite",
+ "pin-project-lite 0.1.11",
  "route-recognizer",
  "serde 1.0.117",
  "serde_json",
@@ -2498,13 +2504,13 @@ dependencies = [
 
 [[package]]
 name = "tracing"
-version = "0.1.21"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b0987850db3733619253fe60e17cb59b82d37c7e6c0236bb81e4d6b87c879f27"
+checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "log",
- "pin-project-lite",
+ "pin-project-lite 0.2.0",
  "tracing-attributes",
  "tracing-core",
 ]

docs/Writing_Steps.adoc

@@ -0,0 +1,21 @@
+= Writing steps for Otto
+
+
+
+== Caching
+
+If the manifest contains `cache: true`, the agent will create a cache directory
+for this step to be used by all invocations. The location of the step's caching
+directory will be available in the invocation file:
+
+.Example invocation file snippet for the `git` step
+[source,yaml]
+----
+configuration:
+  cache: '/home/otto/caches/git'
+parameters:
+----
+
+The location of this caching directory should **not** be treated as
+deterministic, and can change at any time. Steps should always rely on the
+`cache` key in the invocation file's `configuration` block.

stdlib/git/Cargo.toml

@@ -9,3 +9,5 @@ git2 = "~0.13.12"
 otto-agent = { path = "../../crates/agent" }
 serde = {version = "~1.0.117", features = ["derive"]}
 url = "~2.2.0"
+# Used for managing the cached reference directories
+sha2 = "~0.9.2"

stdlib/git/src/main.rs

@@ -4,6 +4,7 @@
 
 use otto_agent::step::*;
 use serde::Deserialize;
+use std::path::PathBuf;
 use url::Url;
 
 #[derive(Clone, Debug, Deserialize)]
@@ -26,13 +27,110 @@ fn repo_from_url(repo_url: &Url) -> Option<String> {
     None
 }
 
-fn main() -> std::io::Result<()> {
-    use std::path::Path;
+/**
+ * Generate the reference repo path from the given Url
+ */
+fn locate_reference_for(url: &Url, cache_dir: &PathBuf) -> PathBuf {
+    use sha2::{Digest, Sha256};
+    let mut hasher = Sha256::new();
+    hasher.update(url.as_str());
+    let result = hasher.finalize();
+    cache_dir.join(format!("{:x}", result))
+}
 
+/**
+ * Clone a Git repository
+ */
+fn clone(
+    repo: String,
+    into: &PathBuf,
+    branch: Option<String>,
+    bare: Option<bool>,
+) -> std::io::Result<()> {
+    let mut builder = git2::build::RepoBuilder::new();
+
+    if let Some(branch) = branch {
+        builder.branch(&branch);
+    }
+
+    if let Some(bare) = bare {
+        // https://github.com/rust-lang/git2-rs/issues/521
+        builder
+            .bare(bare)
+            .remote_create(|repo, name, url| repo.remote_with_fetch(name, url, "+refs/*:refs/*"));
+    }
+
+    println!("Cloning {} into {:?}", repo, into);
+
+    let _repo = match builder.clone(&repo, into) {
+        Ok(repo) => repo,
+        Err(e) => panic!("failed to clone {} to {:?}: {}", repo, into, e),
+    };
+    Ok(())
+}
+
+/**
+ * Fetch all remotes in the given repository
+ */
+fn fetch(repo_path: &PathBuf, refs: Vec<String>, bare: bool) {
+    println!("Fetching updates for {:?} - {:?}", repo_path, refs);
+    let repo = match bare {
+        true => git2::Repository::open_bare(&repo_path).expect("Failed to open repo"),
+        false => git2::Repository::open(&repo_path).expect("Failed to open repo"),
+    };
+
+    if let Ok(remotes) = repo.remotes() {
+        for remote in remotes.iter() {
+            if let Ok(mut remote) = repo.find_remote(remote.unwrap()) {
+                remote.fetch(&refs, None, None).expect("Failed to fetch");
+            }
+        }
+    }
+}
+
+/**
+ * Return the String of the URL of the repo that should be relied upon for cloning
+ */
+fn reference_or_upstream_repo(invoke: &Invocation<Parameters>) -> String {
+    let url = &invoke.parameters.url;
+
+    if let Some(cache) = &invoke.configuration.cache {
+        /*
+         * When a cache directory is present, the step should create a new cached clone
+         * for this repo, or update the existing one and return the path
+         */
+        let ref_repo = locate_reference_for(url, cache);
+
+        if ref_repo.as_path().is_dir() {
+            let refs = match &invoke.parameters.branch {
+                Some(branch) => vec![branch.clone()],
+                None => vec![],
+            };
+            fetch(&ref_repo, refs, true);
+        } else {
+            clone(
+                url.clone().into_string(),
+                &ref_repo,
+                invoke.parameters.branch.clone(),
+                Some(true),
+            );
+        }
+        ref_repo.as_path().to_string_lossy().to_string()
+    } else {
+        /*
+         * In the cases where the cache directory isn't known, the step is just
+         * going to have to clone the source repo
+         */
+        url.clone().into_string()
+    }
+}
+
+fn main() -> std::io::Result<()> {
     let args = std::env::args().collect();
     let invoke: Invocation<Parameters> =
         invocation_from_args(&args).expect("Failed to deserialize the invocation for the step");
 
+    let repo_url = reference_or_upstream_repo(&invoke);
     let clone_path = match invoke.parameters.into {
         Some(into) => into,
         None => {
@@ -40,18 +138,12 @@ fn main() -> std::io::Result<()> {
         }
     };
 
-    println!("Clone!");
-    let mut builder = git2::build::RepoBuilder::new();
-
-    if let Some(branch) = &invoke.parameters.branch {
-        builder.branch(&branch);
-    }
-
-    let _repo = match builder.clone(&invoke.parameters.url.into_string(), Path::new(&clone_path)) {
-        Ok(repo) => repo,
-        Err(e) => panic!("failed to clone: {}", e),
-    };
-
+    clone(
+        repo_url,
+        &PathBuf::from(clone_path),
+        invoke.parameters.branch,
+        None,
+    );
     Ok(())
 }
 
@@ -71,4 +163,16 @@ mod tests {
         let u = Url::parse("https://example.com/repo").expect("Failed to parse");
         assert_eq!(repo_from_url(&u).unwrap(), "repo");
     }
+
+    #[test]
+    fn test_location_reference_for() {
+        use std::path::PathBuf;
+        let pb = PathBuf::from("/tmp/");
+        let url = Url::parse("https://example.com").expect("Failed to parse url");
+        let result = locate_reference_for(&url, &pb);
+        assert_eq!(
+            PathBuf::from("/tmp/0f115db062b7c0dd030b16878c99dea5c354b49dc37b38eb8846179c7783e9d7"),
+            result
+        );
+    }
 }

stdlib/git/tests/simple.sh

@@ -28,7 +28,7 @@ EOF
     rm -rf otto-test-repository
 }
 
-test_clone_ref_tag() {
+test_clone_ref_branch() {
     cat > $INVOCATION_FILE<<EOF
     {
     "configuration" : {
@@ -46,7 +46,6 @@ test_clone_ref_tag() {
 EOF
 
     output=$(git-step $INVOCATION_FILE)
-    echo $output
     assertTrue "step should be able to clone the given url: ${output}" $?
     assertTrue "step should have cloned the repo" "test -d otto-test-repository"
     assertTrue "step should have cloned the repo to the branch" "test -f otto-test-repository/this-is-a-branch"
@@ -76,7 +75,9 @@ EOF
         assertTrue "step should be able to clone the given url: ${output}" $?
         assertTrue "step should have cloned the repo into $PWD" "test -f README.adoc"
     popd
+    rm -rf work-dir
 }
+
 test_clone_with_cache() {
     cache_dir="$PWD/caches"
 
@@ -102,6 +103,67 @@ EOF
         assertTrue "step should be able to clone the given url: ${output}" $?
     popd
 
+    assertTrue "Reference repository should exist", "test -d ${cache_dir}/0884584c5aa4d28cbc4779fbc4cc9566625597528ee92e0092603e823057c1aa"
+    rm -rf work-dir
+}
+
+test_repeat_clone_with_cache() {
+    cache_dir="$PWD/caches"
+
+    cat > $INVOCATION_FILE<<EOF
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "cache" : "${cache_dir}",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+        }
+    },
+    "parameters" : {
+        "url" : "https://git.brokenco.de/rtyler/otto-test-repository"
+    }
+}
+EOF
+
+    # Clone into one working directory with the "main" refspec
+
+    mkdir work-dir
+    pushd work-dir
+        output=$(git-step $INVOCATION_FILE)
+        assertTrue "step should be able to clone the given url: ${output}" $?
+        assertTrue "step should have cloned the repo" "test -d otto-test-repository"
+    popd
+    rm -rf work-dir
+
+    # Now that we're confident that the cache is primed, try to clone
+    # a branch from that cached bare reference repo
+
+    cat > $INVOCATION_FILE<<EOF
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "cache" : "${cache_dir}",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+        }
+    },
+    "parameters" : {
+        "url" : "https://git.brokenco.de/rtyler/otto-test-repository",
+        "branch" : "test-branch"
+    }
+}
+EOF
+
+    mkdir work-dir
+    pushd work-dir
+        output=$(git-step $INVOCATION_FILE)
+        assertTrue "step should be able to clone the given url: ${output}" $?
+        assertTrue "step should have cloned the repo" "test -d otto-test-repository"
+        assertTrue "step should have cloned the repo to the branch" "test -f otto-test-repository/this-is-a-branch"
+    popd
+    rm -rf work-dir
 }
 
 . $(dirname $0)/../../../contrib/shunit2/shunit2