Finish out the basic implementation of the git step

Right now this won't support SSH URLs, but I'm okay with that. Fixes #32
Implement the most basic version of the git step which can just clone a URL
2020-11-15 20:29:44 -08:00 · 2020-11-15 16:12:12 -08:00 · 2020-11-15 16:11:32 -08:00 · 2020-11-15 15:10:08 -08:00 · 2020-11-15 14:45:43 -08:00 · 2020-11-15 14:45:23 -08:00
22 changed files with 473 additions and 100 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -132,9 +132,9 @@ dependencies = [

 [[package]]
 name = "async-executor"
-version = "1.3.0"
+version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d373d78ded7d0b3fa8039375718cde0aace493f2e34fb60f51cbf567562ca801"
+checksum = "eb877970c7b440ead138f6321a3b5395d6061183af779340b65e20c0fede9146"
 dependencies = [
 "async-task",
 "concurrent-queue",
@ -175,9 +175,9 @@ dependencies = [

 [[package]]
 name = "async-io"
-version = "1.1.10"
+version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d54bc4c1c7292475efb2253227dbcfad8fe1ca4c02bc62c510cc2f3da5c4704e"
+checksum = "40a0b2bb8ae20fede194e779150fe283f65a4a08461b496de546ec366b174ad9"
 dependencies = [
 "concurrent-queue",
 "fastrand",
@ -455,6 +455,9 @@ name = "cc"
 version = "1.0.62"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40"
+dependencies = [
+ "jobserver",
+]

 [[package]]
 name = "cfg-if"
@ -879,24 +882,24 @@ dependencies = [

 [[package]]
 name = "futures-channel"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0448174b01148032eed37ac4aed28963aaaa8cfa93569a08e5b479bbc6c2c151"
+checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64"
 dependencies = [
 "futures-core",
 ]

 [[package]]
 name = "futures-core"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "18eaa56102984bed2c88ea39026cff3ce3b4c7f508ca970cedf2450ea10d4e46"
+checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748"

 [[package]]
 name = "futures-io"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e1798854a4727ff944a7b12aa999f58ce7aa81db80d2dfaaf2ba06f065ddd2b"
+checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb"

 [[package]]
 name = "futures-lite"
@ -915,9 +918,9 @@ dependencies = [

 [[package]]
 name = "futures-macro"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e36fccf3fc58563b4a14d265027c627c3b665d7fed489427e88e7cc929559efe"
+checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556"
 dependencies = [
 "proc-macro-hack",
 "proc-macro2",
@ -927,18 +930,18 @@ dependencies = [

 [[package]]
 name = "futures-task"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96d502af37186c4fef99453df03e374683f8a1eec9dcc1e66b3b82dc8278ce3c"
+checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d"
 dependencies = [
 "once_cell",
 ]

 [[package]]
 name = "futures-util"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abcb44342f62e6f3e8ac427b8aa815f724fd705dfad060b18ac7866c15bb8e34"
+checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2"
 dependencies = [
 "futures-core",
 "futures-io",
@ -991,6 +994,31 @@ dependencies = [
 "polyval",
 ]

+[[package]]
+name = "git-step"
+version = "0.1.0"
+dependencies = [
+ "git2",
+ "otto-agent",
+ "serde 1.0.117",
+ "url",
+]
+
+[[package]]
+name = "git2"
+version = "0.13.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca6f1a0238d7f8f8fd5ee642f4ebac4dbc03e03d1f78fbe7a3ede35dcf7e2224"
+dependencies = [
+ "bitflags",
+ "libc",
+ "libgit2-sys",
+ "log",
+ "openssl-probe",
+ "openssl-sys",
+ "url",
+]
+
 [[package]]
 name = "glob"
 version = "0.3.0"
@ -1097,9 +1125,9 @@ dependencies = [

 [[package]]
 name = "http-types"
-version = "2.7.0"
+version = "2.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffa4e35764a650ce8e709c50e985431eb8963eee7fc793d923134d4aa8e530b4"
+checksum = "6f316f6a06306570e899238d3b85375f350cfceda60ec47807c4164d6e169e58"
 dependencies = [
 "anyhow",
 "async-channel",
@ -1160,9 +1188,9 @@ dependencies = [

 [[package]]
 name = "isahc"
-version = "0.9.11"
+version = "0.9.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0cf84ee8215bcaa999a24870485ef49d051cd858985a80d123e56be2d921d811"
+checksum = "003be15733203fb6e7a926e4a23517b0dbc45e4ab3ead5bbc24c7ad84faae681"
 dependencies = [
 "bytes",
 "crossbeam-channel",
@ -1187,6 +1215,15 @@ version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"

+[[package]]
+name = "jobserver"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c71313ebb9439f74b00d9d2dcec36440beaf57a6aa0623068441dd7cd81a7f2"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "js-sys"
 version = "0.3.45"
@ -1230,6 +1267,20 @@ version = "0.2.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614"

+[[package]]
+name = "libgit2-sys"
+version = "0.12.14+1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f25af58e6495f7caf2919d08f212de550cfa3ed2f5e744988938ea292b9f549"
+dependencies = [
+ "cc",
+ "libc",
+ "libssh2-sys",
+ "libz-sys",
+ "openssl-sys",
+ "pkg-config",
+]
+
 [[package]]
 name = "libnghttp2-sys"
 version = "0.1.4+1.41.0"
@ -1240,6 +1291,20 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "libssh2-sys"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca46220853ba1c512fc82826d0834d87b06bcd3c2a42241b7de72f3d2fe17056"
+dependencies = [
+ "cc",
+ "libc",
+ "libz-sys",
+ "openssl-sys",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "libz-sys"
 version = "1.1.2"
@ -1329,9 +1394,9 @@ dependencies = [

 [[package]]
 name = "native-tls"
-version = "0.2.5"
+version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a1cda389c26d6b88f3d2dc38aa1b750fe87d298cc5d795ec9e975f402f00372"
+checksum = "6fcc7939b5edc4e4f86b1b4a04bb1498afaaf871b1a6691838ed06fcb48d3a3f"
 dependencies = [
 "lazy_static",
 "libc",
@ -1406,9 +1471,9 @@ dependencies = [

 [[package]]
 name = "once_cell"
-version = "1.4.1"
+version = "1.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad"
+checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"

 [[package]]
 name = "opaque-debug"
@ -1487,9 +1552,10 @@ dependencies = [
 "otto-models",
 "pretty_env_logger 0.4.0",
 "serde 1.0.117",
+ "serde_json",
 "serde_yaml",
 "tempfile",
- "tide",
+ "tide 0.14.0",
 "url",
 "uuid",
 ]
@ -1523,7 +1589,7 @@ name = "otto-models"
 version = "0.1.0"
 dependencies = [
 "serde 1.0.117",
- "serde_yaml",
+ "serde_json",
 "uuid",
 ]

@ -1534,7 +1600,7 @@ dependencies = [
 "async-std",
 "log",
 "pretty_env_logger 0.4.0",
- "tide",
+ "tide 0.15.0",
 ]

 [[package]]
@ -1548,8 +1614,7 @@ dependencies = [
 "pest_derive",
 "pretty_env_logger 0.4.0",
 "serde_json",
- "serde_yaml",
- "tide",
+ "tide 0.15.0",
 "uuid",
 ]

@ -2044,21 +2109,20 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"

 [[package]]
 name = "sluice"
-version = "0.5.2"
+version = "0.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed13b7cb46f13a15db2c4740f087a848acc8b31af89f95844d40137451f89b1"
+checksum = "8e24ed1edc8e774f2ec098b0650eec82bfc7c59ddd16cd0e17797bdc92ce2bf1"
 dependencies = [
 "futures-channel",
 "futures-core",
 "futures-io",
- "futures-util",
 ]

 [[package]]
 name = "socket2"
-version = "0.3.15"
+version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1fa70dc5c8104ec096f4fe7ede7a221d35ae13dcd19ba1ad9a81d2cab9a1c44"
+checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d"
 dependencies = [
 "cfg-if 0.1.10",
 "libc",
@ -2277,6 +2341,29 @@ dependencies = [
 "serde_json",
 ]

+[[package]]
+name = "tide"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c832ca099a0645f6446b6c8900b09e3b76978dedc0d7383438698b32ff07f734"
+dependencies = [
+ "async-h1",
+ "async-session",
+ "async-sse",
+ "async-std",
+ "async-trait",
+ "femme",
+ "futures-util",
+ "http-client",
+ "http-types",
+ "kv-log-macro",
+ "log",
+ "pin-project-lite",
+ "route-recognizer",
+ "serde 1.0.117",
+ "serde_json",
+]
+
 [[package]]
 name = "time"
 version = "0.1.44"
@ -2328,9 +2415,18 @@ dependencies = [

 [[package]]
 name = "tinyvec"
-version = "0.3.4"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117"
+checksum = "b78a366903f506d2ad52ca8dc552102ffdd3e937ba8a227f024dc1d1eae28575"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"

 [[package]]
 name = "toml"
@ -2426,9 +2522,9 @@ dependencies = [

 [[package]]
 name = "unicode-normalization"
-version = "0.1.13"
+version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977"
+checksum = "b7f98e67a4d84f730d343392f9bfff7d21e3fca562b9cb7a43b768350beeddc6"
 dependencies = [
 "tinyvec",
 ]
@ -2457,10 +2553,11 @@ dependencies = [

 [[package]]
 name = "url"
-version = "2.1.1"
+version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb"
+checksum = "5909f2b0817350449ed73e8bcd81c8c3c8d9a7a5d8acba4b27db277f1868976e"
 dependencies = [
+ "form_urlencoded",
 "idna",
 "matches",
 "percent-encoding",
--- a/Cargo.toml
+++ b/Cargo.toml
@ -13,6 +13,7 @@ members = [
    "stdlib/archive",
    "stdlib/dir",
    "stdlib/error",
+    "stdlib/git",
    "stdlib/sh",
    "stdlib/unarchive",
 ]
--- a/9
+++ b/9
@ -21,10 +21,15 @@ steps: release
 	done;

 test: contrib/shunit2/shunit2 ## Run the acceptance tests for steps
-	@for t in $$(find stdlib -iname "tests" -type d); do \
+	set -e
+	@for t in $$(find $(PWD)/stdlib -iname "tests" -type d); do \
 		echo ">> Running acceptance tests for $$t"; \
 		for f in $$(find $$t -iname "*.sh" -type f); do \
-			PATH="target/debug:$(PATH)" "$$f"; \
+			DIR="$(PWD)/tmp/test-run-$${RANDOM}"; \
+			echo ">> Using $${DIR} for $$f"; \
+			mkdir -p $$DIR; \
+			(cd $$DIR && \
+			PATH="$(PWD)/target/debug:$(PATH)" "$$f"); \
 		done; \
 	done;

--- a/agent/Cargo.toml
+++ b/agent/Cargo.toml
@ -19,9 +19,11 @@ otto-models = { path = "../models" }
 osp = { path = "../osp" }
 os_pipe = "~0.9.2"
 pretty_env_logger = "~0.4.0"
+serde_json = "~1.0.59"
+# Needed for reading manifest yamls
 serde_yaml = "~0.8.13"
 serde = {version = "~1.0.117", features = ["rc", "derive"]}
 tempfile = "~3.1.0"
 tide = "~0.14.0"
-url = "~2.1.1"
+url = "~2.2.0"
 uuid = { version = "~0.8.1", features = ["v4", "serde"]}
--- a/agent/src/lib.rs
+++ b/agent/src/lib.rs
@ -149,7 +149,7 @@ pub fn run(
                parameters: step.parameters.clone(),
            };

-            serde_yaml::to_writer(&mut file, &invocation)
+            serde_json::to_writer(&mut file, &invocation)
                .expect("Failed to write temporary file for script");

            use os_pipe::pipe;
@ -228,7 +228,7 @@ mod tests {

    #[test]
    fn load_manifests_stdlib() {
-        let params = serde_yaml::Value::Null;
+        let params = serde_json::Value::Null;
        let step = Step {
            symbol: "echo".to_string(),
            uuid: otto_models::generate_uuid(),
--- a/agent/src/main.rs
+++ b/agent/src/main.rs
@ -40,7 +40,7 @@ async fn main() -> std::io::Result<()> {
    let file = File::open(&args[1])?;
    let (sender, receiver) = channel(MAX_CONTROL_MSGS);

-    match serde_yaml::from_reader::<File, Invocation>(file) {
+    match serde_json::from_reader::<File, Invocation>(file) {
        Err(e) => {
            panic!("Failed to parse parameters file: {:#?}", e);
        }
--- a/agent/src/step.rs
+++ b/agent/src/step.rs
@ -64,7 +64,7 @@ pub fn invocation_from_args<P: serde::de::DeserializeOwned>(

    let file = File::open(&args[1])?;

-    match serde_yaml::from_reader::<File, Invocation<P>>(file) {
+    match serde_json::from_reader::<File, Invocation<P>>(file) {
        Err(e) => {
            error!("Failed to deserialize invocation file: {:#?}", e);
            Err(Error::new(ErrorKind::InvalidData, e))
--- a/models/Cargo.toml
+++ b/models/Cargo.toml
@ -6,5 +6,5 @@ edition = "2018"

 [dependencies]
 serde = {version = "~1.0.117", features = ["rc", "derive"]}
-serde_yaml = "~0.8.13"
+serde_json = "~1.0.59"
 uuid = { version = "~0.8.1", features = ["v4", "serde"]}
--- a/models/src/lib.rs
+++ b/models/src/lib.rs
@ -1,5 +1,5 @@
 use serde::{Deserialize, Serialize};
-use serde_yaml::Value;
+use serde_json::Value;
 use std::collections::HashMap;
 use uuid::Uuid;

@ -166,12 +166,13 @@ mod tests {
    #[test]
    fn deserialize_positional() {
        let buf = r#"
-    symbol: sh
-    uuid: '5599cffb-f23a-4e0f-a0b9-f74654641b2b'
-    context: '3ce1f6fb-79ca-4564-a47e-98265f53ef7f'
-    parameters:
-      - 'ls -lah | tail -n 5'"#;
-        let step = serde_yaml::from_str::<Step>(&buf).expect("Failed to deserialize");
+        {"symbol":"sh",
+        "uuid":"5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "context":"3ce1f6fb-79ca-4564-a47e-98265f53ef7f",
+        "parameters" : [
+            "ls -lah | tail -n 5"
+        ]}"#;
+        let step = serde_json::from_str::<Step>(&buf).expect("Failed to deserialize");

        assert_eq!(step.symbol, "sh");
    }
@ -179,12 +180,13 @@ mod tests {
    #[test]
    fn deserialize_kwargs() {
        let buf = r#"
-    symbol: sh
-    uuid: '5599cffb-f23a-4e0f-a0b9-f74654641b2b'
-    context: '3ce1f6fb-79ca-4564-a47e-98265f53ef7f'
-    parameters:
-      script: 'ls -lah | tail -n 5'"#;
-        let step = serde_yaml::from_str::<Step>(&buf).expect("Failed to deserialize");
+        {"symbol":"sh",
+        "uuid":"5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "context":"3ce1f6fb-79ca-4564-a47e-98265f53ef7f",
+        "parameters" : {
+            "script" : "ls -lah | tail -n 5"
+        }}"#;
+        let step = serde_json::from_str::<Step>(&buf).expect("Failed to deserialize");

        assert_eq!(step.symbol, "sh");
    }
--- a/services/object-store/Cargo.toml
+++ b/services/object-store/Cargo.toml
@ -17,4 +17,4 @@ path = "src/main.rs"
 async-std = { version = "~1.7", features = ["attributes"]}
 log = "~0.4.11"
 pretty_env_logger = "~0.4.0"
-tide = "~0.14.0"
+tide = "~0.15.0"
--- a/services/parser/Cargo.toml
+++ b/services/parser/Cargo.toml
@ -20,6 +20,5 @@ pest = "~2.1.3"
 pest_derive = "~2.1.0"
 pretty_env_logger = "~0.4.0"
 serde_json = "~1.0.59"
-serde_yaml = "~0.8.13"
-tide = "~0.14.0"
+tide = "~0.15.0"
 uuid = { version = "~0.8.1", features = ["v4", "serde"]}
--- a/services/parser/src/lib.rs
+++ b/services/parser/src/lib.rs
@ -87,7 +87,7 @@ fn parse_steps(parser: &mut Pairs<Rule>, uuid: Uuid) -> Vec<Step> {
            let symbol = parts[0].as_str().to_string();
            let command = parse_str(&mut parts.pop().unwrap());

-            let parameters = serde_yaml::Value::String(command);
+            let parameters = serde_json::Value::String(command);
            let parameters = StepParameters::Positional(vec![parameters]);
            let step = Step::new(uuid, symbol, parameters);
            steps.push(step);
--- a/stdlib/archive/README.adoc
+++ b/stdlib/archive/README.adoc
@ -3,4 +3,24 @@
 The `archive` step will store files or file patterns associated with the
 running pipeline in the Otto object store.

-This can be helpful for persisting logs, or built artifacts.
+This can be helpful for persisting logs or built artifacts.
+
+
+[NOTE]
+====
+File patterns are matched using the link:https://docs.rs/glob/0.3.0[glob]
+crate. For questions about its file pattern syntax, refer to its documentation.
+====
+
+
+.Single archive
+[source]
+----
+archive 'my-release.tar.gz'
+----
+
+.Pattern archive
+[source]
+----
+archive 'target/release/*.tar.gz'
+---
--- a/stdlib/archive/tests/invalid_cases.sh
+++ b/stdlib/archive/tests/invalid_cases.sh
@ -1,6 +1,6 @@
 #!/bin/sh

-INVOCATION_FILE=tmp_archivetest_invocation_file.yml
+INVOCATION_FILE=tmp_archivetest_invocation_file.json

 oneTimeTearDown() {
    rm -f $INVOCATION_FILE
@ -12,20 +12,28 @@ test_fail_with_no_file() {

 test_fail_with_malformed_file() {
    cat > $INVOCATION_FILE <<EOF
---
-# This is an invalid yaml file
+    {}
 EOF
    assertFalse "step should fail when invoked with a bad invocation file" "archive-step $INVOCATION_FILE"
 }

 test_fail_with_absolute_path() {
    cat > $INVOCATION_FILE<<EOF
---
-configuration:
-  ipc: unix:///dev/null
-  endpoints: {}
-parameters:
-  artifacts: /etc/passwd
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+            "objects" : {
+                "url" : "http://example.com"
+            }
+        }
+    },
+    "parameters" : {
+        "artifacts" : "/etc/passwd"
+    }
+}
 EOF

    assertFalse "step should fail when invoked with an absolute path" "archive-step $INVOCATION_FILE"
@ -33,12 +41,21 @@ EOF

 test_fail_with_path_traversal() {
    cat > $INVOCATION_FILE<<EOF
---
-configuration:
-  ipc: unix:///dev/null
-  endpoints: {}
-parameters:
-  artifacts: ../../../
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+            "objects" : {
+                "url" : "http://example.com"
+            }
+        }
+    },
+    "parameters" : {
+        "artifacts" : "../../../"
+    }
+}
 EOF

    assertFalse "step should fail when invoked with an absolute path" "archive-step $INVOCATION_FILE"
--- a/stdlib/archive/tests/valid_cases.sh
+++ b/stdlib/archive/tests/valid_cases.sh
@ -1,40 +1,62 @@
 #!/bin/sh

-INVOCATION_FILE=tmp_archivetest_invocation_file.yml
-TAR_NAME=tmp_archivetest_tar
+INVOCATION_FILE=tmp_archivetest_invocation_file.json

 oneTimeTearDown() {
    rm -f $INVOCATION_FILE
-    rm -f "${TAR_NAME}.tar.gz"
-
 }

-test_fail_with_file() {
+test_pass_with_file() {
    cat > $INVOCATION_FILE<<EOF
---
-configuration:
-  ipc: unix:///dev/null
-  endpoints: {}
-parameters:
-  artifacts: Cargo.toml
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+            "objects" : {
+                "url" : "http://example.com"
+            }
+        }
+    },
+    "parameters" : {
+        "artifacts" : "Cargo.toml"
+    }
+}
 EOF

+    echo "Hello world" > Cargo.toml
+
    assertTrue "step should do nothing with a single file" "archive-step $INVOCATION_FILE"
 }

-test_fail_with_dir() {
+test_pass_with_dir() {
+    TAR_NAME=tmp_archivetest_tar
    cat > $INVOCATION_FILE<<EOF
---
-configuration:
-  ipc: unix:///dev/null
-  endpoints: {}
-parameters:
-  artifacts: $(dirname $0)
-  name: ${TAR_NAME}
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+            "objects" : {
+                "url" : "http://example.com"
+            }
+        }
+    },
+    "parameters" : {
+        "artifacts": "release",
+        "name": "${TAR_NAME}"
+    }
+    }
 EOF
+    mkdir release
+    touch release/one
+    touch release/two

    assertTrue "step should create tarball with a directory" "archive-step $INVOCATION_FILE"
    assertTrue "file name ${TAR_NAME}.tar.gz not found" "test -f ${TAR_NAME}.tar.gz"
+    rm -f "${TAR_NAME}.tar.gz"
 }

 . $(dirname $0)/../../../contrib/shunit2/shunit2
--- a/stdlib/git/.gitignore
+++ b/stdlib/git/.gitignore
@ -0,0 +1 @@
+target/
--- a/stdlib/git/Cargo.toml
+++ b/stdlib/git/Cargo.toml
@ -0,0 +1,11 @@
+[package]
+name = "git-step"
+version = "0.1.0"
+authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
+edition = "2018"
+
+[dependencies]
+git2 = "~0.13.12"
+otto-agent = { path = "../../agent" }
+serde = {version = "~1.0.117", features = ["derive"]}
+url = "~2.2.0"
--- a/stdlib/git/README.adoc
+++ b/stdlib/git/README.adoc
@ -0,0 +1,13 @@
+= git
+
+
+The `git` step provides a simple mechanism for checking out code from a git
+repository.
+
+
+.Simple clone
+[source]
+----
+git url: 'https://github.com/rtyler/otto', branch: 'main'
+----
+
--- a/stdlib/git/manifest.yml
+++ b/stdlib/git/manifest.yml
@ -0,0 +1,25 @@
+---
+symbol: git
+description: |
+  The `git` step will clone a given url.
+
+includes:
+  - name: target/release/git-step
+    flatten: true
+
+entrypoint:
+  path: git-step
+  multiarch: false
+
+parameters:
+  - name: url
+    required: true
+    type: string
+    description: |
+      The URL to pass to the underlying Git library
+
+  - name: branch
+    required: false
+    type: string
+    description: |
+      A git branch to clone instead of the default.
--- a/stdlib/git/src/main.rs
+++ b/stdlib/git/src/main.rs
@ -0,0 +1,69 @@
+/*
+ * The git step does simple clones of git repositories
+ */
+
+use otto_agent::step::*;
+use serde::Deserialize;
+use url::Url;
+
+#[derive(Clone, Debug, Deserialize)]
+struct Parameters {
+    url: Url,
+    branch: Option<String>,
+}
+
+/**
+ * This funciton is a simple function which will just tease out the local directory
+ * path which should be used for cloning this repository URL
+ */
+fn repo_from_url(repo_url: &Url) -> Option<String> {
+    if let Some(segments) = repo_url.path_segments() {
+        if let Some(last) = segments.last() {
+            return Some(last.replace(".git", ""));
+        }
+    }
+    None
+}
+
+fn main() -> std::io::Result<()> {
+    use std::path::Path;
+
+    let args = std::env::args().collect();
+    let invoke: Invocation<Parameters> =
+        invocation_from_args(&args).expect("Failed to deserialize the invocation for the step");
+
+    if let Some(path) = repo_from_url(&invoke.parameters.url) {
+        println!("Clone!");
+        let mut builder = git2::build::RepoBuilder::new();
+
+        if let Some(branch) = &invoke.parameters.branch {
+            builder.branch(&branch);
+        }
+        let _repo = match builder.clone(&invoke.parameters.url.into_string(), Path::new(&path)) {
+            Ok(repo) => repo,
+            Err(e) => panic!("failed to clone: {}", e),
+        };
+    } else {
+        println!("Failed to determine the right local path to clone the repository into");
+        std::process::exit(1);
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use url::Url;
+
+    #[test]
+    fn test_simple_url() {
+        let u = Url::parse("https://example.com/repo.git").expect("Failed to parse");
+        assert_eq!(repo_from_url(&u).unwrap(), "repo");
+    }
+
+    #[test]
+    fn test_simple_url_no_dot_git() {
+        let u = Url::parse("https://example.com/repo").expect("Failed to parse");
+        assert_eq!(repo_from_url(&u).unwrap(), "repo");
+    }
+}
--- a/stdlib/git/tests/simple.sh
+++ b/stdlib/git/tests/simple.sh
@ -0,0 +1,55 @@
+#!/bin/sh
+
+INVOCATION_FILE=tmp_gittest_invocation_file.json
+
+oneTimeTearDown() {
+    rm -f $INVOCATION_FILE
+}
+
+test_clone_simple() {
+    cat > $INVOCATION_FILE<<EOF
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+        }
+    },
+    "parameters" : {
+        "url" : "https://git.brokenco.de/rtyler/otto-test-repository"
+    }
+}
+EOF
+
+    output=$(git-step $INVOCATION_FILE)
+    assertTrue "step should be able to clone the given url: ${output}" $?
+    assertTrue "step should have cloned the repo" "test -d otto-test-repository"
+    rm -rf otto-test-repository
+}
+
+test_clone_ref_tag() {
+    cat > $INVOCATION_FILE<<EOF
+    {
+    "configuration" : {
+        "pipeline" : "2265b5d0-1f70-46de-bf50-f1050e9fac9a",
+        "uuid" : "5599cffb-f23a-4e0f-a0b9-f74654641b2b",
+        "ipc" : "unix:///dev/null",
+        "endpoints" : {
+        }
+    },
+    "parameters" : {
+        "url" : "https://git.brokenco.de/rtyler/otto-test-repository",
+        "branch" : "test-branch"
+    }
+}
+EOF
+
+    output=$(git-step $INVOCATION_FILE)
+    assertTrue "step should be able to clone the given url: ${output}" $?
+    assertTrue "step should have cloned the repo" "test -d otto-test-repository"
+    assertTrue "step should have cloned the repo to the branch" "test -f otto-test-repository/this-is-a-branch"
+    rm -rf otto-test-repository
+}
+
+. $(dirname $0)/../../../contrib/shunit2/shunit2
--- a/stdlib/unarchive/README.adoc
+++ b/stdlib/unarchive/README.adoc
@ -0,0 +1,34 @@
+= unarchive
+
+The `unarchive` step can help retrieve archived artifacts into the current
+agent's executing workspace. This pattern allows using one agent for building
+artifacts, and retrieving them on other agents for further use in the pipeline.
+
+
+.Simple archive/unarchive
+[source]
+----
+stage {
+  name = 'Build'
+  steps {
+    sh 'make release'
+    archive 'build/release/hello-world'
+  }
+}
+stage {
+  name = 'Test'
+  steps {
+    unarchive 'hello-world'
+    sh 'chmod +x hello-world && ./hello-world'
+  }
+}
+----
+
+.Unarchiving patterns
+[source]
+----
+archive artifacts: 'target/release/*.tar.gz', name: 'release-builds'
+
+
+unarchive 'release-builds'
+---
Author	SHA1	Message	Date
R Tyler Croy	4b6f99a099	Finish out the basic implementation of the git step Right now this won't support SSH URLs, but I'm okay with that. Fixes #32	2020-11-15 20:29:44 -08:00
R Tyler Croy	28513c7a0b	Implement the most basic version of the git step which can just clone a URL This doesn't yet support the `ref` parameter, and is frankly pretty dumb Fixes #32	2020-11-15 16:12:12 -08:00
R Tyler Croy	610b16d542	Restructure the shunit2 integration tests to use a test directory for each run This will help avoid conflicts	2020-11-15 16:11:32 -08:00
R Tyler Croy	0d17491f97	Make the invalid cases for the archive step fail again for the right reasons	2020-11-15 15:10:08 -08:00
R Tyler Croy	8d5d2b7cea	cargo update	2020-11-15 14:45:43 -08:00
R Tyler Croy	e4e3660d3a	Upgrade to the latest tide release 👏	2020-11-15 14:45:23 -08:00
R Tyler Croy	bea61697fa	Ensure that `make test` will return a non-zero on shunit2 test failures GitHub Actions were in fact catching the failure. Oops	2020-11-15 14:40:59 -08:00
R Tyler Croy	273a06ad94	Fix failing integration tests for the archive step These had been failing for a few commits, oops	2020-11-15 14:39:14 -08:00
R Tyler Croy	1b6ebdd3db	Work in progress on the git step See #32	2020-11-15 14:39:00 -08:00
R Tyler Croy	43540a5dd6	Switch the invocation file formats from YAML to JSON YAML is a bonkers format with a lot of leeway for bugs, JSON is dumb but very predictable. Since these are machine-read files, they can be JSON. The human-editable step library manifests will remain YAML	2020-11-14 20:15:16 -08:00