cargo update

Still not sure what this API should look like, but will start playing around
with it.

Cargo.lock

@@ -285,9 +285,9 @@ checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0"
 
 [[package]]
 name = "async-trait"
-version = "0.1.41"
+version = "0.1.42"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0"
+checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -452,9 +452,9 @@ checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba"
 
 [[package]]
 name = "cc"
-version = "1.0.62"
+version = "1.0.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f1770ced377336a88a67c473594ccc14eca6f4559217c34f64aac8f83d641b40"
+checksum = "95752358c8f7552394baf48cd82695b345628ad3f170d607de3ca03b8dacca15"
 dependencies = [
  "jobserver",
 ]
@@ -560,7 +560,7 @@ dependencies = [
  "percent-encoding",
  "rand",
  "sha2",
- "time 0.2.22",
+ "time 0.2.23",
  "version_check",
 ]
 
@@ -841,9 +841,9 @@ dependencies = [
 
 [[package]]
 name = "flume"
-version = "0.9.1"
+version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9e818efa7776f4dd7df0e542f877f7a5a87bddd6a1a10f59a7732b71ffb9d55"
+checksum = "1bebadab126f8120d410b677ed95eee4ba6eb7c6dd8e34a5ec88a08050e26132"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -955,7 +955,7 @@ dependencies = [
  "futures-macro",
  "futures-task",
  "memchr",
- "pin-project 1.0.1",
+ "pin-project 1.0.2",
  "pin-utils",
  "proc-macro-hack",
  "proc-macro-nested",
@@ -1186,9 +1186,9 @@ checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac"
 
 [[package]]
 name = "instant"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb1fc4429a33e1f80d41dc9fea4d108a88bec1de8053878898ae448a0b52f613"
+checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
 dependencies = [
  "cfg-if 1.0.0",
 ]
@@ -1342,9 +1342,9 @@ checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
 
 [[package]]
 name = "lock_api"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
+checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
 dependencies = [
  "scopeguard",
 ]
@@ -1600,6 +1600,16 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "otto-local-orchestrator"
+version = "0.1.0"
+dependencies = [
+ "async-std",
+ "log",
+ "pretty_env_logger 0.4.0",
+ "tide 0.15.0",
+]
+
 [[package]]
 name = "otto-models"
 version = "0.1.0"
@@ -1700,11 +1710,11 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee41d838744f60d959d7074e3afb6b35c7456d0f61cad38a24e35e6553f73841"
+checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7"
 dependencies = [
- "pin-project-internal 1.0.1",
+ "pin-project-internal 1.0.2",
 ]
 
 [[package]]
@@ -1720,9 +1730,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-internal"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81a4ffa594b66bff340084d4081df649a7dc049ac8d7fc458d8e628bfbbb2f86"
+checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2142,11 +2152,11 @@ dependencies = [
 
 [[package]]
 name = "socket2"
-version = "0.3.16"
+version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fd8b795c389288baa5f355489c65e71fd48a02104600d15c4cfbc561e9e429d"
+checksum = "2c29947abdee2a218277abeca306f25789c938e500ea5a9d4b12a5a504466902"
 dependencies = [
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "libc",
  "redox_syscall",
  "winapi",
@@ -2169,9 +2179,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
 [[package]]
 name = "standback"
-version = "0.2.11"
+version = "0.2.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f4e0831040d2cf2bdfd51b844be71885783d489898a192f254ae25d57cce725c"
+checksum = "cf906c8b8fc3f6ecd1046e01da1d8ddec83e48c8b08b84dcc02b585a6bedf5a8"
 dependencies = [
  "version_check",
 ]
@@ -2267,9 +2277,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "1.0.48"
+version = "1.0.50"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc371affeffc477f42a221a1e4297aedcea33d47d19b61455588bd9d8f6b19ac"
+checksum = "443b4178719c5a851e1bde36ce12da21d74a0e60b4d982ec3385a933c812f0f6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2304,9 +2314,9 @@ dependencies = [
 
 [[package]]
 name = "termcolor"
-version = "1.1.0"
+version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
+checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4"
 dependencies = [
  "winapi-util",
 ]
@@ -2408,9 +2418,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.2.22"
+version = "0.2.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "55b7151c9065e80917fbf285d9a5d1432f60db41d170ccafc749a136b41a93af"
+checksum = "bcdaeea317915d59b2b4cd3b5efcd156c309108664277793f5351700c02ce98b"
 dependencies = [
  "const_fn",
  "libc",
@@ -2446,9 +2456,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.0.1"
+version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b78a366903f506d2ad52ca8dc552102ffdd3e937ba8a227f024dc1d1eae28575"
+checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -2553,9 +2563,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-normalization"
-version = "0.1.14"
+version = "0.1.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7f98e67a4d84f730d343392f9bfff7d21e3fca562b9cb7a43b768350beeddc6"
+checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606"
 dependencies = [
  "tinyvec",
 ]

Cargo.toml

@@ -7,6 +7,7 @@ members = [
 
     "services/auctioneer",
     "services/eventbus",
+    "services/local-orchestrator",
     "services/object-store",
     "services/parser",
 

Makefile

@@ -20,6 +20,10 @@ steps: release
 		./target/release/osp $$dir; \
 	done;
 
+apispecs:
+	schemathesis run ./services/parser/apispec.yml --base-url=http://localhost:7672 --checks all
+	schemathesis run ./services/local-orchestrator/apispec.yml --base-url=http://localhost:7673 --checks all
+
 test: contrib/shunit2/shunit2 ## Run the acceptance tests for steps
 	set -e
 	@for t in $$(find $(PWD)/stdlib -iname "tests" -type d); do \
@@ -48,4 +52,4 @@ contrib/shunit2/shunit2:
 
 ################################################################################
 
-.PHONY: clean diagram help steps release
+.PHONY: apispecs clean diagram help steps release

Procfile

@@ -3,5 +3,7 @@
 #
 
 object-store: OTTO_OBJECT_DIR=tmp/objects RUST_LOG=debug ./target/debug/otto-object-store
+orchestrator: RUST_LOG=debug ./target/debug/otto-local-orchestrator
+parser: RUST_LOG=debug ./target/debug/otto-parser
 
 # vim: ft=sh

docs/Developing_Services.adoc

@@ -23,5 +23,9 @@ This document captures some details around developing these Otto services.
 | Parser
 | Service which parses .otto files and spits out the Otto intermediate execution format.
 
+| 7673
+| Orchestrator
+| Service which can provision environments capable of executing agents.
+
 |===
 

services/local-orchestrator/.gitignore

@@ -0,0 +1 @@
+target/

services/local-orchestrator/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "otto-local-orchestrator"
+version = "0.1.0"
+authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
+edition = "2018"
+
+[dependencies]
+async-std = { version = "~1.7", features = ["attributes"]}
+log = "~0.4.11"
+pretty_env_logger = "~0.4.0"
+tide = "~0.15.0"

services/local-orchestrator/README.adoc

@@ -0,0 +1,12 @@
+= Local Orchestrator
+
+The Otto Local Orchestrator is the simplest implementation of an orchestrator
+in that it simply receives the orchestration payload and launches the
+`otto-agent` locally.
+
+
+.Environnment Variables
+|===
+| Name | Default | Description
+
+|===

services/local-orchestrator/apispec.yml

@@ -0,0 +1,34 @@
+---
+openapi: 3.0.3
+info:
+  title: Otto Orchestrator Service
+  description: |
+    This specification describes the Local Orchestrator which doesn't really do
+    much other than run an `otto-agent` locally.
+  version: '1.0.0'
+  contact:
+    name: R Tyler Croy
+    email: 'rtyler@brokenco.de'
+    x-twitter: agentdero
+  license:
+    name: 'GNU AGPL 3.0'
+    url: 'https://www.gnu.org/licenses/agpl-3.0.en.html'
+externalDocs:
+  description: 'Find out more about Otto'
+  url: 'https://github.com/rtyler/otto'
+servers:
+  - url: 'http://localhost:7673'
+    description: 'Local dev server'
+paths:
+  '/health':
+    get:
+      operationId: GetHealth
+      description: |
+        The health endpoint helps indicate whether the service is healthy or not.
+        Any non-200 response is unhealthy.
+      responses:
+        '200':
+          description: 'A successful healthcheck'
+          content:
+            application/json: {}
+

services/local-orchestrator/src/lib.rs

@@ -0,0 +1,75 @@
+use log::*;
+use std::path::PathBuf;
+use tide::Request;
+
+#[derive(Clone, Debug)]
+pub struct State {
+    pub upload_dir: PathBuf,
+}
+
+async fn put_object(req: Request<State>) -> tide::Result {
+    use async_std::{fs::OpenOptions, io};
+    let key = req.url().path();
+
+    info!("Uploading: {:#?} into {:#?}", key, req.state().upload_dir);
+
+    /*
+     * A path will normally come in like /SomeFile.xlsx and Path::push will see
+     * that as a new absolute file which doesn't _join_ but instead overwrites
+     */
+    let key = key.strip_prefix("/").unwrap_or(key);
+    let fs_path = req.state().upload_dir.join(key);
+
+    /*
+     * In the case of nested keys, we need to create the layout on disk
+     */
+    if let Some(parent) = fs_path.parent() {
+        std::fs::create_dir_all(parent)?;
+    }
+
+    let file = OpenOptions::new()
+        .create(true)
+        .write(true)
+        .open(&fs_path)
+        .await?;
+
+    let bytes_written = io::copy(req, file).await?;
+
+    tide::log::info!("file written", {
+        bytes: bytes_written,
+        path: fs_path.canonicalize()?.to_str()
+    });
+
+    Ok("{}".into())
+}
+
+async fn get_object(req: Request<State>) -> tide::Result {
+    use tide::{Body, Response};
+
+    let key = req.url().path();
+    info!("Fetching: {:#?} from{:#?}", key, req.state().upload_dir);
+
+    let key = key.strip_prefix("/").unwrap_or(key);
+    let fs_path = req.state().upload_dir.join(key);
+
+    if fs_path.exists() {
+        Ok(Response::builder(200)
+            .body(Body::from_file(&fs_path).await?)
+            .build())
+    } else {
+        Err(tide::Error::from_str(404, "Failed to locate key"))
+    }
+}
+
+pub fn app(mut upload_dir: PathBuf) -> tide::Server<State> {
+    upload_dir = std::fs::canonicalize(upload_dir).expect("Unable to canonicalize the upload_dir");
+    let state = State { upload_dir };
+    let mut app = tide::with_state(state);
+    app.at("/*").put(put_object);
+    app.at("/*").get(get_object);
+    app.at("/").get(|_| async { Ok("Hello, world!") });
+    app
+}
+
+#[cfg(test)]
+mod tests {}

services/local-orchestrator/src/main.rs

@@ -0,0 +1,27 @@
+/*
+ * The local orchestrator doesn't do much
+ */
+use tide::Request;
+
+async fn healthcheck(_req: Request<()>) -> tide::Result {
+    Ok(tide::Response::builder(200)
+        .body("{}")
+        .content_type("application/json")
+        .build())
+}
+
+#[async_std::main]
+async fn main() -> std::io::Result<()> {
+    use std::{env, net::TcpListener, os::unix::io::FromRawFd};
+    tide::log::start();
+
+    let mut app = tide::new();
+    app.at("/health").get(healthcheck);
+
+    if let Some(fd) = env::var("LISTEN_FD").ok().and_then(|fd| fd.parse().ok()) {
+        app.listen(unsafe { TcpListener::from_raw_fd(fd) }).await?;
+    } else {
+        app.listen("http://localhost:7673").await?;
+    }
+    Ok(())
+}

services/parser/apispec.yml

@@ -84,8 +84,7 @@ components:
       type: object
       required:
         - uuid
-        - contexts
-        - steps
+        - batches
       properties:
         uuid:
           type: string
@@ -98,16 +97,18 @@ components:
         summary: 'A simplistic Pipeline'
         value:
           uuid: '9edc4483-a78a-480f-8e06-2726db1ddf24'
-          contexts:
-            - uuid: '8109f601-12e8-4621-96c6-11baff409d93'
-              properties:
-                name: 'Build'
-          steps:
-            - uuid: '6193b9b1-c6be-4c18-9bb8-1aeead5e7d14'
-              context: '8109f601-12e8-4621-96c6-11baff409d93'
-              symbol: 'sh'
-              parameters:
-                - 'ls'
+          batches:
+            - mode: Linear
+              contexts:
+                - uuid: '8109f601-12e8-4621-96c6-11baff409d93'
+                  properties:
+                    name: 'Build'
+                  steps:
+                    - uuid: '6193b9b1-c6be-4c18-9bb8-1aeead5e7d14'
+                      context: '8109f601-12e8-4621-96c6-11baff409d93'
+                      symbol: 'sh'
+                      parameters:
+                        - 'ls'
 
     ParsePipelineFailure:
       type: object

stdlib/archive/tests/invalid_cases.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 INVOCATION_FILE=tmp_archivetest_invocation_file.json
 

stdlib/archive/tests/valid_cases.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env bash
 
 INVOCATION_FILE=tmp_archivetest_invocation_file.json
 

stdlib/git/tests/simple.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 INVOCATION_FILE=$PWD/tmp_gittest_invocation_file.json