Work in progress for checking to see if we have layers to contaminate

With this commit, Contaminate is effectively able to act as a fairly dumb docker
registry proxy

Cargo.lock

@@ -184,6 +184,8 @@ dependencies = [
  "config 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)",
  "surf 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "tide 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -1115,6 +1117,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 name = "serde"
 version = "1.0.104"
 source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)",
+]
 
 [[package]]
 name = "serde-hjson"
@@ -1128,6 +1133,16 @@ dependencies = [
  "serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "serde_derive"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "serde_json"
 version = "1.0.45"
@@ -1790,6 +1805,7 @@ dependencies = [
 "checksum serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8"
 "checksum serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "414115f25f818d7dfccec8ee535d76949ae78584fc4f79a6f45a904bf8ab4449"
 "checksum serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8"
+"checksum serde_derive 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)" = "128f9e303a5a29922045a830221b8f78ec74a5f544944f3d5984f8ec3895ef64"
 "checksum serde_json 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)" = "eab8f15f15d6c41a154c1b128a22f2dfabe350ef53c40953d84e36155c91192b"
 "checksum serde_qs 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d43eef44996bbe16e99ac720e1577eefa16f7b76b5172165c98ced20ae9903e1"
 "checksum serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5"

Cargo.toml

@@ -4,6 +4,10 @@ version = "0.1.0"
 authors = ["R. Tyler Croy <rtyler@brokenco.de>"]
 edition = "2018"
 
+[[bin]]
+name = "test-registry"
+path = "src/test_registry/main.rs"
+
 [dependencies]
 # Used for serving the website
 tide = "~0.5.1"
@@ -18,3 +22,6 @@ config = { version = "~0.10.1", features = ["yaml"] }
 
 # Used for making http requests to the upstream docker registry
 surf = "~1.0.3"
+
+serde = { version = "~1.0.104", features = ["derive"] }
+serde_json = "~1.0.0"

README.adoc

@@ -40,12 +40,30 @@ environment variables which can override configuration values.
 | `warn`
 | Log level for Contaminate logs to be printed
 
-| `CT_LAYERS_DIR`
+| `CT_layers_dir`
 | `./layers.d`
 | A directory containing the layers to override on images passing through Contaminate.
 
-| `CT_REGISTRY`
+| `CT_registry`
 | https://registry-1.docker.io
 | A Registry HTTP V2 compliant URL, reachable by Contaminate.
 
 |===
+
+== Hacking
+
+Ensure that your local Docker daemon can access your Contaminate instance
+without requiring HTTPs:
+
+./etc/sysconfig/docker
+[source, sh]
+----
+DOCKER_OPTS="--insecure-registry=localhost:5000 --insecure-registry=localhost:9090"
+----
+
+Running a local Docker registry to contaminate:
+
+[source, sh]
+----
+docker run --rm -ti -e REGISTRY_HTTP_SECRET=secret -p 5000:5000 registry:2
+----

src/main.rs

@@ -4,14 +4,25 @@
 
 extern crate config;
 extern crate pretty_env_logger;
+#[macro_use]
+extern crate serde;
+#[macro_use]
+extern crate serde_json;
+extern crate surf;
+extern crate tide;
 
 use async_std::task;
 use log::*;
+use tide::{Request, Response};
+
+use std::path::Path;
+
+mod models;
 
 /**
  * Load the settings based on the hierarchy.
  *
- * First we load the configuration file (contaminate.yml) if it exists
+ * First we load the configuration file `contaminate.yml` if it exists
  * Then we look at environment variables.
  */
 fn load_settings() -> config::Config {
@@ -27,16 +38,156 @@ fn load_settings() -> config::Config {
         .merge(config::Environment::with_prefix("CT"))
         .expect("Failed to load settings defined by CT_* env vars");
 
+    debug!("Loaded configuration: {:?}", settings);
+
     return settings;
 }
 
+/**
+ * AppState is a simple struct to carry information into request handlers
+ */
+struct AppState {
+    conf: config::Config,
+    upstream: String,
+}
+
+impl AppState {
+    /**
+     * This function returns a true if the configured `layers_dir` has an override
+     * for the given triplet of org/image:digest
+     *
+     * For example, if we have a `<layers_dir>/library/alpine/latest/` directory
+     * with `*.tar.gz` files within it, then the function would return true.
+     */
+    fn override_exists(&self, org: String, image: String, digest: String) -> bool {
+        let layers_dir = self.conf.get_str("layers_dir")
+            .expect("Unable to access `layers_dir` conf variable");
+
+        info!("Looking in directory: {}", layers_dir);
+        let layers_dir = Path::new(&layers_dir);
+
+
+        return false;
+    }
+}
+
+/**
+ * Proxy the given response to the upstream registry and return the response
+ * back to the client request it.
+ */
+async fn proxy_upstream(req: Request<AppState>) -> Response {
+    let full_url = format!("{}{}", req.state().upstream, req.uri());
+    info!("Proxying request upstream to {}", full_url);
+     /*
+      * We need to send the Authorization header along as well, otherwise
+      * the upstream repository might complain that we're not authorized
+      */
+    let token = req.header("Authorization").unwrap_or("");
+    let accepts = req.header("Accept").unwrap_or("");
+
+    let outbound = surf::get(full_url)
+        .set_header("Authorization", token)
+        .set_header("Accept", accepts);
+
+    if let Ok(mut u_res) = outbound.await {
+        let status = u_res.status().as_u16();
+        let body = u_res.body_string().await;
+        match body {
+            Ok(body) => {
+                /*
+                 * If we don't explicitly set the content type here, the client will think
+                 * that we're sending back a v1 manifest schema and complain about a "missing
+                 * signature key"
+                 */
+                debug!("upstream headers: {:?}", u_res.headers());
+                debug!("upstream response for {}:\n{}", req.uri(), body);
+                let content_type = u_res.header("Content-Type").unwrap_or("text/plain");
+                Response::new(status)
+                    .set_header("Content-Type", content_type)
+                    .body_string(body)
+            },
+            Err(err) => {
+                error!("Failed to make upstream request: {:?}", err);
+                Response::new(500)
+            },
+        }
+    }
+    else {
+        error!("Failed to make request upstream to {}", req.uri());
+        Response::new(500)
+    }
+
+}
+
+/**
+ * This function will fetch and manipulate the upstream manifest, typically
+ * located at `/v2/myorg/myimage/manifests/latest`
+ *
+ * This will return a Response to the client which conforms to the manifest
+ * specification.
+ */
+async fn fetch_digest(req: Request<AppState>) -> Response {
+    let org: String = req.param("org").unwrap_or("".to_string());
+    let image: String = req.param("image").unwrap_or("".to_string());
+    let digest: String = req.param("digest").unwrap_or("".to_string());
+    if req.state().override_exists(org, image, digest) {
+        error!("We should not proxy");
+        Response::new(200)
+    }
+    else {
+        error!("We SHOULD proxy");
+        Response::new(200)
+    }
+}
+
+async fn fetch_blob(req: Request<AppState>) -> Response {
+    info!("fetch_blob: {}", req.uri());
+    Response::new(200)
+}
+
 fn main() -> Result<(), std::io::Error> {
     pretty_env_logger::init();
+    let conf = load_settings();
+    let upstream_url = conf.get_str("registry")
+        .expect("`registry` not properly configured, must be a string");
+
+    info!("Starting with the following upstream: {}", upstream_url);
+
+    let layers_dir = conf.get_str("layers_dir")
+        .expect("`layers_dir` not properly configured, must be a string");
+    let layers_dir = Path::new(&layers_dir);
+
+    if ! layers_dir.is_dir() {
+        error!("The `layers_dir` ({}) does not appear to be a directory", layers_dir.display());
+        panic!("`layers_dir` must be a directory");
+    }
+
+    let state = AppState {
+        conf: conf,
+        upstream: upstream_url,
+    };
 
     task::block_on(async {
-        let mut app = tide::new();
+        let mut app = tide::with_state(state);
         app.at("/").get(|_| async move { "Hello, world!" });
-        app.listen("127.0.0.1:9000").await?;
+        /*
+         * This route works for "normal" images, which have name of org/image
+         */
+        app.at("/v2/:org/:image/manifests/:digest").get(fetch_digest);
+        /*
+         * This route works handles images which look like "official" images,
+         * such as `alpine:latest`, which _actually_ maps to `library/alpine:latest`
+         * in DockerHub
+         */
+        //app.at("/v2/:image/manifests/:digest").get(fetch_digest);
+        app.at("/v2/:org/:image/blobs/:sha").get(fetch_blob);
+        /*
+         * The catch-all for the remainder of the v2 API calls should proxy to
+         * the upstream repository, since Contaminate does not implement a full
+         * registry API
+         */
+        app.at("/v2/*").get(proxy_upstream);
+        app.listen("127.0.0.1:9090").await?;
         Ok(())
     })
 }

src/models.rs

@@ -0,0 +1,77 @@
+/**
+ * The models module contains all the serde structs for the Docker registry
+ * requests and responses that we care about
+ */
+
+extern crate serde;
+extern crate serde_json;
+
+use serde::{Deserialize, Serialize};
+use serde_json::Result;
+
+/**
+ * Manifest format, retrieved from: /v2/alpine/manifests/latest
+
+{
+   "schemaVersion": 1,
+   "name": "alpine",
+   "tag": "latest",
+   "architecture": "amd64",
+   "fsLayers": [
+      {
+         "blobSum": "sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4"
+      },
+      {
+         "blobSum": "sha256:6c40cc604d8e4c121adcb6b0bfe8bb038815c350980090e74aa5a6423f8f82c0"
+      }
+   ],
+   "history": [
+      {
+         "v1Compatibility": "{\"architecture\":\"amd64\",\"config\":{\"Hostname\":\"\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\"],\"ArgsEscaped\":true,\"Image\":\"sha256:ce244ca5cf823254a1dff4ea35589dcdbe540266820f401a86b7b8dc9eda8f19\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":null},\"container\":\"35bf94cc91dd11f6bd36502cefc82fd4515b20e0181b49e7c316bd78ff7c75d6\",\"container_config\":{\"Hostname\":\"35bf94cc91dd\",\"Domainname\":\"\",\"User\":\"\",\"AttachStdin\":false,\"AttachStdout\":false,\"AttachStderr\":false,\"Tty\":false,\"OpenStdin\":false,\"StdinOnce\":false,\"Env\":[\"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\"],\"Cmd\":[\"/bin/sh\",\"-c\",\"#(nop) \",\"CMD [\\\"/bin/sh\\\"]\"],\"ArgsEscaped\":true,\"Image\":\"sha256:ce244ca5cf823254a1dff4ea35589dcdbe540266820f401a86b7b8dc9eda8f19\",\"Volumes\":null,\"WorkingDir\":\"\",\"Entrypoint\":null,\"OnBuild\":null,\"Labels\":{}},\"created\":\"2019-01-30T22:19:52.734509838Z\",\"docker_version\":\"18.06.1-ce\",\"id\":\"02f7a7ef96f88a71b565eae4fd329ae31942b036f9deec4489c53540c2a18b6d\",\"os\":\"linux\",\"parent\":\"92bdbc97504bab151c3cf7451f2664797538d1fbe2fa0c8b92a218a97cc079df\",\"throwaway\":true}"
+      },
+      {
+         "v1Compatibility": "{\"id\":\"92bdbc97504bab151c3cf7451f2664797538d1fbe2fa0c8b92a218a97cc079df\",\"created\":\"2019-01-30T22:19:52.585366638Z\",\"container_config\":{\"Cmd\":[\"/bin/sh -c #(nop) ADD file:2a1fc9351afe35698918545b2d466d9805c2e8afcec52f916785ee65bbafeced in / \"]}}"
+      }
+   ],
+   "signatures": [
+      {
+         "header": {
+            "jwk": {
+               "crv": "P-256",
+               "kid": "GNNE:U4EY:Q2RK:62PO:3FIP:TAHB:XHLQ:IMOC:LYHD:HRH3:QJ2I:VJVZ",
+               "kty": "EC",
+               "x": "8q3mDSgd7V3wjnwTlGpsuS4f7XVGRwcWJBGkfTj5C2g",
+               "y": "E2wP8yAe8iLLuVF3_QbGndah-9_O9FkhXOE1nuzvAPE"
+            },
+            "alg": "ES256"
+         },
+         "signature": "ZZdKYM_K9PWoMZ1EZQwiRg7J1dYWPNYmy7gxIX37eDErZ-8E6gfvibcxVIzsvTpV6a2v-kKOLwl_qmAAif-_FA",
+         "protected": "eyJmb3JtYXRMZW5ndGgiOjIxMzMsImZvcm1hdFRhaWwiOiJDbjAiLCJ0aW1lIjoiMjAyMC0wMi0wMVQxNjozODo1OFoifQ"
+      }
+   ]
+}
+ */ 
+#[derive(Serialize, Deserialize)]
+struct Manifest {
+    #[serde(rename = "schemaVersion")]
+    schema_version: u16,
+    name: String,
+    tag: String,
+    architecture: String,
+    #[serde(rename = "fsLayers")]
+    fs_layers: Vec<Layer>,
+    signatures: Vec<Signature>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Layer {
+    #[serde(rename = "blobSum")]
+    blob_sum: String,
+}
+
+#[derive(Serialize, Deserialize)]
+struct Signature {
+    signature: String,
+    protected: String,
+}
+

src/test_registry/main.rs

@@ -0,0 +1,33 @@
+/**
+ * The test_registry is a simple service which just responds with some canned
+ * JSON responses
+ */
+
+extern crate pretty_env_logger;
+extern crate tide;
+
+use async_std::task;
+use log::*;
+use tide::Request;
+
+
+async fn generic_json(req: Request<()>) -> String {
+    info!("Received request to: {}", req.uri());
+    format!(r#"{{"url" : "{}"}}"#, req.uri())
+}
+
+fn main() -> Result<(), std::io::Error> {
+    pretty_env_logger::init();
+
+    task::block_on(async {
+        let mut app = tide::new();
+        app.at("/").get(generic_json);
+        app.at("*").get(generic_json);
+        app.listen("127.0.0.1:2345").await?;
+        Ok(())
+    })
+}
+
+#[cfg(test)]
+mod tests {
+}

tools/shoreman

@@ -0,0 +1,130 @@
+#!/bin/bash
+
+# [shoreman](https://github.com/chrismytton/shoreman) is an
+# implementation of the **Procfile** format. Inspired by the original
+# [foreman](http://ddollar.github.com/foreman/) tool for ruby, as
+# well as [norman](https://github.com/josh/norman) for node.js.
+
+# Make sure that any errors cause the script to exit immediately.
+set -eo pipefail
+[[ "$TRACE" ]] && set -x
+
+# ## Usage
+
+# Usage message that is displayed when `--help` is given as an argument.
+usage() {
+  echo "Usage: shoreman [procfile|Procfile] [envfile|.env]"
+  echo "Run Procfiles using shell."
+  echo
+  echo "The shoreman script reads commands from [procfile] and starts up the"
+  echo "processes that it describes."
+}
+
+# ## Logging
+
+# For logging we want to prefix each entry with the current time, as well
+# as the process name. This takes two arguments, the name of the process
+# with its index, and then reads data from stdin, formats it, and sends it
+# to stdout.
+log() {
+  local index="$2"
+  local format="%s %s\t| %s"
+
+  # We add colors when output is a terminal. `SHOREMAN_COLORS` can override it.
+  if [ -t 1 -o "$SHOREMAN_COLORS" == "always" ] \
+     && [ "$SHOREMAN_COLORS" != "never" ]; then
+    # Bash colors start from 31 up to 37. We calculate what color the process
+    # gets based on its index.
+    local color="$((31 + (index % 7)))"
+    format="\033[0;${color}m%s %s\t|\033[0m %s"
+  fi
+
+  while read -r data
+  do
+    printf "$format\n" "$(date +"%H:%M:%S")" "$1" "$data"
+  done
+}
+
+# ## Running commands
+
+# When a process is started, we want to keep track of its pid so we can
+# `kill` it when the parent process receives a signal, and so we can `wait`
+# for it to finish before exiting the parent process.
+store_pid() {
+  pids="$pids $1"
+}
+
+# This starts a command asynchronously and stores its pid in a list for use
+# later on in the script.
+start_command() {
+  bash -c "$1" 2>&1 | log "$2" "$3" &
+  pid="$(jobs -p %%)"
+  store_pid "$pid"
+}
+
+# ## Reading the .env file
+
+# The .env file needs to be a list of assignments like in a shell script.
+# Shell-style comments are permitted.
+load_env_file() {
+  local env_file=${1:-'.env'}
+
+  # Set a default port before loading the .env file
+  export PORT=${PORT:-5000}
+
+  if [[ -f "$env_file" ]]; then
+    export $(grep "^[^#]*=.*" "$env_file" | xargs)
+  fi
+}
+
+# ## Reading the Procfile
+
+# The Procfile needs to be parsed to extract the process names and commands.
+# The file is given on stdin, see the `<` at the end of this while loop.
+run_procfile() {
+  local procfile=${1:-'Procfile'}
+  # We give each process an index to track its color. We start with 1,
+  # because it corresponds to green which is easier on the eye than red (0).
+  local index=1
+  while read line || [[ -n "$line" ]]; do
+    if [[ -z "$line" ]] || [[ "$line" == \#* ]]; then continue; fi
+    local name="${line%%:*}"
+    local command="${line#*:[[:space:]]}"
+    start_command "$command" "${name}" "$index"
+    echo "'${command}' started with pid $pid" | log "${name}" "$index"
+    index=$((index + 1))
+  done < "$procfile"
+}
+
+# ## Cleanup
+
+# When a `SIGINT`, `SIGTERM` or `EXIT` is received, this action is run, killing the
+# child processes. The sleep stops STDOUT from pouring over the prompt, it
+# should probably go at some point.
+onexit() {
+  echo "SIGINT received"
+  echo "sending SIGTERM to all processes"
+  kill $pids
+  sleep 1
+}
+
+main() {
+  local procfile="$1"
+  local env_file="$2"
+
+  # If the --help option is given, show the usage message and exit.
+  expr -- "$*" : ".*--help" >/dev/null && {
+    usage
+    exit 0
+  }
+
+  load_env_file "$env_file"
+  run_procfile "$procfile"
+
+  trap onexit INT TERM
+
+  # Wait for the children to finish executing before exiting.
+  wait $pids
+}
+
+main "$@"