rtyler
/
contaminate
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Clean up a lot of the older files

This commit is contained in:
R Tyler Croy 2020-09-12 10:31:52 -07:00
parent 70b5a5371c
commit 7e883d84f4
9 changed files with 2 additions and 3975 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
HACKING.adoc
View File

@ -1,6 +1,6 @@
= Hacking Ahab
= Hacking contaminate
This document contains a loose collection of notes on how to hack Ahab.
This document contains a loose collection of notes on how to hack Contaminate.
== Example Responses

271
index.js
View File

@ -1,271 +0,0 @@
/**
* This file is the main server for Ahab and contains all the basic request
* routing logic for proxying and mutating results from a Docker registry
*/
const express = require('express');
const fs = require('fs');
const NodeCache = require('node-cache');
const path = require('path');
const request = require('request');
const rp = require('request-promise');
const { LAYERS_CHECKSUM_CACHE, Layer } = require('./src/layers');
const logger = require('./src/logger');
const UPSTREAM_REGISTRY = process.env.UPSTREAM_REGISTRY || 'https://registry-1.docker.io';
const LAYERS_DIR = path.resolve(process.env.LAYERS_DIR || './layers.d/');
logger.info(`Using the layers directory: ${LAYERS_DIR}`);
/*
* This cache is to be used for any of the digests we spot in manifests fetched
* from the upstream. For the caches where these digests will represent
* something which needs to be overridden, we need to know such that the blob
* response handler can trigger the appropriate override behavior
*/
const OVERRIDDEN_BLOB_CACHE = new NodeCache({ stdTTL: 86400 });
/**
* Return the computed override path for the given image:tag
*/
function overridePathFor(org, image, tag) {
return path.join(LAYERS_DIR, org, image, tag);
}
/**
* Return true if we have an override directory for the image
*/
function shouldOverrideImage(org, image, tag) {
const computedPath = overridePathFor(org, image, tag);
logger.debug(`Checking to presence of override dir: ${computedPath}`);
return fs.existsSync(computedPath);
}
/**
* Collect the layers to use for override
*
* @return Array of strings with full file paths
*/
function collectLayersFor(org, image, tag) {
// Just to make sure we're never called with bad data
if (!shouldOverrideImage(org, image, tag)) {
return [];
}
const computedPath = overridePathFor(org, image, tag);
return fs.readdirSync(computedPath)
.filter((filename) => {
return filename.endsWith('.tar.gz');
})
.sort()
.map((filename) => {
const key = Layer.keyFor(org, image, tag, filename);
if (!LAYERS_CHECKSUM_CACHE.get(key)) {
logger.debug(`Computing a new layer for key ${key}`);
const layer = new Layer(org, image, tag, path.join(computedPath, filename));
layer.process();
LAYERS_CHECKSUM_CACHE.set(key, layer);
}
return LAYERS_CHECKSUM_CACHE.get(key);
});
}
/**
* Proxy the given request directly to the upstream
*
* @param req An Express Request object
* @param res An Express Response object
*/
function proxyToUpstream(req, res) {
logger.info(`Passing this request along upstream (${req.originalUrl})`);
return request({
url: `${UPSTREAM_REGISTRY}${req.originalUrl}`,
method: req.method,
headers: {
/*
* We need to send the Authorization header along as well, otherwise
* the upstream repository might complain that we're not authorized
*/
'Authorization' : req.get('Authorization'),
'Accept' : req.get('Accept'),
},
}).pipe(res);
}
const app = express();
const port = 9090;
/*
* The manifests API is defined in the "Pulling an Image" part of the HTTP
* registry API documentation, see here:
* <https://docs.docker.com/registry/spec/api/#pulling-an-image>
*
* This express route will:
* - fetch the upstream manifest in order to fetch the image configuration
* - fetch the image configuration, which must have its diff_ids overwritten
* - re-compute the digest of the image configuration
* - grab all the layers which must be inserted into the manifest
* - generate the updated manifest
*/
app.get('/v2/:org/:image/manifests/:digest', (req, res) => {
const { org, image, digest } = req.params;
if (!shouldOverrideImage(org, image, digest)) {
return proxyToUpstream(req, res);
}
/*
* Begin request to upstream in order to get the original manifest which Ahab
* will then manipulate
*/
const manifestReq = rp({
url: `${UPSTREAM_REGISTRY}${req.originalUrl}`,
headers: {
'Authorization' : req.get('Authorization'),
'Accept' : req.get('Accept'),
},
}).then((response) => {
const parsed = JSON.parse(response);
if (parsed.manifests) {
// This is a manifest list!
logger.info('Received manifest list, caching and returning');
parsed.manifests.map((manifest) => {
/*
* Shove this into the cache
*/
})
return res.send(response);
} else {
// This is just a manifest then
}
});
return;
/*
* If we don't explicitly set the content type here, the client will think
* that we're sending back a v1 manifest schema and complain about a "missing
* signature key"
*/
res.set('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json');
logger.info(`Overriding the pull for ${org}/${image}:${digest}`);
const layers = collectLayersFor(org, image, digest);
logger.info(layers);
/*
* if we get a manifest list, we basically need to cache all the referenced
* digests in relation to our current image:tag reference since the client
* will be quickly making subsequent requests for those digests, which must
* themselves be overwritten
*/
res.status(500);
res.send('Fail');
return
rp({
url: `${UPSTREAM_REGISTRY}/v2/${org}/${image}/blobs/${digest}`,
headers: {
/*
* We need to send the Authorization header along as well, otherwise
* the upstream repository might complain that we're not authorized
*/
'Authorization' : req.get('Authorization'),
},
})
.then((response) => {
image = JSON.parse(response);
//image = JSON.parse(image);
//image.rootfs.diff_ids.push(
// 'sha256:5f9da7cc9d8d83c96245ac27854466f6ed89fbfade5dd8a4f307861bfb72d1b8',
//);
//console.log(image.rootfs);
//const checksum = crypto.createHash('sha256').update(JSON.stringify(image), 'utf8').digest('hex');
//console.log(JSON.stringify(image));
//console.log(`Sending content with a checksum of ${checksum}`);
//console.log(`Client is expecting ${req.params.sha}`);
//res.send(image);
//console.log(req.originalUrl);
//const checksum = crypto.createHash('sha256').update(JSON.stringify(response), 'utf8').digest('hex');
//res.set('Docker-Content-Digest', `sha256:${checksum}`);
})
.catch((error) => {
logger.error(`Failed to hit the blob API for ${org}/${image}:${digest} - ${error}`);
res.status(500);
res.send(error);
});
});
/*
* Serve up a custom blob if we've got one!
*/
app.get('/v2/:org/:image/blobs/:sha', (req, res) => {
console.log(`Hitting ${req.originalUrl}`);
if (req.params.sha == 'sha256:cf98438b1781c2be1fde41967959140379c715b75a85723501d0bca82f215a76') {
/*
* Requesting our custom blob layer itself
*/
const filename = 'layer.tar.gz';
console.log('hitting our custom blob');
fs.createReadStream(filename).pipe(res);
} else if (req.params.sha == imageConfig) {
/*
* Requesting our image configuration
*/
console.log('hitting our image configuration')
let image = '';
/*
* We need to modify the diff_ids of the image configuration to include the
* checksum of the gunzipped version of our custom layer
*
* https://medium.com/@saschagrunert/demystifying-containers-part-iii-container-images-244865de6fef
*
*/
const upstreamBlob = 'sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e'
request({
url: `${UPSTREAM_REGISTRY}/v2/${req.params.org}/${req.parmas.name}/blobs/${upstreamBlob}`,
method: req.method,
headers: {
/*
* We need to send the Authorization header along as well, otherwise
* the upstream repository might complain that we're not authorized
*/
'Authorization' : req.get('Authorization'),
},
})
.on('response', (response) => {
console.log(`Hitting upstream gave a ${response.statusCode}`);
})
.on('data', (chunk) => {
image += chunk;
})
.on('end', () => {
image = JSON.parse(image);
image.rootfs.diff_ids.push(
'sha256:5f9da7cc9d8d83c96245ac27854466f6ed89fbfade5dd8a4f307861bfb72d1b8',
);
console.log(image.rootfs);
const checksum = crypto.createHash('sha256').update(JSON.stringify(image), 'utf8').digest('hex');
console.log(JSON.stringify(image));
console.log(`Sending content with a checksum of ${checksum}`);
console.log(`Client is expecting ${req.params.sha}`);
res.send(image);
});
} else {
console.log(`Piping blob upstream for ${req.originalUrl}`);
proxyToUpstream(req, res);
}
});
/*
* Pass all other v2 API requests on to the upstream Docker registry
*/
app.get('/v2/*', (req, res) => proxyToUpstream(req, res));
app.listen(port, () => logger.info(`Ahab is now hunting whales on port ${port}`));

3416
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

21
package.json
View File

@ -1,21 +0,0 @@
{
"name": "ahab",
"version": "1.0.0",
"description": "Hunting the white whale of a mutable immutable infrastructure",
"main": "index.js",
"scripts": {
"test": "jest"
},
"author": "R Tyler Cory",
"license": "AGPL-3.0",
"dependencies": {
"express": "^4.17.1",
"node-cache": "^4.2.1",
"request": "^2.88.0",
"request-promise": "^4.2.4",
"winston": "^3.2.1"
},
"devDependencies": {
"nodemon": "^1.19.4"
}
}

18
src/checksum.js
View File

@ -1,18 +0,0 @@
const crypto = require('crypto');
module.exports = {};
/**
* Create a SHA256 checksum (hex encoded) of the given object
*
* Any non-string objects will be JSON encoded before checksumming
*/
module.exports.sha256sum = (obj) => {
let buffer = obj;
if (!obj instanceof String) {
buffer = JSON.stringify(obj);
}
return crypto.createHash('sha256')
.update(buffer, 'utf8')
.digest('hex');
}

54
src/layers.js
View File

@ -1,54 +0,0 @@
const fs = require('fs');
const NodeCache = require('node-cache');
const path = require('path');
const zlib = require('zlib');
const { sha256sum } = require('./checksum');
const logger = require('./logger');
module.exports = {};
/**
* This cache is to ensure that we're not processing files found in LAYERS_DIR
* and is intended to hold a key-value mapping from a layer `key` to a `Layer`
* class
*
*/
module.exports.LAYERS_CHECKSUM_CACHE = new NodeCache({ stdTTL: 300 });
class Layer {
constructor(org, image, tag, filePath) {
this.filePath = filePath;
this.key = Layer.keyFor(org, image, tag, path.basename(filePath));
this.digest = null;
this.tarDigest = null;
}
static keyFor(org, image, tag, filename) {
return `${org}/${image}:${tag}|${filename}`;
}
/**
* The process function will ensure that the file has been considered and
* checksums have been created
*/
process() {
logger.info(`Processing ${this} with ${this.filePath}`);
if ((this.digest == null) || (this.tarDigest == null)) {
// We'll need the contents of the layer for checksumming and gunziping
const layerContents = fs.readFileSync(this.filePath);
logger.debug(`Computing digest for ${this}`);
this.digest = sha256sum(layerContents);
logger.debug(`Computing tarDigest for ${this}`);
this.tarDigest = sha256sum(zlib.gunzipSync(layerContents));
}
}
toString() {
return `Layer(${this.key})`;
}
}
module.exports.Layer = Layer;

18
src/logger.js
View File

@ -1,18 +0,0 @@
/*
* Simple logger module to preconfigure winston'
*/
const { createLogger, format, transports } = require('winston');
module.exports = createLogger({
// To see more detailed errors, change this to 'debug'
level: process.env.LOG_LEVEL || 'info',
format: format.combine(
format.splat(),
format.simple()
),
transports: [
new transports.Console()
],
});

45
tools/ahab.sh
View File

@ -1,45 +0,0 @@
#!/bin/sh
image=library/alpine
digest=latest
#digest="sha256:3085eec036f1a96b2f1cfbdc7f12ff6e38fde621324fae5608154db221c16d04"
#digest="sha256:92c7f9c92844bbbb5d0a101b22f7c2a7949e40f8ea90c8b3bc396879d95e899a"
registryBase='https://registry-1.docker.io'
authBase='https://auth.docker.io'
authService='registry.docker.io'
blob="sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
imageblob="sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
curl -fvSL \
-H "Authorization: Bearer $token" \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
"$registryBase/v2/$image/manifests/$digest" | jq
blob="sha256:acd3ca9941a85e8ed16515bfc5328e4e2f8c128caa72959a58a127b7801ee01f"
curl -fvSL \
-H "Authorization: Bearer $token" \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
"$registryBase/v2/$image/manifests/$blob" | jq
exit
curl -fvSL \
-H "Authorization: Bearer $token" \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
"$registryBase/v2/$image/blobs/$blob" | jq
exit
curl -fvSL \
-H "Authorization: Bearer $token" \
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
"$registryBase/v2/$image/blobs/$imageblob" > hello-world-image.json

130
tools/shoreman
View File

@ -1,130 +0,0 @@
#!/bin/bash
# [shoreman](https://github.com/chrismytton/shoreman) is an
# implementation of the **Procfile** format. Inspired by the original
# [foreman](http://ddollar.github.com/foreman/) tool for ruby, as
# well as [norman](https://github.com/josh/norman) for node.js.
# Make sure that any errors cause the script to exit immediately.
set -eo pipefail
[[ "$TRACE" ]] && set -x
# ## Usage
# Usage message that is displayed when `--help` is given as an argument.
usage() {
echo "Usage: shoreman [procfile|Procfile] [envfile|.env]"
echo "Run Procfiles using shell."
echo
echo "The shoreman script reads commands from [procfile] and starts up the"
echo "processes that it describes."
}
# ## Logging
# For logging we want to prefix each entry with the current time, as well
# as the process name. This takes two arguments, the name of the process
# with its index, and then reads data from stdin, formats it, and sends it
# to stdout.
log() {
local index="$2"
local format="%s %s\t| %s"
# We add colors when output is a terminal. `SHOREMAN_COLORS` can override it.
if [ -t 1 -o "$SHOREMAN_COLORS" == "always" ] \
&& [ "$SHOREMAN_COLORS" != "never" ]; then
# Bash colors start from 31 up to 37. We calculate what color the process
# gets based on its index.
local color="$((31 + (index % 7)))"
format="\033[0;${color}m%s %s\t|\033[0m %s"
fi
while read -r data
do
printf "$format\n" "$(date +"%H:%M:%S")" "$1" "$data"
done
}
# ## Running commands
# When a process is started, we want to keep track of its pid so we can
# `kill` it when the parent process receives a signal, and so we can `wait`
# for it to finish before exiting the parent process.
store_pid() {
pids="$pids $1"
}
# This starts a command asynchronously and stores its pid in a list for use
# later on in the script.
start_command() {
bash -c "$1" 2>&1 | log "$2" "$3" &
pid="$(jobs -p %%)"
store_pid "$pid"
}
# ## Reading the .env file
# The .env file needs to be a list of assignments like in a shell script.
# Shell-style comments are permitted.
load_env_file() {
local env_file=${1:-'.env'}
# Set a default port before loading the .env file
export PORT=${PORT:-5000}
if [[ -f "$env_file" ]]; then
export $(grep "^[^#]*=.*" "$env_file" | xargs)
fi
}
# ## Reading the Procfile
# The Procfile needs to be parsed to extract the process names and commands.
# The file is given on stdin, see the `<` at the end of this while loop.
run_procfile() {
local procfile=${1:-'Procfile'}
# We give each process an index to track its color. We start with 1,
# because it corresponds to green which is easier on the eye than red (0).
local index=1
while read line || [[ -n "$line" ]]; do
if [[ -z "$line" ]] || [[ "$line" == \#* ]]; then continue; fi
local name="${line%%:*}"
local command="${line#*:[[:space:]]}"
start_command "$command" "${name}" "$index"
echo "'${command}' started with pid $pid" | log "${name}" "$index"
index=$((index + 1))
done < "$procfile"
}
# ## Cleanup
# When a `SIGINT`, `SIGTERM` or `EXIT` is received, this action is run, killing the
# child processes. The sleep stops STDOUT from pouring over the prompt, it
# should probably go at some point.
onexit() {
echo "SIGINT received"
echo "sending SIGTERM to all processes"
kill $pids
sleep 1
}
main() {
local procfile="$1"
local env_file="$2"
# If the --help option is given, show the usage message and exit.
expr -- "$*" : ".*--help" >/dev/null && {
usage
exit 0
}
load_env_file "$env_file"
run_procfile "$procfile"
trap onexit INT TERM
# Wait for the children to finish executing before exiting.
wait $pids
}
main "$@"