Clean up a lot of the older files
This commit is contained in:
parent
70b5a5371c
commit
7e883d84f4
|
@ -1,6 +1,6 @@
|
|||
= Hacking Ahab
|
||||
= Hacking contaminate
|
||||
|
||||
This document contains a loose collection of notes on how to hack Ahab.
|
||||
This document contains a loose collection of notes on how to hack Contaminate.
|
||||
|
||||
|
||||
== Example Responses
|
||||
|
|
271
index.js
271
index.js
|
@ -1,271 +0,0 @@
|
|||
/**
|
||||
* This file is the main server for Ahab and contains all the basic request
|
||||
* routing logic for proxying and mutating results from a Docker registry
|
||||
*/
|
||||
|
||||
const express = require('express');
|
||||
const fs = require('fs');
|
||||
const NodeCache = require('node-cache');
|
||||
const path = require('path');
|
||||
const request = require('request');
|
||||
const rp = require('request-promise');
|
||||
|
||||
const { LAYERS_CHECKSUM_CACHE, Layer } = require('./src/layers');
|
||||
const logger = require('./src/logger');
|
||||
|
||||
const UPSTREAM_REGISTRY = process.env.UPSTREAM_REGISTRY || 'https://registry-1.docker.io';
|
||||
const LAYERS_DIR = path.resolve(process.env.LAYERS_DIR || './layers.d/');
|
||||
logger.info(`Using the layers directory: ${LAYERS_DIR}`);
|
||||
|
||||
/*
|
||||
* This cache is to be used for any of the digests we spot in manifests fetched
|
||||
* from the upstream. For the caches where these digests will represent
|
||||
* something which needs to be overridden, we need to know such that the blob
|
||||
* response handler can trigger the appropriate override behavior
|
||||
*/
|
||||
const OVERRIDDEN_BLOB_CACHE = new NodeCache({ stdTTL: 86400 });
|
||||
|
||||
|
||||
/**
|
||||
* Return the computed override path for the given image:tag
|
||||
*/
|
||||
function overridePathFor(org, image, tag) {
|
||||
return path.join(LAYERS_DIR, org, image, tag);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true if we have an override directory for the image
|
||||
*/
|
||||
function shouldOverrideImage(org, image, tag) {
|
||||
const computedPath = overridePathFor(org, image, tag);
|
||||
logger.debug(`Checking to presence of override dir: ${computedPath}`);
|
||||
return fs.existsSync(computedPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect the layers to use for override
|
||||
*
|
||||
* @return Array of strings with full file paths
|
||||
*/
|
||||
function collectLayersFor(org, image, tag) {
|
||||
// Just to make sure we're never called with bad data
|
||||
if (!shouldOverrideImage(org, image, tag)) {
|
||||
return [];
|
||||
}
|
||||
const computedPath = overridePathFor(org, image, tag);
|
||||
return fs.readdirSync(computedPath)
|
||||
.filter((filename) => {
|
||||
return filename.endsWith('.tar.gz');
|
||||
})
|
||||
.sort()
|
||||
.map((filename) => {
|
||||
const key = Layer.keyFor(org, image, tag, filename);
|
||||
|
||||
if (!LAYERS_CHECKSUM_CACHE.get(key)) {
|
||||
logger.debug(`Computing a new layer for key ${key}`);
|
||||
const layer = new Layer(org, image, tag, path.join(computedPath, filename));
|
||||
layer.process();
|
||||
LAYERS_CHECKSUM_CACHE.set(key, layer);
|
||||
}
|
||||
return LAYERS_CHECKSUM_CACHE.get(key);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Proxy the given request directly to the upstream
|
||||
*
|
||||
* @param req An Express Request object
|
||||
* @param res An Express Response object
|
||||
*/
|
||||
function proxyToUpstream(req, res) {
|
||||
logger.info(`Passing this request along upstream (${req.originalUrl})`);
|
||||
return request({
|
||||
url: `${UPSTREAM_REGISTRY}${req.originalUrl}`,
|
||||
method: req.method,
|
||||
headers: {
|
||||
/*
|
||||
* We need to send the Authorization header along as well, otherwise
|
||||
* the upstream repository might complain that we're not authorized
|
||||
*/
|
||||
'Authorization' : req.get('Authorization'),
|
||||
'Accept' : req.get('Accept'),
|
||||
},
|
||||
}).pipe(res);
|
||||
}
|
||||
|
||||
const app = express();
|
||||
const port = 9090;
|
||||
|
||||
/*
|
||||
* The manifests API is defined in the "Pulling an Image" part of the HTTP
|
||||
* registry API documentation, see here:
|
||||
* <https://docs.docker.com/registry/spec/api/#pulling-an-image>
|
||||
*
|
||||
* This express route will:
|
||||
* - fetch the upstream manifest in order to fetch the image configuration
|
||||
* - fetch the image configuration, which must have its diff_ids overwritten
|
||||
* - re-compute the digest of the image configuration
|
||||
* - grab all the layers which must be inserted into the manifest
|
||||
* - generate the updated manifest
|
||||
*/
|
||||
app.get('/v2/:org/:image/manifests/:digest', (req, res) => {
|
||||
const { org, image, digest } = req.params;
|
||||
|
||||
if (!shouldOverrideImage(org, image, digest)) {
|
||||
return proxyToUpstream(req, res);
|
||||
}
|
||||
|
||||
/*
|
||||
* Begin request to upstream in order to get the original manifest which Ahab
|
||||
* will then manipulate
|
||||
*/
|
||||
const manifestReq = rp({
|
||||
url: `${UPSTREAM_REGISTRY}${req.originalUrl}`,
|
||||
headers: {
|
||||
'Authorization' : req.get('Authorization'),
|
||||
'Accept' : req.get('Accept'),
|
||||
},
|
||||
}).then((response) => {
|
||||
const parsed = JSON.parse(response);
|
||||
if (parsed.manifests) {
|
||||
// This is a manifest list!
|
||||
logger.info('Received manifest list, caching and returning');
|
||||
|
||||
parsed.manifests.map((manifest) => {
|
||||
/*
|
||||
* Shove this into the cache
|
||||
*/
|
||||
})
|
||||
return res.send(response);
|
||||
} else {
|
||||
// This is just a manifest then
|
||||
}
|
||||
});
|
||||
|
||||
return;
|
||||
|
||||
/*
|
||||
* If we don't explicitly set the content type here, the client will think
|
||||
* that we're sending back a v1 manifest schema and complain about a "missing
|
||||
* signature key"
|
||||
*/
|
||||
res.set('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json');
|
||||
|
||||
logger.info(`Overriding the pull for ${org}/${image}:${digest}`);
|
||||
|
||||
const layers = collectLayersFor(org, image, digest);
|
||||
logger.info(layers);
|
||||
|
||||
/*
|
||||
* if we get a manifest list, we basically need to cache all the referenced
|
||||
* digests in relation to our current image:tag reference since the client
|
||||
* will be quickly making subsequent requests for those digests, which must
|
||||
* themselves be overwritten
|
||||
*/
|
||||
|
||||
res.status(500);
|
||||
res.send('Fail');
|
||||
return
|
||||
|
||||
rp({
|
||||
url: `${UPSTREAM_REGISTRY}/v2/${org}/${image}/blobs/${digest}`,
|
||||
headers: {
|
||||
/*
|
||||
* We need to send the Authorization header along as well, otherwise
|
||||
* the upstream repository might complain that we're not authorized
|
||||
*/
|
||||
'Authorization' : req.get('Authorization'),
|
||||
},
|
||||
})
|
||||
.then((response) => {
|
||||
image = JSON.parse(response);
|
||||
//image = JSON.parse(image);
|
||||
//image.rootfs.diff_ids.push(
|
||||
// 'sha256:5f9da7cc9d8d83c96245ac27854466f6ed89fbfade5dd8a4f307861bfb72d1b8',
|
||||
//);
|
||||
//console.log(image.rootfs);
|
||||
//const checksum = crypto.createHash('sha256').update(JSON.stringify(image), 'utf8').digest('hex');
|
||||
//console.log(JSON.stringify(image));
|
||||
//console.log(`Sending content with a checksum of ${checksum}`);
|
||||
//console.log(`Client is expecting ${req.params.sha}`);
|
||||
//res.send(image);
|
||||
//console.log(req.originalUrl);
|
||||
//const checksum = crypto.createHash('sha256').update(JSON.stringify(response), 'utf8').digest('hex');
|
||||
//res.set('Docker-Content-Digest', `sha256:${checksum}`);
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error(`Failed to hit the blob API for ${org}/${image}:${digest} - ${error}`);
|
||||
res.status(500);
|
||||
res.send(error);
|
||||
});
|
||||
});
|
||||
|
||||
/*
|
||||
* Serve up a custom blob if we've got one!
|
||||
*/
|
||||
app.get('/v2/:org/:image/blobs/:sha', (req, res) => {
|
||||
console.log(`Hitting ${req.originalUrl}`);
|
||||
|
||||
if (req.params.sha == 'sha256:cf98438b1781c2be1fde41967959140379c715b75a85723501d0bca82f215a76') {
|
||||
/*
|
||||
* Requesting our custom blob layer itself
|
||||
*/
|
||||
const filename = 'layer.tar.gz';
|
||||
console.log('hitting our custom blob');
|
||||
fs.createReadStream(filename).pipe(res);
|
||||
} else if (req.params.sha == imageConfig) {
|
||||
/*
|
||||
* Requesting our image configuration
|
||||
*/
|
||||
console.log('hitting our image configuration')
|
||||
let image = '';
|
||||
/*
|
||||
* We need to modify the diff_ids of the image configuration to include the
|
||||
* checksum of the gunzipped version of our custom layer
|
||||
*
|
||||
* https://medium.com/@saschagrunert/demystifying-containers-part-iii-container-images-244865de6fef
|
||||
*
|
||||
*/
|
||||
|
||||
const upstreamBlob = 'sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e'
|
||||
request({
|
||||
url: `${UPSTREAM_REGISTRY}/v2/${req.params.org}/${req.parmas.name}/blobs/${upstreamBlob}`,
|
||||
method: req.method,
|
||||
headers: {
|
||||
/*
|
||||
* We need to send the Authorization header along as well, otherwise
|
||||
* the upstream repository might complain that we're not authorized
|
||||
*/
|
||||
'Authorization' : req.get('Authorization'),
|
||||
},
|
||||
})
|
||||
.on('response', (response) => {
|
||||
console.log(`Hitting upstream gave a ${response.statusCode}`);
|
||||
})
|
||||
.on('data', (chunk) => {
|
||||
image += chunk;
|
||||
})
|
||||
.on('end', () => {
|
||||
image = JSON.parse(image);
|
||||
image.rootfs.diff_ids.push(
|
||||
'sha256:5f9da7cc9d8d83c96245ac27854466f6ed89fbfade5dd8a4f307861bfb72d1b8',
|
||||
);
|
||||
console.log(image.rootfs);
|
||||
const checksum = crypto.createHash('sha256').update(JSON.stringify(image), 'utf8').digest('hex');
|
||||
console.log(JSON.stringify(image));
|
||||
console.log(`Sending content with a checksum of ${checksum}`);
|
||||
console.log(`Client is expecting ${req.params.sha}`);
|
||||
res.send(image);
|
||||
});
|
||||
} else {
|
||||
console.log(`Piping blob upstream for ${req.originalUrl}`);
|
||||
proxyToUpstream(req, res);
|
||||
}
|
||||
});
|
||||
|
||||
/*
|
||||
* Pass all other v2 API requests on to the upstream Docker registry
|
||||
*/
|
||||
app.get('/v2/*', (req, res) => proxyToUpstream(req, res));
|
||||
|
||||
app.listen(port, () => logger.info(`Ahab is now hunting whales on port ${port}`));
|
File diff suppressed because it is too large
Load Diff
21
package.json
21
package.json
|
@ -1,21 +0,0 @@
|
|||
{
|
||||
"name": "ahab",
|
||||
"version": "1.0.0",
|
||||
"description": "Hunting the white whale of a mutable immutable infrastructure",
|
||||
"main": "index.js",
|
||||
"scripts": {
|
||||
"test": "jest"
|
||||
},
|
||||
"author": "R Tyler Cory",
|
||||
"license": "AGPL-3.0",
|
||||
"dependencies": {
|
||||
"express": "^4.17.1",
|
||||
"node-cache": "^4.2.1",
|
||||
"request": "^2.88.0",
|
||||
"request-promise": "^4.2.4",
|
||||
"winston": "^3.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^1.19.4"
|
||||
}
|
||||
}
|
|
@ -1,18 +0,0 @@
|
|||
const crypto = require('crypto');
|
||||
|
||||
module.exports = {};
|
||||
|
||||
/**
|
||||
* Create a SHA256 checksum (hex encoded) of the given object
|
||||
*
|
||||
* Any non-string objects will be JSON encoded before checksumming
|
||||
*/
|
||||
module.exports.sha256sum = (obj) => {
|
||||
let buffer = obj;
|
||||
if (!obj instanceof String) {
|
||||
buffer = JSON.stringify(obj);
|
||||
}
|
||||
return crypto.createHash('sha256')
|
||||
.update(buffer, 'utf8')
|
||||
.digest('hex');
|
||||
}
|
|
@ -1,54 +0,0 @@
|
|||
const fs = require('fs');
|
||||
const NodeCache = require('node-cache');
|
||||
const path = require('path');
|
||||
const zlib = require('zlib');
|
||||
|
||||
const { sha256sum } = require('./checksum');
|
||||
const logger = require('./logger');
|
||||
|
||||
module.exports = {};
|
||||
/**
|
||||
* This cache is to ensure that we're not processing files found in LAYERS_DIR
|
||||
* and is intended to hold a key-value mapping from a layer `key` to a `Layer`
|
||||
* class
|
||||
*
|
||||
*/
|
||||
module.exports.LAYERS_CHECKSUM_CACHE = new NodeCache({ stdTTL: 300 });
|
||||
|
||||
class Layer {
|
||||
constructor(org, image, tag, filePath) {
|
||||
this.filePath = filePath;
|
||||
this.key = Layer.keyFor(org, image, tag, path.basename(filePath));
|
||||
this.digest = null;
|
||||
this.tarDigest = null;
|
||||
}
|
||||
|
||||
static keyFor(org, image, tag, filename) {
|
||||
return `${org}/${image}:${tag}|${filename}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The process function will ensure that the file has been considered and
|
||||
* checksums have been created
|
||||
*/
|
||||
process() {
|
||||
logger.info(`Processing ${this} with ${this.filePath}`);
|
||||
|
||||
if ((this.digest == null) || (this.tarDigest == null)) {
|
||||
// We'll need the contents of the layer for checksumming and gunziping
|
||||
const layerContents = fs.readFileSync(this.filePath);
|
||||
|
||||
logger.debug(`Computing digest for ${this}`);
|
||||
this.digest = sha256sum(layerContents);
|
||||
|
||||
logger.debug(`Computing tarDigest for ${this}`);
|
||||
this.tarDigest = sha256sum(zlib.gunzipSync(layerContents));
|
||||
}
|
||||
}
|
||||
|
||||
toString() {
|
||||
return `Layer(${this.key})`;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports.Layer = Layer;
|
|
@ -1,18 +0,0 @@
|
|||
/*
|
||||
* Simple logger module to preconfigure winston'
|
||||
*/
|
||||
|
||||
const { createLogger, format, transports } = require('winston');
|
||||
|
||||
module.exports = createLogger({
|
||||
// To see more detailed errors, change this to 'debug'
|
||||
level: process.env.LOG_LEVEL || 'info',
|
||||
format: format.combine(
|
||||
format.splat(),
|
||||
format.simple()
|
||||
),
|
||||
transports: [
|
||||
new transports.Console()
|
||||
],
|
||||
});
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
image=library/alpine
|
||||
digest=latest
|
||||
#digest="sha256:3085eec036f1a96b2f1cfbdc7f12ff6e38fde621324fae5608154db221c16d04"
|
||||
#digest="sha256:92c7f9c92844bbbb5d0a101b22f7c2a7949e40f8ea90c8b3bc396879d95e899a"
|
||||
|
||||
registryBase='https://registry-1.docker.io'
|
||||
authBase='https://auth.docker.io'
|
||||
authService='registry.docker.io'
|
||||
|
||||
blob="sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
|
||||
imageblob="sha256:fce289e99eb9bca977dae136fbe2a82b6b7d4c372474c9235adc1741675f587e"
|
||||
|
||||
token="$(curl -fsSL "$authBase/token?service=$authService&scope=repository:$image:pull" | jq --raw-output '.token')"
|
||||
|
||||
curl -fvSL \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
|
||||
"$registryBase/v2/$image/manifests/$digest" | jq
|
||||
|
||||
blob="sha256:acd3ca9941a85e8ed16515bfc5328e4e2f8c128caa72959a58a127b7801ee01f"
|
||||
curl -fvSL \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
|
||||
"$registryBase/v2/$image/manifests/$blob" | jq
|
||||
|
||||
exit
|
||||
curl -fvSL \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
|
||||
"$registryBase/v2/$image/blobs/$blob" | jq
|
||||
exit
|
||||
curl -fvSL \
|
||||
-H "Authorization: Bearer $token" \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
-H 'Accept: application/vnd.docker.distribution.manifest.v1+json' \
|
||||
"$registryBase/v2/$image/blobs/$imageblob" > hello-world-image.json
|
130
tools/shoreman
130
tools/shoreman
|
@ -1,130 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# [shoreman](https://github.com/chrismytton/shoreman) is an
|
||||
# implementation of the **Procfile** format. Inspired by the original
|
||||
# [foreman](http://ddollar.github.com/foreman/) tool for ruby, as
|
||||
# well as [norman](https://github.com/josh/norman) for node.js.
|
||||
|
||||
# Make sure that any errors cause the script to exit immediately.
|
||||
set -eo pipefail
|
||||
[[ "$TRACE" ]] && set -x
|
||||
|
||||
# ## Usage
|
||||
|
||||
# Usage message that is displayed when `--help` is given as an argument.
|
||||
usage() {
|
||||
echo "Usage: shoreman [procfile|Procfile] [envfile|.env]"
|
||||
echo "Run Procfiles using shell."
|
||||
echo
|
||||
echo "The shoreman script reads commands from [procfile] and starts up the"
|
||||
echo "processes that it describes."
|
||||
}
|
||||
|
||||
# ## Logging
|
||||
|
||||
# For logging we want to prefix each entry with the current time, as well
|
||||
# as the process name. This takes two arguments, the name of the process
|
||||
# with its index, and then reads data from stdin, formats it, and sends it
|
||||
# to stdout.
|
||||
log() {
|
||||
local index="$2"
|
||||
local format="%s %s\t| %s"
|
||||
|
||||
# We add colors when output is a terminal. `SHOREMAN_COLORS` can override it.
|
||||
if [ -t 1 -o "$SHOREMAN_COLORS" == "always" ] \
|
||||
&& [ "$SHOREMAN_COLORS" != "never" ]; then
|
||||
# Bash colors start from 31 up to 37. We calculate what color the process
|
||||
# gets based on its index.
|
||||
local color="$((31 + (index % 7)))"
|
||||
format="\033[0;${color}m%s %s\t|\033[0m %s"
|
||||
fi
|
||||
|
||||
while read -r data
|
||||
do
|
||||
printf "$format\n" "$(date +"%H:%M:%S")" "$1" "$data"
|
||||
done
|
||||
}
|
||||
|
||||
# ## Running commands
|
||||
|
||||
# When a process is started, we want to keep track of its pid so we can
|
||||
# `kill` it when the parent process receives a signal, and so we can `wait`
|
||||
# for it to finish before exiting the parent process.
|
||||
store_pid() {
|
||||
pids="$pids $1"
|
||||
}
|
||||
|
||||
# This starts a command asynchronously and stores its pid in a list for use
|
||||
# later on in the script.
|
||||
start_command() {
|
||||
bash -c "$1" 2>&1 | log "$2" "$3" &
|
||||
pid="$(jobs -p %%)"
|
||||
store_pid "$pid"
|
||||
}
|
||||
|
||||
# ## Reading the .env file
|
||||
|
||||
# The .env file needs to be a list of assignments like in a shell script.
|
||||
# Shell-style comments are permitted.
|
||||
load_env_file() {
|
||||
local env_file=${1:-'.env'}
|
||||
|
||||
# Set a default port before loading the .env file
|
||||
export PORT=${PORT:-5000}
|
||||
|
||||
if [[ -f "$env_file" ]]; then
|
||||
export $(grep "^[^#]*=.*" "$env_file" | xargs)
|
||||
fi
|
||||
}
|
||||
|
||||
# ## Reading the Procfile
|
||||
|
||||
# The Procfile needs to be parsed to extract the process names and commands.
|
||||
# The file is given on stdin, see the `<` at the end of this while loop.
|
||||
run_procfile() {
|
||||
local procfile=${1:-'Procfile'}
|
||||
# We give each process an index to track its color. We start with 1,
|
||||
# because it corresponds to green which is easier on the eye than red (0).
|
||||
local index=1
|
||||
while read line || [[ -n "$line" ]]; do
|
||||
if [[ -z "$line" ]] || [[ "$line" == \#* ]]; then continue; fi
|
||||
local name="${line%%:*}"
|
||||
local command="${line#*:[[:space:]]}"
|
||||
start_command "$command" "${name}" "$index"
|
||||
echo "'${command}' started with pid $pid" | log "${name}" "$index"
|
||||
index=$((index + 1))
|
||||
done < "$procfile"
|
||||
}
|
||||
|
||||
# ## Cleanup
|
||||
|
||||
# When a `SIGINT`, `SIGTERM` or `EXIT` is received, this action is run, killing the
|
||||
# child processes. The sleep stops STDOUT from pouring over the prompt, it
|
||||
# should probably go at some point.
|
||||
onexit() {
|
||||
echo "SIGINT received"
|
||||
echo "sending SIGTERM to all processes"
|
||||
kill $pids
|
||||
sleep 1
|
||||
}
|
||||
|
||||
main() {
|
||||
local procfile="$1"
|
||||
local env_file="$2"
|
||||
|
||||
# If the --help option is given, show the usage message and exit.
|
||||
expr -- "$*" : ".*--help" >/dev/null && {
|
||||
usage
|
||||
exit 0
|
||||
}
|
||||
|
||||
load_env_file "$env_file"
|
||||
run_procfile "$procfile"
|
||||
|
||||
trap onexit INT TERM
|
||||
|
||||
# Wait for the children to finish executing before exiting.
|
||||
wait $pids
|
||||
}
|
||||
|
||||
main "$@"
|
Loading…
Reference in New Issue