Refactoring to do on-the-fly processing of layers
This will allow the layers.d/ directory to be full of tarballs and Ahab can figure out whatever intermediate checksums it might need at runtime. With this in place, the request dance with the upstream manifest API can begin!
This commit is contained in:
parent
fe173e0e6c
commit
888fee1be2
241
index.js
241
index.js
|
|
@ -1,24 +1,67 @@
|
|||
const crypto = require('crypto');
|
||||
const express = require('express');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const request = require('request');
|
||||
/**
|
||||
* This file is the main server for Ahab and contains all the basic request
|
||||
* routing logic for proxying and mutating results from a Docker registry
|
||||
*/
|
||||
|
||||
const logger = require('./src/logger');
|
||||
const express = require('express');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const request = require('request');
|
||||
const rp = require('request-promise');
|
||||
|
||||
const { LAYERS_CHECKSUM_CACHE, Layer } = require('./src/layers');
|
||||
const logger = require('./src/logger');
|
||||
|
||||
const UPSTREAM_REGISTRY = process.env.UPSTREAM_REGISTRY || 'https://registry-1.docker.io';
|
||||
const LAYERS_DIR = path.resolve('./layers.d/');
|
||||
const LAYERS_DIR = path.resolve(process.env.LAYERS_DIR || './layers.d/');
|
||||
logger.info(`Using the layers directory: ${LAYERS_DIR}`);
|
||||
|
||||
|
||||
/**
|
||||
* Return the computed override path for the given image:tag
|
||||
*/
|
||||
function overridePathFor(org, image, tag) {
|
||||
return path.join(LAYERS_DIR, org, image, tag);
|
||||
}
|
||||
|
||||
/**
|
||||
* Return true if we have an override directory for the image
|
||||
*/
|
||||
function shouldOverrideImage(org, name, tag) {
|
||||
const computedPath = path.join(LAYERS_DIR, org, name, tag);
|
||||
function shouldOverrideImage(org, image, tag) {
|
||||
const computedPath = overridePathFor(org, image, tag);
|
||||
logger.debug(`Checking to presence of override dir: ${computedPath}`);
|
||||
return fs.existsSync(computedPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Collect the layers to use for override
|
||||
*
|
||||
* @return Array of strings with full file paths
|
||||
*/
|
||||
function collectLayersFor(org, image, tag) {
|
||||
// Just to make sure we're never called with bad data
|
||||
if (!shouldOverrideImage(org, image, tag)) {
|
||||
return [];
|
||||
}
|
||||
const computedPath = overridePathFor(org, image, tag);
|
||||
return fs.readdirSync(computedPath)
|
||||
.filter((filename) => {
|
||||
return filename.endsWith('.tar.gz');
|
||||
})
|
||||
.sort()
|
||||
.map((filename) => {
|
||||
const key = Layer.keyFor(org, image, tag, filename);
|
||||
|
||||
if (!LAYERS_CHECKSUM_CACHE[key]) {
|
||||
logger.debug(`Computing a new layer for key ${key}`);
|
||||
const layer = new Layer(org, image, tag, path.join(computedPath, filename));
|
||||
layer.process();
|
||||
LAYERS_CHECKSUM_CACHE[key] = layer;
|
||||
}
|
||||
return LAYERS_CHECKSUM_CACHE[key];
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Proxy the given request directly to the upstream
|
||||
*
|
||||
|
|
@ -40,133 +83,8 @@ function proxyToUpstream(req, res) {
|
|||
}).pipe(res);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a SHA256 checksum (hex encoded) of the given object
|
||||
*
|
||||
* The object will be JSON encoded before checksumming
|
||||
*/
|
||||
function sha256Sum(obj) {
|
||||
return crypto.createHash('sha256')
|
||||
.update(JSON.stringify(obj), 'utf8')
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
const app = express();
|
||||
const port = 9090;
|
||||
const imageConfig = 'sha256:b0b17eacf99071d7178bbf12d31a556bb3f05b429564c7fc4db33899d3eda85b';
|
||||
|
||||
let alpineResponse = {
|
||||
"manifests": [
|
||||
{
|
||||
"digest": "sha256:acd3ca9941a85e8ed16515bfc5328e4e2f8c128caa72959a58a127b7801ee01f",
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"platform": {
|
||||
"architecture": "amd64",
|
||||
"os": "linux"
|
||||
},
|
||||
"size": 528
|
||||
},
|
||||
{
|
||||
"digest": "sha256:0489474da8ea22426ece86ace6c1c0026ab2fd3cdfbbd62b7e94650266c37d9a",
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"platform": {
|
||||
"architecture": "arm",
|
||||
"os": "linux",
|
||||
"variant": "v6"
|
||||
},
|
||||
"size": 528
|
||||
},
|
||||
{
|
||||
"digest": "sha256:1316a4e4b2361242457f4136cc409c38e8a48ffda76e4e1be31896006e5fc4a2",
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"platform": {
|
||||
"architecture": "arm",
|
||||
"os": "linux",
|
||||
"variant": "v7"
|
||||
},
|
||||
"size": 528
|
||||
},
|
||||
{
|
||||
"digest": "sha256:db7f3dcef3d586f7dd123f107c93d7911515a5991c4b9e51fa2a43e46335a43e",
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"platform": {
|
||||
"architecture": "arm64",
|
||||
"os": "linux",
|
||||
"variant": "v8"
|
||||
},
|
||||
"size": 528
|
||||
},
|
||||
{
|
||||
"digest": "sha256:499416c8a5bcb311d75e12fb4667886e32b43aaf11003be2a334cbe265e81ce4",
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"platform": {
|
||||
"architecture": "386",
|
||||
"os": "linux"
|
||||
},
|
||||
"size": 528
|
||||
},
|
||||
{
|
||||
"digest": "sha256:5abbfe2915ad8c466bf6c9f33d03622cde0298c36cd88f55d16a3aa3d9c2c35e",
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"platform": {
|
||||
"architecture": "ppc64le",
|
||||
"os": "linux"
|
||||
},
|
||||
"size": 528
|
||||
},
|
||||
{
|
||||
"digest": "sha256:43955d6857268cc948ae9b370b221091057de83c4962da0826f9a2bdc9bd6b44",
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"platform": {
|
||||
"architecture": "s390x",
|
||||
"os": "linux"
|
||||
},
|
||||
"size": 528
|
||||
}
|
||||
],
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"schemaVersion": 2
|
||||
}
|
||||
|
||||
let alpineImage = {
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.docker.container.image.v1+json",
|
||||
"size": 1512,
|
||||
"digest": "sha256:961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4"
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 2789669,
|
||||
"digest": "sha256:9d48c3bd43c520dc2784e868a780e976b207cbf493eaff8c6596eb871cbd9609"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
let response = {
|
||||
"schemaVersion": 2,
|
||||
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
|
||||
"config": {
|
||||
"mediaType": "application/vnd.docker.container.image.v1+json",
|
||||
"size": 1585,
|
||||
"digest": imageConfig,
|
||||
},
|
||||
"layers": [
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 977,
|
||||
"digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced"
|
||||
},
|
||||
{
|
||||
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
|
||||
"size": 5808040,
|
||||
"digest": "sha256:cf98438b1781c2be1fde41967959140379c715b75a85723501d0bca82f215a76"
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
|
||||
/*
|
||||
* The manifests API is defined in the "Pulling an Image" part of the HTTP
|
||||
|
|
@ -183,26 +101,57 @@ let response = {
|
|||
app.get('/v2/:org/:image/manifests/:digest', (req, res) => {
|
||||
const { org, image, digest } = req.params;
|
||||
|
||||
if (shouldOverrideImage(org, image, digest)) {
|
||||
logger.info(`Overriding the pull for ${org}/${image}:${digest}`);
|
||||
if (!shouldOverrideImage(org, image, digest)) {
|
||||
return proxyToUpstream(req, res);
|
||||
}
|
||||
|
||||
res.status(500);
|
||||
res.send('Fail');
|
||||
return
|
||||
|
||||
|
||||
console.log(req.originalUrl);
|
||||
const checksum = crypto.createHash('sha256').update(JSON.stringify(response), 'utf8').digest('hex');
|
||||
res.set('Docker-Content-Digest', `sha256:${checksum}`);
|
||||
|
||||
/*
|
||||
* If we don't explicitly set the content type here, the client will think
|
||||
* that we're sending back a v1 manifest schema and complain about a "missing
|
||||
* signature key"
|
||||
*/
|
||||
res.set('Content-Type', 'application/vnd.docker.distribution.manifest.v2+json');
|
||||
res.send(response);
|
||||
|
||||
logger.info(`Overriding the pull for ${org}/${image}:${digest}`);
|
||||
|
||||
const layers = collectLayersFor(org, image, digest);
|
||||
logger.info(layers);
|
||||
|
||||
res.status(500);
|
||||
res.send('Fail');
|
||||
return
|
||||
|
||||
rp({
|
||||
url: `${UPSTREAM_REGISTRY}/v2/${org}/${image}/blobs/${digest}`,
|
||||
headers: {
|
||||
/*
|
||||
* We need to send the Authorization header along as well, otherwise
|
||||
* the upstream repository might complain that we're not authorized
|
||||
*/
|
||||
'Authorization' : req.get('Authorization'),
|
||||
},
|
||||
})
|
||||
.then((response) => {
|
||||
image = JSON.parse(response);
|
||||
//image = JSON.parse(image);
|
||||
//image.rootfs.diff_ids.push(
|
||||
// 'sha256:5f9da7cc9d8d83c96245ac27854466f6ed89fbfade5dd8a4f307861bfb72d1b8',
|
||||
//);
|
||||
//console.log(image.rootfs);
|
||||
//const checksum = crypto.createHash('sha256').update(JSON.stringify(image), 'utf8').digest('hex');
|
||||
//console.log(JSON.stringify(image));
|
||||
//console.log(`Sending content with a checksum of ${checksum}`);
|
||||
//console.log(`Client is expecting ${req.params.sha}`);
|
||||
//res.send(image);
|
||||
//console.log(req.originalUrl);
|
||||
//const checksum = crypto.createHash('sha256').update(JSON.stringify(response), 'utf8').digest('hex');
|
||||
//res.set('Docker-Content-Digest', `sha256:${checksum}`);
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.error(`Failed to hit the blob API for ${org}/${image}:${digest} - ${error}`);
|
||||
res.status(500);
|
||||
res.send(error);
|
||||
});
|
||||
});
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
const crypto = require('crypto');
|
||||
|
||||
module.exports = {};
|
||||
|
||||
/**
|
||||
* Create a SHA256 checksum (hex encoded) of the given object
|
||||
*
|
||||
* Any non-string objects will be JSON encoded before checksumming
|
||||
*/
|
||||
module.exports.sha256sum = (obj) => {
|
||||
let buffer = obj;
|
||||
if (!obj instanceof String) {
|
||||
buffer = JSON.stringify(obj);
|
||||
}
|
||||
return crypto.createHash('sha256')
|
||||
.update(buffer, 'utf8')
|
||||
.digest('hex');
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const zlib = require('zlib');
|
||||
|
||||
const { sha256sum } = require('./checksum');
|
||||
const logger = require('./logger');
|
||||
|
||||
module.exports = {};
|
||||
/**
|
||||
* This cache is to ensure that we're not processing files found in LAYERS_DIR
|
||||
* and is intended to hold a key-value mapping from a layer `key` to a `Layer`
|
||||
* class
|
||||
*
|
||||
*/
|
||||
module.exports.LAYERS_CHECKSUM_CACHE = {};
|
||||
|
||||
class Layer {
|
||||
constructor(org, image, tag, filePath) {
|
||||
this.filePath = filePath;
|
||||
this.key = Layer.keyFor(org, image, tag, path.basename(filePath));
|
||||
this.digest = null;
|
||||
this.tarDigest = null;
|
||||
}
|
||||
|
||||
static keyFor(org, image, tag, filename) {
|
||||
return `${org}/${image}:${tag}|${filename}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The process function will ensure that the file has been considered and
|
||||
* checksums have been created
|
||||
*/
|
||||
process() {
|
||||
logger.info(`Processing ${this} with ${this.filePath}`);
|
||||
|
||||
if ((this.digest == null) || (this.tarDigest == null)) {
|
||||
// We'll need the contents of the layer for checksumming and gunziping
|
||||
const layerContents = fs.readFileSync(this.filePath);
|
||||
|
||||
logger.debug(`Computing digest for ${this}`);
|
||||
this.digest = sha256sum(layerContents);
|
||||
|
||||
logger.debug(`Computing tarDigest for ${this}`);
|
||||
this.tarDigest = sha256sum(zlib.gunzipSync(layerContents));
|
||||
}
|
||||
}
|
||||
|
||||
toString() {
|
||||
return `Layer(${this.key})`;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports.Layer = Layer;
|
||||
Loading…
Reference in New Issue