Add network container tests

This commit is contained in:
Eric Huss 2023-01-14 14:48:53 -08:00
parent 88f14290f2
commit 4cb9ac35bf
19 changed files with 1136 additions and 2 deletions

View File

@ -34,6 +34,7 @@ jobs:
CARGO_PROFILE_DEV_DEBUG: 1
CARGO_PROFILE_TEST_DEBUG: 1
CARGO_INCREMENTAL: 0
CARGO_PUBLIC_NETWORK_TESTS: 1
strategy:
matrix:
include:
@ -77,6 +78,9 @@ jobs:
- run: sudo apt update -y && sudo apt install gcc-multilib libsecret-1-0 libsecret-1-dev -y
if: matrix.os == 'ubuntu-latest'
- run: rustup component add rustfmt || echo "rustfmt not available"
- name: Configure extra test environment
run: echo CARGO_CONTAINER_TESTS=1 >> $GITHUB_ENV
if: matrix.os == 'ubuntu-latest'
# Deny warnings on CI to avoid warnings getting into the codebase.
- run: cargo test --features 'deny-warnings'

View File

@ -55,6 +55,23 @@ pub fn cargo_test(attr: TokenStream, item: TokenStream) -> TokenStream {
"does not work on windows-gnu"
);
}
"container_test" => {
// These tests must be opt-in because they require docker.
set_ignore!(
option_env!("CARGO_CONTAINER_TESTS").is_none(),
"CARGO_CONTAINER_TESTS must be set"
);
}
"public_network_test" => {
// These tests must be opt-in because they touch the public
// network. The use of these should be **EXTREMELY RARE**, and
// should only touch things which would nearly certainly work
// in CI (like github.com).
set_ignore!(
option_env!("CARGO_PUBLIC_NETWORK_TESTS").is_none(),
"CARGO_PUBLIC_NETWORK_TESTS must be set"
);
}
"nightly" => {
requires_reason = true;
set_ignore!(is_not_nightly, "requires nightly");

View File

@ -3,4 +3,5 @@ fn main() {
"cargo:rustc-env=NATIVE_ARCH={}",
std::env::var("TARGET").unwrap()
);
println!("cargo:rerun-if-changed=build.rs");
}

View File

@ -0,0 +1,26 @@
FROM httpd:2.4-alpine
RUN apk add --no-cache git git-daemon openssl
COPY bar /repos/bar
WORKDIR /repos/bar
RUN git config --global user.email "testuser@example.com" &&\
git config --global user.name "Test User" &&\
git init -b master . &&\
git add Cargo.toml src &&\
git commit -m "Initial commit" &&\
mv .git ../bar.git &&\
cd ../bar.git &&\
git config --bool core.bare true &&\
rm -rf ../bar
WORKDIR /
EXPOSE 443
WORKDIR /usr/local/apache2/conf
COPY httpd-cargo.conf .
RUN cat httpd-cargo.conf >> httpd.conf
RUN openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \
-keyout server.key -out server.crt \
-subj "/emailAddress=webmaster@example.com/C=US/ST=California/L=San Francisco/O=Rust/OU=Cargo/CN=127.0.0.1"
WORKDIR /

View File

@ -0,0 +1,4 @@
[package]
name = "bar"
version = "1.0.0"
edition = "2021"

View File

@ -0,0 +1 @@
// Intentionally blank.

View File

@ -0,0 +1,12 @@
SetEnv GIT_PROJECT_ROOT /repos
SetEnv GIT_HTTP_EXPORT_ALL
ScriptAlias /repos /usr/libexec/git-core/git-http-backend/
LoadModule cgid_module modules/mod_cgid.so
<Files "git-http-backend">
Require all granted
</Files>
Include conf/extra/httpd-ssl.conf
LoadModule ssl_module modules/mod_ssl.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so

View File

@ -0,0 +1,29 @@
FROM alpine:3.17
RUN apk add --no-cache openssh git
RUN ssh-keygen -A
RUN addgroup -S testuser && adduser -S testuser -G testuser -s /bin/ash
# NOTE: Ideally the password should be set to *, but I am uncertain how to do
# that in alpine. It shouldn't matter since PermitEmptyPasswords is "no".
RUN passwd -u testuser
RUN mkdir /repos && chown testuser /repos
COPY --chown=testuser:testuser bar /repos/bar
USER testuser
WORKDIR /repos/bar
RUN git config --global user.email "testuser@example.com" &&\
git config --global user.name "Test User" &&\
git init -b master . &&\
git add Cargo.toml src &&\
git commit -m "Initial commit" &&\
mv .git ../bar.git &&\
cd ../bar.git &&\
git config --bool core.bare true &&\
rm -rf ../bar
WORKDIR /
USER root
EXPOSE 22
ENTRYPOINT ["/usr/sbin/sshd", "-D", "-E", "/var/log/auth.log"]

View File

@ -0,0 +1,4 @@
[package]
name = "bar"
version = "1.0.0"
edition = "2021"

View File

@ -0,0 +1 @@
// Intentionally blank.

View File

@ -0,0 +1,285 @@
//! Support for testing using Docker containers.
//!
//! The [`Container`] type is a builder for configuring a container to run.
//! After you call `launch`, you can use the [`ContainerHandle`] to interact
//! with the running container.
//!
//! Tests using containers must use `#[cargo_test(container_test)]` to disable
//! them unless the CARGO_CONTAINER_TESTS environment variable is set.
use cargo_util::ProcessBuilder;
use std::collections::HashMap;
use std::io::Read;
use std::path::PathBuf;
use std::process::Command;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Mutex;
use tar::Header;
/// A builder for configuring a container to run.
pub struct Container {
/// The host directory that forms the basis of the Docker image.
build_context: PathBuf,
/// Files to copy over to the image.
files: Vec<MkFile>,
}
/// A handle to a running container.
///
/// You can use this to interact with the container.
pub struct ContainerHandle {
/// The name of the container.
name: String,
/// The IP address of the container.
///
/// NOTE: This is currently unused, but may be useful so I left it in.
/// This can only be used on Linux. macOS and Windows docker doesn't allow
/// direct connection to the container.
pub ip_address: String,
/// Port mappings of container_port to host_port for ports exposed via EXPOSE.
pub port_mappings: HashMap<u16, u16>,
}
impl Container {
pub fn new(context_dir: &str) -> Container {
assert!(std::env::var_os("CARGO_CONTAINER_TESTS").is_some());
let mut build_context = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
build_context.push("containers");
build_context.push(context_dir);
Container {
build_context,
files: Vec::new(),
}
}
/// Adds a file to be copied into the container.
pub fn file(mut self, file: MkFile) -> Self {
self.files.push(file);
self
}
/// Starts the container.
pub fn launch(mut self) -> ContainerHandle {
static NEXT_ID: AtomicUsize = AtomicUsize::new(0);
let id = NEXT_ID.fetch_add(1, Ordering::SeqCst);
let name = format!("cargo_test_{id}");
remove_if_exists(&name);
self.create_container(&name);
self.copy_files(&name);
self.start_container(&name);
let info = self.container_inspect(&name);
let ip_address = if cfg!(target_os = "linux") {
info[0]["NetworkSettings"]["IPAddress"]
.as_str()
.unwrap()
.to_string()
} else {
// macOS and Windows can't make direct connections to the
// container. It only works through exposed ports or mapped ports.
"127.0.0.1".to_string()
};
let port_mappings = self.port_mappings(&info);
self.wait_till_ready(&port_mappings);
ContainerHandle {
name,
ip_address,
port_mappings,
}
}
fn create_container(&self, name: &str) {
static BUILD_LOCK: Mutex<()> = Mutex::new(());
let image_base = self.build_context.file_name().unwrap();
let image_name = format!("cargo-test-{}", image_base.to_str().unwrap());
let _lock = BUILD_LOCK.lock().unwrap();
ProcessBuilder::new("docker")
.args(&["build", "--tag", image_name.as_str()])
.arg(&self.build_context)
.exec_with_output()
.unwrap();
ProcessBuilder::new("docker")
.args(&[
"container",
"create",
"--publish-all",
"--rm",
"--name",
name,
])
.arg(image_name)
.exec_with_output()
.unwrap();
}
fn copy_files(&mut self, name: &str) {
if self.files.is_empty() {
return;
}
let mut ar = tar::Builder::new(Vec::new());
let files = std::mem::replace(&mut self.files, Vec::new());
for mut file in files {
ar.append_data(&mut file.header, &file.path, file.contents.as_slice())
.unwrap();
}
let ar = ar.into_inner().unwrap();
ProcessBuilder::new("docker")
.args(&["cp", "-"])
.arg(format!("{name}:/"))
.stdin(ar)
.exec_with_output()
.unwrap();
}
fn start_container(&self, name: &str) {
ProcessBuilder::new("docker")
.args(&["container", "start"])
.arg(name)
.exec_with_output()
.unwrap();
}
fn container_inspect(&self, name: &str) -> serde_json::Value {
let output = ProcessBuilder::new("docker")
.args(&["inspect", name])
.exec_with_output()
.unwrap();
serde_json::from_slice(&output.stdout).unwrap()
}
/// Returns the mapping of container_port->host_port for ports that were
/// exposed with EXPOSE.
fn port_mappings(&self, info: &serde_json::Value) -> HashMap<u16, u16> {
info[0]["NetworkSettings"]["Ports"]
.as_object()
.unwrap()
.iter()
.map(|(key, value)| {
let key = key
.strip_suffix("/tcp")
.expect("expected TCP only ports")
.parse()
.unwrap();
let values = value.as_array().unwrap();
let value = values
.iter()
.find(|value| value["HostIp"].as_str().unwrap() == "0.0.0.0")
.expect("expected localhost IP");
let host_port = value["HostPort"].as_str().unwrap().parse().unwrap();
(key, host_port)
})
.collect()
}
fn wait_till_ready(&self, port_mappings: &HashMap<u16, u16>) {
for port in port_mappings.values() {
let mut ok = false;
for _ in 0..30 {
match std::net::TcpStream::connect(format!("127.0.0.1:{port}")) {
Ok(_) => {
ok = true;
break;
}
Err(e) => {
if e.kind() != std::io::ErrorKind::ConnectionRefused {
panic!("unexpected localhost connection error: {e:?}");
}
std::thread::sleep(std::time::Duration::new(1, 0));
}
}
}
if !ok {
panic!("no listener on localhost port {port}");
}
}
}
}
impl ContainerHandle {
/// Executes a program inside a running container.
pub fn exec(&self, args: &[&str]) -> std::process::Output {
ProcessBuilder::new("docker")
.args(&["container", "exec", &self.name])
.args(args)
.exec_with_output()
.unwrap()
}
/// Returns the contents of a file inside the container.
pub fn read_file(&self, path: &str) -> String {
let output = ProcessBuilder::new("docker")
.args(&["cp", &format!("{}:{}", self.name, path), "-"])
.exec_with_output()
.unwrap();
let mut ar = tar::Archive::new(output.stdout.as_slice());
let mut entry = ar.entries().unwrap().next().unwrap().unwrap();
let mut contents = String::new();
entry.read_to_string(&mut contents).unwrap();
contents
}
}
impl Drop for ContainerHandle {
fn drop(&mut self) {
// To help with debugging, this will keep the container alive.
if std::env::var_os("CARGO_CONTAINER_TEST_KEEP").is_some() {
return;
}
remove_if_exists(&self.name);
}
}
fn remove_if_exists(name: &str) {
if let Err(e) = Command::new("docker")
.args(&["container", "rm", "--force", name])
.output()
{
panic!("failed to run docker: {e}");
}
}
/// Builder for configuring a file to copy into a container.
pub struct MkFile {
path: String,
contents: Vec<u8>,
header: Header,
}
impl MkFile {
/// Defines a file to add to the container.
///
/// This should be passed to `Container::file`.
///
/// The path is the path inside the container to create the file.
pub fn path(path: &str) -> MkFile {
MkFile {
path: path.to_string(),
contents: Vec::new(),
header: Header::new_gnu(),
}
}
pub fn contents(mut self, contents: impl Into<Vec<u8>>) -> Self {
self.contents = contents.into();
self.header.set_size(self.contents.len() as u64);
self
}
pub fn mode(mut self, mode: u32) -> Self {
self.header.set_mode(mode);
self
}
pub fn uid(mut self, uid: u64) -> Self {
self.header.set_uid(uid);
self
}
pub fn gid(mut self, gid: u64) -> Self {
self.header.set_gid(gid);
self
}
}

View File

@ -70,6 +70,7 @@ pub fn panic_error(what: &str, err: impl Into<anyhow::Error>) -> ! {
pub use cargo_test_macro::cargo_test;
pub mod compare;
pub mod containers;
pub mod cross_compile;
mod diff;
pub mod git;
@ -1227,6 +1228,8 @@ pub trait TestEnv: Sized {
// should hopefully not surprise us as we add cargo features over time and
// cargo rides the trains.
.env("__CARGO_TEST_CHANNEL_OVERRIDE_DO_NOT_USE_THIS", "stable")
// Keeps cargo within its sandbox.
.env("__CARGO_TEST_DISABLE_GLOBAL_KNOWN_HOST", "1")
// For now disable incremental by default as support hasn't ridden to the
// stable channel yet. Once incremental support hits the stable compiler we
// can switch this to one and then fix the tests.
@ -1247,11 +1250,15 @@ pub trait TestEnv: Sized {
.env_remove("GIT_AUTHOR_EMAIL")
.env_remove("GIT_COMMITTER_NAME")
.env_remove("GIT_COMMITTER_EMAIL")
.env_remove("SSH_AUTH_SOCK") // ensure an outer agent is never contacted
.env_remove("MSYSTEM"); // assume cmd.exe everywhere on windows
if cfg!(target_os = "macos") {
// Work-around a bug in macOS 10.15, see `link_or_copy` for details.
self = self.env("__CARGO_COPY_DONT_LINK_DO_NOT_USE_THIS", "1");
}
if cfg!(windows) {
self = self.env("USERPROFILE", paths::home());
}
self
}

View File

@ -358,7 +358,8 @@ fn check_ssh_known_hosts_loaded(
/// Returns a list of files to try loading OpenSSH-formatted known hosts.
fn known_host_files() -> Vec<PathBuf> {
let mut result = Vec::new();
if cfg!(unix) {
if std::env::var_os("__CARGO_TEST_DISABLE_GLOBAL_KNOWN_HOST").is_some() {
} else if cfg!(unix) {
result.push(PathBuf::from("/etc/ssh/ssh_known_hosts"));
} else if cfg!(windows) {
// The msys/cygwin version of OpenSSH uses `/etc` from the posix root

View File

@ -55,7 +55,7 @@ fn maybe_spurious(err: &Error) -> bool {
git2::ErrorClass::Net
| git2::ErrorClass::Os
| git2::ErrorClass::Zlib
| git2::ErrorClass::Http => return true,
| git2::ErrorClass::Http => return git_err.code() != git2::ErrorCode::Certificate,
_ => (),
}
}

View File

@ -39,3 +39,30 @@ the `CARGO_RUN_BUILD_STD_TESTS=1` environment variable and running `cargo test
--test build-std`. This requires the nightly channel, and also requires the
`rust-src` component installed with `rustup component add rust-src
--toolchain=nightly`.
## Running public network tests
Some (very rare) tests involve connecting to the public internet.
These tests are disabled by default,
but you can run them by setting the `CARGO_PUBLIC_NETWORK_TESTS=1` environment variable.
Additionally our CI suite has a smoke test for fetching dependencies.
For most contributors, you will never need to bother with this.
## Running container tests
Tests marked with `container_test` involve running Docker to test more complex configurations.
These tests are disabled by default,
but you can run them by setting the `CARGO_CONTAINER_TESTS=1` environment variable.
You will need to have Docker installed and running to use these.
> Note: Container tests mostly do not work on Windows.
> * The SSH tests require ssh-agent, but the two versions of ssh-agent
> on Windows are not suitable for testing.
> * The Microsoft version of ssh-agent runs as a global service, and can't be isolated per test.
> * The mingw/cygwin one can't be accessed from a Windows executable like cargo.
> * Pageant similarly does not seem to have a way to isolate it (and I'm not certain it can be driven completely from the command-line).
>
> The tests also can't run on Windows CI because the Docker that is preinstalled doesn't support Linux containers, and setting up Windows containers is a pain.
>
> macOS should work with Docker installed and running,
> but unfortunately the tests are not run on CI because Docker is not available.

View File

@ -92,6 +92,19 @@ The options it supports are:
* `>=1.64` — This indicates that the test will only run with the given version of `rustc` or newer.
This can be used when a new `rustc` feature has been stabilized that the test depends on.
If this is specified, a `reason` is required to explain why it is being checked.
* `public_network_test` — This tests contacts the public internet.
These tests are disabled unless the `CARGO_PUBLIC_NETWORK_TESTS` environment variable is set.
Use of this should be *extremely rare*, please avoid using it if possible.
The hosts it contacts should have a relatively high confidence that they are reliable and stable (such as github.com), especially in CI.
The tests should be carefully considered for developer security and privacy as well.
* `container_test` — This indicates that it is a test that uses Docker.
These tests are disabled unless the `CARGO_CONTAINER_TESTS` environment variable is set.
This requires that you have Docker installed.
The SSH tests also assume that you have OpenSSH installed.
These should work on Linux, macOS, and Windows where possible.
Unfortunately these tests are not run in CI for macOS or Windows (no Docker on macOS, and Windows does not support Linux images).
See [`crates/cargo-test-support/src/containers.rs`](https://github.com/rust-lang/cargo/blob/master/crates/cargo-test-support/src/containers.rs) for more on writing these tests.
#### Testing Nightly Features

152
tests/testsuite/https.rs Normal file
View File

@ -0,0 +1,152 @@
//! Network tests for https transport.
//!
//! Note that these tests will generally require setting CARGO_CONTAINER_TESTS
//! or CARGO_PUBLIC_NETWORK_TESTS.
use cargo_test_support::containers::Container;
use cargo_test_support::project;
#[cargo_test(container_test)]
fn self_signed_should_fail() {
// Cargo should not allow a connection to a self-signed certificate.
let apache = Container::new("apache").launch();
let port = apache.port_mappings[&443];
let url = format!("https://127.0.0.1:{port}/repos/bar.git");
let p = project()
.file(
"Cargo.toml",
&format!(
r#"
[package]
name = "foo"
version = "0.1.0"
[dependencies]
bar = {{ git = "{url}" }}
"#
),
)
.file("src/lib.rs", "")
.build();
// I think the text here depends on the curl backend.
let err_msg = if cfg!(target_os = "macos") {
"untrusted connection error; class=Ssl (16); code=Certificate (-17)"
} else if cfg!(unix) {
"the SSL certificate is invalid; class=Ssl (16); code=Certificate (-17)"
} else if cfg!(windows) {
"user cancelled certificate check; class=Http (34); code=Certificate (-17)"
} else {
panic!("target not supported");
};
p.cargo("fetch")
.with_status(101)
.with_stderr(&format!(
"\
[UPDATING] git repository `https://127.0.0.1:[..]/repos/bar.git`
error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)`
Caused by:
failed to load source for dependency `bar`
Caused by:
Unable to update https://127.0.0.1:[..]/repos/bar.git
Caused by:
failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..]
Caused by:
network failure seems to have happened
if a proxy or similar is necessary `net.git-fetch-with-cli` may help here
https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli
Caused by:
{err_msg}
"
))
.run();
}
#[cargo_test(container_test)]
fn self_signed_with_cacert() {
// When using cainfo, that should allow a connection to a self-signed cert.
if cfg!(target_os = "macos") {
// This test only seems to work with the
// curl-sys/force-system-lib-on-osx feature enabled. For some reason
// SecureTransport doesn't seem to like the self-signed certificate.
// It works if the certificate is manually approved via Keychain
// Access. The system libcurl is built with a LibreSSL fallback which
// is used when CAINFO is set, which seems to work correctly. This
// could use some more investigation. The official Rust binaries use
// curl-sys/force-system-lib-on-osx so it is mostly an issue for local
// testing.
//
// The error is:
// [60] SSL peer certificate or SSH remote key was not OK (SSL:
// certificate verification failed (result: 5)); class=Net (12)
let curl_v = curl::Version::get();
if curl_v.vendored() {
eprintln!(
"vendored curl not supported on macOS, \
set curl-sys/force-system-lib-on-osx to enable"
);
return;
}
}
let apache = Container::new("apache").launch();
let port = apache.port_mappings[&443];
let url = format!("https://127.0.0.1:{port}/repos/bar.git");
let server_crt = apache.read_file("/usr/local/apache2/conf/server.crt");
let p = project()
.file(
"Cargo.toml",
&format!(
r#"
[package]
name = "foo"
version = "0.1.0"
[dependencies]
bar = {{ git = "{url}" }}
"#
),
)
.file("src/lib.rs", "")
.file(
".cargo/config.toml",
&format!(
r#"
[http]
cainfo = "server.crt"
"#
),
)
.file("server.crt", &server_crt)
.build();
p.cargo("fetch")
.with_stderr("[UPDATING] git repository `https://127.0.0.1:[..]/repos/bar.git`")
.run();
}
#[cargo_test(public_network_test)]
fn github_works() {
// Check that an https connection to github.com works.
let p = project()
.file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.1.0"
[dependencies]
bitflags = { git = "https://github.com/rust-lang/bitflags.git", tag="1.3.2" }
"#,
)
.file("src/lib.rs", "")
.build();
p.cargo("fetch")
.with_stderr("[UPDATING] git repository `https://github.com/rust-lang/bitflags.git`")
.run();
}

View File

@ -61,6 +61,7 @@ mod git_auth;
mod git_gc;
mod glob_targets;
mod help;
mod https;
mod inheritable_workspace_fields;
mod init;
mod install;
@ -121,6 +122,7 @@ mod rustflags;
mod search;
mod shell_quoting;
mod source_replacement;
mod ssh;
mod standard_lib;
mod test;
mod timings;

548
tests/testsuite/ssh.rs Normal file
View File

@ -0,0 +1,548 @@
//! Network tests for SSH connections.
//!
//! Note that these tests will generally require setting CARGO_CONTAINER_TESTS
//! or CARGO_PUBLIC_NETWORK_TESTS.
//!
//! NOTE: The container tests almost certainly won't work on Windows.
use cargo_test_support::containers::{Container, ContainerHandle, MkFile};
use cargo_test_support::{paths, process, project, Project};
use std::fs;
use std::io::Write;
use std::path::PathBuf;
fn ssh_repo_url(container: &ContainerHandle, name: &str) -> String {
let port = container.port_mappings[&22];
format!("ssh://testuser@127.0.0.1:{port}/repos/{name}.git")
}
/// The path to the client's private key.
fn key_path() -> PathBuf {
paths::home().join(".ssh/id_ed25519")
}
/// Generates the SSH keys for authenticating into the container.
fn gen_ssh_keys() -> String {
let path = key_path();
process("ssh-keygen")
.args(&["-t", "ed25519", "-N", "", "-f"])
.arg(&path)
.exec_with_output()
.unwrap();
let pub_key = path.with_extension("pub");
fs::read_to_string(pub_key).unwrap()
}
/// Handler for running ssh-agent for SSH authentication.
///
/// Be sure to set `SSH_AUTH_SOCK` when running a process in order to use the
/// agent. Keys will need to be copied into the container with the
/// `authorized_keys()` method.
struct Agent {
sock: PathBuf,
pid: String,
ssh_dir: PathBuf,
pub_key: String,
}
impl Agent {
fn launch() -> Agent {
let ssh_dir = paths::home().join(".ssh");
fs::create_dir(&ssh_dir).unwrap();
let pub_key = gen_ssh_keys();
let sock = paths::root().join("agent");
let output = process("ssh-agent")
.args(&["-s", "-a"])
.arg(&sock)
.exec_with_output()
.unwrap();
let stdout = std::str::from_utf8(&output.stdout).unwrap();
let start = stdout.find("SSH_AGENT_PID=").unwrap() + 14;
let end = &stdout[start..].find(';').unwrap();
let pid = (&stdout[start..start + end]).to_string();
eprintln!("SSH_AGENT_PID={pid}");
process("ssh-add")
.arg(key_path())
.env("SSH_AUTH_SOCK", &sock)
.exec_with_output()
.unwrap();
Agent {
sock,
pid,
ssh_dir,
pub_key,
}
}
/// Returns a `MkFile` which can be passed into the `Container` builder to
/// copy an `authorized_keys` file containing this agent's public key.
fn authorized_keys(&self) -> MkFile {
MkFile::path("home/testuser/.ssh/authorized_keys")
.contents(self.pub_key.as_bytes())
.mode(0o600)
.uid(100)
.gid(101)
}
}
impl Drop for Agent {
fn drop(&mut self) {
if let Err(e) = process("ssh-agent")
.args(&["-k", "-a"])
.arg(&self.sock)
.env("SSH_AGENT_PID", &self.pid)
.exec_with_output()
{
eprintln!("failed to stop ssh-agent: {e:?}");
}
}
}
/// Common project used for several tests.
fn foo_bar_project(url: &str) -> Project {
project()
.file(
"Cargo.toml",
&format!(
r#"
[package]
name = "foo"
version = "0.1.0"
[dependencies]
bar = {{ git = "{url}" }}
"#
),
)
.file("src/lib.rs", "")
.build()
}
#[cargo_test(container_test)]
fn no_known_host() {
// When host is not known, it should show an error.
let sshd = Container::new("sshd").launch();
let url = ssh_repo_url(&sshd, "bar");
let p = foo_bar_project(&url);
p.cargo("fetch")
.with_status(101)
.with_stderr(
"\
[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`
error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)`
Caused by:
failed to load source for dependency `bar`
Caused by:
Unable to update ssh://testuser@127.0.0.1:[..]/repos/bar.git
Caused by:
failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..]
Caused by:
error: unknown SSH host key
The SSH host key for `[127.0.0.1]:[..]` is not known and cannot be validated.
To resolve this issue, add the host key to the `net.ssh.known-hosts` array in \
your Cargo configuration (such as [ROOT]/home/.cargo/config.toml) or in your \
OpenSSH known_hosts file at [ROOT]/home/.ssh/known_hosts
The key to add is:
[127.0.0.1]:[..] ecdsa-sha2-nistp256 AAAA[..]
The ECDSA key fingerprint is: SHA256:[..]
This fingerprint should be validated with the server administrator that it is correct.
See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts \
for more information.
",
)
.run();
}
#[cargo_test(container_test)]
fn known_host_works() {
// The key displayed in the error message should work when added to known_hosts.
let agent = Agent::launch();
let sshd = Container::new("sshd")
.file(agent.authorized_keys())
.launch();
let url = ssh_repo_url(&sshd, "bar");
let p = foo_bar_project(&url);
let output = p
.cargo("fetch")
.env("SSH_AUTH_SOCK", &agent.sock)
.build_command()
.output()
.unwrap();
let stderr = std::str::from_utf8(&output.stderr).unwrap();
// Validate the fingerprint while we're here.
let fingerprint = stderr
.lines()
.find(|line| line.starts_with(" The ECDSA key fingerprint"))
.unwrap()
.trim();
let fingerprint = &fingerprint[30..];
let finger_out = sshd.exec(&["ssh-keygen", "-l", "-f", "/etc/ssh/ssh_host_ecdsa_key.pub"]);
let gen_finger = std::str::from_utf8(&finger_out.stdout).unwrap();
// <key-size> <fingerprint> <comments…>
let gen_finger = gen_finger.split_whitespace().nth(1).unwrap();
assert_eq!(fingerprint, gen_finger);
// Add the key to known_hosts, and try again.
let key = stderr
.lines()
.find(|line| line.starts_with(" [127.0.0.1]:"))
.unwrap()
.trim();
fs::write(agent.ssh_dir.join("known_hosts"), key).unwrap();
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &agent.sock)
.with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`")
.run();
}
#[cargo_test(container_test)]
fn same_key_different_hostname() {
// The error message should mention if an identical key was found.
let agent = Agent::launch();
let sshd = Container::new("sshd").launch();
let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub");
let known_hosts = format!("example.com {hostkey}");
fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap();
let url = ssh_repo_url(&sshd, "bar");
let p = foo_bar_project(&url);
p.cargo("fetch")
.with_status(101)
.with_stderr(
"\
[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`
error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)`
Caused by:
failed to load source for dependency `bar`
Caused by:
Unable to update ssh://testuser@127.0.0.1:[..]/repos/bar.git
Caused by:
failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..]
Caused by:
error: unknown SSH host key
The SSH host key for `[127.0.0.1]:[..]` is not known and cannot be validated.
To resolve this issue, add the host key to the `net.ssh.known-hosts` array in \
your Cargo configuration (such as [ROOT]/home/.cargo/config.toml) or in your \
OpenSSH known_hosts file at [ROOT]/home/.ssh/known_hosts
The key to add is:
[127.0.0.1]:[..] ecdsa-sha2-nistp256 AAAA[..]
The ECDSA key fingerprint is: SHA256:[..]
This fingerprint should be validated with the server administrator that it is correct.
Note: This host key was found, but is associated with a different host:
[ROOT]/home/.ssh/known_hosts line 1: example.com
See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts \
for more information.
",
)
.run();
}
#[cargo_test(container_test)]
fn known_host_without_port() {
// A known_host entry without a port should match a connection to a non-standard port.
let agent = Agent::launch();
let sshd = Container::new("sshd")
.file(agent.authorized_keys())
.launch();
let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub");
// The important part of this test is that this line does not have a port.
let known_hosts = format!("127.0.0.1 {hostkey}");
fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap();
let url = ssh_repo_url(&sshd, "bar");
let p = foo_bar_project(&url);
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &agent.sock)
.with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`")
.run();
}
#[cargo_test(container_test)]
fn hostname_case_insensitive() {
// hostname checking should be case-insensitive.
let agent = Agent::launch();
let sshd = Container::new("sshd")
.file(agent.authorized_keys())
.launch();
// Consider using `gethostname-rs` instead?
let hostname = process("hostname").exec_with_output().unwrap();
let hostname = std::str::from_utf8(&hostname.stdout).unwrap().trim();
let inv_hostname = if hostname.chars().any(|c| c.is_lowercase()) {
hostname.to_uppercase()
} else {
// There should be *some* chars in the name.
assert!(hostname.chars().any(|c| c.is_uppercase()));
hostname.to_lowercase()
};
eprintln!("converted {hostname} to {inv_hostname}");
let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub");
let known_hosts = format!("{inv_hostname} {hostkey}");
fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap();
let port = sshd.port_mappings[&22];
let url = format!("ssh://testuser@{hostname}:{port}/repos/bar.git");
let p = foo_bar_project(&url);
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &agent.sock)
.with_stderr(&format!(
"[UPDATING] git repository `ssh://testuser@{hostname}:{port}/repos/bar.git`"
))
.run();
}
#[cargo_test(container_test)]
fn invalid_key_error() {
// An error when a known_host value doesn't match.
let agent = Agent::launch();
let sshd = Container::new("sshd")
.file(agent.authorized_keys())
.launch();
let port = sshd.port_mappings[&22];
let known_hosts = format!(
"[127.0.0.1]:{port} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLqLMclVr7MDuaVsm3sEnnq2OrGxTFiHSw90wd6N14BU8xVC9cZldC3rJ58Wmw6bEVKPjk7foNG0lHwS5bCKX+U=\n"
);
fs::write(agent.ssh_dir.join("known_hosts"), known_hosts).unwrap();
let url = ssh_repo_url(&sshd, "bar");
let p = foo_bar_project(&url);
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &agent.sock)
.with_status(101)
.with_stderr(&format!("\
[UPDATING] git repository `ssh://testuser@127.0.0.1:{port}/repos/bar.git`
error: failed to get `bar` as a dependency of package `foo v0.1.0 ([ROOT]/foo)`
Caused by:
failed to load source for dependency `bar`
Caused by:
Unable to update ssh://testuser@127.0.0.1:{port}/repos/bar.git
Caused by:
failed to clone into: [ROOT]/home/.cargo/git/db/bar-[..]
Caused by:
error: SSH host key has changed for `[127.0.0.1]:{port}`
*********************************
* WARNING: HOST KEY HAS CHANGED *
*********************************
This may be caused by a man-in-the-middle attack, or the server may have changed its host key.
The ECDSA fingerprint for the key from the remote host is:
SHA256:[..]
You are strongly encouraged to contact the server administrator for `[127.0.0.1]:{port}` \
to verify that this new key is correct.
If you can verify that the server has a new key, you can resolve this error by \
removing the old ecdsa-sha2-nistp256 key for `[127.0.0.1]:{port}` located at \
[ROOT]/home/.ssh/known_hosts line 1, and adding the new key to the \
`net.ssh.known-hosts` array in your Cargo configuration (such as \
[ROOT]/home/.cargo/config.toml) or in your OpenSSH known_hosts file at \
[ROOT]/home/.ssh/known_hosts
The key provided by the remote host is:
[127.0.0.1]:{port} ecdsa-sha2-nistp256 [..]
See https://doc.rust-lang.org/stable/cargo/appendix/git-authentication.html#ssh-known-hosts for more information.
"))
.run();
// Add the key, it should work even with the old key left behind.
let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub");
let known_hosts_path = agent.ssh_dir.join("known_hosts");
let mut f = fs::OpenOptions::new()
.append(true)
.open(known_hosts_path)
.unwrap();
write!(f, "[127.0.0.1]:{port} {hostkey}").unwrap();
drop(f);
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &agent.sock)
.with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`")
.run();
}
#[cargo_test(public_network_test)]
fn invalid_github_key() {
// A key for github.com in known_hosts should override the built-in key.
// This uses a bogus key which should result in an error.
let ssh_dir = paths::home().join(".ssh");
fs::create_dir(&ssh_dir).unwrap();
let known_hosts = "\
github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLqLMclVr7MDuaVsm3sEnnq2OrGxTFiHSw90wd6N14BU8xVC9cZldC3rJ58Wmw6bEVKPjk7foNG0lHwS5bCKX+U=\n\
github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDgi+8rMcyFCBq5y7BXrb2aaYGhMjlU3QDy7YDvtNL5KSecYOsaqQHaXr87Bbx0EEkgbhK4kVMkmThlCoNITQS9Vc3zIMQ+Tg6+O4qXx719uCzywl50Tb5tDqPGMj54jcq3VUiu/dvse0yeehyvzoPNWewgGWLx11KI4A4wOwMnc6guhculEWe9DjGEjUQ34lPbmdfu/Hza7ZVu/RhgF/wc43uzXWB2KpMEqtuY1SgRlCZqTASoEtfKZi0AuM7AEdOwE5aTotS4CQZHWimb1bMFpF4DAq92CZ8Jhrm4rWETbO29WmjviCJEA3KNQyd3oA7H9AE9z/22PJaVEmjiZZ+wyLgwyIpOlsnHYNEdGeQMQ4SgLRkARLwcnKmByv1AAxsBW4LI3Os4FpwxVPdXHcBebydtvxIsbtUVkkq99nbsIlnSRFSTvb0alrdzRuKTdWpHtN1v9hagFqmeCx/kJfH76NXYBbtaWZhSOnxfEbhLYuOb+IS4jYzHAIkzy9FjVuk=\n\
ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEeMB6BUAW6FfvfLxRO3kGASe0yXnrRT4kpqncsup2b2\n";
fs::write(ssh_dir.join("known_hosts"), known_hosts).unwrap();
let p = project()
.file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.1.0"
[dependencies]
bitflags = { git = "ssh://git@github.com/rust-lang/bitflags.git", tag = "1.3.2" }
"#,
)
.file("src/lib.rs", "")
.build();
p.cargo("fetch")
.with_status(101)
.with_stderr_contains(" error: SSH host key has changed for `github.com`")
.run();
}
#[cargo_test(public_network_test)]
fn bundled_github_works() {
// The bundled key for github.com works.
//
// Use a bogus auth sock to force an authentication error.
// On Windows, if the agent service is running, it could allow a
// successful authentication.
//
// If the bundled hostkey did not work, it would result in an "unknown SSH
// host key" instead.
let bogus_auth_sock = paths::home().join("ssh_auth_sock");
let p = project()
.file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.1.0"
[dependencies]
bitflags = { git = "ssh://git@github.com/rust-lang/bitflags.git", tag = "1.3.2" }
"#,
)
.file("src/lib.rs", "")
.build();
let err = if cfg!(windows) {
"error authenticating: unable to connect to agent pipe; class=Ssh (23)"
} else {
"error authenticating: failed connecting with agent; class=Ssh (23)"
};
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &bogus_auth_sock)
.with_status(101)
.with_stderr(&format!(
"\
[UPDATING] git repository `ssh://git@github.com/rust-lang/bitflags.git`
error: failed to get `bitflags` as a dependency of package `foo v0.1.0 ([ROOT]/foo)`
Caused by:
failed to load source for dependency `bitflags`
Caused by:
Unable to update ssh://git@github.com/rust-lang/bitflags.git?tag=1.3.2
Caused by:
failed to clone into: [ROOT]/home/.cargo/git/db/bitflags-[..]
Caused by:
failed to authenticate when downloading repository
* attempted ssh-agent authentication, but no usernames succeeded: `git`
if the git CLI succeeds then `net.git-fetch-with-cli` may help here
https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli
Caused by:
{err}
"
))
.run();
// Explicit :22 should also work with bundled.
p.change_file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.1.0"
[dependencies]
bitflags = { git = "ssh://git@github.com:22/rust-lang/bitflags.git", tag = "1.3.2" }
"#,
);
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &bogus_auth_sock)
.with_status(101)
.with_stderr(&format!(
"\
[UPDATING] git repository `ssh://git@github.com:22/rust-lang/bitflags.git`
error: failed to get `bitflags` as a dependency of package `foo v0.1.0 ([ROOT]/foo)`
Caused by:
failed to load source for dependency `bitflags`
Caused by:
Unable to update ssh://git@github.com:22/rust-lang/bitflags.git?tag=1.3.2
Caused by:
failed to clone into: [ROOT]/home/.cargo/git/db/bitflags-[..]
Caused by:
failed to authenticate when downloading repository
* attempted ssh-agent authentication, but no usernames succeeded: `git`
if the git CLI succeeds then `net.git-fetch-with-cli` may help here
https://doc.rust-lang.org/cargo/reference/config.html#netgit-fetch-with-cli
Caused by:
{err}
"
))
.run();
}
#[cargo_test(container_test)]
fn ssh_key_in_config() {
// known_host in config works.
let agent = Agent::launch();
let sshd = Container::new("sshd")
.file(agent.authorized_keys())
.launch();
let hostkey = sshd.read_file("/etc/ssh/ssh_host_ecdsa_key.pub");
let url = ssh_repo_url(&sshd, "bar");
let p = foo_bar_project(&url);
p.change_file(
".cargo/config.toml",
&format!(
r#"
[net.ssh]
known-hosts = ['127.0.0.1 {}']
"#,
hostkey.trim()
),
);
p.cargo("fetch")
.env("SSH_AUTH_SOCK", &agent.sock)
.with_stderr("[UPDATING] git repository `ssh://testuser@127.0.0.1:[..]/repos/bar.git`")
.run();
}