Refactor the acceptance tests to be separated into offline and online

Offline meaning that a server isn't necessary to run the tests, although one may
be running

.dockerignore

@@ -1,5 +1,4 @@
-apps/
 client/test
-jenkins.home/
+services/
 tools/
-updatesrv/
+tests/

Dockerfile

@@ -53,7 +53,8 @@ RUN apk add --no-cache git \
                         bash \
                         supervisor \
                         nodejs \
-                        ttf-dejavu
+                        ttf-dejavu \
+                        curl
 
 # TODO: add a checksum check?
 RUN cd /tmp && \

Makefile

@@ -22,6 +22,7 @@ check: lint
 container-prereqs: build/jenkins-support build/jenkins.sh
 
 container-check: shunit2 ./tests/tests.sh containers
+	./tests/offline-tests.sh
 	./tests/tests.sh
 
 container: container-prereqs Dockerfile configuration/supervisord.conf

tests/offline-tests.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Note: would have used set -euo pipefail, but ./shunit2 unfortunately fails hard with this :-(.
+
+current_directory=$(dirname "$0")
+export PATH="$current_directory/../tools:$PATH"
+
+JENKINS_HOME=to_override
+
+# shellcheck source=tests/utilities
+. "$current_directory/utilities"
+
+
+# JENKINS-49864
+test_docker_CLI_available() {
+  docker exec "$container_under_test" which docker > /dev/null
+  assertEquals "docker found in the PATH" 0 $?
+
+  # Check that not only something called docker can be found on the PATH
+  # but is actually looking more like it using a specific command call
+  output=$( docker exec "$container_under_test" docker version 2>&1 )
+  assertEquals "error is expected since no Docker daemon $?" 1 $?
+
+  echo "$output" | \
+      grep "Cannot connect to the Docker daemon" > /dev/null
+  assertEquals "expected message about daemon unavailable" 0 $?
+}
+
+# JENKINS-50195
+test_not_root() {
+  username=$( docker exec "$container_under_test" whoami )
+  assertEquals "jenkins" "$username"
+
+  for process_user in $( docker exec "$container_under_test" ps -o user | grep -v USER)
+  do
+    assertEquals "jenkins" "$process_user"
+  done
+}
+
+
+# Check NPM is 5+ to make sure we do check the integrity values
+# https://github.com/jenkins-infra/evergreen/pull/60#discussion_r182666012
+test_npm_5_plus() {
+  result=$( docker exec "$container_under_test" npm --version )
+  assertEquals "5." "${result:0:2}"
+}
+
+# Ensure that we can successfully connect to only Let's Encrypt authorized
+# sites. See JEP-307
+test_jep_307() {
+  result=$( docker exec "$container_under_test" curl -s https://jenkins.io/ )
+  assertEquals "jenkins.io should be OK" "0" "$?"
+
+  # Incrementals, like https://repo.jenkins-ci.org/incrementals/org/jenkins-ci/main/jenkins-war/maven-metadata.xml
+  result=$( docker exec "$container_under_test" curl -s https://repo.jenkins-ci.org )
+  assertEquals "Incrementals repo should be OK" "0" "$?"
+
+  result=$( docker exec "$container_under_test" curl -s https://sonic.com/ )
+  assertEquals "everything else should not validate" "60" "$?"
+}
+
+. ./shunit2/shunit2

tests/tests.sh

@@ -9,18 +9,6 @@ JENKINS_HOME=to_override
 # shellcheck source=tests/utilities
 . "$current_directory/utilities"
 
-# trick to silence shellcheck which does not handle very well variables coming from sourced file
-export container_under_test=${container_under_test:?}
-
-oneTimeSetUp() {
-  setup_container_under_test
-  # shellcheck disable=SC2016
-  JENKINS_HOME="$( docker exec "$container_under_test" bash -c 'echo $JENKINS_HOME' )"
-}
-
-oneTimeTearDown() {
-  cleanup
-}
 
 test_smoke() {
   docker exec "$container_under_test" ps aux | grep npm > /dev/null
@@ -71,17 +59,6 @@ test_no_executor() {
   assertEquals "<numExecutors>0</numExecutors>" "$numExecutors"
 }
 
-# JENKINS-50195
-test_not_root() {
-  username=$( docker exec "$container_under_test" whoami )
-  assertEquals "jenkins" "$username"
-
-  for process_user in $( docker exec "$container_under_test" ps -o user | grep -v USER)
-  do
-    assertEquals "jenkins" "$process_user"
-  done
-}
-
 # JENKINS-49406 check data segregation
 test_plugins_are_not_exploded_under_jenkins_home() {
   # shellcheck disable=SC2016
@@ -151,7 +128,6 @@ test_metrics_health_check() {
 
 # JENKINS-49811
 test_logs_are_propagated() {
-
   result=$( $COMPOSE exec -T instance curl -s http://backend:3030/errorTelemetry | \
               jq -r '.[0].log' )
   assertEquals "$result should be not empty and JSON" "0" "$?"
@@ -164,28 +140,6 @@ test_logs_are_propagated() {
   # Depends on https://github.com/jenkinsci/essentials-plugin/blob/0d7ee52820db08f5790d79c189a88e2237cfe902/src/main/java/io/jenkins/plugins/essentials/logging/EssentialsLoggingConfigurer.java#L34 being the first
   echo "$result" | grep EssentialsLoggingConfigurer > /dev/null
   assertEquals "$result should contain the log from the Essentials Jenkins plugin" "0" "$?"
-
-}
-
-# Check NPM is 5+ to make sure we do check the integrity values
-# https://github.com/jenkins-infra/evergreen/pull/60#discussion_r182666012
-test_npm_5_plus() {
-  result=$( docker exec "$container_under_test" npm --version )
-  assertEquals "5." "${result:0:2}"
-}
-
-# Ensure that we can successfully connect to only Let's Encrypt authorized
-# sites. See JEP-307
-test_jep_307() {
-  result=$( docker exec "$container_under_test" curl -s https://jenkins.io/ )
-  assertEquals "jenkins.io should be OK" "0" "$?"
-
-  # Incrementals, like https://repo.jenkins-ci.org/incrementals/org/jenkins-ci/main/jenkins-war/maven-metadata.xml
-  result=$( docker exec "$container_under_test" curl -s https://repo.jenkins-ci.org )
-  assertEquals "Incrementals repo should be OK" "0" "$?"
-
-  result=$( docker exec "$container_under_test" curl -s https://sonic.com/ )
-  assertEquals "everything else should be KO" "60" "$?"
 }
 
 # Test everything under /evergreen is owned by the jenkins user

tests/utilities

@@ -2,7 +2,6 @@ container_under_test_prefix=evergreen-testing
 container_under_test=$container_under_test_prefix-$RANDOM
 COMPOSE="./tools/compose"
 
-
 RED='\033[0;31m'
 NC='\033[0m' # No Color
 
@@ -55,14 +54,20 @@ setup_container_under_test() {
   info "Start containers under test (port=$TEST_PORT) and wait a bit for its startup:"
   #docker run --rm --name $container_under_test -p $TEST_PORT:8080 -d jenkins/evergreen:latest
   $COMPOSE up -d
+  sleep 3
+  determine_container_name
+}
 
+determine_container_name() {
+  docker ps
+  container_under_test=$( docker ps --filter "name=instance" --format='{{.Names}}' )
+}
+
+wait_for_jenkins() {
   # FIXME: FLAKY matching
   # evergreen_instance_1 could be not the only one, also what do we do if preexisting containers are running?
   echo "Running containers (beware, ideally there should be none to avoid issues)"
-  docker ps
-
-  container_under_test=$( docker ps --filter "name=instance" --format='{{.Names}}' )
-  sleep 5
+  determine_container_name
 
   # FIXME: have to wait pretty long because plugin installations
   # Possibly we'll want a special mode to accelerate testing, maybe by downloading plugins
@@ -84,3 +89,18 @@ setup_container_under_test() {
     sleep $cur_attempts
   done
 }
+
+# trick to silence shellcheck which does not handle very well variables coming from sourced file
+export container_under_test=${container_under_test:?}
+
+oneTimeSetUp() {
+  setup_container_under_test
+  # shellcheck disable=SC2016
+  JENKINS_HOME="$( docker exec "$container_under_test" bash -c 'echo $JENKINS_HOME' )"
+}
+
+oneTimeTearDown() {
+  cleanup
+}
+
+# vim: ft=sh