Add rollback test (only test)

hence expected to fail

Jenkinsfile

@@ -142,6 +142,17 @@ pipeline {
                       }
                   }
                 }
+                stage('Rollback testing') {
+                  agent { label 'linux' }
+                  steps {
+                      sh 'make -C distribution clean rollback-check'
+                  }
+                  post {
+                      always {
+                          archiveArtifacts artifacts: '**/build/tests-run*/**'
+                      }
+                  }
+                }
                 stage('AWS Cloud image (smokes)') {
                   agent { label 'linux' }
                   steps {

distribution/Makefile

@@ -82,7 +82,11 @@ container-check-prereqs: shunit2 ./tests/tests.sh ./tests/offline-tests.sh creat
 	$(MAKE) -C ../services lint
 	$(MAKE) -C ../services dump generate-ingest
 
-container-check: docker-cloud-container-check aws-cloud-container-check
+container-check: docker-cloud-container-check aws-cloud-container-check rollback-check
+
+# Only checking it on docker-cloud.
+rollback-check: containers container-check-prereqs
+	ENVIRONMENT=docker-cloud ./tests/rollback-tests.sh
 
 docker-cloud-container-check: containers container-check-prereqs
 	ENVIRONMENT=docker-cloud ./tests/offline-tests.sh

distribution/docker-compose.yml

@@ -33,6 +33,7 @@ services:
       - 'LOG_LEVEL=debug'
       - 'INSECURE_SHOW_ADMIN_PASSWORD=true'
       - 'DEVELOPMENT=true'
+      - 'PROCESS_RETRY_OVERRIDE=10'
     ports:
       - '8080:80'
     depends_on:

distribution/tests/rollback-tests.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+# Note: would have used set -euo pipefail, but ./shunit2 unfortunately fails hard with this :-(.
+
+current_directory=$(dirname "$0")
+# TODO: use the $( cd blah ; ... ) trick to un-relativize path below
+export PATH="$current_directory/../../tools:$PATH"
+
+echo "Debugging: PATH=***$PATH***"
+
+JENKINS_HOME=to_override
+
+# shellcheck source=tests/utilities
+. "$current_directory/utilities"
+
+oneTimeSetUp() {
+  setup_container_under_test
+
+  echo "PWD=$( pwd )"
+
+  upload_update_level "./tests/rollback/1-ingest-ok.json"
+
+  wait_for_jenkins
+  # shellcheck disable=SC2016
+  JENKINS_HOME="$( docker exec "$container_under_test" bash -c 'echo $JENKINS_HOME' )"
+}
+
+test_rollback() {
+
+  # Check UL is the correct one (UL 1 or UL 2?!)
+  docker exec "$container_under_test" bash -c 'ls $EVERGREEN_DATA'
+  # extract 2 from `"level":2` from the updates.json file
+  correctUL=$( docker exec "$container_under_test" bash -c 'cat $EVERGREEN_DATA/updates.json' | \
+                                    grep --only-matching '"level":.' | \
+                                    cut -d : -f 2  )
+  assertEquals "Command should have succeeded" 0 "$?"
+
+  assertEquals "Should be UL 2" 2 "$correctUL"
+
+  # upload borked update level to backend
+  sleep 5
+
+  echo "UPLOAD BORKED UL, FFS"
+  upload_update_level "./tests/rollback/2-ingest-borked.json"
+
+  # wait enough until upgrade happens, then rollback: check UL is the same as before
+  wait_for_jenkins
+
+  # FIXME: fixed sleep time is crappy
+  sleep 120
+  assertEquals "Should be UL 2" 2 "$correctUL"
+
+  # let's now check the upload and upgrade attempt to borked UL3 *actually* happened
+  # because if this didn't, then we'd still on UL2, but not testing there was a rollback somewhere
+  beforeLastUpdate=$( docker exec "$container_under_test" bash -c 'cat $EVERGREEN_DATA/updates.auditlog' | tail -2 | head -1 | jq -r .updateLevel )
+  lastUpdate=$( docker exec "$container_under_test" bash -c 'cat $EVERGREEN_DATA/updates.auditlog' | tail -1 | jq -r .updateLevel )
+
+  assertEquals "Previous UL should be 3, the one expected to be rolled back" 3 "$beforeLastUpdate"
+  assertEquals "UL should be 2 (gone back to 2 from borked 3)" 2 "$lastUpdate"
+
+}
+
+. ./shunit2/shunit2