[JENKINS-53499] Improve tests to avoid race condition for updates during startup

The change here aims at making sure the client will start only when:
* the backend has started, and
* it has already received a first UL

So, we do the following:
* delay the client startup by a few seconds, only in development mode
  (by adding a startup wrapper script that will handle this)
* the test framework tries to upload the UL every seconds, instead of
  every N seconds, with N growing by one on each failed attempt.

distribution/config/supervisord.conf

@@ -9,7 +9,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 
 [program:evergreen-client]
 environment=HOME=%(ENV_EVERGREEN_HOME)s # Needed for Git or Node JENKINS-53856
-command=/usr/local/bin/npm run client
+command=/evergreen/scripts/start-client.sh
 directory=%(ENV_EVERGREEN_HOME)s/client
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

distribution/docker-compose.yml

@@ -32,6 +32,7 @@ services:
       - 'EVERGREEN_ENDPOINT=http://backend:3030'
       - 'LOG_LEVEL=debug'
       - 'INSECURE_SHOW_ADMIN_PASSWORD=true'
+      - 'DEVELOPMENT=true'
     ports:
       - '8080:80'
     depends_on:

distribution/environments/docker-cloud/config/supervisord.conf

@@ -9,7 +9,7 @@ supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
 
 [program:evergreen-client]
 environment=HOME=%(ENV_EVERGREEN_HOME)s # Needed for Git or Node JENKINS-53856
-command=/usr/local/bin/npm run client
+command=/evergreen/scripts/start-client.sh
 directory=%(ENV_EVERGREEN_HOME)s/client
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

distribution/packaging-list.scripts.txt

@@ -1,3 +1,4 @@
 jenkins-evergreen.sh
 jenkins-support
 jenkins.sh
+start-client.sh

distribution/scripts/start-client.sh

@@ -0,0 +1,27 @@
+#!/bin/sh
+
+# Wrapper to force the client to wait for the backend to become available
+# Should only be useful during development
+set -euo pipefail
+
+sleepTime=8
+if [ ! -z ${DEVELOPMENT:-} ]; then
+  echo "DEVELOPMENT MODE: client will wait $sleepTime before starting, to give time to the backend to start and receive a first UL"
+  sleep $sleepTime
+else
+  echo "Client is starting up"
+fi
+
+maxAttempts=30
+
+until curl -s $EVERGREEN_ENDPOINT --output /dev/null ; do
+  maxAttempts=$(( $maxAttempts - 1 ))
+  if [[ $maxAttempts <= 0 ]]; then
+    >&2 echo "Maximum number of attempts reached: exiting"
+    exit 1
+  fi
+  >&2 echo "Backend is unavailable - sleeping for some more time"
+  sleep 1
+done
+
+exec /usr/local/bin/npm run client

distribution/tests/tests.sh

@@ -16,7 +16,7 @@ JENKINS_HOME=to_override
 # Retries a few times in case of error
 upload_update_level() {
   n=0
-  until [ $n -ge 20 ]
+  until [ $n -ge 30 ]
   do
     echo "Uploading Update Level to /update service (attempt #$n):"
     curl --data-raw "{\"commit\":\"container-tests\",\"manifest\":$(cat ../services/ingest.json)}" \
@@ -26,7 +26,7 @@ upload_update_level() {
        && break
 
     n=$((n+1))
-    sleep $n
+    sleep 1
   done
 }
 
@@ -268,10 +268,8 @@ test_blueocean_default_redirect() {
 test_git_history_is_present() {
   commitCount=$( docker exec -w "$JENKINS_HOME" "$container_under_test" git rev-list --count HEAD )
   assertEquals "git call to count commits should have succeeded" 0 "$?"
-  # Depending on if ingest.json is pushed to backend before or after the client first
-  # polls the backend, we'll get 3 or 4 commits...
-  # See JENKINS-53499
-  assertTrue "[ $commitCount -ge 3 ]"
+
+  assertEquals 3  "$commitCount"
 
   docker exec -w "$JENKINS_HOME" "$container_under_test" git log --pretty=format:%s HEAD~..HEAD | \
                 grep 'Snapshot after downloads completed, before Jenkins restart' > /dev/null