Delete old versions after deployment (#4527)

* Delete old versions after deployment

This changes the order in which deletion of old versions of wpt.fyi is
deleted from the infrastructure. Prior to this change the deletion
happened at the beginning of the deployment. The deletion itself takes
many minutes per service, with prompts inbetween, so the time from start
of deplyment to the new image actually being available in the
infrastructure was longer than it needed to be. With this change, the
deployment starts right away, and once the deployment is done, we ask
whether old versions should be deleted and do those deletions w/o
prompting per service, meaning the person doing the deployment doesn't
need to sit and wait etc.

* Fix nesting of if statement.

Author: jstenback

util/deploy-production.sh

@@ -17,13 +17,6 @@ usage() {
   echo "${USAGE}"
 }
 
-# Deletes the service passed as a parameter.
-delete_oldest_version() {
-  OLDEST_REV=$(gcloud app --project=wptdashboard versions list --sort-by=last_deployed_time --filter="service=$1" --limit=1 --format=json | jq -r '.[] | .id')
-  echo "Deleting $1 service version $OLDEST_REV"
-  gcloud app --project=wptdashboard versions delete --service=$SERVICE ${QUIET:+--quiet} $OLDEST_REV
-}
-
 while getopts ':bfqh' flag; do
   case "${flag}" in
     b) SKIP_ISSUE_CREATION='true' ;;
@@ -94,22 +87,22 @@ EOF
   fi
 fi
 
-# Confirm there are 3 versions for each service and delete the oldest version.
+# Confirm there are no more than two versions for each service to make sure
+# there's room for the ones we're about to push. If there are more than two
+# versions available, something didn't go as planned in the previous
+# deployment. If so, delete old versions manually in the cloud console.
 SERVICES="default processor searchcache"
 for SERVICE in $SERVICES
 do
   VERSIONS=$(gcloud app --project=wptdashboard versions list --filter="service=$SERVICE" --format=list | wc -l)
-  if [[ "${VERSIONS}" -eq "3"  ]];
+  if ((${VERSIONS} > 2));
   then
-    echo "Found 3 versions for service $SERVICE, will delete the oldest"
-    delete_oldest_version $SERVICE
-  elif [[ "${VERSIONS}" -lt "3"  ]];
-  then
-    echo -e "\n$VERSIONS versions found for service $SERVICE"
-  else
-    echo -e "\n$VERSIONS versions found for service $SERVICE!"
+    echo -e "Found more than 2 versions ($VERSIONS) for service $SERVICE.\nPlease make sure there are no more than 2 versions of each service and try\nagain."
+
     exit 3
   fi
+
+  echo "Found $VERSIONS versions for service $SERVICE. Good to proceed."
 done
 
 # Start a docker instance.
@@ -143,3 +136,31 @@ fi
 # Update and close deployment bug.
 LAST_DEPLOYMENT_ISSUE=$(gh issue list --state open --label "$PROD_LABEL" --label "$RELEASE_LABEL" --limit 1 --json number --jq '.[] | .number')
 gh issue close "$LAST_DEPLOYMENT_ISSUE" -c "Deployment is now complete."
+
+# Check if there are more more than two versions of the default service left
+# after we're done with this deplyment to make sure there's room for the next
+# deployment. If there are, ask to delete the oldest default service version,
+# and also delete the same version from the other services which will also exist
+# if all went well during the deployment. This check isn't fail safe, but
+# combined with the check we do before doing any deployments earlier in this
+# script, this should leave us in a good state.
+
+VERSIONS=$(gcloud app --project=wptdashboard versions list --filter="service=default" --format=list | wc -l)
+
+if (($VERSIONS == 3)); then
+  echo -e "Please ensure the deployment was successful. If so, we can go ahead and\ndelete the oldest version of all services if necessary, leaving the one just\ndeployed and the one running before this deployment. This will ensure we leave\nroom for the next deployment.\n"
+
+  read -p "Delete oldest version of all services to leave room for the next deplyment? (y/n): " DELETE
+
+  if [[ $DELETE == "y" ]]; then
+    echo "Found $VERSIONS for the default service, deleting the oldest version of all services."
+
+    OLDEST_REV=$(gcloud app --project=wptdashboard versions list --sort-by=last_deployed_time --filter="service=$SERVICE" --limit=1 --format=json | jq -r '.[] | .id')
+    for SERVICE in $SERVICES; do
+      echo "Deleting $SERVICE service version $OLDEST_REV"
+      gcloud app --project=wptdashboard versions delete --service=$SERVICE --quiet $OLDEST_REV
+    done
+  fi
+elif (($VERSIONS > 3)); then
+  echo -e "\nUnexpectedly found $VERSIONS versions for the default service.\nPlease delete old versions for all services manually until there are no more than two left."
+fi