From def11937b8a8e8b372783336ec57f5e2cf840961 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Wed, 24 Aug 2022 15:51:19 -0600 Subject: [PATCH 1/2] improve rollback logic --- functions/update_apps.sh | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 8a0c353f..a72f54d9 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -192,7 +192,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then while true do status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}') - if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report.. + if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report.. [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..") before_loop=$(head -n 1 all_app_status) current_loop=0 @@ -229,10 +229,18 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then echo_array+=("If this is a slow starting application, set a higher timeout with -t") echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration") echo_array+=("Reverting update..") - midclt call chart.release.rollback "$app_name" "{\"item_version\": \"$rollback_version\"}" &> /dev/null || { echo_array+=("Error: Failed to rollback $app_name") ; break ; } - [[ "$startstatus" == "STOPPED" ]] && failed="true" && after_update_actions #run back after_update_actions function if the app was stopped prior to update echo "$app_name,$new_full_ver" >> failed - break + if rollback_app ; then + echo_array+=("Rolled Back") + else + echo_array+=("Error: Failed to rollback $app_name\nAbandoning") + echo_array + return 1 + fi + failed="true" + SECONDS=0 + count=0 + continue #run back after_update_actions function if the app was stopped prior to update else echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") echo_array+=("The application failed to be ACTIVE even after a rollback") @@ -251,6 +259,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then echo_array+=("If this is a slow starting application, set a higher timeout with -t") echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration") echo_array+=("Manual intervention is required\nStopping, then Abandoning") + echo "$app_name,$new_full_ver" >> failed if stop_app ; then echo_array+=("Stopped") else @@ -272,6 +281,26 @@ echo_array export -f after_update_actions +rollback_app(){ +count=0 +while [[ $update_avail != "true" ]] +do + update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}') + if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell + return 1 + elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then + before_loop=$(head -n 1 all_app_status) + ((count++)) + until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing + do + sleep 1 + done + else + break + fi +done +} + echo_array(){ #Dump the echo_array, ensures all output is in a neat order. for i in "${echo_array[@]}" From fbdcf88a9995a31f27e5e6720c4cb4a89d3df970 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Wed, 24 Aug 2022 16:16:34 -0600 Subject: [PATCH 2/2] fix rollback_app logic --- functions/update_apps.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index a72f54d9..d695e4a2 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -283,7 +283,8 @@ export -f after_update_actions rollback_app(){ count=0 -while [[ $update_avail != "true" ]] +update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}') +while [[ $update_avail == "false" ]] do update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}') if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell