From def11937b8a8e8b372783336ec57f5e2cf840961 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 24 Aug 2022 15:51:19 -0600
Subject: [PATCH 1/2] improve rollback logic

---
 functions/update_apps.sh | 37 +++++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/functions/update_apps.sh b/functions/update_apps.sh
index 8a0c353f..a72f54d9 100644
--- a/functions/update_apps.sh
+++ b/functions/update_apps.sh
@@ -192,7 +192,7 @@ if [[ $rollback == "true" || "$startstatus"  ==  "STOPPED" ]]; then
     while true
     do
         status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
-        if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then                # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
+        if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then  # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
             [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
             before_loop=$(head -n 1 all_app_status)
             current_loop=0
@@ -229,10 +229,18 @@ if [[ $rollback == "true" || "$startstatus"  ==  "STOPPED" ]]; then
                     echo_array+=("If this is a slow starting application, set a higher timeout with -t")
                     echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
                     echo_array+=("Reverting update..")
-                    midclt call chart.release.rollback "$app_name" "{\"item_version\": \"$rollback_version\"}" &> /dev/null || { echo_array+=("Error: Failed to rollback $app_name") ; break ; }
-                    [[ "$startstatus"  ==  "STOPPED" ]] && failed="true" && after_update_actions #run back after_update_actions function if the app was stopped prior to update
                     echo "$app_name,$new_full_ver" >> failed
-                    break
+                    if rollback_app ; then
+                        echo_array+=("Rolled Back")
+                    else
+                        echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
+                        echo_array
+                        return 1
+                    fi                    
+                    failed="true"
+                    SECONDS=0
+                    count=0
+                    continue #run back after_update_actions function if the app was stopped prior to update
                 else
                     echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
                     echo_array+=("The application failed to be ACTIVE even after a rollback")
@@ -251,6 +259,7 @@ if [[ $rollback == "true" || "$startstatus"  ==  "STOPPED" ]]; then
                 echo_array+=("If this is a slow starting application, set a higher timeout with -t")
                 echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
                 echo_array+=("Manual intervention is required\nStopping, then Abandoning")
+                echo "$app_name,$new_full_ver" >> failed
                 if stop_app ; then
                     echo_array+=("Stopped")
                 else
@@ -272,6 +281,26 @@ echo_array
 export -f after_update_actions
 
 
+rollback_app(){
+count=0
+while [[ $update_avail != "true" ]]
+do
+    update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
+    if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
+        return 1
+    elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
+        before_loop=$(head -n 1 all_app_status)
+        ((count++))
+        until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
+        do
+            sleep 1
+        done
+    else 
+        break
+    fi
+done
+}
+
 echo_array(){
 #Dump the echo_array, ensures all output is in a neat order. 
 for i in "${echo_array[@]}"

From fbdcf88a9995a31f27e5e6720c4cb4a89d3df970 Mon Sep 17 00:00:00 2001
From: Heavybullets8 <heavybullets8@gmail.com>
Date: Wed, 24 Aug 2022 16:16:34 -0600
Subject: [PATCH 2/2] fix rollback_app logic

---
 functions/update_apps.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/functions/update_apps.sh b/functions/update_apps.sh
index a72f54d9..d695e4a2 100644
--- a/functions/update_apps.sh
+++ b/functions/update_apps.sh
@@ -283,7 +283,8 @@ export -f after_update_actions
 
 rollback_app(){
 count=0
-while [[ $update_avail != "true" ]]
+update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
+while [[ $update_avail == "false" ]]
 do
     update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
     if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell