From de5ef940fd289a63e4b3c2e2d8be5ac9e9a42cb5 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 01:44:09 -0600 Subject: [PATCH 1/8] parse exeptions first --- functions/update_apps.sh | 41 +++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 44e19f98..bacb008e 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -9,6 +9,31 @@ echo "Asynchronous Updates: $update_limit" [[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout" [[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips" pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r") + +index=0 +for app in "${array[@]}" +do + app_name=$(echo "$app" | awk -F ',' '{print $1}') #print out first catagory, name. + old_app_ver=$(echo "$app" | awk -F ',' '{print $4}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #previous/current Application MAJOR Version + new_app_ver=$(echo "$app" | awk -F ',' '{print $5}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #new Application MAJOR Version + old_chart_ver=$(echo "$app" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # Old Chart MAJOR version + new_chart_ver=$(echo "$app" | awk -F ',' '{print $5}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # New Chart MAJOR version + diff_app=$(diff <(echo "$old_app_ver") <(echo "$new_app_ver")) #caluclating difference in major app versions + diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclating difference in Chart versions + + #Skip application if its on ignore list + if printf '%s\0' "${ignore[@]}" | grep -iFxqz "${app_name}" ; then + echo -e "\n$app_name\nIgnored, skipping" + unset "array[$index]" + #Skip appliaction if major update and not ignoreing major versions + elif [[ "$diff_app" != "$diff_chart" && $update_apps == "true" ]] ; then + echo -e "\n$app_name\nMajor Release, update manually" + unset "array[$index]" + fi + ((index++)) +done + + it=0 while_count=0 rm deploying 2>/dev/null @@ -56,15 +81,7 @@ export -f commander pre_process(){ app_name=$(echo "${array[$it]}" | awk -F ',' '{print $1}') #print out first catagory, name. -printf '%s\0' "${ignore[@]}" | grep -iFxqz "${app_name}" && echo -e "\n$app_name\nIgnored, skipping" && final_check && return 0 #If application is on ignore list, skip -old_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #previous/current Application MAJOR Version -new_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #new Application MAJOR Version -old_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # Old Chart MAJOR version -new_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # New Chart MAJOR version startstatus=$(echo "${array[$it]}" | awk -F ',' '{print $2}') #status of the app: STOPPED / DEPLOYING / ACTIVE -diff_app=$(diff <(echo "$old_app_ver") <(echo "$new_app_ver")) #caluclating difference in major app versions -diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclating difference in Chart versions -[[ "$diff_app" != "$diff_chart" && $update_apps == "true" ]] && echo -e "\n$app_name\nMajor Release, update manually" && final_check && return old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}') @@ -191,13 +208,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then fi elif [[ "$SECONDS" -ge "$timeout" ]]; then if [[ $rollback == "true" ]]; then - if [[ $old_full_ver == "$new_full_ver" ]]; then - echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") - echo_array+=("This is the result of a container image update..") - echo_array+=("Reverting is not possible, Abandoning") - echo_array - return 1 - elif [[ "$failed" != "true" ]]; then + if [[ "$failed" != "true" ]]; then echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") echo_array+=("If this is a slow starting application, set a higher timeout with -t") echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration") From a9a23633845885035e3c73792ff57d3776f3c6b9 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 01:44:29 -0600 Subject: [PATCH 2/8] add skip prev failed --- functions/update_apps.sh | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index bacb008e..89753cca 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -29,6 +29,16 @@ do elif [[ "$diff_app" != "$diff_chart" && $update_apps == "true" ]] ; then echo -e "\n$app_name\nMajor Release, update manually" unset "array[$index]" + # Skip update if application previously failed on this exact update version + elif grep -qs "^$app_name," failed 2>/dev/null; then + failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}') + if [[ "$failed_ver" == "$new_full_ver" ]] ; then + echo -e "\n$app_name" + echo -e "Skipping previously failed version:\n$new_full_ver" + else + sed -i /"$app_name",/d failed + fi + unset "array[$index]" fi ((index++)) done @@ -86,18 +96,6 @@ old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}') -# Skip update if application previously failed on this exact update version -if grep -qs "^$app_name," failed 2>/dev/null; then - failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}') - if [[ "$failed_ver" == "$new_full_ver" ]] ; then - echo -e "\n$app_name" - echo -e "Skipping previously failed version:\n$new_full_ver" - final_check - return 0 - else - sed -i /"$app_name",/d failed - fi -fi # Check if app is external services, append outcome to external_services file [[ ! -e external_services ]] && touch external_services From fc8af22d07c20f1bb40368c6681f34e471647b9e Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 01:51:35 -0600 Subject: [PATCH 3/8] if nothing remains in array, return --- functions/update_apps.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 89753cca..545dfd6b 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -43,6 +43,7 @@ do ((index++)) done +[[ ${#array[@]} == 0 ]] && return it=0 while_count=0 From bb9e332a34fef98622a7224befc17bcd410f0dd9 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 02:46:41 -0600 Subject: [PATCH 4/8] rebuild array --- functions/update_apps.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 545dfd6b..a48551d4 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -45,6 +45,8 @@ done [[ ${#array[@]} == 0 ]] && return +array=("${array[@]}") + it=0 while_count=0 rm deploying 2>/dev/null From 3b502bd93a2e54dd0104fae8593a5db001063066 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 12:29:45 -0600 Subject: [PATCH 5/8] reassign array rather than use proccount --- functions/update_apps.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index a48551d4..a95b67ef 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -51,7 +51,7 @@ it=0 while_count=0 rm deploying 2>/dev/null rm finished 2>/dev/null -while [[ $proc_count != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]] +while [[ ${#processes[@]} != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]] do if while_status=$(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,container_images_update_available,status' 2>/dev/null) ; then ((while_count++)) @@ -69,14 +69,14 @@ do echo "Middlewared timed out. Consider setting a lower number for async applications" continue fi - proc_count=${#processes[@]} count=0 for proc in "${processes[@]}" do - kill -0 "$proc" &> /dev/null || { unset "processes[$count]"; ((proc_count--)); } + kill -0 "$proc" &> /dev/null || unset "processes[$count]" ((count++)) done - if [[ $it -lt ${#array[@]} && "$proc_count" -lt "$update_limit" ]]; then + processes=("${processes[@]}") + if [[ $it -lt ${#array[@]} && "${#processes[@]}" -lt "$update_limit" ]]; then pre_process "${array[$it]}" & processes+=($!) ((it++)) From ad0c8e07f51952fce53262f72c5687e763dffd32 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 12:38:01 -0600 Subject: [PATCH 6/8] reassign array prior to element check --- functions/update_apps.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index a95b67ef..1ad97377 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -42,10 +42,9 @@ do fi ((index++)) done - +array=("${array[@]}") [[ ${#array[@]} == 0 ]] && return -array=("${array[@]}") it=0 while_count=0 From c05570885080d3ed92c8bcaf5acc3eecf6f381a9 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 12:43:01 -0600 Subject: [PATCH 7/8] rename it to index --- functions/update_apps.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 1ad97377..01962ff8 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -43,10 +43,10 @@ do ((index++)) done array=("${array[@]}") -[[ ${#array[@]} == 0 ]] && return +[[ ${#array[@]} == 0 ]] && echo && echo && return -it=0 +index=0 while_count=0 rm deploying 2>/dev/null rm finished 2>/dev/null @@ -75,10 +75,10 @@ do ((count++)) done processes=("${processes[@]}") - if [[ $it -lt ${#array[@]} && "${#processes[@]}" -lt "$update_limit" ]]; then - pre_process "${array[$it]}" & + if [[ $index -lt ${#array[@]} && "${#processes[@]}" -lt "$update_limit" ]]; then + pre_process "${array[$index]}" & processes+=($!) - ((it++)) + ((index++)) else sleep 3 fi From 317377705acd501b9a39ea2d29fe9ffae869bd9e Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Mon, 5 Sep 2022 13:09:24 -0600 Subject: [PATCH 8/8] also rename indexes in pre_process --- functions/update_apps.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 01962ff8..812f4123 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -92,11 +92,11 @@ export -f commander pre_process(){ -app_name=$(echo "${array[$it]}" | awk -F ',' '{print $1}') #print out first catagory, name. -startstatus=$(echo "${array[$it]}" | awk -F ',' '{print $2}') #status of the app: STOPPED / DEPLOYING / ACTIVE -old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From -new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To -rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}') +app_name=$(echo "${array[$index]}" | awk -F ',' '{print $1}') #print out first catagory, name. +startstatus=$(echo "${array[$index]}" | awk -F ',' '{print $2}') #status of the app: STOPPED / DEPLOYING / ACTIVE +old_full_ver=$(echo "${array[$index]}" | awk -F ',' '{print $4}') #Upgraded From +new_full_ver=$(echo "${array[$index]}" | awk -F ',' '{print $5}') #Upraded To +rollback_version=$(echo "${array[$index]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}') # Check if app is external services, append outcome to external_services file