#!/bin/bash commander(){ mapfile -t array < <(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,container_images_update_available,status' | tr -d " \t\r" | grep -E ",true($|,)" | sort) echo -e "🅄 🄿 🄳 🄰 🅃 🄴 🅂" [[ -z ${array[*]} ]] && echo "There are no updates available" && echo -e "\n" && return 0 || echo "Update(s) Available: ${#array[@]}" echo "Asynchronous Updates: $update_limit" [[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout" [[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips" it=0 while_count=0 while true do if while_status=$(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,status' 2>/dev/null) ; then ((while_count++)) [[ -z $while_status ]] && continue || echo -e "$while_count\n$while_status" > temp.txt else echo "Middlewared timed out. Consider setting a lower number for async applications" continue fi proc_count=${#processes[@]} count=0 for proc in "${processes[@]}" do kill -0 "$proc" &> /dev/null || { unset "processes[$count]"; ((proc_count--)); } ((count++)) done if [[ "$proc_count" -ge "$update_limit" ]]; then sleep 3 elif [[ $it -lt ${#array[@]} ]]; then # loop=0 # until [[ $loop -ge 2 || $it -ge ${#array[@]} ]]; # do update_apps "${array[$it]}" & processes+=($!) ((it++)) # ((loop++)) # done elif [[ $proc_count != 0 ]]; then # Wait for all processes to finish sleep 3 else # All processes must be completed, break out of loop break fi done rm temp.txt echo echo } export -f commander update_apps(){ app_name=$(echo "${array[$it]}" | awk -F ',' '{print $1}') #print out first catagory, name. printf '%s\0' "${ignore[@]}" | grep -iFxqz "${app_name}" && echo -e "\n$app_name\nIgnored, skipping" && return 0 #If application is on ignore list, skip old_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #previous/current Application MAJOR Version new_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #new Application MAJOR Version old_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # Old Chart MAJOR version new_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # New Chart MAJOR version startstatus=$(echo "${array[$it]}" | awk -F ',' '{print $2}') #status of the app: STOPPED / DEPLOYING / ACTIVE diff_app=$(diff <(echo "$old_app_ver") <(echo "$new_app_ver")) #caluclating difference in major app versions diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclating difference in Chart versions old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}') if grep -qs "^$app_name," failed.txt ; then failed_ver=$(grep "^$app_name," failed.txt | awk -F ',' '{print $2}') if [[ "$failed_ver" == "$new_full_ver" ]] ; then echo -e "\n$app_name" echo -e "Skipping previously failed version:\n$new_full_ver" return 0 else sed -i /"$app_name",/d failed.txt fi fi if [[ "$diff_app" == "$diff_chart" || "$update_all_apps" == "true" ]]; then #continue to update if [[ $stop_before_update == "true" ]]; then # Check to see if user is using -S or not if [[ "$startstatus" == "STOPPED" ]]; then # if status is already stopped, skip while loop echo_array+=("\n$app_name") else # if status was not STOPPED, stop the app prior to updating echo_array+=("\n$app_name") [[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..") if stop_app ; then echo_array+=("Stopped") else echo_array+=("Error: Failed to stop $app_name") return 1 fi fi else #user must not be using -S, just update echo_array+=("\n$app_name") fi else echo -e "\n$app_name\nMajor Release, update manually" return 0 fi [[ "$verbose" == "true" ]] && echo_array+=("Updating..") if update ;then echo_array+=("Updated\n$old_full_ver\n$new_full_ver") else echo_array+=("Failed to update") return fi after_update_actions } export -f update_apps update(){ current_loop=0 while true do update_avail=$(grep "^$app_name," temp.txt | awk -F ',' '{print $3}') if [[ $update_avail == "true" ]]; then if ! cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null ; then echo "Fail Trigger - Debugging" before_loop=$(head -n 1 temp.txt) current_loop=0 until [[ "$(grep "^$app_name," temp.txt | awk -F ',' '{print $3}')" != "$update_avail" ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by. do if [[ $current_loop -gt 2 ]]; then cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null || return 1 # After waiting, attempt an update once more, if fails, return error code elif ! echo -e "$(head -n 1 temp.txt)" | grep -qs ^"$before_loop" ; then # The file has been updated, but nothing changed specifically for the app. before_loop=$(head -n 1 temp.txt) ((current_loop++)) fi sleep 1 done fi break elif [[ $update_avail == "false" ]]; then break else sleep 3 fi done } export -f update stop_app(){ count=0 while [[ "$status" != "STOPPED" ]] do status=$( grep "^$app_name," temp.txt | awk -F ',' '{print $2}') if [[ $count -gt 2 ]]; then # If failed to stop app 3 times, return failure to parent shell return 1 elif ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' &> /dev/null ; then echo "Fail Trigger Stop - Debugging" before_loop=$(head -n 1 temp.txt) ((count++)) until [[ $(head -n 1 temp.txt) != "$before_loop" ]] # Upon failure, wait for status update before continuing do sleep 1 done else break fi done } export -f stop_app after_update_actions(){ SECONDS=0 count=0 if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then while true do status=$( grep "^$app_name," temp.txt | awk -F ',' '{print $2}') if [[ $count -lt 1 && $status != "Deploying" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report.. [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..") old_status=$status before_loop=$(head -n 1 temp.txt) current_loop=0 until [[ "$status" != "$old_status" || $current_loop -gt 3 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by. do status=$( grep "^$app_name," temp.txt | awk -F ',' '{print $2}') sleep 1 if ! echo -e "$(head -n 1 temp.txt)" | grep -qs ^"$before_loop" ; then before_loop=$(head -n 1 temp.txt) ((current_loop++)) fi done fi (( count++ )) if [[ "$status" == "ACTIVE" ]]; then if [[ "$startstatus" == "STOPPED" ]]; then [[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..") if stop_app ; then echo_array+=("Stopped") else echo_array+=("Error: Failed to stop $app_name") return 1 fi break else echo_array+=("Active") break fi elif [[ "$status" == "STOPPED" ]]; then echo_array+=("Stopped") break elif [[ "$SECONDS" -ge "$timeout" && "$status" == "DEPLOYING" ]]; then if [[ $rollback == "true" ]]; then if [[ "$failed" != "true" ]]; then echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") echo_array+=("If this is a slow starting application, set a higher timeout with -t") echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration") echo_array+=("Reverting update..") midclt call chart.release.rollback "$app_name" "{\"item_version\": \"$rollback_version\"}" &> /dev/null || { echo_array+=("Error: Failed to rollback $app_name") ; break ; } [[ "$startstatus" == "STOPPED" ]] && failed="true" && after_update_actions #run back after_update_actions function if the app was stopped prior to update echo "$app_name,$new_full_ver" >> failed.txt break else echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") echo_array+=("The application failed to be ACTIVE even after a rollback") echo_array+=("Manual intervention is required\nAbandoning") break fi else echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") echo_array+=("If this is a slow starting application, set a higher timeout with -t") echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration") break fi else [[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE") sleep 5 continue fi done fi #Dump the echo_array, ensures all output is in a neat order. for i in "${echo_array[@]}" do echo -e "$i" done } export -f after_update_actions