scale-catalog/functions/update_apps.sh

162 lines
9.7 KiB
Bash
Raw Normal View History

2022-07-25 02:29:58 +00:00
#!/bin/bash
2022-07-25 20:16:08 +00:00
2022-07-25 04:11:04 +00:00
commander(){
2022-07-25 04:03:57 +00:00
mapfile -t array < <(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,container_images_update_available,status' | tr -d " \t\r" | grep -E ",true($|,)" | sort)
2022-07-25 02:29:58 +00:00
[[ -z $array ]] && echo -e "\nThere are no updates available" && return 0 || echo -e "\n${#array[@]} update(s) available"
[[ -z $timeout ]] && echo -e "\nDefault Timeout: 500" && timeout=500 || echo -e "\nCustom Timeout: $timeout"
[[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips"
2022-07-26 04:25:14 +00:00
echo "Asynchronous Updates: $update_limit"
2022-07-25 20:16:08 +00:00
2022-07-25 21:02:47 +00:00
it=0
2022-07-25 22:46:05 +00:00
2022-07-25 21:02:47 +00:00
while [[ $it -lt ${#array[@]} ]]
2022-07-25 20:42:57 +00:00
do
2022-07-25 21:02:47 +00:00
jobs=$(jobs -p | wc -l)
if [[ "$jobs" -ge "$update_limit" ]]; then
2022-07-25 23:45:35 +00:00
sleep 1
2022-07-25 20:42:57 +00:00
else
2022-07-25 22:48:17 +00:00
update_apps "${array[$it]}" &
2022-07-25 21:02:47 +00:00
processes+=($!)
((it++))
2022-07-25 20:42:57 +00:00
fi
done
2022-07-25 22:04:43 +00:00
2022-07-25 22:05:37 +00:00
for proc in "${processes[@]}"
2022-07-25 22:04:43 +00:00
do
2022-07-25 22:05:37 +00:00
wait "$proc"
2022-07-25 22:04:43 +00:00
done
2022-07-25 20:42:57 +00:00
2022-07-25 22:05:37 +00:00
2022-07-25 04:11:04 +00:00
}
export -f commander
2022-07-25 04:03:57 +00:00
update_apps(){
2022-07-25 21:20:32 +00:00
app_name=$(echo "${array[$it]}" | awk -F ',' '{print $1}') #print out first catagory, name.
old_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #previous/current Application MAJOR Version
new_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #new Application MAJOR Version
old_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # Old Chart MAJOR version
new_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # New Chart MAJOR version
status=$(echo "${array[$it]}" | awk -F ',' '{print $2}') #status of the app: STOPPED / DEPLOYING / ACTIVE
2022-07-25 04:03:57 +00:00
startstatus=$status
diff_app=$(diff <(echo "$old_app_ver") <(echo "$new_app_ver")) #caluclating difference in major app versions
diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclating difference in Chart versions
2022-07-25 21:20:32 +00:00
old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From
new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To
rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}')
2022-07-25 22:56:40 +00:00
printf '%s\0' "${ignore[@]}" | grep -iFxqz "${app_name}" && echo -e "\n$app_name\nIgnored, skipping" && return #If application is on ignore list, skip
2022-07-25 02:29:58 +00:00
if [[ "$diff_app" == "$diff_chart" || "$update_all_apps" == "true" ]]; then #continue to update
if [[ $stop_before_update == "true" ]]; then # Check to see if user is using -S or not
if [[ "$status" == "STOPPED" ]]; then # if status is already stopped, skip while loop
2022-07-25 23:06:59 +00:00
echo_array+=("\n$app_name")
[[ "$verbose" == "true" ]] && echo_array+=("Updating..")
cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null && echo_array+=("Updated\n$old_full_ver\n$new_full_ver") && after_update_actions || echo_array+=("FAILED")
2022-07-25 04:03:57 +00:00
return
2022-07-25 02:29:58 +00:00
else # if status was not STOPPED, stop the app prior to updating
2022-07-25 23:06:59 +00:00
echo_array+=("\n$app_name")
[[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..")
midclt call chart.release.scale "$app_name" '{"replica_count": 0}' &> /dev/null && SECONDS=0 || echo_array+=("FAILED")
2022-07-25 02:29:58 +00:00
while [[ "$status" != "STOPPED" ]]
do
status=$(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,status' | grep "^$app_name," | awk -F ',' '{print $2}')
if [[ "$status" == "STOPPED" ]]; then
2022-07-25 23:06:59 +00:00
echo_array+=("Stopped")
[[ "$verbose" == "true" ]] && echo_array+=("Updating..")
2022-07-25 23:11:56 +00:00
cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null && echo_array+=("Updated\n$old_full_ver\n$new_full_ver") && after_update_actions || echo_array+=("Failed to update")
2022-07-25 02:29:58 +00:00
break
elif [[ "$SECONDS" -ge "$timeout" ]]; then
2022-07-25 23:06:59 +00:00
echo_array+=("Error: Run Time($SECONDS) has exceeded Timeout($timeout)")
2022-07-25 02:29:58 +00:00
break
elif [[ "$status" != "STOPPED" ]]; then
2022-07-25 23:06:59 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be STOPPED")
2022-07-25 02:29:58 +00:00
sleep 10
fi
done
fi
else #user must not be using -S, just update
2022-07-25 23:06:59 +00:00
echo_array+=("\n$app_name")
[[ "$verbose" == "true" ]] && echo_array+=("Updating..")
2022-07-25 23:11:56 +00:00
cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null && echo_array+=("Updated\n$old_full_ver\n$new_full_ver") && after_update_actions || echo_array+=("FAILED")
2022-07-25 02:29:58 +00:00
fi
else
2022-07-25 23:06:59 +00:00
echo_array+=("\n$app_name\nMajor Release, update manually")
2022-07-25 04:03:57 +00:00
return
2022-07-25 02:29:58 +00:00
fi
2022-07-25 22:52:51 +00:00
2022-07-25 22:50:44 +00:00
}
2022-07-25 02:29:58 +00:00
export -f update_apps
after_update_actions(){
SECONDS=0
count=0
if [[ $rollback == "true" ]]; then
while true
do
(( count++ ))
status=$(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,status' | grep "^$app_name," | awk -F ',' '{print $2}')
if [[ "$status" == "ACTIVE" && "$startstatus" == "STOPPED" ]]; then
2022-07-25 23:11:56 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
midclt call chart.release.scale "$app_name" '{"replica_count": 0}' &> /dev/null && echo_array+=("Stopped")|| echo_array+=("FAILED")
2022-07-25 02:29:58 +00:00
break
elif [[ "$SECONDS" -ge "$timeout" && "$status" == "DEPLOYING" && "$failed" != "true" ]]; then
2022-07-25 23:11:56 +00:00
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)\nIf this is a slow starting application, set a higher timeout with -t\nIf this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration\nReverting update..")
2022-07-25 02:29:58 +00:00
midclt call chart.release.rollback "$app_name" "{\"item_version\": \"$rollback_version\"}" &> /dev/null
[[ "$startstatus" == "STOPPED" ]] && failed="true" && after_update_actions && unset failed #run back after_update_actions function if the app was stopped prior to update
break
elif [[ "$SECONDS" -ge "$timeout" && "$status" == "DEPLOYING" && "$failed" == "true" ]]; then
2022-07-25 23:11:56 +00:00
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)\nThe application failed to be ACTIVE even after a rollback,\nManual intervention is required\nAbandoning")
2022-07-25 02:29:58 +00:00
break
elif [[ "$status" == "STOPPED" ]]; then
2022-07-25 23:11:56 +00:00
[[ "$count" -le 1 && "$verbose" == "true" ]] && echo_array+=("Verifying Stopped..") && sleep 15 && continue #if reports stopped on FIRST time through loop, double check
2022-07-25 02:29:58 +00:00
[[ "$count" -le 1 && -z "$verbose" ]] && sleep 15 && continue #if reports stopped on FIRST time through loop, double check
2022-07-25 23:11:56 +00:00
echo_array+=("Stopped") && break #if reports stopped any time after the first loop, assume its extermal services.
2022-07-25 02:29:58 +00:00
elif [[ "$status" == "ACTIVE" ]]; then
2022-07-25 23:11:56 +00:00
[[ "$count" -le 1 && "$verbose" == "true" ]] && echo_array+=("Verifying Active..") && sleep 15 && continue #if reports active on FIRST time through loop, double check
2022-07-25 02:29:58 +00:00
[[ "$count" -le 1 && -z "$verbose" ]] && sleep 15 && continue #if reports active on FIRST time through loop, double check
2022-07-25 23:40:05 +00:00
echo_array+=("Active") && break #if reports active any time after the first loop, assume actually active.
2022-07-25 02:29:58 +00:00
else
2022-07-25 23:11:56 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
2022-07-25 02:29:58 +00:00
sleep 15
continue
fi
done
else
if [[ "$startstatus" == "STOPPED" ]]; then
while true #using a constant while loop, then breaking out of the loop with break commands below.
do
(( count++ ))
status=$(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,status' | grep "^$app_name," | awk -F ',' '{print $2}')
if [[ "$status" == "STOPPED" ]]; then
2022-07-25 23:11:56 +00:00
[[ "$count" -le 1 && "$verbose" == "true" ]] && echo_array+=("Verifying Stopped..") && sleep 15 && continue #if reports stopped on FIRST time through loop, double check
2022-07-25 02:29:58 +00:00
[[ "$count" -le 1 && -z "$verbose" ]] && sleep 15 && continue #if reports stopped on FIRST time through loop, double check
2022-07-25 23:11:56 +00:00
echo_array+=("Stopped") && break #assume actually stopped anytime AFTER the first loop
2022-07-25 02:29:58 +00:00
break
elif [[ "$status" == "ACTIVE" ]]; then
2022-07-25 23:11:56 +00:00
[[ "$count" -le 1 && "$verbose" == "true" ]] && echo_array+=("Verifying Active..") && sleep 15 && continue #if reports active on FIRST time through loop, double check
2022-07-25 02:29:58 +00:00
[[ "$count" -le 1 && -z "$verbose" ]] && sleep 15 && continue #if reports active on FIRST time through loop, double check
2022-07-25 23:11:56 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
midclt call chart.release.scale "$app_name" '{"replica_count": 0}' &> /dev/null && echo_array+=("Stopped")|| echo_array+=("FAILED")
2022-07-25 02:29:58 +00:00
break
elif [[ "$SECONDS" -ge "$timeout" ]]; then
2022-07-25 23:11:56 +00:00
echo_array+=("Error: Run Time($SECONDS) has exceeded Timeout($timeout)")
2022-07-25 02:29:58 +00:00
break
else
2022-07-25 23:11:56 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
2022-07-25 02:29:58 +00:00
sleep 10
continue
fi
done
fi
fi
2022-07-25 23:21:41 +00:00
2022-07-25 23:38:10 +00:00
#Dump the echo_array, ensures all output is in a neat order.
2022-07-25 23:21:41 +00:00
for i in "${echo_array[@]}"
do
echo -e "$i"
done
2022-07-25 02:29:58 +00:00
}
export -f after_update_actions