scale-catalog/functions/update_apps.sh

319 lines
13 KiB
Bash
Raw Normal View History

2022-07-25 02:29:58 +00:00
#!/bin/bash
2022-07-25 20:16:08 +00:00
2022-07-25 04:11:04 +00:00
commander(){
2022-07-25 04:03:57 +00:00
mapfile -t array < <(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,container_images_update_available,status' | tr -d " \t\r" | grep -E ",true($|,)" | sort)
2022-07-31 19:03:45 +00:00
echo -e "🅄 🄿 🄳 🄰 🅃 🄴 🅂"
2022-07-31 19:16:40 +00:00
[[ -z ${array[*]} ]] && echo "There are no updates available" && echo -e "\n" && return 0 || echo "Update(s) Available: ${#array[@]}"
2022-07-27 22:40:44 +00:00
echo "Asynchronous Updates: $update_limit"
2022-07-27 00:00:03 +00:00
[[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout"
2022-07-25 02:29:58 +00:00
[[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips"
2022-08-13 16:37:48 +00:00
pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
2022-08-24 11:21:43 +00:00
rm external_services 2>/dev/null # TODO remove later
2022-08-11 03:46:08 +00:00
it=0
2022-08-11 04:51:58 +00:00
while_count=0
rm deploying 2>/dev/null
rm finished 2>/dev/null
2022-07-27 20:13:40 +00:00
while true
2022-07-25 20:42:57 +00:00
do
2022-08-13 21:26:50 +00:00
if while_status=$(cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,status' 2>/dev/null) ; then
2022-08-12 04:42:43 +00:00
((while_count++))
2022-08-13 23:32:44 +00:00
[[ -z $while_status ]] && continue || echo -e "$while_count\n$while_status" > all_app_status
mapfile -t deploying_check < <(grep ",DEPLOYING," all_app_status)
2022-08-13 21:26:50 +00:00
for i in "${deploying_check[@]}"
2022-08-13 21:07:06 +00:00
do
app_name=$(echo "$i" | awk -F ',' '{print $1}')
2022-08-13 21:26:50 +00:00
[[ ! -e deploying ]] && touch deploying
if ! grep -qs "$app_name,DEPLOYING" deploying; then
echo "$app_name,DEPLOYING" >> deploying
2022-08-13 21:07:06 +00:00
fi
done
2022-08-11 03:46:08 +00:00
else
echo "Middlewared timed out. Consider setting a lower number for async applications"
continue
2022-08-09 05:47:34 +00:00
fi
2022-08-05 01:52:08 +00:00
proc_count=${#processes[@]}
2022-07-26 05:44:09 +00:00
count=0
2022-07-26 05:38:29 +00:00
for proc in "${processes[@]}"
do
2022-07-26 05:44:55 +00:00
kill -0 "$proc" &> /dev/null || { unset "processes[$count]"; ((proc_count--)); }
2022-07-27 21:07:53 +00:00
((count++))
2022-07-26 05:38:29 +00:00
done
2022-07-27 19:47:44 +00:00
if [[ "$proc_count" -ge "$update_limit" ]]; then
2022-08-13 21:41:50 +00:00
sleep 3
2022-07-27 20:13:40 +00:00
elif [[ $it -lt ${#array[@]} ]]; then
# loop=0
# until [[ $loop -ge 2 || $it -ge ${#array[@]} ]];
# do
2022-08-12 16:33:54 +00:00
pre_process "${array[$it]}" &
processes+=($!)
((it++))
2022-08-11 22:37:19 +00:00
# ((loop++))
# done
elif [[ $proc_count != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]]; then # Wait for all processes to finish
2022-08-13 21:41:50 +00:00
sleep 3
2022-07-27 21:07:53 +00:00
else # All processes must be completed, break out of loop
2022-07-27 20:50:52 +00:00
break
2022-07-25 20:42:57 +00:00
fi
done
rm deploying 2>/dev/null
rm finished 2>/dev/null
2022-07-31 19:03:45 +00:00
echo
echo
2022-07-25 04:11:04 +00:00
}
export -f commander
2022-07-25 04:03:57 +00:00
2022-07-27 01:04:27 +00:00
2022-08-12 16:33:54 +00:00
pre_process(){
2022-07-25 21:20:32 +00:00
app_name=$(echo "${array[$it]}" | awk -F ',' '{print $1}') #print out first catagory, name.
printf '%s\0' "${ignore[@]}" | grep -iFxqz "${app_name}" && echo -e "\n$app_name\nIgnored, skipping" && final_check && return 0 #If application is on ignore list, skip
2022-07-25 21:20:32 +00:00
old_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #previous/current Application MAJOR Version
new_app_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}') #new Application MAJOR Version
old_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # Old Chart MAJOR version
new_chart_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}') # New Chart MAJOR version
2022-07-27 21:47:11 +00:00
startstatus=$(echo "${array[$it]}" | awk -F ',' '{print $2}') #status of the app: STOPPED / DEPLOYING / ACTIVE
2022-07-25 04:03:57 +00:00
diff_app=$(diff <(echo "$old_app_ver") <(echo "$new_app_ver")) #caluclating difference in major app versions
diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclating difference in Chart versions
[[ "$diff_app" != "$diff_chart" && $update_apps == "true" ]] && echo -e "\n$app_name\nMajor Release, update manually" && final_check && return
2022-07-25 21:20:32 +00:00
old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From
new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To
rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}')
2022-08-14 23:43:07 +00:00
if grep -qs "^$app_name," failed 2>/dev/null; then
failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}')
2022-08-04 20:46:00 +00:00
if [[ "$failed_ver" == "$new_full_ver" ]] ; then
2022-08-04 20:20:40 +00:00
echo -e "\n$app_name"
2022-08-04 22:08:45 +00:00
echo -e "Skipping previously failed version:\n$new_full_ver"
final_check
2022-08-04 20:14:32 +00:00
return 0
2022-08-04 20:46:00 +00:00
else
2022-08-14 23:43:07 +00:00
sed -i /"$app_name",/d failed
2022-08-04 20:14:32 +00:00
fi
fi
2022-08-13 16:37:48 +00:00
[[ ! -e external_services ]] && touch external_services
if ! grep -qs "^$app_name," external_services ; then
2022-08-24 11:09:59 +00:00
if ! grep -qs "/external-service" /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/"$(find /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/ -maxdepth 1 -type d -printf '%P\n' | sort -r | head -n 1)"/Chart.yaml; then
2022-08-13 16:37:48 +00:00
echo "$app_name,false" >> external_services
else
echo "$app_name,true" >> external_services
fi
fi
2022-08-12 16:44:32 +00:00
echo_array+=("\n$app_name")
if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # Check to see if user is using -S or not
[[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..")
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
final_check
return 1
2022-07-25 02:29:58 +00:00
fi
2022-08-01 21:47:08 +00:00
fi
2022-08-12 12:53:35 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Updating..")
2022-08-12 16:33:54 +00:00
if update_app ;then
2022-08-12 12:53:35 +00:00
echo_array+=("Updated\n$old_full_ver\n$new_full_ver")
else
2022-08-23 22:08:08 +00:00
echo_array+=("Failed to update\nManual intervention may be required")
echo_array
final_check
return
2022-08-12 12:53:35 +00:00
fi
if grep -qs "^$app_name,true" external_services ; then
echo_array
final_check
return
else
after_update_actions
final_check
fi
2022-07-25 22:50:44 +00:00
}
2022-08-12 16:33:54 +00:00
export -f pre_process
2022-07-25 02:29:58 +00:00
2022-08-12 16:33:54 +00:00
update_app(){
current_loop=0
2022-08-11 22:37:19 +00:00
while true
2022-08-09 23:32:47 +00:00
do
2022-08-13 23:32:44 +00:00
update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
2022-08-12 02:41:31 +00:00
if [[ $update_avail == "true" ]]; then
2022-08-10 04:47:59 +00:00
if ! cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null ; then
2022-08-13 23:32:44 +00:00
before_loop=$(head -n 1 all_app_status)
2022-08-12 02:30:49 +00:00
current_loop=0
2022-08-13 23:32:44 +00:00
until [[ "$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')" != "$update_avail" ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
2022-08-12 02:30:49 +00:00
do
if [[ $current_loop -gt 2 ]]; then
cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null || return 1 # After waiting, attempt an update once more, if fails, return error code
2022-08-13 23:32:44 +00:00
elif ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then # The file has been updated, but nothing changed specifically for the app.
before_loop=$(head -n 1 all_app_status)
2022-08-12 02:30:49 +00:00
((current_loop++))
fi
2022-08-12 02:41:31 +00:00
sleep 1
2022-08-12 02:30:49 +00:00
done
2022-08-10 04:47:59 +00:00
fi
break
elif [[ $update_avail == "false" ]]; then
2022-08-09 23:32:47 +00:00
break
2022-08-12 02:41:31 +00:00
else
sleep 3
2022-08-09 23:32:47 +00:00
fi
done
}
2022-08-12 16:33:54 +00:00
export -f update_app
2022-08-09 23:32:47 +00:00
2022-08-18 06:36:54 +00:00
stop_app(){
count=0
while [[ "$status" != "STOPPED" ]]
do
2022-08-13 23:32:44 +00:00
status=$( grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
if [[ $count -gt 2 ]]; then # If failed to stop app 3 times, return failure to parent shell
return 1
elif ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' &> /dev/null ; then
2022-08-13 23:32:44 +00:00
before_loop=$(head -n 1 all_app_status)
((count++))
2022-08-13 23:32:44 +00:00
until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
do
sleep 1
done
else
break
fi
done
}
export -f stop_app
2022-07-25 02:29:58 +00:00
after_update_actions(){
SECONDS=0
count=0
2022-08-04 22:48:55 +00:00
if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then
2022-07-25 02:29:58 +00:00
while true
do
2022-08-13 23:32:44 +00:00
status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
2022-08-24 21:51:19 +00:00
if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
2022-08-12 13:33:13 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
2022-08-13 23:32:44 +00:00
before_loop=$(head -n 1 all_app_status)
2022-08-12 02:30:49 +00:00
current_loop=0
until [[ "$status" != "ACTIVE" || $current_loop -gt 4 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
2022-08-11 05:27:11 +00:00
do
2022-08-14 23:43:07 +00:00
status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
2022-08-11 05:37:53 +00:00
sleep 1
2022-08-13 23:32:44 +00:00
if ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then
before_loop=$(head -n 1 all_app_status)
((current_loop++))
fi
2022-08-11 05:27:11 +00:00
done
fi
(( count++ ))
2022-08-04 23:19:44 +00:00
if [[ "$status" == "ACTIVE" ]]; then
if [[ "$startstatus" == "STOPPED" ]]; then
[[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
2022-08-04 23:19:44 +00:00
break
else
echo_array+=("Active")
2022-08-12 13:33:13 +00:00
break
2022-08-04 23:19:44 +00:00
fi
elif [[ "$SECONDS" -ge "$timeout" ]]; then
2022-08-04 22:48:55 +00:00
if [[ $rollback == "true" ]]; then
if [[ "$failed" != "true" ]]; then
2022-08-04 23:19:44 +00:00
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("If this is a slow starting application, set a higher timeout with -t")
echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
echo_array+=("Reverting update..")
2022-08-14 23:43:07 +00:00
echo "$app_name,$new_full_ver" >> failed
2022-08-24 21:51:19 +00:00
if rollback_app ; then
echo_array+=("Rolled Back")
else
echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
echo_array
return 1
fi
failed="true"
SECONDS=0
count=0
continue #run back after_update_actions function if the app was stopped prior to update
2022-08-04 23:19:44 +00:00
else
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("The application failed to be ACTIVE even after a rollback")
2022-08-21 22:44:22 +00:00
echo_array+=("Manual intervention is required\nStopping, then Abandoning")
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
2022-08-04 22:48:55 +00:00
break
fi
else
2022-08-04 23:19:44 +00:00
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("If this is a slow starting application, set a higher timeout with -t")
echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
2022-08-21 22:44:22 +00:00
echo_array+=("Manual intervention is required\nStopping, then Abandoning")
2022-08-24 21:51:19 +00:00
echo "$app_name,$new_full_ver" >> failed
2022-08-21 22:44:22 +00:00
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
2022-08-21 22:45:53 +00:00
break
2022-08-04 22:48:55 +00:00
fi
2022-07-25 02:29:58 +00:00
else
2022-07-25 23:11:56 +00:00
[[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
sleep 5
2022-07-25 02:29:58 +00:00
continue
fi
done
fi
echo_array
}
export -f after_update_actions
2022-07-25 23:21:41 +00:00
2022-08-04 22:48:55 +00:00
2022-08-24 21:51:19 +00:00
rollback_app(){
count=0
2022-08-24 22:16:34 +00:00
update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
while [[ $update_avail == "false" ]]
2022-08-24 21:51:19 +00:00
do
update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
return 1
elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
before_loop=$(head -n 1 all_app_status)
((count++))
until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
do
sleep 1
done
else
break
fi
done
}
echo_array(){
2022-07-25 23:38:10 +00:00
#Dump the echo_array, ensures all output is in a neat order.
2022-07-25 23:21:41 +00:00
for i in "${echo_array[@]}"
do
echo -e "$i"
done
2022-07-25 02:29:58 +00:00
}
export -f echo_array
final_check(){
[[ ! -e finished ]] && touch finished
echo "$app_name,finished" >> finished
}