2022-07-25 02:29:58 +00:00
#!/bin/bash
2022-07-25 20:16:08 +00:00
2022-07-25 04:11:04 +00:00
commander( ) {
2022-07-25 04:03:57 +00:00
mapfile -t array < <( cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,container_images_update_available,status' | tr -d " \t\r" | grep -E " ,true( $|,) " | sort)
2022-07-31 19:03:45 +00:00
echo -e "🅄 🄿 🄳 🄰 🅃 🄴 🅂"
2022-07-31 19:16:40 +00:00
[ [ -z ${ array [*] } ] ] && echo "There are no updates available" && echo -e "\n" && return 0 || echo " Update(s) Available: ${# array [@] } "
2022-07-27 22:40:44 +00:00
echo " Asynchronous Updates: $update_limit "
2022-07-27 00:00:03 +00:00
[ [ -z $timeout ] ] && echo "Default Timeout: 500" && timeout = 500 || echo " Custom Timeout: $timeout "
2022-07-25 02:29:58 +00:00
[ [ " $timeout " -le 120 ] ] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips"
2022-08-13 16:37:48 +00:00
pool = $( cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r" )
2022-08-24 11:21:43 +00:00
rm external_services 2>/dev/null # TODO remove later
2022-08-11 03:46:08 +00:00
it = 0
2022-08-11 04:51:58 +00:00
while_count = 0
2022-08-13 23:03:32 +00:00
rm deploying 2>/dev/null
2022-08-21 23:54:10 +00:00
rm finished 2>/dev/null
2022-07-27 20:13:40 +00:00
while true
2022-07-25 20:42:57 +00:00
do
2022-08-13 21:26:50 +00:00
if while_status = $( cli -m csv -c 'app chart_release query name,update_available,human_version,human_latest_version,status' 2>/dev/null) ; then
2022-08-12 04:42:43 +00:00
( ( while_count++) )
2022-08-13 23:32:44 +00:00
[ [ -z $while_status ] ] && continue || echo -e " $while_count \n $while_status " > all_app_status
mapfile -t deploying_check < <( grep ",DEPLOYING," all_app_status)
2022-08-13 21:26:50 +00:00
for i in " ${ deploying_check [@] } "
2022-08-13 21:07:06 +00:00
do
app_name = $( echo " $i " | awk -F ',' '{print $1}' )
2022-08-13 21:26:50 +00:00
[ [ ! -e deploying ] ] && touch deploying
if ! grep -qs " $app_name ,DEPLOYING " deploying; then
echo " $app_name ,DEPLOYING " >> deploying
2022-08-13 21:07:06 +00:00
fi
done
2022-08-11 03:46:08 +00:00
else
echo "Middlewared timed out. Consider setting a lower number for async applications"
continue
2022-08-09 05:47:34 +00:00
fi
2022-08-05 01:52:08 +00:00
proc_count = ${# processes [@] }
2022-07-26 05:44:09 +00:00
count = 0
2022-07-26 05:38:29 +00:00
for proc in " ${ processes [@] } "
do
2022-07-26 05:44:55 +00:00
kill -0 " $proc " & > /dev/null || { unset " processes[ $count ] " ; ( ( proc_count--) ) ; }
2022-07-27 21:07:53 +00:00
( ( count++) )
2022-07-26 05:38:29 +00:00
done
2022-07-27 19:47:44 +00:00
if [ [ " $proc_count " -ge " $update_limit " ] ] ; then
2022-08-13 21:41:50 +00:00
sleep 3
2022-07-27 20:13:40 +00:00
elif [ [ $it -lt ${# array [@] } ] ] ; then
2022-08-11 04:35:30 +00:00
# loop=0
# until [[ $loop -ge 2 || $it -ge ${#array[@]} ]];
# do
2022-08-12 16:33:54 +00:00
pre_process " ${ array [ $it ] } " &
2022-08-11 04:35:30 +00:00
processes += ( $! )
( ( it++) )
2022-08-11 22:37:19 +00:00
# ((loop++))
2022-08-11 04:35:30 +00:00
# done
2022-08-21 23:54:10 +00:00
elif [ [ $proc_count != 0 || $( wc -l finished 2>/dev/null | awk '{ print $1 }' ) -lt " ${# array [@] } " ] ] ; then # Wait for all processes to finish
2022-08-13 21:41:50 +00:00
sleep 3
2022-07-27 21:07:53 +00:00
else # All processes must be completed, break out of loop
2022-07-27 20:50:52 +00:00
break
2022-07-25 20:42:57 +00:00
fi
done
2022-08-13 23:03:32 +00:00
rm deploying 2>/dev/null
2022-08-21 23:54:10 +00:00
rm finished 2>/dev/null
2022-07-31 19:03:45 +00:00
echo
echo
2022-07-25 04:11:04 +00:00
}
export -f commander
2022-07-25 04:03:57 +00:00
2022-07-27 01:04:27 +00:00
2022-08-12 16:33:54 +00:00
pre_process( ) {
2022-07-25 21:20:32 +00:00
app_name = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $1}' ) #print out first catagory, name.
2022-08-21 23:54:10 +00:00
printf '%s\0' " ${ ignore [@] } " | grep -iFxqz " ${ app_name } " && echo -e " \n $app_name \nIgnored, skipping " && final_check && return 0 #If application is on ignore list, skip
2022-07-25 21:20:32 +00:00
old_app_ver = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $4}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}' ) #previous/current Application MAJOR Version
new_app_ver = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $5}' | awk -F '_' '{print $1}' | awk -F '.' '{print $1}' ) #new Application MAJOR Version
old_chart_ver = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $4}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}' ) # Old Chart MAJOR version
new_chart_ver = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $5}' | awk -F '_' '{print $2}' | awk -F '.' '{print $1}' ) # New Chart MAJOR version
2022-07-27 21:47:11 +00:00
startstatus = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $2}' ) #status of the app: STOPPED / DEPLOYING / ACTIVE
2022-07-25 04:03:57 +00:00
diff_app = $( diff <( echo " $old_app_ver " ) <( echo " $new_app_ver " ) ) #caluclating difference in major app versions
diff_chart = $( diff <( echo " $old_chart_ver " ) <( echo " $new_chart_ver " ) ) #caluclating difference in Chart versions
2022-08-21 23:54:10 +00:00
[ [ " $diff_app " != " $diff_chart " && $update_apps = = "true" ] ] && echo -e " \n $app_name \nMajor Release, update manually " && final_check && return
2022-07-25 21:20:32 +00:00
old_full_ver = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $4}' ) #Upgraded From
new_full_ver = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $5}' ) #Upraded To
rollback_version = $( echo " ${ array [ $it ] } " | awk -F ',' '{print $4}' | awk -F '_' '{print $2}' )
2022-08-14 23:43:07 +00:00
if grep -qs " ^ $app_name , " failed 2>/dev/null; then
failed_ver = $( grep " ^ $app_name , " failed | awk -F ',' '{print $2}' )
2022-08-04 20:46:00 +00:00
if [ [ " $failed_ver " = = " $new_full_ver " ] ] ; then
2022-08-04 20:20:40 +00:00
echo -e " \n $app_name "
2022-08-04 22:08:45 +00:00
echo -e " Skipping previously failed version:\n $new_full_ver "
2022-08-21 23:54:10 +00:00
final_check
2022-08-04 20:14:32 +00:00
return 0
2022-08-04 20:46:00 +00:00
else
2022-08-14 23:43:07 +00:00
sed -i /" $app_name " ,/d failed
2022-08-04 20:14:32 +00:00
fi
fi
2022-08-13 16:37:48 +00:00
[ [ ! -e external_services ] ] && touch external_services
if ! grep -qs " ^ $app_name , " external_services ; then
2022-08-24 11:09:59 +00:00
if ! grep -qs "/external-service" /mnt/" $pool " /ix-applications/releases/" $app_name " /charts/" $( find /mnt/" $pool " /ix-applications/releases/" $app_name " /charts/ -maxdepth 1 -type d -printf '%P\n' | sort -r | head -n 1) " /Chart.yaml; then
2022-08-13 16:37:48 +00:00
echo " $app_name ,false " >> external_services
else
echo " $app_name ,true " >> external_services
fi
fi
2022-08-12 16:44:32 +00:00
echo_array += ( " \n $app_name " )
2022-08-12 17:08:51 +00:00
if [ [ $stop_before_update = = "true" && " $startstatus " != "STOPPED" ] ] ; then # Check to see if user is using -S or not
[ [ " $verbose " = = "true" ] ] && echo_array += ( "Stopping prior to update.." )
if stop_app ; then
echo_array += ( "Stopped" )
else
echo_array += ( " Error: Failed to stop $app_name " )
echo_array
2022-08-21 23:54:10 +00:00
final_check
2022-08-12 17:08:51 +00:00
return 1
2022-07-25 02:29:58 +00:00
fi
2022-08-01 21:47:08 +00:00
fi
2022-08-12 12:53:35 +00:00
[ [ " $verbose " = = "true" ] ] && echo_array += ( "Updating.." )
2022-08-12 16:33:54 +00:00
if update_app ; then
2022-08-12 12:53:35 +00:00
echo_array += ( " Updated\n $old_full_ver \n $new_full_ver " )
else
2022-08-23 22:08:08 +00:00
echo_array += ( "Failed to update\nManual intervention may be required" )
2022-08-12 17:08:51 +00:00
echo_array
2022-08-21 23:54:10 +00:00
final_check
2022-08-12 17:08:51 +00:00
return
2022-08-12 12:53:35 +00:00
fi
2022-08-13 16:49:47 +00:00
if grep -qs " ^ $app_name ,true " external_services ; then
echo_array
2022-08-21 23:54:10 +00:00
final_check
2022-08-13 16:49:47 +00:00
return
else
after_update_actions
2022-08-21 23:54:10 +00:00
final_check
2022-08-13 16:49:47 +00:00
fi
2022-07-25 22:50:44 +00:00
}
2022-08-12 16:33:54 +00:00
export -f pre_process
2022-07-25 02:29:58 +00:00
2022-08-12 16:33:54 +00:00
update_app( ) {
2022-08-12 12:37:30 +00:00
current_loop = 0
2022-08-11 22:37:19 +00:00
while true
2022-08-09 23:32:47 +00:00
do
2022-08-13 23:32:44 +00:00
update_avail = $( grep " ^ $app_name , " all_app_status | awk -F ',' '{print $3}' )
2022-08-12 02:41:31 +00:00
if [ [ $update_avail = = "true" ] ] ; then
2022-08-10 04:47:59 +00:00
if ! cli -c 'app chart_release upgrade release_name=' '"' " $app_name " '"' & > /dev/null ; then
2022-08-13 23:32:44 +00:00
before_loop = $( head -n 1 all_app_status)
2022-08-12 02:30:49 +00:00
current_loop = 0
2022-08-13 23:32:44 +00:00
until [ [ " $( grep " ^ $app_name , " all_app_status | awk -F ',' '{print $3}' ) " != " $update_avail " ] ] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
2022-08-12 02:30:49 +00:00
do
2022-08-12 12:37:30 +00:00
if [ [ $current_loop -gt 2 ] ] ; then
2022-08-12 04:52:45 +00:00
cli -c 'app chart_release upgrade release_name=' '"' " $app_name " '"' & > /dev/null || return 1 # After waiting, attempt an update once more, if fails, return error code
2022-08-13 23:32:44 +00:00
elif ! echo -e " $( head -n 1 all_app_status) " | grep -qs ^" $before_loop " ; then # The file has been updated, but nothing changed specifically for the app.
before_loop = $( head -n 1 all_app_status)
2022-08-12 02:30:49 +00:00
( ( current_loop++) )
fi
2022-08-12 02:41:31 +00:00
sleep 1
2022-08-12 02:30:49 +00:00
done
2022-08-10 04:47:59 +00:00
fi
break
elif [ [ $update_avail = = "false" ] ] ; then
2022-08-09 23:32:47 +00:00
break
2022-08-12 02:41:31 +00:00
else
sleep 3
2022-08-09 23:32:47 +00:00
fi
done
}
2022-08-12 16:33:54 +00:00
export -f update_app
2022-08-09 23:32:47 +00:00
2022-08-18 06:36:54 +00:00
2022-08-12 12:37:30 +00:00
stop_app( ) {
count = 0
while [ [ " $status " != "STOPPED" ] ]
do
2022-08-13 23:32:44 +00:00
status = $( grep " ^ $app_name , " all_app_status | awk -F ',' '{print $2}' )
2022-08-12 12:37:30 +00:00
if [ [ $count -gt 2 ] ] ; then # If failed to stop app 3 times, return failure to parent shell
return 1
elif ! cli -c 'app chart_release scale release_name=' \" " $app_name " \" \ 'scale_options={"replica_count": 0}' & > /dev/null ; then
2022-08-13 23:32:44 +00:00
before_loop = $( head -n 1 all_app_status)
2022-08-12 12:37:30 +00:00
( ( count++) )
2022-08-13 23:32:44 +00:00
until [ [ $( head -n 1 all_app_status) != " $before_loop " ] ] # Upon failure, wait for status update before continuing
2022-08-12 12:37:30 +00:00
do
sleep 1
done
else
break
fi
done
}
export -f stop_app
2022-07-25 02:29:58 +00:00
after_update_actions( ) {
SECONDS = 0
count = 0
2022-08-04 22:48:55 +00:00
if [ [ $rollback = = "true" || " $startstatus " = = "STOPPED" ] ] ; then
2022-07-25 02:29:58 +00:00
while true
do
2022-08-13 23:32:44 +00:00
status = $( grep " ^ $app_name , " all_app_status | awk -F ',' '{print $2}' )
2022-08-24 21:51:19 +00:00
if [ [ $count -lt 1 && $status = = "ACTIVE" && " $( grep " ^ $app_name , " deploying 2>/dev/null | awk -F ',' '{print $2}' ) " != "DEPLOYING" ] ] ; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
2022-08-12 13:33:13 +00:00
[ [ " $verbose " = = "true" ] ] && echo_array += ( " Verifying $status .. " )
2022-08-13 23:32:44 +00:00
before_loop = $( head -n 1 all_app_status)
2022-08-12 02:30:49 +00:00
current_loop = 0
2022-08-13 23:36:19 +00:00
until [ [ " $status " != "ACTIVE" || $current_loop -gt 4 ] ] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
2022-08-11 05:27:11 +00:00
do
2022-08-14 23:43:07 +00:00
status = $( grep " ^ $app_name , " all_app_status | awk -F ',' '{print $2}' )
2022-08-11 05:37:53 +00:00
sleep 1
2022-08-13 23:32:44 +00:00
if ! echo -e " $( head -n 1 all_app_status) " | grep -qs ^" $before_loop " ; then
before_loop = $( head -n 1 all_app_status)
2022-08-11 06:18:53 +00:00
( ( current_loop++) )
fi
2022-08-11 05:27:11 +00:00
done
fi
( ( count++ ) )
2022-08-04 23:19:44 +00:00
if [ [ " $status " = = "ACTIVE" ] ] ; then
if [ [ " $startstatus " = = "STOPPED" ] ] ; then
[ [ " $verbose " = = "true" ] ] && echo_array += ( "Returing to STOPPED state.." )
2022-08-12 12:37:30 +00:00
if stop_app ; then
echo_array += ( "Stopped" )
else
echo_array += ( " Error: Failed to stop $app_name " )
2022-08-12 17:08:51 +00:00
echo_array
2022-08-12 12:37:30 +00:00
return 1
fi
2022-08-04 23:19:44 +00:00
break
else
echo_array += ( "Active" )
2022-08-12 13:33:13 +00:00
break
2022-08-04 23:19:44 +00:00
fi
2022-08-13 23:07:27 +00:00
elif [ [ " $SECONDS " -ge " $timeout " ] ] ; then
2022-08-04 22:48:55 +00:00
if [ [ $rollback = = "true" ] ] ; then
if [ [ " $failed " != "true" ] ] ; then
2022-08-04 23:19:44 +00:00
echo_array += ( " Error: Run Time( $SECONDS ) for $app_name has exceeded Timeout( $timeout ) " )
echo_array += ( "If this is a slow starting application, set a higher timeout with -t" )
echo_array += ( "If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration" )
echo_array += ( "Reverting update.." )
2022-08-14 23:43:07 +00:00
echo " $app_name , $new_full_ver " >> failed
2022-08-24 21:51:19 +00:00
if rollback_app ; then
echo_array += ( "Rolled Back" )
else
echo_array += ( " Error: Failed to rollback $app_name \nAbandoning " )
echo_array
return 1
fi
failed = "true"
SECONDS = 0
count = 0
continue #run back after_update_actions function if the app was stopped prior to update
2022-08-04 23:19:44 +00:00
else
echo_array += ( " Error: Run Time( $SECONDS ) for $app_name has exceeded Timeout( $timeout ) " )
echo_array += ( "The application failed to be ACTIVE even after a rollback" )
2022-08-21 22:44:22 +00:00
echo_array += ( "Manual intervention is required\nStopping, then Abandoning" )
if stop_app ; then
echo_array += ( "Stopped" )
else
echo_array += ( " Error: Failed to stop $app_name " )
echo_array
return 1
fi
2022-08-04 22:48:55 +00:00
break
fi
else
2022-08-04 23:19:44 +00:00
echo_array += ( " Error: Run Time( $SECONDS ) for $app_name has exceeded Timeout( $timeout ) " )
echo_array += ( "If this is a slow starting application, set a higher timeout with -t" )
echo_array += ( "If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration" )
2022-08-21 22:44:22 +00:00
echo_array += ( "Manual intervention is required\nStopping, then Abandoning" )
2022-08-24 21:51:19 +00:00
echo " $app_name , $new_full_ver " >> failed
2022-08-21 22:44:22 +00:00
if stop_app ; then
echo_array += ( "Stopped" )
else
echo_array += ( " Error: Failed to stop $app_name " )
echo_array
return 1
fi
2022-08-21 22:45:53 +00:00
break
2022-08-04 22:48:55 +00:00
fi
2022-07-25 02:29:58 +00:00
else
2022-07-25 23:11:56 +00:00
[ [ " $verbose " = = "true" ] ] && echo_array += ( " Waiting $(( timeout-SECONDS)) more seconds for $app_name to be ACTIVE " )
2022-08-12 03:20:04 +00:00
sleep 5
2022-07-25 02:29:58 +00:00
continue
fi
done
fi
2022-08-12 17:08:51 +00:00
echo_array
}
export -f after_update_actions
2022-07-25 23:21:41 +00:00
2022-08-04 22:48:55 +00:00
2022-08-24 21:51:19 +00:00
rollback_app( ) {
count = 0
2022-08-24 22:16:34 +00:00
update_avail = $( grep " ^ $app_name , " all_app_status | awk -F ',' '{print $3}' )
while [ [ $update_avail = = "false" ] ]
2022-08-24 21:51:19 +00:00
do
update_avail = $( grep " ^ $app_name , " all_app_status | awk -F ',' '{print $3}' )
if [ [ $count -gt 2 ] ] ; then # If failed to rollback app 3 times, return failure to parent shell
return 1
elif ! cli -c " app chart_release rollback release_name=\" $app_name \" rollback_options={\"item_version\": \" $rollback_version \"} " & > /dev/null ; then
before_loop = $( head -n 1 all_app_status)
( ( count++) )
until [ [ $( head -n 1 all_app_status) != " $before_loop " ] ] # Upon failure, wait for status update before continuing
do
sleep 1
done
else
break
fi
done
}
2022-08-12 17:08:51 +00:00
echo_array( ) {
2022-07-25 23:38:10 +00:00
#Dump the echo_array, ensures all output is in a neat order.
2022-07-25 23:21:41 +00:00
for i in " ${ echo_array [@] } "
do
echo -e " $i "
done
2022-08-12 17:08:51 +00:00
2022-07-25 02:29:58 +00:00
}
2022-08-12 17:08:51 +00:00
export -f echo_array
2022-08-21 23:54:10 +00:00
final_check( ) {
[ [ ! -e finished ] ] && touch finished
echo " $app_name ,finished " >> finished
}