diff --git a/functions/backup.sh b/functions/backup.sh index 642b58f5..0ca69069 100644 --- a/functions/backup.sh +++ b/functions/backup.sh @@ -30,7 +30,6 @@ echo export -f backup - deleteBackup(){ while true do @@ -170,4 +169,4 @@ do done done } -export -f restore +export -f restore \ No newline at end of file diff --git a/functions/menu.sh b/functions/menu.sh index 5b86ca5f..a8677314 100644 --- a/functions/menu.sh +++ b/functions/menu.sh @@ -1,5 +1,6 @@ #!/bin/bash + menu(){ clear -x title @@ -45,7 +46,6 @@ case $selection in 8) script_create ;; - 9) cmd_to_container ;; @@ -55,4 +55,4 @@ case $selection in esac echo } -export -f menu +export -f menu \ No newline at end of file diff --git a/functions/misc.sh b/functions/misc.sh index 33b4d4c0..ebff4ff8 100644 --- a/functions/misc.sh +++ b/functions/misc.sh @@ -15,6 +15,7 @@ echo } export -f sync + prune(){ echo -e "🄿 🅁 🅄 🄽 🄴" echo "Pruned Docker Images" @@ -22,6 +23,7 @@ docker image prune -af | grep "^Total" || echo "Failed to Prune Docker Images" } export -f prune + title(){ echo ' _ _ _____ _ _ ' echo '| | | | / ___| (_) | | ' @@ -36,6 +38,7 @@ echo } export -f title + help(){ [[ $help == "true" ]] && clear -x diff --git a/functions/mount.sh b/functions/mount.sh index 26198633..c915407e 100644 --- a/functions/mount.sh +++ b/functions/mount.sh @@ -1,5 +1,6 @@ #!/bin/bash + mount(){ while true do diff --git a/functions/script_create.sh b/functions/script_create.sh index b1448218..de761cc6 100644 --- a/functions/script_create.sh +++ b/functions/script_create.sh @@ -1,5 +1,6 @@ #!/bin/bash + script_create(){ while true do diff --git a/functions/self_update.sh b/functions/self_update.sh index a0ee1d01..144157d4 100644 --- a/functions/self_update.sh +++ b/functions/self_update.sh @@ -1,5 +1,6 @@ #!/bin/bash + args=("$@") self_update() { diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 61d670a6..50629e2e 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -9,7 +9,6 @@ echo "Asynchronous Updates: $update_limit" [[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout" [[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips" pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r") -rm external_services 2>/dev/null # TODO remove later it=0 while_count=0 rm deploying 2>/dev/null @@ -42,14 +41,9 @@ do if [[ "$proc_count" -ge "$update_limit" ]]; then sleep 3 elif [[ $it -lt ${#array[@]} ]]; then - # loop=0 - # until [[ $loop -ge 2 || $it -ge ${#array[@]} ]]; - # do pre_process "${array[$it]}" & processes+=($!) ((it++)) - # ((loop++)) - # done elif [[ $proc_count != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]]; then # Wait for all processes to finish sleep 3 else # All processes must be completed, break out of loop @@ -78,6 +72,8 @@ diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclatin old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}') + +# Skip update if application previously failed on this exact update version if grep -qs "^$app_name," failed 2>/dev/null; then failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}') if [[ "$failed_ver" == "$new_full_ver" ]] ; then @@ -90,6 +86,7 @@ if grep -qs "^$app_name," failed 2>/dev/null; then fi fi +# Check if app is external services, append outcome to external_services file [[ ! -e external_services ]] && touch external_services if ! grep -qs "^$app_name," external_services ; then if ! grep -qs "/external-service" /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/"$(find /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/ -maxdepth 1 -type d -printf '%P\n' | sort -r | head -n 1)"/Chart.yaml; then @@ -99,6 +96,7 @@ if ! grep -qs "^$app_name," external_services ; then fi fi +# If user is using -S, stop app prior to updating echo_array+=("\n$app_name") if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # Check to see if user is using -S or not [[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..") @@ -110,6 +108,8 @@ if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # C return 1 fi fi + +# Send app through update function [[ "$verbose" == "true" ]] && echo_array+=("Updating..") if update_app ;then echo_array+=("Updated\n$old_full_ver\n$new_full_ver") @@ -118,70 +118,19 @@ else echo_array return fi + +# If app is external services, do not send for post processing if grep -qs "^$app_name,true" external_services ; then echo_array return else - after_update_actions + post_process fi } export -f pre_process -update_app(){ -current_loop=0 -while true -do - update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}') - if [[ $update_avail =~ "true" ]]; then - if ! cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null ; then - before_loop=$(head -n 1 all_app_status) - current_loop=0 - until [[ "$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}')" != "$update_avail" ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by. - do - if [[ $current_loop -gt 2 ]]; then - cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null || return 1 # After waiting, attempt an update once more, if fails, return error code - elif ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then # The file has been updated, but nothing changed specifically for the app. - before_loop=$(head -n 1 all_app_status) - ((current_loop++)) - fi - sleep 1 - done - fi - break - elif [[ ! $update_avail =~ "true" ]]; then - break - else - sleep 3 - fi -done -} -export -f update_app - - -stop_app(){ -count=0 -while [[ "$status" != "STOPPED" ]] -do - status=$( grep "^$app_name," all_app_status | awk -F ',' '{print $2}') - if [[ $count -gt 2 ]]; then # If failed to stop app 3 times, return failure to parent shell - return 1 - elif ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' &> /dev/null ; then - before_loop=$(head -n 1 all_app_status) - ((count++)) - until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing - do - sleep 1 - done - else - break - fi -done -} -export -f stop_app - - -after_update_actions(){ +post_process(){ SECONDS=0 count=0 if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then @@ -242,7 +191,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then failed="true" SECONDS=0 count=0 - continue #run back after_update_actions function if the app was stopped prior to update + continue #run back post_process function if the app was stopped prior to update else echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") echo_array+=("The application failed to be ACTIVE even after a rollback") @@ -280,7 +229,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then fi echo_array } -export -f after_update_actions +export -f post_process rollback_app(){ @@ -304,6 +253,60 @@ do done } + +update_app(){ +current_loop=0 +while true +do + update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}') + if [[ $update_avail =~ "true" ]]; then + if ! cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null ; then + before_loop=$(head -n 1 all_app_status) + current_loop=0 + until [[ "$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}')" != "$update_avail" ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by. + do + if [[ $current_loop -gt 2 ]]; then + cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null || return 1 # After waiting, attempt an update once more, if fails, return error code + elif ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then # The file has been updated, but nothing changed specifically for the app. + before_loop=$(head -n 1 all_app_status) + ((current_loop++)) + fi + sleep 1 + done + fi + break + elif [[ ! $update_avail =~ "true" ]]; then + break + else + sleep 3 + fi +done +} +export -f update_app + + +stop_app(){ +count=0 +while [[ "$status" != "STOPPED" ]] +do + status=$( grep "^$app_name," all_app_status | awk -F ',' '{print $2}') + if [[ $count -gt 2 ]]; then # If failed to stop app 3 times, return failure to parent shell + return 1 + elif ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' &> /dev/null ; then + before_loop=$(head -n 1 all_app_status) + ((count++)) + until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing + do + sleep 1 + done + else + break + fi +done +} +export -f stop_app + + echo_array(){ #Dump the echo_array, ensures all output is in a neat order. for i in "${echo_array[@]}" @@ -318,4 +321,4 @@ export -f echo_array final_check(){ [[ ! -e finished ]] && touch finished echo "$app_name,finished" >> finished -} +} \ No newline at end of file diff --git a/heavy_script.sh b/heavy_script.sh index 7a687b51..6858ed4d 100644 --- a/heavy_script.sh +++ b/heavy_script.sh @@ -1,5 +1,6 @@ #!/bin/bash + # cd to script, this ensures the script can find the source scripts below, even when ran from a seperate directory script=$(readlink -f "$0") script_path=$(dirname "$script") @@ -20,13 +21,10 @@ source functions/cmd_to_container.sh source functions/script_create.sh - - #If no argument is passed, kill the script. [[ -z "$*" || "-" == "$*" || "--" == "$*" ]] && menu - # Parse script options while getopts ":si:rb:t:uUpSRv-:" opt do @@ -121,10 +119,6 @@ do v) verbose="true" ;; - \?) - echo -e "Invalid Option \"-$OPTARG\"\n" - help - ;; *) echo -e "Invalid Option \"-$OPTARG\"\n" help @@ -133,7 +127,6 @@ do done - #exit if incompatable functions are called [[ "$update_all_apps" == "true" && "$update_apps" == "true" ]] && echo -e "-U and -u cannot BOTH be called" && exit