Merge pull request #17 from Heavybullets8/general_improvements

General improvements

README.md

@@ -27,7 +27,7 @@
 |---------------	|------------------------	|-----------------	|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------	|
 | -U            	| -U <br>-U 5            	| None or Integer 	| Update applications, ignoring major version changes<br>_Optionally, you can supply a number after the argument to update multiple applications at once_                                                                    	|
 | -u            	| -u<br>-u 5             	| None or Integer 	| Update applications, do NOT update if there was a major version change<br>_Optionally, you can supply a number after the argument to update multiple applications at once_                                                 	|
-| -b            	| -b 14                  	| Integer         	| Backup `ix-appliactions` dataset<br>_Creates backups up to the number you've chosen_                                                                                                                                       	|
+| -b            	| -b 14                  	| Integer         	| Backup `ix-applications` dataset<br>_Creates backups up to the number you've chosen_                                                                                                                                       	|
 | -i            	| -i nextcloud -i sonarr 	| String          	| Applications listed will be ignored during updating<br>_List one application after another as shown in the example_                                                                                                        	|
 | -r            	| -r                     	| None            	| Monitors applications after they update<br>If the app does not become "ACTIVE" after either:<br>The custom Timeout, or Default Timeout,<br>rollback the application.                                                       	|
 | -v            	| -v                     	| None            	| Verbose Output<br>_Look at the bottom of this page for an example_                                                                                                                                                         	|

functions/backup.sh

@@ -30,7 +30,6 @@ echo
 export -f backup
 
 
-
 deleteBackup(){
 while true
 do

functions/menu.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+
 menu(){
 clear -x
 title
@@ -31,7 +32,6 @@ case $selection in
         ;;
     4)
         read -rt 120 -p "What is the maximun number of backups you would like?: " number_of_backups || { echo -e "\nFailed to make a selection in time" ; exit; }
-        backup="true"
         ;;
     5)
         restore
@@ -45,7 +45,6 @@ case $selection in
     8)
         script_create
         ;;
-
     9)
         cmd_to_container
         ;;

functions/misc.sh

@@ -15,6 +15,7 @@ echo
 }
 export -f sync
 
+
 prune(){
 echo -e "🄿 🅁 🅄 🄽 🄴"  
 echo "Pruned Docker Images"
@@ -22,6 +23,7 @@ docker image prune -af | grep "^Total" || echo "Failed to Prune Docker Images"
 }
 export -f prune
 
+
 title(){
 echo ' _   _                        _____           _       _   '
 echo '| | | |                      /  ___|         (_)     | | '
@@ -36,6 +38,7 @@ echo
 }
 export -f title
 
+
 help(){
 [[ $help == "true" ]] && clear -x
 

functions/mount.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+
 mount(){
 while true
 do
@@ -50,7 +51,7 @@ do
                 mapfile -t full_path < <(zfs list | grep "$volume_name" | awk '{print $1}')
                 if [[  "${#full_path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup.
                         echo "$app is using the same volume identifier on more than one pool.. attempting to use your current kubernetes apps pool"
-                        pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
+                        pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")
                         full_path=$(zfs list | grep "$volume_name" | grep "$pool" | awk '{print $1}')
                 fi
                 echo -e "\nMounting\n$full_path\nTo\n/mnt/heavyscript/$data_name"
@@ -89,7 +90,7 @@ do
                 mapfile -t path < <(find /mnt/*/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-)
                 if [[  "${#path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup.
                     echo "$i is a name used on more than one pool.. attempting to use your current kubernetes apps pool"
-                    pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
+                    pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")
                     full_path=$(find /mnt/"$pool"/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-)
                     zfs set mountpoint=legacy "$full_path""$pvc" 
                     echo "$i unmounted" && rmdir /mnt/heavyscript/"$i" || echo "failed to unmount $i"

functions/script_create.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+
 script_create(){
 while true 
 do

functions/self_update.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
+
 args=("$@")
 self_update() {
 
 git fetch --tags &>/dev/null
+git reset --hard
 latest_ver=$(git describe --tags "$(git rev-list --tags --max-count=1)")
 echo "🅂 🄴 🄻 🄵"
 echo "🅄 🄿 🄳 🄰 🅃 🄴"

functions/update_apps.sh

@@ -9,7 +9,6 @@ echo "Asynchronous Updates: $update_limit"
 [[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout"
 [[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips"
 pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
-rm external_services 2>/dev/null # TODO remove later
 it=0
 while_count=0
 rm deploying 2>/dev/null
@@ -42,14 +41,9 @@ do
     if [[ "$proc_count" -ge "$update_limit" ]]; then
         sleep 3
     elif [[ $it -lt ${#array[@]} ]]; then
-        # loop=0
-        # until [[ $loop -ge 2 || $it -ge ${#array[@]} ]];
-        # do
         pre_process "${array[$it]}" &
         processes+=($!)
         ((it++))
-        # ((loop++))
-        # done
     elif [[ $proc_count != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]]; then # Wait for all processes to finish
         sleep 3
     else # All processes must be completed, break out of loop
@@ -78,6 +72,8 @@ diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclatin
 old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From
 new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To
 rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}')
+
+# Skip update if application previously failed on this exact update version
 if  grep -qs "^$app_name," failed 2>/dev/null; then
     failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}')
     if [[ "$failed_ver" == "$new_full_ver" ]] ; then
@@ -90,6 +86,7 @@ if  grep -qs "^$app_name," failed 2>/dev/null; then
     fi
 fi
 
+# Check if app is external services, append outcome to external_services file
 [[ ! -e external_services ]] && touch external_services
 if ! grep -qs "^$app_name," external_services ; then 
     if ! grep -qs "/external-service" /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/"$(find /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/ -maxdepth 1 -type d -printf '%P\n' | sort -r | head -n 1)"/Chart.yaml; then
@@ -99,6 +96,7 @@ if ! grep -qs "^$app_name," external_services ; then
     fi
 fi
 
+# If user is using -S, stop app prior to updating
 echo_array+=("\n$app_name")
 if [[ $stop_before_update == "true" && "$startstatus" !=  "STOPPED" ]]; then # Check to see if user is using -S or not
     [[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..")
@@ -110,6 +108,8 @@ if [[ $stop_before_update == "true" && "$startstatus" !=  "STOPPED" ]]; then # C
         return 1
     fi
 fi
+
+# Send app through update function
 [[ "$verbose" == "true" ]] && echo_array+=("Updating..")
 if update_app ;then
     echo_array+=("Updated\n$old_full_ver\n$new_full_ver")
@@ -118,16 +118,145 @@ else
     echo_array
     return
 fi
+
+# If app is external services, do not send for post processing
 if grep -qs "^$app_name,true" external_services ; then
     echo_array
     return
 else
-    after_update_actions
+    post_process
 fi
 }
 export -f pre_process
 
 
+post_process(){
+SECONDS=0
+count=0
+if [[ $rollback == "true" || "$startstatus"  ==  "STOPPED" ]]; then
+    while true
+    do
+        
+        # If app reports ACTIVE right away, assume its a false positive and wait for it to change, or trust it after 5 updates to all_app_status
+        status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
+        if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then  # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
+            [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
+            before_loop=$(head -n 1 all_app_status)
+            current_loop=0
+            until [[ "$status" != "ACTIVE" || $current_loop -gt 4 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
+            do
+                status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
+                sleep 1
+                if ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then
+                    before_loop=$(head -n 1 all_app_status)
+                    ((current_loop++))
+                fi
+            done
+        fi
+        (( count++ ))
+
+        if [[ "$status"  ==  "ACTIVE" ]]; then
+            if [[ "$startstatus"  ==  "STOPPED" ]]; then
+                [[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
+                if stop_app ; then
+                    echo_array+=("Stopped")
+                else
+                    echo_array+=("Error: Failed to stop $app_name")
+                    echo_array
+                    return 1
+                fi
+                break
+            else
+                echo_array+=("Active")
+                break 
+            fi
+        elif [[ "$SECONDS" -ge "$timeout" ]]; then
+            if [[ $rollback == "true" ]]; then
+                if [[ $old_full_ver == "$new_full_ver" ]]; then
+                    echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+                    echo_array+=("This is the result of a container image update..")
+                    echo_array+=("Reverting is not possible, Abandoning")
+                    echo_array
+                    return 1
+                elif [[ "$failed" != "true" ]]; then
+                    echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+                    echo_array+=("If this is a slow starting application, set a higher timeout with -t")
+                    echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
+                    echo_array+=("Reverting update..")
+                    echo "$app_name,$new_full_ver" >> failed
+                    if rollback_app ; then
+                        echo_array+=("Rolled Back")
+                    else
+                        echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
+                        echo_array
+                        return 1
+                    fi                    
+                    failed="true"
+                    SECONDS=0
+                    count=0
+                    continue #run back post_process function if the app was stopped prior to update
+                else
+                    echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+                    echo_array+=("The application failed to be ACTIVE even after a rollback")
+                    echo_array+=("Manual intervention is required\nStopping, then Abandoning")
+                    if stop_app ; then
+                        echo_array+=("Stopped")
+                    else
+                        echo_array+=("Error: Failed to stop $app_name")
+                        echo_array
+                        return 1
+                    fi
+                    break
+                fi
+            else
+                echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+                echo_array+=("If this is a slow starting application, set a higher timeout with -t")
+                echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
+                echo_array+=("Manual intervention is required\nStopping, then Abandoning")
+                echo "$app_name,$new_full_ver" >> failed
+                if stop_app ; then
+                    echo_array+=("Stopped")
+                else
+                    echo_array+=("Error: Failed to stop $app_name")
+                    echo_array
+                    return 1
+                fi
+                break
+            fi
+        else
+            [[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
+            sleep 5
+            continue
+        fi
+    done
+fi
+echo_array
+}
+export -f post_process
+
+
+rollback_app(){
+count=0
+app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
+while [[ $app_update_avail == "false" ]]
+do
+    app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
+    if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
+        return 1
+    elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
+        before_loop=$(head -n 1 all_app_status)
+        ((count++))
+        until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
+        do
+            sleep 1
+        done
+    else 
+        break
+    fi
+done
+}
+
+
 update_app(){
 current_loop=0
 while true
@@ -181,129 +310,6 @@ done
 export -f stop_app
 
 
-after_update_actions(){
-SECONDS=0
-count=0
-if [[ $rollback == "true" || "$startstatus"  ==  "STOPPED" ]]; then
-    while true
-    do
-        status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
-        if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then  # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
-            [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
-            before_loop=$(head -n 1 all_app_status)
-            current_loop=0
-            until [[ "$status" != "ACTIVE" || $current_loop -gt 4 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
-            do
-                status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
-                sleep 1
-                if ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then
-                    before_loop=$(head -n 1 all_app_status)
-                    ((current_loop++))
-                fi
-            done
-        fi
-        (( count++ ))
-        if [[ "$status"  ==  "ACTIVE" ]]; then
-            if [[ "$startstatus"  ==  "STOPPED" ]]; then
-                [[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
-                if stop_app ; then
-                    echo_array+=("Stopped")
-                else
-                    echo_array+=("Error: Failed to stop $app_name")
-                    echo_array
-                    return 1
-                fi
-                break
-            else
-                echo_array+=("Active")
-                break 
-            fi
-        elif [[ "$SECONDS" -ge "$timeout" ]]; then
-            if [[ $rollback == "true" ]]; then
-                if [[ $old_full_ver == "$new_full_ver" ]]; then
-                    echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
-                    echo_array+=("This is the result of a container image update..")
-                    echo_array+=("Reverting is not possible, Abandoning")
-                    echo_array
-                    return 1
-                elif [[ "$failed" != "true" ]]; then
-                    echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
-                    echo_array+=("If this is a slow starting application, set a higher timeout with -t")
-                    echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
-                    echo_array+=("Reverting update..")
-                    echo "$app_name,$new_full_ver" >> failed
-                    if rollback_app ; then
-                        echo_array+=("Rolled Back")
-                    else
-                        echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
-                        echo_array
-                        return 1
-                    fi                    
-                    failed="true"
-                    SECONDS=0
-                    count=0
-                    continue #run back after_update_actions function if the app was stopped prior to update
-                else
-                    echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
-                    echo_array+=("The application failed to be ACTIVE even after a rollback")
-                    echo_array+=("Manual intervention is required\nStopping, then Abandoning")
-                    if stop_app ; then
-                        echo_array+=("Stopped")
-                    else
-                        echo_array+=("Error: Failed to stop $app_name")
-                        echo_array
-                        return 1
-                    fi
-                    break
-                fi
-            else
-                echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
-                echo_array+=("If this is a slow starting application, set a higher timeout with -t")
-                echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
-                echo_array+=("Manual intervention is required\nStopping, then Abandoning")
-                echo "$app_name,$new_full_ver" >> failed
-                if stop_app ; then
-                    echo_array+=("Stopped")
-                else
-                    echo_array+=("Error: Failed to stop $app_name")
-                    echo_array
-                    return 1
-                fi
-                break
-            fi
-        else
-            [[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
-            sleep 5
-            continue
-        fi
-    done
-fi
-echo_array
-}
-export -f after_update_actions
-
-
-rollback_app(){
-count=0
-app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
-while [[ $app_update_avail == "false" ]]
-do
-    app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
-    if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
-        return 1
-    elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
-        before_loop=$(head -n 1 all_app_status)
-        ((count++))
-        until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
-        do
-            sleep 1
-        done
-    else 
-        break
-    fi
-done
-}
-
 echo_array(){
 #Dump the echo_array, ensures all output is in a neat order. 
 for i in "${echo_array[@]}"

heavy_script.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
+
 # cd to script, this ensures the script can find the source scripts below, even when ran from a seperate directory
 script=$(readlink -f "$0")
 script_path=$(dirname "$script")
@@ -20,13 +21,10 @@ source functions/cmd_to_container.sh
 source functions/script_create.sh
 
 
-
-
 #If no argument is passed, kill the script.
 [[ -z "$*" || "-" == "$*" || "--" == "$*"  ]] && menu
 
 
-
 # Parse script options
 while getopts ":si:rb:t:uUpSRv-:" opt
 do
@@ -68,7 +66,6 @@ do
         number_of_backups=$OPTARG
         ! [[ $OPTARG =~ ^[0-9]+$  ]] && echo -e "Error: -b needs to be assigned an interger\n\"""$number_of_backups""\" is not an interger" >&2 && exit
         [[ "$number_of_backups" -le 0 ]] && echo "Error: Number of backups is required to be at least 1" && exit
-        backup="true"
         ;;
       r)
         rollback="true"
@@ -121,10 +118,6 @@ do
       v)
         verbose="true"
         ;;
-      \?)
-        echo -e "Invalid Option \"-$OPTARG\"\n"
-        help
-        ;;
       *)
         echo -e "Invalid Option \"-$OPTARG\"\n"
         help
@@ -133,7 +126,6 @@ do
 done
 
 
-
 #exit if incompatable functions are called 
 [[ "$update_all_apps" == "true" && "$update_apps" == "true" ]] && echo -e "-U and -u cannot BOTH be called" && exit
 
@@ -145,18 +137,18 @@ done
 [[ "$dns" == "true" ]] && dns && exit
 [[ "$restore" == "true" ]] && restore && exit
 [[ "$mount" == "true" ]] && mount && exit
-if [[ "$backup" == "true" && "$sync" == "true" ]]; then # Run backup and sync at the same time
+if [[ "$number_of_backups" -gt 1 && "$sync" == "true" ]]; then # Run backup and sync at the same time
     echo "🅃 🄰 🅂 🄺 🅂 :"
     echo -e "-Backing up ix-applications Dataset\n-Syncing catalog(s)"
     echo -e "This can take a LONG time, Please Wait For Both Output..\n\n"
     backup &
     sync &
     wait
-elif [[ "$backup" == "true" && -z "$sync" ]]; then # If only backup is true, run it
+elif [[ "$number_of_backups" -gt 1 && -z "$sync" ]]; then # If only backup is true, run it
     echo "🅃 🄰 🅂 🄺 :"
     echo -e "-Backing up \"ix-applications\" Dataset\nPlease Wait..\n\n"
     backup
-elif [[ "$sync" == "true" && -z "$backup" ]]; then # If only sync is true, run it
+elif [[ "$sync" == "true" && -z "$number_of_backups" ]]; then # If only sync is true, run it
     echo "🅃 🄰 🅂 🄺 :"
     echo -e "Syncing Catalog(s)\nThis Takes a LONG Time, Please Wait..\n\n"
     sync