diff --git a/README.md b/README.md
index 49ded17b..8d4a3bfe 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@
|--------------- |------------------------ |----------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| -U | -U
-U 5 | None or Integer | Update applications, ignoring major version changes
_Optionally, you can supply a number after the argument to update multiple applications at once_ |
| -u | -u
-u 5 | None or Integer | Update applications, do NOT update if there was a major version change
_Optionally, you can supply a number after the argument to update multiple applications at once_ |
-| -b | -b 14 | Integer | Backup `ix-appliactions` dataset
_Creates backups up to the number you've chosen_ |
+| -b | -b 14 | Integer | Backup `ix-applications` dataset
_Creates backups up to the number you've chosen_ |
| -i | -i nextcloud -i sonarr | String | Applications listed will be ignored during updating
_List one application after another as shown in the example_ |
| -r | -r | None | Monitors applications after they update
If the app does not become "ACTIVE" after either:
The custom Timeout, or Default Timeout,
rollback the application. |
| -v | -v | None | Verbose Output
_Look at the bottom of this page for an example_ |
diff --git a/functions/backup.sh b/functions/backup.sh
index 642b58f5..0ca69069 100644
--- a/functions/backup.sh
+++ b/functions/backup.sh
@@ -30,7 +30,6 @@ echo
export -f backup
-
deleteBackup(){
while true
do
@@ -170,4 +169,4 @@ do
done
done
}
-export -f restore
+export -f restore
\ No newline at end of file
diff --git a/functions/menu.sh b/functions/menu.sh
index 5b86ca5f..921b64f6 100644
--- a/functions/menu.sh
+++ b/functions/menu.sh
@@ -1,5 +1,6 @@
#!/bin/bash
+
menu(){
clear -x
title
@@ -31,7 +32,6 @@ case $selection in
;;
4)
read -rt 120 -p "What is the maximun number of backups you would like?: " number_of_backups || { echo -e "\nFailed to make a selection in time" ; exit; }
- backup="true"
;;
5)
restore
@@ -45,7 +45,6 @@ case $selection in
8)
script_create
;;
-
9)
cmd_to_container
;;
@@ -55,4 +54,4 @@ case $selection in
esac
echo
}
-export -f menu
+export -f menu
\ No newline at end of file
diff --git a/functions/misc.sh b/functions/misc.sh
index 33b4d4c0..ebff4ff8 100644
--- a/functions/misc.sh
+++ b/functions/misc.sh
@@ -15,6 +15,7 @@ echo
}
export -f sync
+
prune(){
echo -e "🄿 🅁 🅄 🄽 🄴"
echo "Pruned Docker Images"
@@ -22,6 +23,7 @@ docker image prune -af | grep "^Total" || echo "Failed to Prune Docker Images"
}
export -f prune
+
title(){
echo ' _ _ _____ _ _ '
echo '| | | | / ___| (_) | | '
@@ -36,6 +38,7 @@ echo
}
export -f title
+
help(){
[[ $help == "true" ]] && clear -x
diff --git a/functions/mount.sh b/functions/mount.sh
index 26198633..c58f8571 100644
--- a/functions/mount.sh
+++ b/functions/mount.sh
@@ -1,5 +1,6 @@
#!/bin/bash
+
mount(){
while true
do
@@ -50,7 +51,7 @@ do
mapfile -t full_path < <(zfs list | grep "$volume_name" | awk '{print $1}')
if [[ "${#full_path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup.
echo "$app is using the same volume identifier on more than one pool.. attempting to use your current kubernetes apps pool"
- pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
+ pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")
full_path=$(zfs list | grep "$volume_name" | grep "$pool" | awk '{print $1}')
fi
echo -e "\nMounting\n$full_path\nTo\n/mnt/heavyscript/$data_name"
@@ -89,7 +90,7 @@ do
mapfile -t path < <(find /mnt/*/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-)
if [[ "${#path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup.
echo "$i is a name used on more than one pool.. attempting to use your current kubernetes apps pool"
- pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
+ pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")
full_path=$(find /mnt/"$pool"/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-)
zfs set mountpoint=legacy "$full_path""$pvc"
echo "$i unmounted" && rmdir /mnt/heavyscript/"$i" || echo "failed to unmount $i"
diff --git a/functions/script_create.sh b/functions/script_create.sh
index b1448218..de761cc6 100644
--- a/functions/script_create.sh
+++ b/functions/script_create.sh
@@ -1,5 +1,6 @@
#!/bin/bash
+
script_create(){
while true
do
diff --git a/functions/self_update.sh b/functions/self_update.sh
index 9c60d2b5..144157d4 100644
--- a/functions/self_update.sh
+++ b/functions/self_update.sh
@@ -1,9 +1,11 @@
#!/bin/bash
+
args=("$@")
self_update() {
-git fetch --tags &>/dev/null
+git fetch --tags &>/dev/null
+git reset --hard
latest_ver=$(git describe --tags "$(git rev-list --tags --max-count=1)")
echo "🅂 🄴 🄻 🄵"
echo "🅄 🄿 🄳 🄰 🅃 🄴"
diff --git a/functions/update_apps.sh b/functions/update_apps.sh
index 61d670a6..ce6a4476 100644
--- a/functions/update_apps.sh
+++ b/functions/update_apps.sh
@@ -9,7 +9,6 @@ echo "Asynchronous Updates: $update_limit"
[[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout"
[[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips"
pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
-rm external_services 2>/dev/null # TODO remove later
it=0
while_count=0
rm deploying 2>/dev/null
@@ -42,14 +41,9 @@ do
if [[ "$proc_count" -ge "$update_limit" ]]; then
sleep 3
elif [[ $it -lt ${#array[@]} ]]; then
- # loop=0
- # until [[ $loop -ge 2 || $it -ge ${#array[@]} ]];
- # do
pre_process "${array[$it]}" &
processes+=($!)
((it++))
- # ((loop++))
- # done
elif [[ $proc_count != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]]; then # Wait for all processes to finish
sleep 3
else # All processes must be completed, break out of loop
@@ -78,6 +72,8 @@ diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclatin
old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From
new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To
rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}')
+
+# Skip update if application previously failed on this exact update version
if grep -qs "^$app_name," failed 2>/dev/null; then
failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}')
if [[ "$failed_ver" == "$new_full_ver" ]] ; then
@@ -90,6 +86,7 @@ if grep -qs "^$app_name," failed 2>/dev/null; then
fi
fi
+# Check if app is external services, append outcome to external_services file
[[ ! -e external_services ]] && touch external_services
if ! grep -qs "^$app_name," external_services ; then
if ! grep -qs "/external-service" /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/"$(find /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/ -maxdepth 1 -type d -printf '%P\n' | sort -r | head -n 1)"/Chart.yaml; then
@@ -99,6 +96,7 @@ if ! grep -qs "^$app_name," external_services ; then
fi
fi
+# If user is using -S, stop app prior to updating
echo_array+=("\n$app_name")
if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # Check to see if user is using -S or not
[[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..")
@@ -110,6 +108,8 @@ if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # C
return 1
fi
fi
+
+# Send app through update function
[[ "$verbose" == "true" ]] && echo_array+=("Updating..")
if update_app ;then
echo_array+=("Updated\n$old_full_ver\n$new_full_ver")
@@ -118,16 +118,145 @@ else
echo_array
return
fi
+
+# If app is external services, do not send for post processing
if grep -qs "^$app_name,true" external_services ; then
echo_array
return
else
- after_update_actions
+ post_process
fi
}
export -f pre_process
+post_process(){
+SECONDS=0
+count=0
+if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then
+ while true
+ do
+
+ # If app reports ACTIVE right away, assume its a false positive and wait for it to change, or trust it after 5 updates to all_app_status
+ status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
+ if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
+ [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
+ before_loop=$(head -n 1 all_app_status)
+ current_loop=0
+ until [[ "$status" != "ACTIVE" || $current_loop -gt 4 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
+ do
+ status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
+ sleep 1
+ if ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then
+ before_loop=$(head -n 1 all_app_status)
+ ((current_loop++))
+ fi
+ done
+ fi
+ (( count++ ))
+
+ if [[ "$status" == "ACTIVE" ]]; then
+ if [[ "$startstatus" == "STOPPED" ]]; then
+ [[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
+ if stop_app ; then
+ echo_array+=("Stopped")
+ else
+ echo_array+=("Error: Failed to stop $app_name")
+ echo_array
+ return 1
+ fi
+ break
+ else
+ echo_array+=("Active")
+ break
+ fi
+ elif [[ "$SECONDS" -ge "$timeout" ]]; then
+ if [[ $rollback == "true" ]]; then
+ if [[ $old_full_ver == "$new_full_ver" ]]; then
+ echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+ echo_array+=("This is the result of a container image update..")
+ echo_array+=("Reverting is not possible, Abandoning")
+ echo_array
+ return 1
+ elif [[ "$failed" != "true" ]]; then
+ echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+ echo_array+=("If this is a slow starting application, set a higher timeout with -t")
+ echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
+ echo_array+=("Reverting update..")
+ echo "$app_name,$new_full_ver" >> failed
+ if rollback_app ; then
+ echo_array+=("Rolled Back")
+ else
+ echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
+ echo_array
+ return 1
+ fi
+ failed="true"
+ SECONDS=0
+ count=0
+ continue #run back post_process function if the app was stopped prior to update
+ else
+ echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+ echo_array+=("The application failed to be ACTIVE even after a rollback")
+ echo_array+=("Manual intervention is required\nStopping, then Abandoning")
+ if stop_app ; then
+ echo_array+=("Stopped")
+ else
+ echo_array+=("Error: Failed to stop $app_name")
+ echo_array
+ return 1
+ fi
+ break
+ fi
+ else
+ echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
+ echo_array+=("If this is a slow starting application, set a higher timeout with -t")
+ echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
+ echo_array+=("Manual intervention is required\nStopping, then Abandoning")
+ echo "$app_name,$new_full_ver" >> failed
+ if stop_app ; then
+ echo_array+=("Stopped")
+ else
+ echo_array+=("Error: Failed to stop $app_name")
+ echo_array
+ return 1
+ fi
+ break
+ fi
+ else
+ [[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
+ sleep 5
+ continue
+ fi
+ done
+fi
+echo_array
+}
+export -f post_process
+
+
+rollback_app(){
+count=0
+app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
+while [[ $app_update_avail == "false" ]]
+do
+ app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
+ if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
+ return 1
+ elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
+ before_loop=$(head -n 1 all_app_status)
+ ((count++))
+ until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
+ do
+ sleep 1
+ done
+ else
+ break
+ fi
+done
+}
+
+
update_app(){
current_loop=0
while true
@@ -181,129 +310,6 @@ done
export -f stop_app
-after_update_actions(){
-SECONDS=0
-count=0
-if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then
- while true
- do
- status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
- if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
- [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
- before_loop=$(head -n 1 all_app_status)
- current_loop=0
- until [[ "$status" != "ACTIVE" || $current_loop -gt 4 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
- do
- status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
- sleep 1
- if ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then
- before_loop=$(head -n 1 all_app_status)
- ((current_loop++))
- fi
- done
- fi
- (( count++ ))
- if [[ "$status" == "ACTIVE" ]]; then
- if [[ "$startstatus" == "STOPPED" ]]; then
- [[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
- if stop_app ; then
- echo_array+=("Stopped")
- else
- echo_array+=("Error: Failed to stop $app_name")
- echo_array
- return 1
- fi
- break
- else
- echo_array+=("Active")
- break
- fi
- elif [[ "$SECONDS" -ge "$timeout" ]]; then
- if [[ $rollback == "true" ]]; then
- if [[ $old_full_ver == "$new_full_ver" ]]; then
- echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
- echo_array+=("This is the result of a container image update..")
- echo_array+=("Reverting is not possible, Abandoning")
- echo_array
- return 1
- elif [[ "$failed" != "true" ]]; then
- echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
- echo_array+=("If this is a slow starting application, set a higher timeout with -t")
- echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
- echo_array+=("Reverting update..")
- echo "$app_name,$new_full_ver" >> failed
- if rollback_app ; then
- echo_array+=("Rolled Back")
- else
- echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
- echo_array
- return 1
- fi
- failed="true"
- SECONDS=0
- count=0
- continue #run back after_update_actions function if the app was stopped prior to update
- else
- echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
- echo_array+=("The application failed to be ACTIVE even after a rollback")
- echo_array+=("Manual intervention is required\nStopping, then Abandoning")
- if stop_app ; then
- echo_array+=("Stopped")
- else
- echo_array+=("Error: Failed to stop $app_name")
- echo_array
- return 1
- fi
- break
- fi
- else
- echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
- echo_array+=("If this is a slow starting application, set a higher timeout with -t")
- echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
- echo_array+=("Manual intervention is required\nStopping, then Abandoning")
- echo "$app_name,$new_full_ver" >> failed
- if stop_app ; then
- echo_array+=("Stopped")
- else
- echo_array+=("Error: Failed to stop $app_name")
- echo_array
- return 1
- fi
- break
- fi
- else
- [[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
- sleep 5
- continue
- fi
- done
-fi
-echo_array
-}
-export -f after_update_actions
-
-
-rollback_app(){
-count=0
-app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
-while [[ $app_update_avail == "false" ]]
-do
- app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
- if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
- return 1
- elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
- before_loop=$(head -n 1 all_app_status)
- ((count++))
- until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
- do
- sleep 1
- done
- else
- break
- fi
-done
-}
-
echo_array(){
#Dump the echo_array, ensures all output is in a neat order.
for i in "${echo_array[@]}"
@@ -318,4 +324,4 @@ export -f echo_array
final_check(){
[[ ! -e finished ]] && touch finished
echo "$app_name,finished" >> finished
-}
+}
\ No newline at end of file
diff --git a/heavy_script.sh b/heavy_script.sh
index 7a687b51..d669aac8 100644
--- a/heavy_script.sh
+++ b/heavy_script.sh
@@ -1,5 +1,6 @@
#!/bin/bash
+
# cd to script, this ensures the script can find the source scripts below, even when ran from a seperate directory
script=$(readlink -f "$0")
script_path=$(dirname "$script")
@@ -20,13 +21,10 @@ source functions/cmd_to_container.sh
source functions/script_create.sh
-
-
#If no argument is passed, kill the script.
[[ -z "$*" || "-" == "$*" || "--" == "$*" ]] && menu
-
# Parse script options
while getopts ":si:rb:t:uUpSRv-:" opt
do
@@ -68,7 +66,6 @@ do
number_of_backups=$OPTARG
! [[ $OPTARG =~ ^[0-9]+$ ]] && echo -e "Error: -b needs to be assigned an interger\n\"""$number_of_backups""\" is not an interger" >&2 && exit
[[ "$number_of_backups" -le 0 ]] && echo "Error: Number of backups is required to be at least 1" && exit
- backup="true"
;;
r)
rollback="true"
@@ -121,10 +118,6 @@ do
v)
verbose="true"
;;
- \?)
- echo -e "Invalid Option \"-$OPTARG\"\n"
- help
- ;;
*)
echo -e "Invalid Option \"-$OPTARG\"\n"
help
@@ -133,7 +126,6 @@ do
done
-
#exit if incompatable functions are called
[[ "$update_all_apps" == "true" && "$update_apps" == "true" ]] && echo -e "-U and -u cannot BOTH be called" && exit
@@ -145,18 +137,18 @@ done
[[ "$dns" == "true" ]] && dns && exit
[[ "$restore" == "true" ]] && restore && exit
[[ "$mount" == "true" ]] && mount && exit
-if [[ "$backup" == "true" && "$sync" == "true" ]]; then # Run backup and sync at the same time
+if [[ "$number_of_backups" -gt 1 && "$sync" == "true" ]]; then # Run backup and sync at the same time
echo "🅃 🄰 🅂 🄺 🅂 :"
echo -e "-Backing up ix-applications Dataset\n-Syncing catalog(s)"
echo -e "This can take a LONG time, Please Wait For Both Output..\n\n"
backup &
sync &
wait
-elif [[ "$backup" == "true" && -z "$sync" ]]; then # If only backup is true, run it
+elif [[ "$number_of_backups" -gt 1 && -z "$sync" ]]; then # If only backup is true, run it
echo "🅃 🄰 🅂 🄺 :"
echo -e "-Backing up \"ix-applications\" Dataset\nPlease Wait..\n\n"
backup
-elif [[ "$sync" == "true" && -z "$backup" ]]; then # If only sync is true, run it
+elif [[ "$sync" == "true" && -z "$number_of_backups" ]]; then # If only sync is true, run it
echo "🅃 🄰 🅂 🄺 :"
echo -e "Syncing Catalog(s)\nThis Takes a LONG Time, Please Wait..\n\n"
sync