From fda0b7ca882be2ba2c8df23c3bf60df16093f4cc Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Thu, 25 Aug 2022 22:16:44 -0600 Subject: [PATCH 1/5] git reset hard to force tagged updates --- functions/self_update.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/self_update.sh b/functions/self_update.sh index 9c60d2b5..a0ee1d01 100644 --- a/functions/self_update.sh +++ b/functions/self_update.sh @@ -3,7 +3,8 @@ args=("$@") self_update() { -git fetch --tags &>/dev/null +git fetch --tags &>/dev/null +git reset --hard latest_ver=$(git describe --tags "$(git rev-list --tags --max-count=1)") echo "🅂 🄴 🄻 🄵" echo "🅄 🄿 🄳 🄰 🅃 🄴" From 72930e2f14a8ec39328849e62eb56a88f3ce7c5b Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Thu, 25 Aug 2022 22:39:26 -0600 Subject: [PATCH 2/5] spacing, function explanation --- functions/backup.sh | 3 +- functions/menu.sh | 4 +- functions/misc.sh | 3 + functions/mount.sh | 1 + functions/script_create.sh | 1 + functions/self_update.sh | 1 + functions/update_apps.sh | 131 +++++++++++++++++++------------------ heavy_script.sh | 9 +-- 8 files changed, 77 insertions(+), 76 deletions(-) diff --git a/functions/backup.sh b/functions/backup.sh index 642b58f5..0ca69069 100644 --- a/functions/backup.sh +++ b/functions/backup.sh @@ -30,7 +30,6 @@ echo export -f backup - deleteBackup(){ while true do @@ -170,4 +169,4 @@ do done done } -export -f restore +export -f restore \ No newline at end of file diff --git a/functions/menu.sh b/functions/menu.sh index 5b86ca5f..a8677314 100644 --- a/functions/menu.sh +++ b/functions/menu.sh @@ -1,5 +1,6 @@ #!/bin/bash + menu(){ clear -x title @@ -45,7 +46,6 @@ case $selection in 8) script_create ;; - 9) cmd_to_container ;; @@ -55,4 +55,4 @@ case $selection in esac echo } -export -f menu +export -f menu \ No newline at end of file diff --git a/functions/misc.sh b/functions/misc.sh index 33b4d4c0..ebff4ff8 100644 --- a/functions/misc.sh +++ b/functions/misc.sh @@ -15,6 +15,7 @@ echo } export -f sync + prune(){ echo -e "🄿 🅁 🅄 🄽 🄴" echo "Pruned Docker Images" @@ -22,6 +23,7 @@ docker image prune -af | grep "^Total" || echo "Failed to Prune Docker Images" } export -f prune + title(){ echo ' _ _ _____ _ _ ' echo '| | | | / ___| (_) | | ' @@ -36,6 +38,7 @@ echo } export -f title + help(){ [[ $help == "true" ]] && clear -x diff --git a/functions/mount.sh b/functions/mount.sh index 26198633..c915407e 100644 --- a/functions/mount.sh +++ b/functions/mount.sh @@ -1,5 +1,6 @@ #!/bin/bash + mount(){ while true do diff --git a/functions/script_create.sh b/functions/script_create.sh index b1448218..de761cc6 100644 --- a/functions/script_create.sh +++ b/functions/script_create.sh @@ -1,5 +1,6 @@ #!/bin/bash + script_create(){ while true do diff --git a/functions/self_update.sh b/functions/self_update.sh index a0ee1d01..144157d4 100644 --- a/functions/self_update.sh +++ b/functions/self_update.sh @@ -1,5 +1,6 @@ #!/bin/bash + args=("$@") self_update() { diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 61d670a6..50629e2e 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -9,7 +9,6 @@ echo "Asynchronous Updates: $update_limit" [[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout" [[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips" pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r") -rm external_services 2>/dev/null # TODO remove later it=0 while_count=0 rm deploying 2>/dev/null @@ -42,14 +41,9 @@ do if [[ "$proc_count" -ge "$update_limit" ]]; then sleep 3 elif [[ $it -lt ${#array[@]} ]]; then - # loop=0 - # until [[ $loop -ge 2 || $it -ge ${#array[@]} ]]; - # do pre_process "${array[$it]}" & processes+=($!) ((it++)) - # ((loop++)) - # done elif [[ $proc_count != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]]; then # Wait for all processes to finish sleep 3 else # All processes must be completed, break out of loop @@ -78,6 +72,8 @@ diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclatin old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}') + +# Skip update if application previously failed on this exact update version if grep -qs "^$app_name," failed 2>/dev/null; then failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}') if [[ "$failed_ver" == "$new_full_ver" ]] ; then @@ -90,6 +86,7 @@ if grep -qs "^$app_name," failed 2>/dev/null; then fi fi +# Check if app is external services, append outcome to external_services file [[ ! -e external_services ]] && touch external_services if ! grep -qs "^$app_name," external_services ; then if ! grep -qs "/external-service" /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/"$(find /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/ -maxdepth 1 -type d -printf '%P\n' | sort -r | head -n 1)"/Chart.yaml; then @@ -99,6 +96,7 @@ if ! grep -qs "^$app_name," external_services ; then fi fi +# If user is using -S, stop app prior to updating echo_array+=("\n$app_name") if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # Check to see if user is using -S or not [[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..") @@ -110,6 +108,8 @@ if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # C return 1 fi fi + +# Send app through update function [[ "$verbose" == "true" ]] && echo_array+=("Updating..") if update_app ;then echo_array+=("Updated\n$old_full_ver\n$new_full_ver") @@ -118,70 +118,19 @@ else echo_array return fi + +# If app is external services, do not send for post processing if grep -qs "^$app_name,true" external_services ; then echo_array return else - after_update_actions + post_process fi } export -f pre_process -update_app(){ -current_loop=0 -while true -do - update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}') - if [[ $update_avail =~ "true" ]]; then - if ! cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null ; then - before_loop=$(head -n 1 all_app_status) - current_loop=0 - until [[ "$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}')" != "$update_avail" ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by. - do - if [[ $current_loop -gt 2 ]]; then - cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null || return 1 # After waiting, attempt an update once more, if fails, return error code - elif ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then # The file has been updated, but nothing changed specifically for the app. - before_loop=$(head -n 1 all_app_status) - ((current_loop++)) - fi - sleep 1 - done - fi - break - elif [[ ! $update_avail =~ "true" ]]; then - break - else - sleep 3 - fi -done -} -export -f update_app - - -stop_app(){ -count=0 -while [[ "$status" != "STOPPED" ]] -do - status=$( grep "^$app_name," all_app_status | awk -F ',' '{print $2}') - if [[ $count -gt 2 ]]; then # If failed to stop app 3 times, return failure to parent shell - return 1 - elif ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' &> /dev/null ; then - before_loop=$(head -n 1 all_app_status) - ((count++)) - until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing - do - sleep 1 - done - else - break - fi -done -} -export -f stop_app - - -after_update_actions(){ +post_process(){ SECONDS=0 count=0 if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then @@ -242,7 +191,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then failed="true" SECONDS=0 count=0 - continue #run back after_update_actions function if the app was stopped prior to update + continue #run back post_process function if the app was stopped prior to update else echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)") echo_array+=("The application failed to be ACTIVE even after a rollback") @@ -280,7 +229,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then fi echo_array } -export -f after_update_actions +export -f post_process rollback_app(){ @@ -304,6 +253,60 @@ do done } + +update_app(){ +current_loop=0 +while true +do + update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}') + if [[ $update_avail =~ "true" ]]; then + if ! cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null ; then + before_loop=$(head -n 1 all_app_status) + current_loop=0 + until [[ "$(grep "^$app_name," all_app_status | awk -F ',' '{print $3","$6}')" != "$update_avail" ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by. + do + if [[ $current_loop -gt 2 ]]; then + cli -c 'app chart_release upgrade release_name=''"'"$app_name"'"' &> /dev/null || return 1 # After waiting, attempt an update once more, if fails, return error code + elif ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then # The file has been updated, but nothing changed specifically for the app. + before_loop=$(head -n 1 all_app_status) + ((current_loop++)) + fi + sleep 1 + done + fi + break + elif [[ ! $update_avail =~ "true" ]]; then + break + else + sleep 3 + fi +done +} +export -f update_app + + +stop_app(){ +count=0 +while [[ "$status" != "STOPPED" ]] +do + status=$( grep "^$app_name," all_app_status | awk -F ',' '{print $2}') + if [[ $count -gt 2 ]]; then # If failed to stop app 3 times, return failure to parent shell + return 1 + elif ! cli -c 'app chart_release scale release_name='\""$app_name"\"\ 'scale_options={"replica_count": 0}' &> /dev/null ; then + before_loop=$(head -n 1 all_app_status) + ((count++)) + until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing + do + sleep 1 + done + else + break + fi +done +} +export -f stop_app + + echo_array(){ #Dump the echo_array, ensures all output is in a neat order. for i in "${echo_array[@]}" @@ -318,4 +321,4 @@ export -f echo_array final_check(){ [[ ! -e finished ]] && touch finished echo "$app_name,finished" >> finished -} +} \ No newline at end of file diff --git a/heavy_script.sh b/heavy_script.sh index 7a687b51..6858ed4d 100644 --- a/heavy_script.sh +++ b/heavy_script.sh @@ -1,5 +1,6 @@ #!/bin/bash + # cd to script, this ensures the script can find the source scripts below, even when ran from a seperate directory script=$(readlink -f "$0") script_path=$(dirname "$script") @@ -20,13 +21,10 @@ source functions/cmd_to_container.sh source functions/script_create.sh - - #If no argument is passed, kill the script. [[ -z "$*" || "-" == "$*" || "--" == "$*" ]] && menu - # Parse script options while getopts ":si:rb:t:uUpSRv-:" opt do @@ -121,10 +119,6 @@ do v) verbose="true" ;; - \?) - echo -e "Invalid Option \"-$OPTARG\"\n" - help - ;; *) echo -e "Invalid Option \"-$OPTARG\"\n" help @@ -133,7 +127,6 @@ do done - #exit if incompatable functions are called [[ "$update_all_apps" == "true" && "$update_apps" == "true" ]] && echo -e "-U and -u cannot BOTH be called" && exit From f77ffd65430755f4a6ab401b340e9b56035b6021 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Thu, 25 Aug 2022 22:40:38 -0600 Subject: [PATCH 3/5] README typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 49ded17b..8d4a3bfe 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ |--------------- |------------------------ |----------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | -U | -U
-U 5 | None or Integer | Update applications, ignoring major version changes
_Optionally, you can supply a number after the argument to update multiple applications at once_ | | -u | -u
-u 5 | None or Integer | Update applications, do NOT update if there was a major version change
_Optionally, you can supply a number after the argument to update multiple applications at once_ | -| -b | -b 14 | Integer | Backup `ix-appliactions` dataset
_Creates backups up to the number you've chosen_ | +| -b | -b 14 | Integer | Backup `ix-applications` dataset
_Creates backups up to the number you've chosen_ | | -i | -i nextcloud -i sonarr | String | Applications listed will be ignored during updating
_List one application after another as shown in the example_ | | -r | -r | None | Monitors applications after they update
If the app does not become "ACTIVE" after either:
The custom Timeout, or Default Timeout,
rollback the application. | | -v | -v | None | Verbose Output
_Look at the bottom of this page for an example_ | From c0f6ed10a83133a8f3dd520fe5089cf4c1ae44f1 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Tue, 30 Aug 2022 21:36:34 -0600 Subject: [PATCH 4/5] Documentation, remove redundant code, spacing --- functions/menu.sh | 1 - functions/mount.sh | 4 ++-- functions/update_apps.sh | 3 +++ heavy_script.sh | 5 ++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/functions/menu.sh b/functions/menu.sh index a8677314..921b64f6 100644 --- a/functions/menu.sh +++ b/functions/menu.sh @@ -32,7 +32,6 @@ case $selection in ;; 4) read -rt 120 -p "What is the maximun number of backups you would like?: " number_of_backups || { echo -e "\nFailed to make a selection in time" ; exit; } - backup="true" ;; 5) restore diff --git a/functions/mount.sh b/functions/mount.sh index c915407e..c58f8571 100644 --- a/functions/mount.sh +++ b/functions/mount.sh @@ -51,7 +51,7 @@ do mapfile -t full_path < <(zfs list | grep "$volume_name" | awk '{print $1}') if [[ "${#full_path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup. echo "$app is using the same volume identifier on more than one pool.. attempting to use your current kubernetes apps pool" - pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r") + pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r") full_path=$(zfs list | grep "$volume_name" | grep "$pool" | awk '{print $1}') fi echo -e "\nMounting\n$full_path\nTo\n/mnt/heavyscript/$data_name" @@ -90,7 +90,7 @@ do mapfile -t path < <(find /mnt/*/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-) if [[ "${#path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup. echo "$i is a name used on more than one pool.. attempting to use your current kubernetes apps pool" - pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r") + pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r") full_path=$(find /mnt/"$pool"/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-) zfs set mountpoint=legacy "$full_path""$pvc" echo "$i unmounted" && rmdir /mnt/heavyscript/"$i" || echo "failed to unmount $i" diff --git a/functions/update_apps.sh b/functions/update_apps.sh index 50629e2e..ce6a4476 100644 --- a/functions/update_apps.sh +++ b/functions/update_apps.sh @@ -136,6 +136,8 @@ count=0 if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then while true do + + # If app reports ACTIVE right away, assume its a false positive and wait for it to change, or trust it after 5 updates to all_app_status status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}') if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report.. [[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..") @@ -152,6 +154,7 @@ if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then done fi (( count++ )) + if [[ "$status" == "ACTIVE" ]]; then if [[ "$startstatus" == "STOPPED" ]]; then [[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..") diff --git a/heavy_script.sh b/heavy_script.sh index 6858ed4d..76d03a9a 100644 --- a/heavy_script.sh +++ b/heavy_script.sh @@ -66,7 +66,6 @@ do number_of_backups=$OPTARG ! [[ $OPTARG =~ ^[0-9]+$ ]] && echo -e "Error: -b needs to be assigned an interger\n\"""$number_of_backups""\" is not an interger" >&2 && exit [[ "$number_of_backups" -le 0 ]] && echo "Error: Number of backups is required to be at least 1" && exit - backup="true" ;; r) rollback="true" @@ -138,14 +137,14 @@ done [[ "$dns" == "true" ]] && dns && exit [[ "$restore" == "true" ]] && restore && exit [[ "$mount" == "true" ]] && mount && exit -if [[ "$backup" == "true" && "$sync" == "true" ]]; then # Run backup and sync at the same time +if [[ "$backup" -gt 1 && "$sync" == "true" ]]; then # Run backup and sync at the same time echo "🅃 🄰 🅂 🄺 🅂 :" echo -e "-Backing up ix-applications Dataset\n-Syncing catalog(s)" echo -e "This can take a LONG time, Please Wait For Both Output..\n\n" backup & sync & wait -elif [[ "$backup" == "true" && -z "$sync" ]]; then # If only backup is true, run it +elif [[ "$backup" -gt 1 && -z "$sync" ]]; then # If only backup is true, run it echo "🅃 🄰 🅂 🄺 :" echo -e "-Backing up \"ix-applications\" Dataset\nPlease Wait..\n\n" backup From a406406002ffc8668edd121bdf9f7b5382792bb6 Mon Sep 17 00:00:00 2001 From: Heavybullets8 Date: Tue, 30 Aug 2022 21:40:08 -0600 Subject: [PATCH 5/5] change backup variable --- heavy_script.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/heavy_script.sh b/heavy_script.sh index 76d03a9a..d669aac8 100644 --- a/heavy_script.sh +++ b/heavy_script.sh @@ -137,18 +137,18 @@ done [[ "$dns" == "true" ]] && dns && exit [[ "$restore" == "true" ]] && restore && exit [[ "$mount" == "true" ]] && mount && exit -if [[ "$backup" -gt 1 && "$sync" == "true" ]]; then # Run backup and sync at the same time +if [[ "$number_of_backups" -gt 1 && "$sync" == "true" ]]; then # Run backup and sync at the same time echo "🅃 🄰 🅂 🄺 🅂 :" echo -e "-Backing up ix-applications Dataset\n-Syncing catalog(s)" echo -e "This can take a LONG time, Please Wait For Both Output..\n\n" backup & sync & wait -elif [[ "$backup" -gt 1 && -z "$sync" ]]; then # If only backup is true, run it +elif [[ "$number_of_backups" -gt 1 && -z "$sync" ]]; then # If only backup is true, run it echo "🅃 🄰 🅂 🄺 :" echo -e "-Backing up \"ix-applications\" Dataset\nPlease Wait..\n\n" backup -elif [[ "$sync" == "true" && -z "$backup" ]]; then # If only sync is true, run it +elif [[ "$sync" == "true" && -z "$number_of_backups" ]]; then # If only sync is true, run it echo "🅃 🄰 🅂 🄺 :" echo -e "Syncing Catalog(s)\nThis Takes a LONG Time, Please Wait..\n\n" sync