julian/scale-catalog
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #17 from Heavybullets8/general_improvements

Browse Source 
General improvements
This commit is contained in:
Heavybullets8 2022-08-31 03:55:25 +00:00 committed by GitHub
commit 151c075372
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 155 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -27,7 +27,7 @@
|--------------- |------------------------ |----------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| -U | -U <br>-U 5 | None or Integer | Update applications, ignoring major version changes<br>_Optionally, you can supply a number after the argument to update multiple applications at once_ |
| -u | -u<br>-u 5 | None or Integer | Update applications, do NOT update if there was a major version change<br>_Optionally, you can supply a number after the argument to update multiple applications at once_ |
| -b | -b 14 | Integer | Backup `ix-appliactions` dataset<br>_Creates backups up to the number you've chosen_ |
| -b | -b 14 | Integer | Backup `ix-applications` dataset<br>_Creates backups up to the number you've chosen_ |
| -i | -i nextcloud -i sonarr | String | Applications listed will be ignored during updating<br>_List one application after another as shown in the example_ |
| -r | -r | None | Monitors applications after they update<br>If the app does not become "ACTIVE" after either:<br>The custom Timeout, or Default Timeout,<br>rollback the application. |
| -v | -v | None | Verbose Output<br>_Look at the bottom of this page for an example_ |

3
functions/backup.sh
View File

@ -30,7 +30,6 @@ echo
export -f backup
deleteBackup(){
while true
do
@ -170,4 +169,4 @@ do
done
done
}
export -f restore
export -f restore

5
functions/menu.sh
View File

@ -1,5 +1,6 @@
#!/bin/bash
menu(){
clear -x
title
@ -31,7 +32,6 @@ case $selection in
;;
4)
read -rt 120 -p "What is the maximun number of backups you would like?: " number_of_backups || { echo -e "\nFailed to make a selection in time" ; exit; }
backup="true"
;;
5)
restore
@ -45,7 +45,6 @@ case $selection in
8)
script_create
;;
9)
cmd_to_container
;;
@ -55,4 +54,4 @@ case $selection in
esac
echo
}
export -f menu
export -f menu

3
functions/misc.sh
View File

@ -15,6 +15,7 @@ echo
}
export -f sync
prune(){
echo -e "🄿 🅁 🅄 🄽 🄴"
echo "Pruned Docker Images"
@ -22,6 +23,7 @@ docker image prune -af | grep "^Total" || echo "Failed to Prune Docker Images"
}
export -f prune
title(){
echo ' _ _ _____ _ _ '
echo '| | | | / ___| (_) | | '
@ -36,6 +38,7 @@ echo
}
export -f title
help(){
[[ $help == "true" ]] && clear -x

5
functions/mount.sh
View File

@ -1,5 +1,6 @@
#!/bin/bash
mount(){
while true
do
@ -50,7 +51,7 @@ do
mapfile -t full_path < <(zfs list | grep "$volume_name" | awk '{print $1}')
if [[ "${#full_path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup.
echo "$app is using the same volume identifier on more than one pool.. attempting to use your current kubernetes apps pool"
pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")
full_path=$(zfs list | grep "$volume_name" | grep "$pool" | awk '{print $1}')
fi
echo -e "\nMounting\n$full_path\nTo\n/mnt/heavyscript/$data_name"
@ -89,7 +90,7 @@ do
mapfile -t path < <(find /mnt/*/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-)
if [[ "${#path[@]}" -gt 1 ]]; then #if there is another app with the same name on another pool, use the current pools application, since the other instance is probably old, or unused, or a backup.
echo "$i is a name used on more than one pool.. attempting to use your current kubernetes apps pool"
pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")
full_path=$(find /mnt/"$pool"/ix-applications/releases/"$app"/volumes/ -maxdepth 0 | cut -c 6-)
zfs set mountpoint=legacy "$full_path""$pvc"
echo "$i unmounted" && rmdir /mnt/heavyscript/"$i" || echo "failed to unmount $i"

1
functions/script_create.sh
View File

@ -1,5 +1,6 @@
#!/bin/bash
script_create(){
while true
do

4
functions/self_update.sh
View File

@ -1,9 +1,11 @@
#!/bin/bash
args=("$@")
self_update() {
git fetch --tags &>/dev/null
git fetch --tags &>/dev/null
git reset --hard
latest_ver=$(git describe --tags "$(git rev-list --tags --max-count=1)")
echo "🅂 🄴 🄻 🄵"
echo "🅄 🄿 🄳 🄰 🅃 🄴"

268
functions/update_apps.sh
View File

@ -9,7 +9,6 @@ echo "Asynchronous Updates: $update_limit"
[[ -z $timeout ]] && echo "Default Timeout: 500" && timeout=500 || echo "Custom Timeout: $timeout"
[[ "$timeout" -le 120 ]] && echo "Warning: Your timeout is set low and may lead to premature rollbacks or skips"
pool=$(cli -c 'app kubernetes config' | grep -E "dataset\s\|" | awk -F '|' '{print $3}' | awk -F '/' '{print $1}' | tr -d " \t\n\r")
rm external_services 2>/dev/null # TODO remove later
it=0
while_count=0
rm deploying 2>/dev/null
@ -42,14 +41,9 @@ do
if [[ "$proc_count" -ge "$update_limit" ]]; then
sleep 3
elif [[ $it -lt ${#array[@]} ]]; then
# loop=0
# until [[ $loop -ge 2 || $it -ge ${#array[@]} ]];
# do
pre_process "${array[$it]}" &
processes+=($!)
((it++))
# ((loop++))
# done
elif [[ $proc_count != 0 || $(wc -l finished 2>/dev/null | awk '{ print $1 }') -lt "${#array[@]}" ]]; then # Wait for all processes to finish
sleep 3
else # All processes must be completed, break out of loop
@ -78,6 +72,8 @@ diff_chart=$(diff <(echo "$old_chart_ver") <(echo "$new_chart_ver")) #caluclatin
old_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $4}') #Upgraded From
new_full_ver=$(echo "${array[$it]}" | awk -F ',' '{print $5}') #Upraded To
rollback_version=$(echo "${array[$it]}" | awk -F ',' '{print $4}' | awk -F '_' '{print $2}')
# Skip update if application previously failed on this exact update version
if grep -qs "^$app_name," failed 2>/dev/null; then
failed_ver=$(grep "^$app_name," failed | awk -F ',' '{print $2}')
if [[ "$failed_ver" == "$new_full_ver" ]] ; then
@ -90,6 +86,7 @@ if grep -qs "^$app_name," failed 2>/dev/null; then
fi
fi
# Check if app is external services, append outcome to external_services file
[[ ! -e external_services ]] && touch external_services
if ! grep -qs "^$app_name," external_services ; then
if ! grep -qs "/external-service" /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/"$(find /mnt/"$pool"/ix-applications/releases/"$app_name"/charts/ -maxdepth 1 -type d -printf '%P\n' | sort -r | head -n 1)"/Chart.yaml; then
@ -99,6 +96,7 @@ if ! grep -qs "^$app_name," external_services ; then
fi
fi
# If user is using -S, stop app prior to updating
echo_array+=("\n$app_name")
if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # Check to see if user is using -S or not
[[ "$verbose" == "true" ]] && echo_array+=("Stopping prior to update..")
@ -110,6 +108,8 @@ if [[ $stop_before_update == "true" && "$startstatus" != "STOPPED" ]]; then # C
return 1
fi
fi
# Send app through update function
[[ "$verbose" == "true" ]] && echo_array+=("Updating..")
if update_app ;then
echo_array+=("Updated\n$old_full_ver\n$new_full_ver")
@ -118,16 +118,145 @@ else
echo_array
return
fi
# If app is external services, do not send for post processing
if grep -qs "^$app_name,true" external_services ; then
echo_array
return
else
after_update_actions
post_process
fi
}
export -f pre_process
post_process(){
SECONDS=0
count=0
if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then
while true
do
# If app reports ACTIVE right away, assume its a false positive and wait for it to change, or trust it after 5 updates to all_app_status
status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
[[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
before_loop=$(head -n 1 all_app_status)
current_loop=0
until [[ "$status" != "ACTIVE" || $current_loop -gt 4 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
do
status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
sleep 1
if ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then
before_loop=$(head -n 1 all_app_status)
((current_loop++))
fi
done
fi
(( count++ ))
if [[ "$status" == "ACTIVE" ]]; then
if [[ "$startstatus" == "STOPPED" ]]; then
[[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
break
else
echo_array+=("Active")
break
fi
elif [[ "$SECONDS" -ge "$timeout" ]]; then
if [[ $rollback == "true" ]]; then
if [[ $old_full_ver == "$new_full_ver" ]]; then
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("This is the result of a container image update..")
echo_array+=("Reverting is not possible, Abandoning")
echo_array
return 1
elif [[ "$failed" != "true" ]]; then
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("If this is a slow starting application, set a higher timeout with -t")
echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
echo_array+=("Reverting update..")
echo "$app_name,$new_full_ver" >> failed
if rollback_app ; then
echo_array+=("Rolled Back")
else
echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
echo_array
return 1
fi
failed="true"
SECONDS=0
count=0
continue #run back post_process function if the app was stopped prior to update
else
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("The application failed to be ACTIVE even after a rollback")
echo_array+=("Manual intervention is required\nStopping, then Abandoning")
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
break
fi
else
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("If this is a slow starting application, set a higher timeout with -t")
echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
echo_array+=("Manual intervention is required\nStopping, then Abandoning")
echo "$app_name,$new_full_ver" >> failed
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
break
fi
else
[[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
sleep 5
continue
fi
done
fi
echo_array
}
export -f post_process
rollback_app(){
count=0
app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
while [[ $app_update_avail == "false" ]]
do
app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
return 1
elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
before_loop=$(head -n 1 all_app_status)
((count++))
until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
do
sleep 1
done
else
break
fi
done
}
update_app(){
current_loop=0
while true
@ -181,129 +310,6 @@ done
export -f stop_app
after_update_actions(){
SECONDS=0
count=0
if [[ $rollback == "true" || "$startstatus" == "STOPPED" ]]; then
while true
do
status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
if [[ $count -lt 1 && $status == "ACTIVE" && "$(grep "^$app_name," deploying 2>/dev/null | awk -F ',' '{print $2}')" != "DEPLOYING" ]]; then # If status shows up as Active or Stopped on the first check, verify that. Otherwise it may be a false report..
[[ "$verbose" == "true" ]] && echo_array+=("Verifying $status..")
before_loop=$(head -n 1 all_app_status)
current_loop=0
until [[ "$status" != "ACTIVE" || $current_loop -gt 4 ]] # Wait for a specific change to app status, or 3 refreshes of the file to go by.
do
status=$(grep "^$app_name," all_app_status | awk -F ',' '{print $2}')
sleep 1
if ! echo -e "$(head -n 1 all_app_status)" | grep -qs ^"$before_loop" ; then
before_loop=$(head -n 1 all_app_status)
((current_loop++))
fi
done
fi
(( count++ ))
if [[ "$status" == "ACTIVE" ]]; then
if [[ "$startstatus" == "STOPPED" ]]; then
[[ "$verbose" == "true" ]] && echo_array+=("Returing to STOPPED state..")
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
break
else
echo_array+=("Active")
break
fi
elif [[ "$SECONDS" -ge "$timeout" ]]; then
if [[ $rollback == "true" ]]; then
if [[ $old_full_ver == "$new_full_ver" ]]; then
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("This is the result of a container image update..")
echo_array+=("Reverting is not possible, Abandoning")
echo_array
return 1
elif [[ "$failed" != "true" ]]; then
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("If this is a slow starting application, set a higher timeout with -t")
echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
echo_array+=("Reverting update..")
echo "$app_name,$new_full_ver" >> failed
if rollback_app ; then
echo_array+=("Rolled Back")
else
echo_array+=("Error: Failed to rollback $app_name\nAbandoning")
echo_array
return 1
fi
failed="true"
SECONDS=0
count=0
continue #run back after_update_actions function if the app was stopped prior to update
else
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("The application failed to be ACTIVE even after a rollback")
echo_array+=("Manual intervention is required\nStopping, then Abandoning")
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
break
fi
else
echo_array+=("Error: Run Time($SECONDS) for $app_name has exceeded Timeout($timeout)")
echo_array+=("If this is a slow starting application, set a higher timeout with -t")
echo_array+=("If this applicaion is always DEPLOYING, you can disable all probes under the Healthcheck Probes Liveness section in the edit configuration")
echo_array+=("Manual intervention is required\nStopping, then Abandoning")
echo "$app_name,$new_full_ver" >> failed
if stop_app ; then
echo_array+=("Stopped")
else
echo_array+=("Error: Failed to stop $app_name")
echo_array
return 1
fi
break
fi
else
[[ "$verbose" == "true" ]] && echo_array+=("Waiting $((timeout-SECONDS)) more seconds for $app_name to be ACTIVE")
sleep 5
continue
fi
done
fi
echo_array
}
export -f after_update_actions
rollback_app(){
count=0
app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
while [[ $app_update_avail == "false" ]]
do
app_update_avail=$(grep "^$app_name," all_app_status | awk -F ',' '{print $3}')
if [[ $count -gt 2 ]]; then # If failed to rollback app 3 times, return failure to parent shell
return 1
elif ! cli -c "app chart_release rollback release_name=\"$app_name\" rollback_options={\"item_version\": \"$rollback_version\"}" &> /dev/null ; then
before_loop=$(head -n 1 all_app_status)
((count++))
until [[ $(head -n 1 all_app_status) != "$before_loop" ]] # Upon failure, wait for status update before continuing
do
sleep 1
done
else
break
fi
done
}
echo_array(){
#Dump the echo_array, ensures all output is in a neat order.
for i in "${echo_array[@]}"
@ -318,4 +324,4 @@ export -f echo_array
final_check(){
[[ ! -e finished ]] && touch finished
echo "$app_name,finished" >> finished
}
}

16
heavy_script.sh
View File

@ -1,5 +1,6 @@
#!/bin/bash
# cd to script, this ensures the script can find the source scripts below, even when ran from a seperate directory
script=$(readlink -f "$0")
script_path=$(dirname "$script")
@ -20,13 +21,10 @@ source functions/cmd_to_container.sh
source functions/script_create.sh
#If no argument is passed, kill the script.
[[ -z "$*" || "-" == "$*" || "--" == "$*" ]] && menu
# Parse script options
while getopts ":si:rb:t:uUpSRv-:" opt
do
@ -68,7 +66,6 @@ do
number_of_backups=$OPTARG
! [[ $OPTARG =~ ^[0-9]+$ ]] && echo -e "Error: -b needs to be assigned an interger\n\"""$number_of_backups""\" is not an interger" >&2 && exit
[[ "$number_of_backups" -le 0 ]] && echo "Error: Number of backups is required to be at least 1" && exit
backup="true"
;;
r)
rollback="true"
@ -121,10 +118,6 @@ do
v)
verbose="true"
;;
\?)
echo -e "Invalid Option \"-$OPTARG\"\n"
help
;;
*)
echo -e "Invalid Option \"-$OPTARG\"\n"
help
@ -133,7 +126,6 @@ do
done
#exit if incompatable functions are called
[[ "$update_all_apps" == "true" && "$update_apps" == "true" ]] && echo -e "-U and -u cannot BOTH be called" && exit
@ -145,18 +137,18 @@ done
[[ "$dns" == "true" ]] && dns && exit
[[ "$restore" == "true" ]] && restore && exit
[[ "$mount" == "true" ]] && mount && exit
if [[ "$backup" == "true" && "$sync" == "true" ]]; then # Run backup and sync at the same time
if [[ "$number_of_backups" -gt 1 && "$sync" == "true" ]]; then # Run backup and sync at the same time
echo "🅃 🄰 🅂 🄺 🅂 :"
echo -e "-Backing up ix-applications Dataset\n-Syncing catalog(s)"
echo -e "This can take a LONG time, Please Wait For Both Output..\n\n"
backup &
sync &
wait
elif [[ "$backup" == "true" && -z "$sync" ]]; then # If only backup is true, run it
elif [[ "$number_of_backups" -gt 1 && -z "$sync" ]]; then # If only backup is true, run it
echo "🅃 🄰 🅂 🄺 :"
echo -e "-Backing up \"ix-applications\" Dataset\nPlease Wait..\n\n"
backup
elif [[ "$sync" == "true" && -z "$backup" ]]; then # If only sync is true, run it
elif [[ "$sync" == "true" && -z "$number_of_backups" ]]; then # If only sync is true, run it
echo "🅃 🄰 🅂 🄺 :"
echo -e "Syncing Catalog(s)\nThis Takes a LONG Time, Please Wait..\n\n"
sync