scale-catalog/functions/backup.sh

#!/bin/bash


backup(){
echo_backup+=("🄱 🄰 🄲 🄺 🅄 🄿 🅂")
echo_backup+=("Number of backups was set to $number_of_backups")
date=$(date '+%Y_%m_%d_%H_%M_%S')
[[ "$verbose" == "true" ]] && cli -c 'app kubernetes backup_chart_releases backup_name=''"'HeavyScript_"$date"'"' &> /dev/null && echo_backup+=(HeavyScript_"$date")
[[ -z "$verbose" ]] && echo_backup+=("\nNew Backup Name:") && cli -c 'app kubernetes backup_chart_releases backup_name=''"'HeavyScript_"$date"'"' | tail -n 1 &> /dev/null && echo_backup+=(HeavyScript_"$date")
mapfile -t list_backups < <(cli -c 'app kubernetes list_backups' | grep -E "HeavyScript_|TrueTool_" | sort -t '_' -Vr -k2,7 | awk -F '|'  '{print $2}'| tr -d " \t\r")
if [[  ${#list_backups[@]}  -gt  "$number_of_backups" ]]; then
    echo_backup+=("\nDeleted the oldest backup(s) for exceeding limit:")
    overflow=$(( ${#list_backups[@]} - "$number_of_backups" ))
    mapfile -t list_overflow < <(cli -c 'app kubernetes list_backups' | grep -E "HeavyScript_|TrueTool_"  | sort -t '_' -V -k2,7 | awk -F '|'  '{print $2}'| tr -d " \t\r" | head -n "$overflow")
    for i in "${list_overflow[@]}"
    do
        cli -c 'app kubernetes delete_backup backup_name=''"'"$i"'"' &> /dev/null || echo_backup+=("Failed to delete $i")
        echo_backup+=("$i")
    done
fi

#Dump the echo_array, ensures all output is in a neat order. 
for i in "${echo_backup[@]}"
do
    echo -e "$i"
done
echo
echo
}
export -f backup


deleteBackup(){
clear -x && echo "pulling all restore points.."
list_backups=$(cli -c 'app kubernetes list_backups' | sort -t '_' -Vr -k2,7 | tr -d " \t\r"  | awk -F '|'  '{print $2}' | nl -s ") " | column -t)
if [[ -z "$list_backups" ]]; then
    echo "No restore points available"
    exit
fi

#Select a restore point
while true
do
    clear -x
    title
    echo -e "Choose a Restore Point to Delete\nThese may be out of order if they are not HeavyScript backups"
    echo "$list_backups"
    echo
    echo "0)  Exit"
    read -rt 240 -p "Please type a number: " selection || { echo -e "\nFailed to make a selection in time" ; exit; }
    restore_point=$(echo "$list_backups" | grep ^"$selection)" | awk '{print $2}')
    if [[ $selection == 0 ]]; then
        echo "Exiting.." 
        exit
    elif [[ -z "$selection" ]]; then 
        echo "Your selection cannot be empty"
        sleep 3
        continue
    elif [[ -z "$restore_point" ]]; then
        echo "Invalid Selection: $selection, was not an option"
        sleep 3
        continue
    fi
    break # Break out of the loop if all of the If statement checks above are untrue
done

#Confirm deletion
while true
do
    clear -x
    echo -e "WARNING:\nYou CANNOT go back after deleting your restore point" 
    echo -e "\n\nYou have chosen:\n$restore_point\n\n"
    read -rt 120 -p "Would you like to proceed with deletion? (y/N): " yesno  || { echo -e "\nFailed to make a selection in time" ; exit; }
    case $yesno in
        [Yy] | [Yy][Ee][Ss])
            echo -e "\nDeleting $restore_point"
            cli -c 'app kubernetes delete_backup backup_name=''"'"$restore_point"'"' &>/dev/null || { echo "Failed to delete backup.."; exit; }
            echo "Sucessfully deleted"
            break
            ;;
        [Nn] | [Nn][Oo])
            echo "Exiting"
            exit
            ;;
        *)
            echo "That was not an option, try again"
            sleep 3
            continue
            ;;
    esac
done

#Check if there are more backups to delete
while true
do
    read -rt 120 -p "Delete more backups? (y/N): " yesno || { echo -e "\nFailed to make a selection in time" ; exit; }
    case $yesno in
        [Yy] | [Yy][Ee][Ss])
            break
            ;;
        [Nn] | [Nn][Oo])
            exit
            ;;
        *)
            echo "$yesno was not an option, try again" 
            sleep 2
            continue
            ;;

    esac

done
}
export -f deleteBackup


restore(){
clear -x && echo "pulling restore points.."
list_backups=$(cli -c 'app kubernetes list_backups' | grep -E "HeavyScript_|TrueTool_" | sort -t '_' -Vr -k2,7 | tr -d " \t\r"  | awk -F '|'  '{print $2}' | nl -s ") " | column -t)
if [[ -z "$list_backups" ]]; then
    echo "No HeavyScript restore points available"
    exit
fi

#Select a restore point
while true
do
    clear -x
    title
    echo "Choose a Restore Point"
    echo "$list_backups"
    echo
    echo "0)  Exit"
    read -rt 240 -p "Please type a number: " selection || { echo -e "\nFailed to make a selection in time" ; exit; }
    restore_point=$(echo "$list_backups" | grep ^"$selection)" | awk '{print $2}')
    if [[ $selection == 0 ]]; then
        echo "Exiting.." 
        exit
    elif [[ -z "$selection" ]]; then 
        echo "Your selection cannot be empty"
        sleep 3
        continue
    elif [[ -z "$restore_point" ]]; then
        echo "Invalid Selection: $selection, was not an option"
        sleep 3
        continue
    fi
    break
done


## Check to see if empty PVC data is present in any of the applications ##

# Find all pv_info.json files two subfolders deep with the restore point name
pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")
files=$(find "$(find /mnt/"$pool"/ix-applications/backups -maxdepth 0 )" -name pv_info.json | grep "$restore_point");

# Iterate over the list of files
for file in $files; do
    # Check if the file only contains {} subfolders
    contents=$(cat "$file")
    if [[ "$contents" == '{}' ]]; then
        # Print the file if it meets the criterion
        file=$(echo "$file" | awk -F '/' '{print $7}')
        borked_array+=("${file}")
    fi
done


# Grab applications that are supposed to have PVC data
mapfile -t apps_with_pvc < <(k3s kubectl get pvc -A | sort -u | awk '{print $1 "\t" $2 "\t" $4}' | sed "s/^0/ /" | awk '{print $1}' | cut -c 4-)


# Iterate over the list of applications with empty PVC data
# Unset the application if it is not supposed to have PVC data
index=0
for app in "${borked_array[@]}"; do
    if ! printf '%s\0' "${apps_with_pvc[@]}" | grep -iFxqz "${app}" ; then
        unset "borked_array[$index]"
    else
        borked=True
    fi
    index+=1
done



# If there is still empty PVC data, exit
if [[ $borked == True ]]; then
    echo "Warning!:"
    echo "The following applications have empty PVC data:"
    for app in "${borked_array[@]}"; do
        echo -e "$app"
    done
    echo "We have no choice but to exit"
    echo "If you were to restore, you would lose all of your application data"
    echo "If you are on Bluefin version: 22.12.0, and have not yet ran the patch, you will need to run it"
    echo "Afterwards you will be able to create backups and restore them"
    echo "This is a known ix-systems bug, and has nothing to do with HeavyScript"
    exit
fi



## Check the restore point, and ensure it is the same version as the current system ##
# Boot Query
boot_query=$(cli -m csv -c 'system bootenv query created,realname')

# Get the date of system version and when it was updated
current_version=$(cli -m csv -c 'system version' | awk -F '-' '{print $3}')
when_updated=$(echo "$boot_query" | grep "$current_version",\
| awk -F ',' '{print $2}' | sed 's/[T|-]/_/g' | sed 's/:/_/g' | awk -F '_' '{print $1 $2 $3 $4 $5}')

# Get the date of the chosen restore point
restore_point_date=$(echo "$restore_point" | awk -F '_' '{print $2 $3 $4 $5 $6}' | tr -d "_")

# Grab previous version
previous_version=$(echo "$boot_query" | sort -nr | grep -A 1 "$current_version," | tail -n 1)

# Compare the dates
while (("$restore_point_date" < "$when_updated" ))
do
    clear -x
    echo "The restore point you have chosen is from an older version of Truenas Scale"
    echo "This is not recommended, as it may cause issues with the system"
    echo "Either that, or your systems date is incorrect.."
    echo
    echo "Current SCALE Information:"
    echo "Version:       $current_version"
    echo "When Updated:  $(echo "$restore_point" | awk -F '_' '{print $2 "-" $3 "-" $4}')"
    echo
    echo "Restore Point SCALE Information:"
    echo "Version:       $(echo "$previous_version" | awk -F ',' '{print $1}')"
    echo "When Updated:  $(echo "$previous_version" | awk -F ',' '{print $2}' | awk -F 'T' '{print $1}')"
    echo
    read -rt 120 -p "Would you like to proceed? (y/N): " yesno || { echo -e "\nFailed to make a selection in time" ; exit; }
        case $yesno in
            [Yy] | [Yy][Ee][Ss])
                echo "Proceeding.."
                sleep 3
                break
                ;;
            [Nn] | [Nn][Oo])
                echo "Exiting"
                exit
                ;;
            *)
                echo "That was not an option, try again"
                sleep 3
                continue
                ;;
        esac
done


#Confirm restore
while true
do
    clear -x
    echo -e "WARNING:\nThis is NOT guranteed to work\nThis is ONLY supposed to be used as a LAST RESORT\nConsider rolling back your applications instead if possible"
    echo -e "\n\nYou have chosen:\n$restore_point\n\n"
    read -rt 120 -p "Would you like to proceed with restore? (y/N): " yesno || { echo -e "\nFailed to make a selection in time" ; exit; }
    case $yesno in
        [Yy] | [Yy][Ee][Ss])
            pool=$(cli -c 'app kubernetes config' | grep -E "pool\s\|" | awk -F '|' '{print $3}' | tr -d " \t\n\r")

            # Set mountpoints to legacy prior to restore, ensures correct properties for the are set
            echo -e "\nSetting correct ZFS properties for application volumes.."
            for pvc in $(zfs list -t filesystem -r "$pool"/ix-applications/releases -o name -H | grep "volumes/pvc")
            do
                if zfs set mountpoint=legacy "$pvc"; then
                    echo "Success for - \"$pvc\""
                else
                    echo "Error: Setting properties for \"$pvc\", failed.."
                fi
            done

            # Ensure readonly is turned off
            if ! zfs set readonly=off "$pool"/ix-applications;then
                echo -e "Error: Failed to set ZFS ReadOnly to \"off\""
                echo -e "After the restore, attempt to run the following command manually:"
                echo "zfs set readonly=off $pool/ix-applications"
            fi

            echo "Finished setting properties.."

            # Beginning snapshot restore
            echo -e "\nStarting restore, this will take a LONG time."
            if ! cli -c 'app kubernetes restore_backup backup_name=''"'"$restore_point"'"'; then
                echo "Restore failed, exiting.."
                exit 1
            fi
            exit
            ;;
        [Nn] | [Nn][Oo])
            echo "Exiting"
            exit
            ;;
        *)
            echo "That was not an option, try again"
            sleep 3
            continue
            ;;
    esac
done

}
export -f restore