#! /bin/bash
# vim: shiftwidth=4 tabstop=4 expandtab

# shellcheck source=/dev/null
source "$LIB_DIR/helpers"

#
# Logging
#

declare -rA LOG_LEVELS=( [TRACE]=1 [DEBUG]=2 [INFO]=3 [WARNING]=4 [ERROR]=5 [FATAL]=5 )
declare -rA LOG_LEVELS_COLOR=(
    [TRACE]=${COLORS[purple]} [DEBUG]=${COLORS[blue]} [INFO]=${COLORS[cyan]}
    [WARNING]=${COLORS[yellow]} [ERROR]=${COLORS[red]}
    [FATAL]="${TEXT_STYLES[bold]};${COLORS[red]}"
)
DEFAULT_LOG_LEVEL=WARNING

# Initialize log level variables, console logging & quiet mode
LOG_LEVEL=$DEFAULT_LOG_LEVEL
CONSOLE_LOG=0
QUIET_MODE=0

function log_enable_quiet_mode() {
    if [[ -z "$LOG" ]]; then
        # No log file, just redirect stdout to /dev/null
        exec >> /dev/null
    else
        # Redirect stdout to log file and stderr to both log file and stderr
        exec >> >(sed "s/^/$(date '+%Y-%m-%d %H:%M:%S') - STDOUT - /" >> "$LOG")
        exec 2> >(sed "s/^/$(date '+%Y-%m-%d %H:%M:%S') - STDERR - /" >> "$LOG")
    fi
    QUIET_MODE=1
}

# Check if console log is active
function console_log() {
    { [[ -z "$LOG" ]] || [[ $CONSOLE_LOG -eq 1 ]]; } && [[ $QUIET_MODE -ne 1 ]] && return 0
    return 1
}

function log(){
    local level=${1^^}
    shift
    if [[ "${LOG_LEVELS[$level]:-null}" == "null" ]]; then
        log ERROR "Invalid log level '$level' supplied, used WARNING as default."
        log WARNING "$@"
        return
    fi
    [[ ${LOG_LEVELS[$level]} -lt ${LOG_LEVELS[$LOG_LEVEL]} ]] && return

    # Compute message
    local msg
    msg="$(date '+%Y-%m-%d %H:%M:%S') - $level - $( implode " " "$@" )"

    # Log to log file
    if [[ -n "$LOG" ]]; then
        if [[ -e "$LOG" ]] && [[ ! -w "$LOG" ]]; then
            echo "FATAL - Log file '$LOG' is not writable" >&2
            exit 3
        fi
        if [[ ! -e "$LOG" ]] && [[ ! -w "$(dirname "$LOG")" ]]; then
            echo "FATAL - Log directory '$( basename "$LOG")' is not writable" >&2
            exit 3
        fi
        # Log file exists
        echo -e "$msg" >> "$LOG"
    fi

    # Log to console
    # Note: always on fatal error and otherwise if no log file or console log and
    # never if quiet mode is enabled.
    if console_log || [[ "$level" == "FATAL" ]]; then
        # Use colors if terminal support it and if we are in interactive mode
        [[ -n "$TERM" ]] && [[ -t 1 ]] && \
            msg="$(date '+%Y-%m-%d %H:%M:%S') - \e[${LOG_LEVELS_COLOR[$level]}m$level\e[${RESET_STYLES[all]}m - " \
            msg+="$( implode " " "$@" )"

        # Displays on STDERR from warning messages
        if [[ ${LOG_LEVELS[$level]} -ge ${LOG_LEVELS["WARNING"]} ]]; then
            [[ "$level" == "FATAL" ]] && echo -e "\n\n" >&2
            echo -e "$msg" >&2
        else
            echo -e "$msg"
        fi
    fi

    # Exit on fatal error
    if [[ "$level" == "FATAL" ]]; then
        [[ -n "$TMP" ]] && [[ -e "$TMP" ]] && rm -f "$TMP"
        log_stop
        exit 3
    fi
}

# Retro-compatibility
function debug() { log DEBUG "$@"; }
function error() { log ERROR "$@"; }
function fatal_error() { log FATAL "$@"; }

function _log_command_name() {
    local names=( "$COMMAND" )
    if [[ -n "$SUBCOMMAND" ]]; then
        names+=( "$SUBCOMMAND" )
        [[ "$COMMAND" == "cluster" ]] && [[ "$SUBCOMMAND" == "backup" ]] && \
            [[ "${SUBCOMMAND_ARGS[0]:-null}" != "null" ]] && \
            names+=( "${SUBCOMMAND_ARGS[0]}" )
        [[ "$COMMAND" == "snapshot" ]] && [[ "$SUBCOMMAND" == "scheduled" ]] && \
            [[ "${SUBCOMMAND_ARGS[0]:-null}" != "null" ]] && \
            names+=( "${SUBCOMMAND_ARGS[0]}" )
    fi
    implode " " "${names[@]}"
}

function log_start() {
    log INFO "-------------------- Start $(_log_command_name) ----------------------"
    START_TIME=$(date +%s)
}

function log_stop() {
    [[ -z "$START_TIME" ]] &&  return
    local duration
    (( duration=$(date +%s)-START_TIME ))
    log INFO "Total duration: $(format_duration $duration)"
    log INFO "-------------------- End of $(_log_command_name) command -----------------------"
}

function log_sigint() {
    log WARNING "SIGINT detected"
    log_stop
    exit 1
}
trap log_sigint SIGINT

# Log and display a message to user (if console logging is inactive)
function display() {
    local log_msg
    mapfile -t log_msg < <( array_filter "$@" -- -n )
    log INFO "${log_msg[@]}"
    console_log || echo -e "$@"
}

#
# Loading configuration
#
CONFIG=/etc/eehyp-tools.conf
[ "$LIB_DIR" != "/usr/lib/eehyp-tools" ] && CONFIG="$LIB_DIR/../../..$CONFIG"
# shellcheck source=/dev/null
[ -e "$CONFIG" ] && CONFIG=$(realpath "$CONFIG") && source "$CONFIG"
CONFIG_DIR="$CONFIG.d"
if [[ -d "$CONFIG_DIR" ]] && [[ -n "$(ls "$CONFIG_DIR"/*.conf 2>/dev/null)" ]]; then
    for file in "$CONFIG_DIR"/*.conf; do
        # shellcheck source=/dev/null
        source "$file"
    done
fi

# Default configuration values
[ "${REAL_VIRSH:-null}" == "null" ] && REAL_VIRSH="/usr/bin/virsh"
[ "${AUTODETECT_HYPS_PATTERN:-null}" == "null" ] && \
    AUTODETECT_HYPS_PATTERN='^[\s0-9a-e\.\:]+\s+hyp-[a-z0-9]+-[0-9]+(\s+.*|$)'
[ "${CACHE_LIFETIME:-null}" == "null" ] && CACHE_LIFETIME=300
[ "${VOLUME_POOLS_PATTERN:-null}" == "null" ] && VOLUME_POOLS_PATTERN='^libvirt-pool'
[ "${DEFAULT_VOLUME_POOL:-null}" == "null" ] && DEFAULT_VOLUME_POOL="libvirt-pool"
[ "${RESTORE_OLD_VOLUMES_NAME_SUFFIX_FORMAT:-null}" == "null" ] && \
    RESTORE_OLD_VOLUMES_NAME_SUFFIX_FORMAT="OLD_RESTORE_$(date +%F_%H-%M)"
[ "${DEFAULT_LOCK_ID_FORMAT:-null}" == "null" ] && DEFAULT_LOCK_ID_FORMAT="manual_lock_on_%F_%H-%M"

[ "${WORKING_DAYS:-null}" == "null" ] && WORKING_DAYS=(1 2 3 4 5 6)
[ "${WORKING_START_HOUR:-null}" == "null" ] && WORKING_START_HOUR=8
[ "${WORKING_END_HOUR:-null}" == "null" ] && WORKING_END_HOUR=21
[ "${WORKING_HOURS_SNAPSHOT_REMOVAL_SIZE_LIMIT:-null}" == "null" ] && \
    WORKING_HOURS_SNAPSHOT_REMOVAL_SIZE_LIMIT=50

[ "${HYP_RESERVED_MEMORY:-null}" == "null" ] && HYP_RESERVED_MEMORY=4096
[ "${CLUSTER_WARNING_ALLOCATED_MEMORY_PERC:-null}" == "null" ] && \
    CLUSTER_WARNING_ALLOCATED_MEMORY_PERC=75
[ "${CLUSTER_CRITICAL_ALLOCATED_MEMORY_PERC:-null}" == "null" ] && \
    CLUSTER_CRITICAL_ALLOCATED_MEMORY_PERC=90

[ "${HYP_RESERVED_VCPUS:-null}" == "null" ] && HYP_RESERVED_VCPUS=1
[ "${VCPU_OVER_PROVISIONING:-null}" == "null" ] && VCPU_OVER_PROVISIONING=2
[ "${WARNING_CPU_THRESHOLD:-null}" == "null" ] && WARNING_CPU_THRESHOLD=5
[ "${CRITICAL_CPU_THRESHOLD:-null}" == "null" ] && CRITICAL_CPU_THRESHOLD=10

[ "${CACHE_DIRECTORY_PATH:-null}" == "null" ] && CACHE_DIRECTORY_PATH=/var/cache/eehyp-tools
[ "${LOG_DIRECTORY_PATH:-null}" == "null" ] && LOG_DIRECTORY_PATH=/var/log/eehyp-tools
[[ "${BACKUP_DIRECTORY_PATH:-null}" == "null" ]] && \
    BACKUP_DIRECTORY_PATH="/var/backups/eehyp-tools"

mkdir -p "$LOG_DIRECTORY_PATH" || log FATAL "Failed to create log directory ($LOG_DIRECTORY_PATH)."
LOG="$LOG_DIRECTORY_PATH/eehyp-tools.log"


function is_master() {
    [[ -z "$VIP" ]] && log FATAL "VIP not configured, can't determine if we are on master node."
    [[ $( /sbin/ip addr show | grep -Ec "inet\s+$VIP/" ) -eq 1 ]] && return 0
    return 1
}

#
# Bash IFS helpers
#

# Backup IFS variable to allow restoring it
IFS_BKP=$IFS
restore_ifs() { declare -g IFS=$IFS_BKP; }

set_newline_ifs() {
    declare -g IFS="
"
}

#
# Cache file helpers
#

# Compute cache file path from filename
# Note: if file exists, check its validity and drop it if outdated.
function get_cache_file_path() {
    local filename=$1
    [[ -z "$filename" ]] && log FATAL "get_cache_file_path: no filename specified"
    local output_var=$2
    [[ -z "$output_var" ]] && log FATAL "get_cache_file_path: no output variable specified"
    mkdir -p "$CACHE_DIRECTORY_PATH" || \
        log FATAL "Failed to create cache directory ($CACHE_DIRECTORY_PATH)"
    local filepath="$CACHE_DIRECTORY_PATH/$filename"
    if [[ $CACHE_LIFETIME -le 0 ]]; then
        debug "Cache disabled, drop '$filepath'"
        rm -f "$filepath"
    elif [[ -f "$filepath" ]]; then
        local cache_age
        (( cache_age=$(date +%s)-$(stat -c %Y "$filepath") ))
        if [[ $cache_age -gt $CACHE_LIFETIME ]]; then
            log DEBUG "Cache file $filepath is outdated, drop it " \
                "(age: $(format_duration $cache_age))."
            rm -f "$filepath"
        else
            log DEBUG "Cache file $filepath is still valid, use it " \
                "(age: $(format_duration $cache_age))."
        fi
    fi
    declare -g "$output_var=$filepath"
}

# shellcheck disable=SC2120
function clean_cache() {
    local filename_pattern=$1
    [[ -z "$filename_pattern" ]] && filename_pattern="vm_ls.*"
    log DEBUG "Clean cache files matching with pattern '$filename_pattern'"
    [[ ! -d "$CACHE_DIRECTORY_PATH" ]] && return 0
    run_external find "$CACHE_DIRECTORY_PATH" -name "$filename_pattern" -delete
}

#
# Working hours helpers
#

function in_working_days() {
    in_array "$(date '+%w')" "${WORKING_DAYS[@]}"
}

function in_working_hours() {
    in_working_days || return 1
    local current_hour
    current_hour="$(date '+%_H')"
    [[ $current_hour -ge $WORKING_START_HOUR ]] && [[ $current_hour -lt $WORKING_END_HOUR ]] && \
        return 0
    return 1
}


#
# Monitoring plugins performance data helpers
#
PERFDATAS="[]"
function add_perfdata() {
    PERFDATAS=$(
        jq ". += [
            $( jo label="$1" cur="$2" warn="$3" crit="$4" min="$5" max="$6" )
        ]" <<< "$PERFDATAS"
    )
}

function format_perfdata() {
    # Perf data: 'label'=value[UOM];[warn];[crit];[min];[max]
    jq -r $'[
        .[]
        | (
            [
                "\'" + .label + "\'=\" + (.cur|tostring),
                (.warn|tostring),
                (.crit|tostring),
                (.min|tostring),
                (.max|tostring)
            ] | join(";")
        )
    ]
    | join(", ")' <<< "$PERFDATAS"
}

#
# VM helpers
#

function _vm_command_path() {
    realpath "$LIB_DIR/../../sbin/vm"
}

function _list_vms() {
    if [[ $CACHE_LIFETIME -eq 0 ]]; then
        "$(_vm_command_path)" ls --all --no-cache "$@"
    else
        "$(_vm_command_path)" ls --all "$@"
    fi
}

function _list_vms_by_state() {
    _list_vms | grep -E "\s$1$"
}

function locate_as_() {
    _list_vms | grep -E "\s$1\s+$2$" | awk '{print $1}'
}

function locate_any() {
    local hyps
    hyps=$(_list_vms | awk '{print $1 " " $3}' | grep -E "\s$1$" | awk '{print $1}')
    [[ $(echo "$hyps"|wc -l) -gt 1 ]] && fatal_error \
        "VM $1 found on multiple hypervisors ($(echo "$hyps"|tr '\n' ','))," \
        "can't determine which one is correct."
    echo "$hyps"
}

function locate() {
    local vm=$1 state=$2 hyp
    case $state in
        running)
            locate_as_ "$vm" running
            ;;
        paused)
            locate_as_ "$vm" paused
            ;;
        *)
            hyp=$(locate_as_ "$vm" running)
            [[ -z "$hyp" ]] && hyp=$(locate_as_ "$vm" paused)
            [[ -z "$hyp" ]] && hyp=$(locate_any "$vm")
            echo "$hyp"
            ;;
    esac
}

function vm_state() {
    local vm=$1 hyp=$2 cmd
    cmd=( "$REAL_VIRSH" dominfo "$vm" )
    if [[ -n "$hyp" ]]; then
        check_hyp "$hyp" || { echo "unknown"; return 1; }
        # shellcheck disable=SC2029
        cmd=( ssh "$hyp" "${cmd[@]}" )
    fi
    LC_ALL=C "${cmd[@]}" | grep -E '^State:' | sed 's/^State: *//'
}

function _is() {
    local state_arg=$1 vm=$2 hyp=$3
    if [[ -n "$hyp" ]]; then
        check_hyp "$hyp" || return 0
        # shellcheck disable=SC2029
        [[ $(LC_ALL=C ssh "$hyp" "$REAL_VIRSH" list --name "$1"|grep -cE "^$vm$") -eq 1 ]] && return 0
    else
        [[ $(LC_ALL=C "$REAL_VIRSH" list --name "$state_arg"|grep -cE "^$vm$") -eq 1 ]] && return 0
    fi
    return 1
}

function is_running() { _is --state-running "$@" && return 0; return 1; }
function is_paused() { _is --state-paused "$@" && return 0; return 1; }
function is_stopped() { _is --state-shutoff "$@" && return 0; return 1; }
function is_inactive() { _is --inactive "$@" && return 0; return 1; }

function is_defined() {
    if [[ -n "$2" ]]; then
        check_hyp "$2" || return 0
        # shellcheck disable=SC2029
        LC_ALL=C ssh "$2" "$REAL_VIRSH" desc "$1" > /dev/null 2>&1
        return $?
    fi
    LC_ALL=C "$REAL_VIRSH" desc "$1" > /dev/null 2>&1
    return $?
}

function have_agent() {
    local vm=$1 hyp=$2
    if [[ -n "$hyp" ]]; then
        check_hyp "$hyp" || return 0
        # shellcheck disable=SC2029
        LC_ALL=C ssh "$hyp" "$REAL_VIRSH" guestvcpus "$vm" > /dev/null 2>&1
        return $?
    fi
    LC_ALL=C "$REAL_VIRSH" guestvcpus "$vm" > /dev/null 2>&1
    return $?
}

function wait_stopped() {
    local vm=$1
    local hyp=${2:-$(locate "$vm")}
    if [[ -n "$GENERATE_SCRIPT" ]]; then
        echo "echo -n 'Wait for the virtual machine $vm to stop" \
            "and press [enter] to continue'" >> "$GENERATE_SCRIPT"
        echo "read" >> "$GENERATE_SCRIPT"
        return 0
    elif [[ $JUST_TRY -eq 1 ]]; then
        [[ $AUTO -eq 1 ]] && log DEBUG "Just-try mode: assume VM $vm is stopped" && return 0
        echo -n "Just-try mode: wait until VM $vm is stopped. Press [enter] to continue..."
        read -r
        return 0
    fi
    display -n "Waiting for the virtual machine $vm to stop on $hyp"
    while ! is_stopped "$vm" "$hyp"; do
        console_log || echo -n "."
        sleep 1
    done
    console_log || echo
    display "Stopped."
}

function stop_and_wait_stopped() {
    local vm=$1
    local hyp=$2
    if [[ -n "$hyp" ]]; then
        display "Stop VM $vm on $hyp..."
        # shellcheck disable=SC2029
        run_external ssh "$hyp" "$REAL_VIRSH" shutdown "$vm" || fatal_error "Failed to stop VM $vm on $hyp!"
    else
        display "Stop VM $vm..."
        run_external "$REAL_VIRSH" shutdown "$vm" || fatal_error "Failed to stop VM $vm!"
    fi
    wait_stopped "$vm" "$hyp"
}

function list_vms() {
    local hyp="${1:-local}"
    local -a args=( list --name "${@:3}" )
    case "${2:-all}" in
        active)
            ;;
        inactive)
            args+=( "--inactive" )
            ;;
        running|paused|shutoff|other)
            args+=( "--state-$2" )
            ;;
        all)
            args+=( "--all" )
            ;;
        *)
            log FATAL "list_vms(): invalid status '$2'"
    esac
    if [[ -n "$hyp" ]] && [[ "$hyp" != "local" ]]; then
        # shellcheck disable=SC2029
        LC_ALL=C ssh "$hyp" "$REAL_VIRSH" "${args[@]}" 2>/dev/null | grep -v "^$"
    else
        LC_ALL=C "$REAL_VIRSH" "${args[@]}" 2>/dev/null "${@:3}" | grep -v "^$"
    fi
}

function check_snapshot() {
    local vm=$1 hyp=$2 snap_name=$3 vol snap
    for vol in $(list_volumes "$vm" "$hyp"); do
        for snap in $(rbd --format json snap ls "$vol" | jq -r '.[].name'); do
            [[ "$snap" == "$snap_name" ]] && return 0
        done
    done
    return 1
}

function default_snap_suffix() {
    local author
    author="${GIT_AUTHOR_EMAIL:-${GIT_COMMITTER_EMAIL:-${DEBEMAIL:-$( id -u -n)}}}"
    echo "manual-snapshot-by-${author/%@*/}"
}

function create_snapshot() {
    local vm=$1
    local hyp=$2
    local snap_name=${3:-$(date +%Y%m%d-%H%M%S)-$(default_snap_suffix)}
    local volumes_pattern=${4:-}
    local scheduled_mode=${5:-0}

    # In scheduled mode, we will need to have access to volume to VM mapping : pre-load it before
    # freezing VM's FS to reduce impact duration.
    [[ $scheduled_mode -eq 1 ]] && load_volume_to_vm_mapping

    local fsfreeze_method="none"
    if is_running "$vm" "$hyp"; then
        debug "VM $vm is active, we need to freeze it's filesystems."
        if have_agent "$vm" "$hyp"; then
            debug "Guest agent is installed, using domfsfreeze & domfsthaw commands..."
            fsfreeze_method="agent"
            confirm "Create snapshot $snap_name of the volumes of the running VM $vm:\n\n" \
                "/!\ We need to freeze it's filesystems using its detected guest-agent.\n\n" \
                "Do you want to continue" || exit 1
        else
            debug "Guest agent is not installed, we have to suspend/resume the VM."
            fsfreeze_method="suspend"
            confirm "Create snapshot $snap_name of the volumes of the running VM $vm:\n\n" \
                "/!\ We need to freeze it's filesystems BUT NO GUEST-AGENT seen to be running" \
                "on this VM.\n" \
                "We have to suspend this VM during the snapshot creation.\n\n" \
                "Do you want to continue" || exit 1
        fi
    else
        debug "VM $vm is inactive, not need to freeze FS"
        confirm "Create snapshot $snap_name of the volumes of the offline VM $vm" || exit 1
    fi

    if [[ "$fsfreeze_method" == "agent" ]]; then
        display "Freeze $vm filesystems..."
        # shellcheck disable=SC2029
        if ! run_external ssh "$hyp" "$REAL_VIRSH" domfsfreeze "$vm" &> /dev/null; then
            log WARNING "Failed to freeze filesystems of $vm!"
            confirm "Suspend/resume the VM as a fallback" || exit 1
            fsfreeze_method="suspend"
            display "Suspend $vm..."
            # shellcheck disable=SC2029
            if ! run_external ssh "$hyp" "$REAL_VIRSH" suspend "$vm"; then
                error "Failed to freeze and also failed to suspend $vm!"
                return 1
            fi
        fi
    elif [[ "$fsfreeze_method" == "suspend" ]]; then
        display "Suspend $vm..."
        # shellcheck disable=SC2029
        if ! run_external ssh "$hyp" "$REAL_VIRSH" suspend "$vm"; then
            error "Failed to suspend $vm!"
            return 1
        fi
    fi

    local return_code=0

    display "Create & protect snapshot $snap_name of $vm volumes..."
    for volume in $(list_volumes "$vm" "$hyp"); do
        if [[ -n "$volumes_pattern" ]] && ! check_regex "$volume" "$volumes_pattern"; then
            log DEBUG "Volume $volume excluded by specified pattern ($volumes_pattern)"
            continue
        fi
        if [[ $scheduled_mode -eq 1 ]]; then
            if is_shared_volume "$volume"; then
                read -ra volume_vms <<< "${SHARED_VOLUMES[$volume]}"
                if [[ "$vm" != "${volume_vms[0]}" ]]; then
                    display "- $volume is a shared by VMs $(implode ', ' "${volume_vms[@]}")," \
                        "managing its snapshots with only VM ${volume_vms[0]}"
                    continue
                else
                    log DEBUG "Volume $volume is a shared by VMs" \
                        "$( implode ', ' "${volume_vms[@]}" ) and VM $vm is the first:" \
                        "handle its snapshots"
                fi
            else
                log DEBUG "Volume $volume is not shared"
            fi
        fi
        display "- $volume"
        local result
        if [[ $scheduled_mode -eq 1 ]]; then
            run_external rbd snap create "$volume"@"$snap_name" 2> >(
                grep -Ev 'Creating snap:.*done.' >&2
            )
            result=$?
        else
            run_external rbd snap create "$volume"@"$snap_name"
            result=$?
        fi
        if [[ $result -ne 0 ]]; then
            error "Failed to create the snapshot $volume@$snap_name!"
            return_code=1
            continue
        fi
        if ! run_external rbd snap protect "$volume"@"$snap_name"; then
            error "Failed to protect the snapshot $volume@$snap_name!"
            return_code=1
        fi
    done
    if [[ $return_code -eq 0 ]]; then
        display "done."
    else
        display "done (but some errors occurred)."
    fi

    if [[ "$fsfreeze_method" == "agent" ]]; then
        display "Unfreeze $vm filesystems..."
        # shellcheck disable=SC2029
        if ! run_external ssh "$hyp" "$REAL_VIRSH" domfsthaw "$vm"; then
            error "Failed to unfreeze filesystems of $vm. Please do it manually by running" \
                "'virsh domfsthaw $vm'."
            return_code=1
        else
            display done.
        fi
    elif [[ "$fsfreeze_method" == "suspend" ]]; then
        display "Resume $vm..."
        # shellcheck disable=SC2029
        if ! run_external ssh "$hyp" "$REAL_VIRSH" resume "$vm"; then
            error "Failed to resume $vm. Please do it manually by running 'virsh resume $vm'."
            return_code=1
        else
            display done.
        fi
    fi
    return $return_code
}

function get_snapshots_info() {
    local vol=$1 output_var=$2 with_size=0 snaps
    [[ "$3" == "with-size" ]] && with_size=1

    # Retrieve snapshots main info
    # Note: in ceph 12, the protected key is not returned
    snaps=$(rbd snap ls "$vol" --format=json | \
        jq -r 'map(
            .timestamp |= (strptime("%a %b %d %H:%M:%S %Y") | strftime("%Y-%m-%d %H:%M:%S")) |
            .protected |= if . == "false" then false elif . == "true" then true else "unknown" end
        | { (.name): . }) | add'
    )
    [[ "$snaps" == "null" ]] && declare -g "$output_var={}" && return
    [[ $with_size -eq 0 ]] && declare -g "$output_var=$snaps" && return

    local snaps_size_data prev_size snap_name

    # Add snapshots "used_size" to main info
    # Note: in ceph 12, the id & snapshot_id keys are not returned
    snaps_size_data=$(rbd du "$vol" --format=json)
    # The size of the newer snapshot is the reported size of the source image
    prev_size=$(
        jq -r '.images[] | select(
            (has("snapshot") | not) and (has("snapshot_id") | not)
        ) | .used_size' <<< "$snaps_size_data"
    )
    for snap_name in $(
        jq -r '[ .[] ] | sort_by(.timestamp) | reverse | .[] | .name' <<< "$snaps"
    ); do
        check_int "$prev_size" || prev_size=null
        snaps=$(
            jq --arg snap_name "$snap_name" --argjson size "$prev_size" \
            '.[$snap_name].used_size = $size' <<< "$snaps"
        )
        # The size of the next snapshot is the size of this one
        prev_size=$(
            jq -r --arg snap_name "$snap_name" \
            '.images[]|select(.snapshot == $snap_name) | .used_size' <<< "$snaps_size_data"
        )
    done

    declare -g "$output_var=$snaps"
}

#
# Volumes helpers
#
function list_volume_pools() {
    local output_var=$1
    declare -ga "$output_var"
    mapfile -t "$output_var" < <( ceph osd pool ls | grep -E "$VOLUME_POOLS_PATTERN" )
}

function list_pool_volumes() {
    rbd ls "$1"
}

function list_volumes() {
    local vm=$1 hyp=$2 list
    if [[ -n "$hyp" ]]; then
        # shellcheck disable=SC2029
        list=$(ssh "$hyp" "$REAL_VIRSH" domblklist "$vm" --details 2>/dev/null)
    else
        list=$("$REAL_VIRSH" domblklist "$vm" --details 2>/dev/null)
    fi
    echo -e "$list" | grep -E '^\s*network.*disk' | awk '{print $4}'
}

function get_volume_size() {
    rbd info "$1"|grep size|sed 's/.*size \(.*\) in .*/\1/'
}

function get_volume_target_from_path() {
    local vm=$1 volume=$2 hyp=$3 list
    if [[ -n "$hyp" ]]; then
        # shellcheck disable=SC2029
        list=$(LC_ALL=C ssh "$hyp" "$REAL_VIRSH" domblklist "$vm" --details 2>/dev/null)
    else
        list=$(LC_ALL=C "$REAL_VIRSH" domblklist "$vm" --details 2>/dev/null)
    fi
    echo -e "$list" | grep -E '^\s*network.*disk'|grep -E "\s$volume$"|awk '{print $3}'
}

function check_volumes_pool() {
    check_regex "$1" "$VOLUME_POOLS_PATTERN" || return 1
    list_volume_pools POOLS
    in_array "$1" "${POOLS[@]}" && return 0
    return 1
}

# Build (or load from cache) volume to VM mapping:
# * volume to VM mapping is loaded in VOLUME_TO_VM associative array (key=volume RBD path, value=VM)
# * shared volumes mapping is loaded in SHARED_VOLUMES associative array (key=volume RBD path,
#   value=VMs list)
function load_volume_to_vm_mapping() {
    [[ "${VOLUME_TO_VM[*]:-null}" != "null" ]] && return
    local verbose=0
    [[ "$1" == "verbose" ]] && verbose=1
    local vms vms_state
    mapfile -t vms < <( _list_vms --name | awk '{print $2}' | sort -u )
    vms_state=$( implode $'\n' "${vms[@]}" | md5sum | awk '{print $1}' )
    get_cache_file_path \
        volume_to_vm."$vms_state".map MAP_FILENAME
    if [[ ! -f ${MAP_FILENAME} ]]; then
        if [[ $verbose -eq 1 ]]; then
            display "Generating volume to VM mapping..."
        else
            log INFO "Generating volume to VM mapping"
        fi
        declare -Ag VOLUME_TO_VM
        declare -Ag SHARED_VOLUMES
        local vm hyp volume
        for vm in "${vms[@]}"; do
            hyp=$(locate "$vm")
            [[ -z "$vm" ]] && log WARNING "VM $vm not found, ignore it" && continue
            log DEBUG "VM $vm found on $hyp"
            for volume in $(list_volumes "$vm" "$hyp"); do
                log DEBUG "VM $vm: volume $volume"
                if [[ "${SHARED_VOLUMES[$volume]:-null}" != "null" ]]; then
                    SHARED_VOLUMES[$volume]+=" $vm"
                    log INFO "Volume $volume is shared by multiple VMs (at least:" \
                        "${SHARED_VOLUMES[$volume]} $vm)"
                elif [[ "${VOLUME_TO_VM[$volume]:-null}" != "null" ]]; then
                    log INFO "Volume $volume is shared by multiple VMs (at least:" \
                        "${VOLUME_TO_VM[$volume]} $vm)"
                    SHARED_VOLUMES[$volume]="${VOLUME_TO_VM[$volume]} $vm"
                    unset 'VOLUME_TO_VM[$volume]'
                else
                    log INFO "Volume $volume owned by VM $vm"
                    VOLUME_TO_VM[$volume]=$vm
                fi
            done
        done
        jo \
            volumes_to_vm="$( dump_assoc_array_to_json VOLUME_TO_VM )" \
            shared_volumes="$( dump_assoc_array_to_json SHARED_VOLUMES )" > "${MAP_FILENAME}"
        log INFO "Volume to VM mapping cache file updated (${MAP_FILENAME})"
    else
        log INFO "Load volume to VM mapping from cache file ($MAP_FILENAME)"
        load_assoc_array_from_json "$( jq -r .volumes_to_vm < "${MAP_FILENAME}" )" VOLUME_TO_VM
        load_assoc_array_from_json "$( jq -r .shared_volumes < "${MAP_FILENAME}" )" SHARED_VOLUMES
    fi
}

# Return the name of the VM that "owns" a given RBD volume
# Note: return null if the volume is owned by no VM (or shared).
function get_volume_vm() {
    local volume=$1 output_var=$2
    load_volume_to_vm_mapping "${@:3}"
    declare -g "$output_var=${VOLUME_TO_VM[$volume]:-null}"
}

# Check if the volume (specify by RBD path) is shared by multiple VMs
# Note: if shared, list of VMs could be retrieve from ${SHARED_VOLUMES[volume]} variable
function is_shared_volume() {
    local volume=$1
    load_volume_to_vm_mapping "${@:2}"
    [[ "${SHARED_VOLUMES[$volume]:-null}" != "null" ]] && return 0
    return 1
}

function list_locks() {
    local locks="[]" volume id locker address
    for volume in $(list_volumes "$@"); do
        volume_locks=$(rbd lock list "$volume" --format=json) || \
            fatal_error "Failed to list $volume locks"

        local i=0 lock
        while true; do
            # ceph v12 outputs '{"[lock ID]": {"locker": "...", }, "[lock ID]": {...} }'
            # ceph v16 outputs '[{"id": "[lock ID]", "locker": "..."}, {"id": "[lock ID]", ...}, ...]'
            # Let's handle both syntaxes...
            if [[ ${volume_locks} = [* ]]; then
                id=$( jq -r ".[$i].id" <<< "$volume_locks" )
                { [[ -n "$id" ]] && [[ "$id" != "null" ]]; } || break
                locker=$( jq -r ".[$i].locker" <<< "$volume_locks" )
                address=$( jq -r ".[$i].address" <<< "$volume_locks" )
            else
                id=$( jq -r "keys[$i]" <<< "$volume_locks" )
                { [[ -n "$id" ]] && [[ "$id" != "null" ]]; } || break
                locker=$( jq -r ".\"$id\".locker" <<< "$volume_locks" )
                address=$( jq -r ".\"$id\".address" <<< "$volume_locks" )
            fi
            lock=$(jo volume="$volume" id="$id" locker="$locker" address="$address")
            locks=$( jq ". += [$lock]" <<< "$locks" )
            ((i++))
        done
    done
    echo "$locks"
}

#
# Command-line parameters stuff
#
COMMAND=""
COMMAND_ARGS=()
GENERATE_SCRIPT=""
function handle_args() {
    local idx=1 opt show_usage=0 force_as_command_args=0
    while [[ $idx -le $# ]]; do
        opt=${!idx}
        if [[ "$COMMAND" == "complete" ]]; then
            COMMAND_ARGS+=("$opt")
        elif [[ $force_as_command_args -eq 1 ]]; then
            COMMAND_ARGS+=("$opt")
        else
            case $opt in
                --)
                    [[ -z "$COMMAND" ]] && usage "You must specified the command to execute before the '--' argument."
                    force_as_command_args=1
                ;;
                -A|--auto)
                    AUTO=1
                    log DEBUG "Auto mode enabled"
                ;;
                -F|--force)
                    FORCE=1
                    log DEBUG "Force mode enabled"
                ;;
                -v|--verbose)
                    LOG_LEVEL=INFO
                    CONSOLE_LOG=1
                ;;
                -d|--debug)
                    LOG_LEVEL=DEBUG
                    CONSOLE_LOG=1
                ;;
                -x|--trace)
                    set -x
                    LOG_LEVEL=TRACE
                    CONSOLE_LOG=1
                ;;
                -L|--log-level)
                    ((idx++))
                    LOG_LEVEL="${!idx}"
                    in_array "$LOG_LEVEL" "${!LOG_LEVELS[@]}" || \
                        usage "Invalid log level '$LOG_LEVEL'"
                    log DEBUG "Log level set to '$LOG_LEVEL'"
                ;;
                -C|--console)
                    CONSOLE_LOG=1
                    log DEBUG "Console logging enabled"
                ;;
                -q|--quiet)
                    log_enable_quiet_mode
                    log DEBUG "Quiet mode enabled"
                ;;
                -j|--just-try)
                    JUST_TRY=1
                    log DEBUG "Just-try mode enabled"
                ;;
                -h|--help|help)
                    show_usage=1
                ;;
                -N|--no-cache)
                    CACHE_LIFETIME=0
                    log DEBUG "Cache disabled"
                ;;
                -X|--generate-script)
                    ((idx++))
                    GENERATE_SCRIPT="${!idx}"
                ;;
                *)
                    if [[ -z "$COMMAND" ]]; then
                        COMMAND="$opt"
                        check_command || usage "Invalid command '$COMMAND'"
                    else
                        COMMAND_ARGS+=("$opt")
                    fi
            esac
        fi
        ((idx++))
    done

    [[ $show_usage -eq 1 ]] && usage
    [[ -z "$COMMAND" ]] && usage

    if [[ -n "$GENERATE_SCRIPT" ]]; then
        if [[ -e "$GENERATE_SCRIPT" ]]; then
            display "Output script '$GENERATE_SCRIPT' already exists."
            confirm "Overwrite" || exit 1
        fi
        echo "#!/bin/bash" > "$GENERATE_SCRIPT"
        echo "set -e" >> "$GENERATE_SCRIPT"
        chmod +x "$GENERATE_SCRIPT"
    fi

    # Check hypervisors list is set
    [[ "${HYPS:-null}" == "null" ]] && \
        ! in_array "$COMMAND" "autodetect-hypervisors" "complete" && \
        fatal_error "Please configure hypervisors in $CONFIG!"

    if [[ ${#COMMAND_ARGS[@]} -eq 0 ]]; then
        log DEBUG "Run command $COMMAND without any args"
    else
        log DEBUG "Run command $COMMAND with args: '$( implode "' '" "${COMMAND_ARGS[@]}" )'"
    fi
    run_command "$COMMAND"
    exit $?
}

IN_USAGE=0
function usage() {
    [[ $IN_USAGE -eq 1 ]] && fatal_error "Usage loop detected, exit"
    IN_USAGE=1
    local error="$1"
    [[ -n "$error" ]] && echo -e "$error\n" >&2
    cat << EOF
Usage : $(basename "$0")
    -A|--auto                    Auto mode: do not ask for confirmation
    -F|--force                   Force mode: allow dangerous action
    -v|--verbose                 Verbose mode (log level=INFO & console logging)
    -d|--debug                   Debug mode (log level=DEBUG & console logging)
    -x|--trace                   Enable bash tracing (=set -x & log level=TRACE & console logging)
    -L|--log-level               Specify log level (default: $DEFAULT_LOG_LEVEL)
                                 Possible values: ${!LOG_LEVELS[@]}
    -C|--console                 Enable logging to console
    -q|--quiet                   Enable quiet mode: redirect all output to log file if provided or
                                 redirect all non-warning messages to /dev/null otherwise.
    -j|--just-try                Enable just-try mode: do not really run action that could change
                                 anything
    -X|--generate-script [path]  Do not run any commands that cause modifications, but instead
                                 produce a script that can be used to perform the requested operation
    -N|--no-cache                Disable cache
    -h|--help|help               Show this message

EOF
    if [[ -n "$COMMAND" ]]; then
        COMMAND_ARGS=( help details "${COMMAND_ARGS[@]}" )
        run_command "$COMMAND"|sed 's/^/    /'
    else
        COMMAND_ARGS=( help "${COMMAND_ARGS[@]}" )
        echo "Available commands:"
        list_available_commands
        for COMMAND in "${AVAILABLE_COMMANDS[@]}"; do
            run_command "$COMMAND"|sed 's/^/    /'
        done
    fi
    [[ -n "$error" ]] && exit 1
    exit 0
}

# List available commands in AVAILABLE_COMMANDS global variable
function list_available_commands() {
    declare -ga AVAILABLE_COMMANDS=()
    mapfile -t AVAILABLE_COMMANDS < <(
        find "$LIB_DIR/commands" ! -name "complete" -type f -printf "%f\n" | sort
    )
}

function check_command() {
    [[ -e "$LIB_DIR/commands/$COMMAND" ]] && return 0
    local possible_commands
    mapfile -t possible_commands < <(
        find "$LIB_DIR/commands" -type f -name "${COMMAND}*" -printf "%f\n" | sort
    )
    if [[ ${#possible_commands[@]} -eq 1 ]] && [[ -n "${possible_commands[0]}" ]]; then
        COMMAND=$(basename "${possible_commands[0]}")
        return 0
    fi
    COMMAND=""
    return 1
}

function run_command() {
    local cmdname=${1}; shift
    # shellcheck source=/dev/null
    source "$LIB_DIR/commands/$cmdname"
}

SUBCOMMAND=""
SUBCOMMAND_ARGS=()
function run_subcommand() {
    local subcommand=$1
    [[ -z "$subcommand" ]] && usage
    in_array "$subcommand" "--help" "-h" && subcommand="help"
    if type "subcommand__$subcommand" >/dev/null 2>&1; then
        SUBCOMMAND="$subcommand"
        SUBCOMMAND_ARGS=( "${@:2}" )
        log DEBUG "Run subcommand $SUBCOMMAND with args:" \
            "'$( implode "' '" "${SUBCOMMAND_ARGS[@]}" )'"
        "subcommand__$SUBCOMMAND" "${SUBCOMMAND_ARGS[@]}"
    else
        usage "Invalid $COMMAND $subcommand command"
    fi
}

JUST_TRY=0
function run_external() {
    local cmd=( "${@:1}" )
    if [[ -n "$GENERATE_SCRIPT" ]]; then
        echo "${cmd[0]} '$(implode "' '" "${cmd[@]:1}")'" >> "$GENERATE_SCRIPT"
        return 0
    fi
    [[ $JUST_TRY -eq 1 ]] && \
        log DEBUG "Just-try mode: do not really run '$( implode "', '" "${cmd[@]}" )'" && \
        return 0
    LC_ALL=C "${cmd[@]}"
    return $?
}

function run_on_hyp() {
    local hyp=$1 cmd=( "${@:2}" )
    if [[ $hyp == "$(hostname)" ]]; then
        LC_ALL=C "${cmd[@]}"
        return $?
    fi
    LC_ALL=C ssh "$hyp" -- "${cmd[@]}"
    return $?
}

function check_just_try_not_supported() {
    [[ $JUST_TRY -eq 1 ]] && fatal_error \
        "The -j/--just-try parameter is not supported by this command."
}

function check_generate_script_not_supported() {
    [[ -n "$GENERATE_SCRIPT" ]] && fatal_error \
        "The -X/--generate-script parameter is not supported by this command."
}

function _simple_command() {
    local args=() check_state="" confirm_question="" dangerous=0 short_usage="" long_help_msg=() \
        in_long_help=0 function="" raw_virsh_command=0 clean_cache_on_success=0 idx=1 opt vm hyp \
        tty=0
    while [[ $idx -le $# ]]; do
        opt=${!idx}
        case $opt in
            --check-state)
                ((idx++))
                check_state=${!idx}
                in_long_help=0
                ;;
            --confirm)
                ((idx++))
                confirm_question=${!idx}
                in_long_help=0
                ;;
            --function)
                ((idx++))
                function=${!idx}
                in_long_help=0
                ;;
            --short-usage)
                ((idx++))
                short_usage="${!idx}"
                in_long_help=0
                ;;
            --long-help)
                ((idx++))
                long_help_msg+=( "${!idx}" )
                in_long_help=1
                ;;
            --raw-virsh-command)
                raw_virsh_command=1
                in_long_help=0
                ;;
            --clean-cache-on-success)
                clean_cache_on_success=1
                in_long_help=0
                ;;
            --dangerous)
                dangerous=1
                in_long_help=0
                ;;
            --tty)
                tty=1
                ;;
            *)
                if [[ $in_long_help -eq 1 ]]; then
                    long_help_msg+=( "$opt" )
                else
                    args+=( "$opt" )
                fi
                ;;
        esac
        ((idx++))
    done

    if in_array "${COMMAND_ARGS[0]}" --help -h help; then
        echo "${short_usage:-$COMMAND [VM]}"
        if [[ "${COMMAND_ARGS[1]}" == "details" ]] && [[ ${#long_help_msg[@]} -gt 0 ]]; then
            implode $'\n' "${long_help_msg[@]}" "" | sed 's/^/  /'
        fi
        return
    fi

    vm=${COMMAND_ARGS[0]}
    [[ -z "$vm" ]] && usage "VM name is missing"

    hyp=$( locate "$vm" )
    check_hyp "$hyp" || fatal_error "VM $vm not found"

    if [[ -n "$check_state" ]] && ! is_"$check_state" "$vm" "$hyp"; then
        display "VM $vm is not $check_state"
        exit 0
    fi

    if [[ $dangerous -eq 1 ]]; then
        [[ $AUTO -eq 1 ]] && [[ $FORCE -ne 1 ]] && fatal_error \
            "Auto mode enabled without force mode, action denied." \
            "Run again this command with -F/--force parameter to force it."
        confirm "$confirm_question VM $vm on $hyp" || exit 1
    fi

    if [[ -n "$function" ]]; then
        $function "$vm" "$hyp" "${args[@]}"
    else
        local cmd=( "ssh" "$hyp" )
        [[ $tty -eq 1 ]] && cmd+=( "-t" )
        cmd+=( "$REAL_VIRSH" "${args[@]}" )
        [[ $raw_virsh_command -eq 0 ]] && cmd+=( "$vm" )
        run_external "${cmd[@]}"
    fi

    local result=$?

    [[ $result -eq 0 ]] && [[ $clean_cache_on_success -eq 1 ]] && clean_cache

    exit $result
}

function _deprecated_command() {
    local cmd=() deprecated_cmd="" force_confirm=0 idx=1 deprecated_cmd
    while [[ $idx -le $# ]]; do
        local opt=${!idx}
        case $opt in
            --deprecated-command)
                ((idx++))
                deprecated_cmd=${!idx}
                ;;
            --force-confirm)
                force_confirm=1
                ;;
            *)
                cmd+=( "$opt" )
                ;;
        esac
        ((idx++))
    done

    [[ -z "$deprecated_cmd" ]] && deprecated_cmd="$(basename "$0")"

    # Only display warning on interactive mode
    if [[ -t 1 ]]; then
        cat << EOF

/!\ DEPRECATED COMMAND /!\\

Please note the command '$deprecated_cmd' is deprecated. You should now use the following command:

    ${cmd[@]}

EOF

        confirm "Run '$( implode "', '" "${cmd[@]}" )'" || exit 1
    elif [[ "${cmd[0]}" == "vm" ]] && [[ $force_confirm -eq 0 ]]; then
        # Add --auto parameter in non-interactive mode
        in_array "--auto" "${cmd[@]}" || cmd+=( "--auto" )
    fi

    "${cmd[@]}"
    exit $?
}
