#! /bin/bash
# vim: shiftwidth=4 tabstop=4 expandtab

[[ "${CLUSTER_SYNC_REMOTE_USER:-null}" == "null" ]] && CLUSTER_SYNC_REMOTE_USER="root"
[[ "${CLUSTER_SYNC_RSYNC_PATH:-null}" == "null" ]] && CLUSTER_SYNC_RSYNC_PATH="/usr/bin/rsync"
[[ "${CLUSTER_SYNC_FILES_AND_DIRS[*]:-null}" == "null" ]] && \
    CLUSTER_SYNC_FILES_AND_DIRS=(
        "/etc/eehyp-tools.conf"
        "/etc/eehyp-tools.conf.d/"
        "${BACKUP_DIRECTORY_PATH}/"
    )
[[ "${CLUSTER_SYNC_RSYNC_ARGS[*]:-null}" == "null" ]] && \
    CLUSTER_SYNC_RSYNC_ARGS=( "-az" "--delete" "--exclude=*.swp" )
[[ "${CLUSTER_SYNC_STATE_FILES_PATH_PREFIX:-null}" == "null" ]] && \
    CLUSTER_SYNC_STATE_FILES_PATH_PREFIX="/var/log/eehyp-tools/eesync_cluster"
[[ "${CLUSTER_SYNC_MANAGED_SERVICES[*]:-null}" == "null" ]] && \
    declare -A CLUSTER_SYNC_MANAGED_SERVICES=()
[[ "${CLUSTER_SYNC_MANAGED_SERVICES_CONFIG_EXCLUSION_PATTERNS[*]:-null}" == "null" ]] && \
    CLUSTER_SYNC_MANAGED_SERVICES_CONFIG_EXCLUSION_PATTERNS=(
        '.*.swp' '*.local' '*.local.*'
    )
[[ "${CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND[*]:-null}" == "null" ]] && \
    declare -A CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND=()

[[ "${LAST_CLUSTER_BACKUP_WARNING_AGE:-null}" == "null" ]] && \
    LAST_CLUSTER_BACKUP_WARNING_AGE=750
[[ "${LAST_CLUSTER_BACKUP_CRITICAL_AGE:-null}" == "null" ]] && \
    LAST_CLUSTER_BACKUP_CRITICAL_AGE=1470

# Auto-complete twik: allow sourcing this file from complete command to get supported args
[[ ${IN_AUTOCOMPLETER:-0} -eq 1 ]] && return

# shellcheck source=/dev/null
source "$LIB_DIR"/stats


function subcommand__help() {
    if [[ -z "$2" ]] || [[ "$2" == "stats" ]]; then
        echo "$COMMAND stats [status] [--json]"
        if [[ "$1" == "details" ]] && [[ "$2" == "stats" ]]; then
            echo "  Show cluster stats"
            echo "    status           Filter on VM status (optional)"
            echo "                     Possible values:"
            echo "                     $( implode ", " "${HYP_STATS_STATUS[@]}" )"
            echo "    --json           JSON output"
        fi
    fi
    if [[ -z "$2" ]] || [[ "$2" == "cpuinfo" ]]; then
        echo "$COMMAND cpuinfo"
        if [[ "$1" == "details" ]] && [[ "$2" == "cpuinfo" ]]; then
            echo "  Show cluster CPU info"
        fi
    fi
    if [[ -z "$2" ]] || [[ "$2" == "sync" ]]; then
        echo "$COMMAND sync [--force]"
        if [[ "$1" == "details" ]] && [[ "$2" == "sync" ]]; then
            echo "  Synchronize cluster configuration from master node to another nodes"
            echo "    -i|--init       Initialize synchronization on all cluster nodes"
            echo "    -a|--all        Run synchronization on all cluster nodes"
            echo
            echo "  Notes:"
            echo "    * Use force mode to run synchronization even if this command is run on the"
            echo "      master node or the PID file is present."
            echo "    * Just-try mode and -X|--generate-script arguments are supported"
            echo "    * Synchronize files and directories:"
            local item
            for item in "${CLUSTER_SYNC_FILES_AND_DIRS[@]}"; do
                echo "      - '$item'"
            done
            if [[ ${#CLUSTER_SYNC_MANAGED_SERVICES[@]} -gt 0 ]]; then
                echo "    * Managed services:"
                for item in "${!CLUSTER_SYNC_MANAGED_SERVICES[@]}"; do
                    echo "      - '$item'"
                done
                if [[ ${#CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND[@]} -gt 0 ]]; then
                    echo "    * Managed services custom reload command:"
                    for item in "${!CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND[@]}"; do
                        echo "      - $item: '${CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND[$item]}'"
                    done
                fi
            else
                echo "    * No managed services"
            fi
        fi
    fi
    if [[ -z "$2" ]] || [[ "$2" == "copy" ]]; then
        echo "$COMMAND copy [path1 path2]"
        if [[ "$1" == "details" ]] && [[ "$2" == "copy" ]]; then
            echo "  Copy local files or directories to another hypervisors (in same emplacement)"
            echo "    [path]          File or directory path to copy"
            echo
            echo "  Note: you could specify more than one path to copy."
        fi
    fi
    if [[ -z "$2" ]] || [[ "$2" == "exec" ]]; then
        echo "$COMMAND exec -- [raw command]"
        if [[ "$1" == "details" ]] && [[ "$2" == "stats" ]]; then
            echo "  Execute an arbitrary command on all hypervisors"
            echo
            echo "  Note: to be sure that specified arguments after the 'exec' one will be handled"
            echo "  as part of the command to execute on hypervisors, you may prefix them with the"
            echo "  '--' keyword."
        fi
    fi
    if [[ -z "$2" ]] || [[ "$2" == "auto-upgrade" ]]; then
        echo "$COMMAND auto-upgrade"
        if [[ "$1" == "details" ]] && [[ "$2" == "stats" ]]; then
            echo "  Upgrade eehyp-tools Debian package on all hypersisors using APT"
        fi
    fi
    if [[ -z "$2" ]] || [[ "$2" == "backup" ]]; then
        echo "$COMMAND backup all|active|inactive [hyps ...]"
        if [[ "$1" == "details" ]] && [[ "$2" == "backup" ]]; then
            echo "  Backup VMs configuration"
            echo "    all                Backup running & inactive VMs (default)"
            echo "    inactive           Backup inactive VMs"
            echo "    running            Backup running VMs"
            echo "    hyps               Limit backups on VMs on specified hypervisors"
        fi
        echo "$COMMAND backup check [-w X] [-c Y]"
        if [[ "$1" == "details" ]] && [[ "$2" == "backup" ]]; then
            echo "  Check last cluster backup age"
            echo "    -w/--warning-age   Warning threshold of last backup age" \
                "(default: $(format_duration "$((LAST_CLUSTER_BACKUP_WARNING_AGE*60))"))"
            echo "    -c/--critical-age  Critical threshold of last backup age" \
                "(default: $(format_duration "$((LAST_CLUSTER_BACKUP_CRITICAL_AGE*60))"))"
        fi
    fi
}

function subcommand__stats() {
    check_generate_script_not_supported
    local status status_arg json=0 stats
    for opt in "${@:1}"; do
        case $opt in
            --json)
                json=1
                ;;
            *)
                if in_array "$opt" "${HYP_STATS_STATUS[@]}"; then
                    status=$opt
                else
                    usage "Unknown parameter '$opt'"
                fi
                ;;
        esac
    done

    [[ "${status:-active}" != "all" ]] && status_arg="--list-${status:-active}"

    stats=$( get_cluster_stats "${status_arg:-}" )

    if [[ $json -eq 1 ]]; then
        jq <<< "$stats"
        exit 0
    fi

    jq -r "$JQ_FORMAT_SIZE"'
        . as $root |
        .hyps |
        to_entries |
        (
            ["Hypervisor", "VMs", "vCPUs", "Allocated memory", "Used memory"],
            ["----------", "---", "-----", "----------------", "-----------"],
            (
                .[] |
                [
                    .key,
                    .value.vm_count,
                    "\(.value.vcpu_allocated) / \(.value.vcpu_allocatable) (\(.value.vcpu_allocated_percent)%)",
                    (
                        "\(.value.memory_allocated | format_size) / \(.value.memory | format_size) "
                        + "(\(.value.memory_allocated_percent)% / real RAM: \(.value.memory_allocated_percent)%)"
                    ),
                    "\(.value.memory_used | format_size) / \(.value.memory | format_size) (\(.value.memory_used_percent)%)"
                ]
            ),
            ["----------"],
            [
                "\($root.hyps | length) hosts",
                ($root.vms | length),
                "\($root.vcpu_allocated) / \($root.vcpu_allocatable) (\($root.vcpu_allocated_percent)%)",
                "\($root.memory_allocated | format_size) / \($root.memory | format_size) (\($root.memory_allocated_percent)%)",
                "\($root.memory_used | format_size) / \($root.memory | format_size) (\($root.memory_used_percent)%)"
            ]
        ) |
        @tsv
    ' <<< "$stats" | column -t -s $'\t'

    echo

    jq -r "$JQ_FORMAT_SIZE"'
        .vms |
        to_entries |
        sort_by(.value.hypervisor, .key) |
        (
            ["Hypervisor", "Name", "State", "vCPUs", "Allocated memory", "Used memory"],
            ["----------", "----", "-----", "-----", "----------------", "-----------"],
            (
                (
                    reduce .[] as $item (
                        {prev_hypervisor: null, rows: []};
                        if .prev_hypervisor and .prev_hypervisor != $item.value.hypervisor then
                            .rows += [["----------", "", "", "", "", ""]]
                        else
                            .
                        end |
                        .rows += [
                            [
                                $item.value.hypervisor,
                                $item.key,
                                $item.value.status,
                                $item.value.vcpu.current,
                                ($item.value.balloon.current | format_size),
                                "\($item.value.balloon.used | format_size) (\($item.value.balloon.used_percent)%)"
                            ]
                        ] |
                        .prev_hypervisor = $item.value.hypervisor
                    )
                ).rows | .[]
            )
        ) |
        @tsv
    ' <<< "$stats" | column -t -s $'\t'
}

function subcommand__cpuinfo() {
    local hyp info cpu_model cpu_flag first=1 core_count min_core_count=0 max_core_count=0 \
        total_core_count=0 hyp_count=0 hyp_min_cpu_freq hyp_max_cpu_freq min_cpu_freq=0 \
        max_cpu_freq=0 hyp_cpu_freqs
    local -a common_flags hyp_cpu_models previous_hyps min_core_count_hosts max_core_count_hosts \
        min_cpu_freq_hosts max_cpu_freq_hosts
    local -A hyps_cpuinfo cpu_models extra_flags
    for hyp in $HYPS; do
        if ! info=$( run_external ssh "$hyp" -- "cat /proc/cpuinfo" ); then
            log ERROR "Failed to retrieve CPU info of $hyp"
            continue
        fi
        hyps_cpuinfo+=( ["$hyp"]="$info" )

        # Core count
        core_count=$( grep -cE "^processor\s+:" <<< "$info" )
        log DEBUG "$core_count CPU cores found on $hyp"
        (( total_core_count+=core_count ))
        if [[ "$min_core_count" -gt "$core_count" ]] || [[ "$min_core_count" -eq 0 ]]; then
            min_core_count=$core_count
            min_core_count_hosts=( "$hyp" )
        elif [[ "$min_core_count" -eq "$core_count" ]]; then
            min_core_count_hosts+=( "$hyp" )
        fi
        if [[ "$max_core_count" -lt "$core_count" ]]; then
            max_core_count=$core_count
            max_core_count_hosts=( "$hyp" )
        elif [[ "$max_core_count" -eq "$core_count" ]]; then
            max_core_count_hosts+=( "$hyp" )
        fi

        # CPU freq
        hyp_cpu_freqs=$( ssh "$hyp" 'cat /sys/devices/system/cpu/cpu*/cpufreq/cpuinfo_max_freq' | sort -un )
        hyp_min_cpu_freq=$( head -n 1 <<< "$hyp_cpu_freqs" )
        hyp_max_cpu_freq=$( tail -n 1 <<< "$hyp_cpu_freqs" )
        log DEBUG "Min/max CPU freq on $hyp: $hyp_min_cpu_freq/$hyp_max_cpu_freq"
        if [[ "$min_cpu_freq" -gt "$hyp_min_cpu_freq" ]] || [[ "$min_cpu_freq" -eq 0 ]]; then
            min_cpu_freq=$hyp_min_cpu_freq
            min_cpu_freq_hosts=( "$hyp" )
        elif [[ "$min_cpu_freq" -eq "$hyp_min_cpu_freq" ]]; then
            min_cpu_freq_hosts+=( "$hyp" )
        fi
        if [[ "$max_cpu_freq" -lt "$hyp_max_cpu_freq" ]]; then
            max_cpu_freq=$hyp_max_cpu_freq
            max_cpu_freq_hosts=( "$hyp" )
        elif [[ "$max_cpu_freq" -eq "$hyp_max_cpu_freq" ]]; then
            max_cpu_freq_hosts+=( "$hyp" )
        fi

        # CPU model
        mapfile -t hyp_cpu_models < <( grep -E "^model name\s+:" <<< "$info" | sort -u | sed 's/.*: //' )
        log DEBUG "${#hyp_cpu_models[@]} CPU models found on $hyp: ${hyp_cpu_models[*]}"
        for cpu_model in "${hyp_cpu_models[@]}"; do
            if [[ "${cpu_models[$cpu_model]:-null}" == "null" ]]; then
                cpu_models+=( ["$cpu_model"]="$hyp" )
            else
                cpu_models+=( ["$cpu_model"]="${cpu_models[$cpu_model]} $hyp" )
            fi
        done

        # Flags
        mapfile -t hyp_cpu_flags < <( grep -E "^flags\s+:" <<< "$info" | sort -u | sed 's/.*: //' | tr ' ' '\n' )
        log DEBUG "${#hyp_cpu_flags[@]} flags supported by $hyp"
        if [[ "$first" -eq 1 ]]; then
            log DEBUG "Initialize common flags with $hyp supported ones"
            common_flags=( "${hyp_cpu_flags[@]}" )
            first=0
        else
            for cpu_flag in "${hyp_cpu_flags[@]}"; do
                in_array "$cpu_flag" "${common_flags[@]}" && continue
                log DEBUG "Extra flag $cpu_flag detected on $hyp"
                if [[ "${extra_flags[$cpu_flag]:-null}" == "null" ]]; then
                    extra_flags+=( ["$cpu_flag"]="$hyp" )
                else
                    extra_flags+=( ["$cpu_flag"]="${extra_flags[$cpu_flag]} $hyp" )
                fi
            done
            for cpu_flag in "${common_flags[@]}"; do
                in_array "$cpu_flag" "${hyp_cpu_flags[@]}" && continue
                mapfile -t common_flags < <( array_filter "${common_flags[@]}" -- "$cpu_flag" )
                log DEBUG "Flag $cpu_flag not supported by $hyp (but supported by ${previous_hyps[*]})"
                extra_flags+=( ["$cpu_flag"]="${previous_hyps[*]}" )
            done
        fi

        previous_hyps+=( "$hyp" )
        ((hyp_count++))
    done

    display "Total core count: $total_core_count"
    display " - Minimum core count: $min_core_count (${min_core_count_hosts[*]})"
    display " - Maximum core count: $max_core_count (${max_core_count_hosts[*]})"
    display " - Medium core count: $(( total_core_count/hyp_count ))"

    display "CPU frequency (MHz):"
    display " - Minimum: $(( min_cpu_freq / 1000 )) (${min_cpu_freq_hosts[*]})"
    display " - Maximum: $(( max_cpu_freq / 1000 )) (${max_cpu_freq_hosts[*]})"

    local cpu_model_names
    mapfile -t cpu_model_names < <( implode $'\n' "${!cpu_models[@]}" | sort )
    mapfile -t common_flags < <( implode $'\n' "${common_flags[@]}" | sort )

    display "CPU models:"
    for cpu_model in "${cpu_model_names[@]}"; do
        display " - $cpu_model (${cpu_models[$cpu_model]})"
    done

    if [[ "${#common_flags[@]}" -eq 0 ]]; then
        display "No common flags"
    else
        display "Common flags: ${common_flags[*]}"
    fi
    if [[ "${#extra_flags[@]}" -eq 0 ]]; then
        display "No extra flags"
    else
        local extra_flag_names
        mapfile -t extra_flag_names < <( implode $'\n' "${!extra_flags[@]}" | sort )
        display "Extra flags:"
        for cpu_flag in "${extra_flag_names[@]}"; do
            display " - $cpu_flag (${extra_flags[$cpu_flag]})"
        done
    fi
}

#
# Sync subcommand
#

function _sync_error() {
    local error
    error="$( implode " " "$@" )"
    ERRORS+=( "$error" )
    log ERROR "$error"
}

function _sync_files_and_dirs() {
    local -a only=( "$@" )
    for file in "${CLUSTER_SYNC_FILES_AND_DIRS[@]}"; do
        [[ "${#only[@]}" -gt 0 ]] && ! in_array "$file" "${only[@]}" && continue
        if [[ -d "$file" ]] || [[ "${file:${#file} - 1}" == "/" ]]; then
            log INFO "Synchronize directory $file"
            [[ "${file:${#file} - 1}" != "/" ]] && file+="/"
            run_external \
                "$CLUSTER_SYNC_RSYNC_PATH" \
                "${CLUSTER_SYNC_RSYNC_ARGS[@]}" \
                "$CLUSTER_SYNC_REMOTE_USER@$VIP:$file" "$file"
            # Allow rsync exit code 24 = transfer incomplete because file has vanished
            in_array "$?" 0 24 || _sync_error "Error syncing directory '$file'"
        else
            log INFO "Synchronize file $file..."
            run_external \
                "$CLUSTER_SYNC_RSYNC_PATH" \
                "${CLUSTER_SYNC_RSYNC_ARGS[@]}" \
                "$CLUSTER_SYNC_REMOTE_USER@$VIP:$file" "$file" || \
                _sync_error "Error syncing file '$file'"
        fi
    done
}

function _update_state_file() {
    log INFO "Update state file ($CLUSTER_SYNC_STATE_FILE)"
    if [[ ${#CLUSTER_SYNC_MANAGED_SERVICES[@]} -gt 0 ]] && [[ ${#CLUSTER_SYNC_MANAGED_SERVICES_STATE[@]} -gt 0 ]]; then
        if dump_assoc_array_to_json_file CLUSTER_SYNC_MANAGED_SERVICES_STATE "$CLUSTER_SYNC_STATE_FILE"; then
            log "DEBUG" "Managed services state stored in state file ($CLUSTER_SYNC_STATE_FILE)"
        else
            _sync_error "Fail to store managed services state in state file ($CLUSTER_SYNC_STATE_FILE)"
        fi
    else
        touch "$CLUSTER_SYNC_STATE_FILE" || \
            _sync_error "Error updating state file ($CLUSTER_SYNC_STATE_FILE)"
    fi
}

# Managed services functions

function _get_manage_service_state() {
    local service_config_paths service=$1 output_var=$2
    IFS=" " read -ra service_config_paths <<< "${CLUSTER_SYNC_MANAGED_SERVICES[$service]}"
    local exclusion_args=( )
    for pattern in "${CLUSTER_SYNC_MANAGED_SERVICES_CONFIG_EXCLUSION_PATTERNS[@]}"; do
        exclusion_args+=( "--exclude=$pattern" )
    done
    local cmd=( tar c "${exclusion_args[@]}" "${service_config_paths[@]}" )
    log DEBUG "Dump service $service configuration to compute its state using the following" \
        "command: '$( implode "' '" "${cmd[@]}" )'"

    # Use C locale and grep common warnings about leading slashes
    declare -g "$output_var=$(
        LC_ALL=C "${cmd[@]}" 2> >( grep -vE 'tar: Removing leading' ) | \
            md5sum | \
            awk '{print $1}'
    )"
    log DEBUG "Service $service configuration state=${!output_var}"
}

function _load_managed_services_previous_state() {
    declare -Ag CLUSTER_SYNC_MANAGED_SERVICES_STATE=()
    if [[ ${#CLUSTER_SYNC_MANAGED_SERVICES[@]} -eq 0 ]]; then
        log DEBUG "No managed service"
        return 0
    fi
    if [[ -f "$CLUSTER_SYNC_STATE_FILE" ]]; then
        log INFO "Load managed services previous state from '$CLUSTER_SYNC_STATE_FILE'"
        if load_assoc_array_from_json_file "$CLUSTER_SYNC_STATE_FILE" CLUSTER_SYNC_MANAGED_SERVICES_STATE; then
            log DEBUG "Managed services state: $( dump_assoc_array_to_json CLUSTER_SYNC_MANAGED_SERVICES_STATE )"
        else
            _sync_error "Fail to load managed services previous state from '$CLUSTER_SYNC_STATE_FILE'"
            declare -Ag CLUSTER_SYNC_MANAGED_SERVICES_STATE=()
        fi
    else
        log INFO "State file not present"
    fi
}

function _reload_managed_services_if_need() {
    for service in "${!CLUSTER_SYNC_MANAGED_SERVICES[@]}"; do
        log DEBUG "Check service $service new configuration state"
        _get_manage_service_state "$service" STATE
        if [[ "$STATE" == "${CLUSTER_SYNC_MANAGED_SERVICES_STATE[$service]:-null}" ]]; then
            log DEBUG "No change detected in service $service configuration"
            continue
        fi
        log DEBUG "Service $service configuration change" \
            "($STATE != ${CLUSTER_SYNC_MANAGED_SERVICES_STATE[$service]:-null})"
        log INFO "Service $service configuration change, reload it"
        if [[ "${CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND[$service]:-null}" == "null" ]]; then
            CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND+=(
                [$service]="/usr/sbin/service $service reload"
            )
            log DEBUG "No custom reload command define for service $service, use default"
        else
            log DEBUG "A custom reload command is defined for service $service"
        fi
        IFS=" " read -ra reload_cmd <<< "${CLUSTER_SYNC_MANAGED_SERVICES_RELOAD_COMMAND[$service]}"
        log DEBUG "Reload command: '$( implode "' '" "${reload_cmd[@]}" )'"
        if run_external "${reload_cmd[@]}"; then
            log INFO "Service $service reloaded"
            CLUSTER_SYNC_MANAGED_SERVICES_STATE+=( [$service]="$STATE" )
        else
            _sync_error "Error occurred reloading service '$service'"
        fi
    done
}

function subcommand__sync() {
    local init=0 all=0
    local -a only=()
    for i in "$@"; do
        case "$i" in
            -i|--init)
                init=1
                ;;
            -a|--all)
                all=1
                ;;
            *)
                if in_array "$i" "${CLUSTER_SYNC_FILES_AND_DIRS[@]}"; then
                    only+=( "$i" )
                else
                    usage "Invalid '$i' parameter"
                fi
        esac
    done

    if [[ $init -eq 1 ]]; then
        local cmd error=0 hyp
        cmd=(
            "ssh-keygen" "-R" "$VIP" ";"
            "ssh" "-o" "StrictHostKeyChecking=accept-new" "$VIP" "hostname"
        )
        log DEBUG "Initialization cluster synchronization command: '$( implode "' '" "${cmd[@]}" )'"

        for hyp in $HYPS; do
            log INFO "Initialize cluster synchronization on hypervisor $hyp..."
            # shellcheck disable=SC2029
            if ssh "$hyp" "${cmd[@]}"; then
                log INFO "done."
            else
                log ERROR "Fail to initialize cluster synchronization on hypervisor $hyp"
                ERROR=1
            fi
        done
        [[ $ERROR -eq 1 ]] && exit 1
        exit 0
    fi

    if [[ $all -eq 1 ]]; then
        local cmd error=0 hyp
        cmd=( "$(_vm_command_path)" "cluster" "sync" )
        [[ $FORCE -eq 1 ]] && cmd+=( "--force" )
        [[ $JUST_TRY -eq 1 ]] && cmd+=( "--just-try" )
        log DEBUG "Cluster synchronization command: '$( implode "' '" "${cmd[@]}" )'"

        for hyp in $HYPS; do
            log INFO "Synchronize hypervisor $hyp..."
            # shellcheck disable=SC2029
            if ssh "$hyp" "${cmd[@]}"; then
                log INFO "done."
            else
                log ERROR "Fail to synchronize cluster on hypervisor $hyp"
                ERROR=1
            fi
        done
        [[ $ERROR -eq 1 ]] && exit 1
        exit 0
    fi

    log_start

    declare -g CLUSTER_SYNC_PID_FILE="${CLUSTER_SYNC_STATE_FILES_PATH_PREFIX}.pid"
    declare -g CLUSTER_SYNC_STATE_FILE="${CLUSTER_SYNC_STATE_FILES_PATH_PREFIX}.state"
    declare -g CLUSTER_SYNC_ERRORS_FILE="${CLUSTER_SYNC_STATE_FILES_PATH_PREFIX}.errors"

    if is_master; then
        if [[ $FORCE -eq 1 ]]; then
            log INFO "Force mode: synchronize configuration even if we are on the master node"
        else
            log DEBUG "Master node, just update the status file"
            touch "$CLUSTER_SYNC_STATE_FILE"
            log_stop
            exit 0
        fi
    fi

    # Check & create PID file
    if [[ -f "$CLUSTER_SYNC_PID_FILE" ]]; then
        if [[ $FORCE -eq 1 ]]; then
            log INFO "Force mode: continue even if PID file is present" \
                "($CLUSTER_SYNC_PID_FILE, content='$( cat "$CLUSTER_SYNC_PID_FILE" )')"
        else
            fatal_error "PID file '$CLUSTER_SYNC_PID_FILE' still present!"
        fi
    fi
    echo "$$" > "$CLUSTER_SYNC_PID_FILE"

    # Initialize errors list
    declare -ga ERRORS=()

    # Load managed services previous states
    _load_managed_services_previous_state

    # Sync files & dirs
    if [[ ${#ERRORS[@]} -eq 0 ]]; then
        _sync_files_and_dirs "${only[@]}"
        _reload_managed_services_if_need
    fi

    # Remove PID file
    rm -f "$CLUSTER_SYNC_PID_FILE" || \
        _sync_error "Error deleting PID file ($CLUSTER_SYNC_PID_FILE)"

    # Update state file
    [[ "${#only[@]}" -eq 0 ]] && _update_state_file

    # Put errors in errors file
    if [[ ${#ERRORS} -gt 0 ]]; then
        log WARNING "Some errors occurred, add them in errors file ($CLUSTER_SYNC_ERRORS_FILE)"
        implode '$\n' "${ERRORS[@]}" >> "$CLUSTER_SYNC_ERRORS_FILE"
    fi

    log_stop
    [[ ${#ERRORS} -gt 0 ]] && exit 1
    exit 0
}

function subcommand__copy() {
    [[ "$#" -eq 0 ]] && \
        usage "You must provide at least one file/directory path to copy on another hypervisors"
    local error=0 me
    me="$( hostname -s )"
    for path in "$@"; do
        if [[ ! -e "$path" ]]; then
            log ERROR "$path: not found"
            error=1
            continue
        fi
        # Ensure to add a trailing slash for directories
        [[ -d "$path" ]] && path="${path/%\//}/"
        for hyp in $HYPS; do
            [[ "$hyp" == "$me" ]] && continue
            if run_external rsync -a "$path" "$hyp:$path"; then
                log INFO "$path: copied on $hyp"
            else
                log ERROR "$path: failed to copy on $hyp"
                error=1
            fi
        done
    done
    exit $error
}

function subcommand__exec() {
    [[ -z "$*" ]] && usage "No command specified"
    for hyp in $HYPS; do
        run_external ssh "$hyp" -- "$@" || log ERROR "Failed to run '$*' command on $hyp"
    done
}

function subcommand__auto-upgrade() {
    for hyp in $HYPS; do
        run_external ssh "$hyp" -- "apt update; apt install --upgrade eehyp-tools"
    done
}

function subcommand__backup() {
    check_generate_script_not_supported
    check_just_try_not_supported

    # shellcheck source=/dev/null
    source "$LIB_DIR"/backup

    if [[ "$1" == "check" ]]; then
        subcommand__backup__check "${@:2}"
        exit $?
    fi

    local status="all" hyp error=0
    for i in "$@"; do
        case "$i" in
            active|inactive|all)
                status="$i"
                ;;
            *)
                check_hyp "$i" || usage "Invalid '$i' parameter"
                hyps_list+=( "$i" )
        esac
    done

    check_is_backup_master cluster backup "$status" "${hyps_list[@]}"

    log_start
    _backup_dir BACKUP_DIR
    set_default_backup_author_name "vm cluster backup $status"

    # Iterate on all hypervisors
    local -Ag ACTIVE_VMS_HYP
    for hyp in $HYPS; do
        # Handle list of hypervisors to handle (if specified)
        [[ ${#hyps_list[@]} -gt 0 ]] && ! in_array "$hyp" "${hyps_list[@]}" && continue

        if backup_hyp \
            --backup-dir "$BACKUP_DIR" \
            --status "$status" \
            --active-vms-hyp ACTIVE_VMS_HYP \
            "$hyp"; then
            log DEBUG "VMs of $hyp configurations inspected"
        else
            log ERROR "Some errors occurred while inspecting VMs of $hyp"
            error=1
        fi
    done

    if [[ "$status" != "inactive" ]]; then
        log INFO "Cleanup old active VM symlinks"

        mapfile -t symlinks < <( find "$BACKUP_DIR" -maxdepth 1 -type l -name '*.xml' ) || \
            log "FATAL" "Failed to list active VM symlinks"
        for symlink in "${symlinks[@]}"; do
            hyp="$( basename "$( dirname "$( readlink -f "$symlink" )" )" )"
            [[ ${#hyps_list[@]} -gt 0 ]] && ! in_array "$hyp" "${hyps_list[@]}" && continue

            vm="$( basename "$symlink" )"
            vm="${vm/%.xml/}"
            [[ "${ACTIVE_VMS_HYP[$vm]:-null}" == "$hyp" ]] && continue

            log INFO "Remove old VM $vm active symlink on $hyp ($symlink)"
            if add_backup_pending_change \
                "$BACKUP_DIR" \
                rm "$symlink"; then
                log INFO "Removal of old VM $vm active symlink on $hyp commit"
            else
                log ERROR "Failed to commit removal of VM $vm active symlink on $hyp ($symlink)"
                error=1
            fi
        done
    fi

    # Update telltale file on full backup success
    if [[ "$status" == "all" ]] && [[ $error -eq 0 ]] && [[ ${#hyps_list[@]} -eq 0 ]]; then
        backup_telltale_path TELLTALE_PATH "$BACKUP_DIR"
        log DEBUG "Backup success, updating telltale file ($TELLTALE_PATH)"
        if date -R > "$TELLTALE_PATH"; then
            log INFO "Telltale file updated ($TELLTALE_PATH)"

            # shellcheck disable=SC2097,SC2098
            GIT_AUTHOR_NAME=$GIT_AUTHOR_NAME GIT_COMMITTER_EMAIL=$GIT_COMMITTER_EMAIL \
                run_external git -C "$BACKUP_DIR" commit -q \
                -m "Successful backup" \
                --author="$GIT_AUTHOR_NAME <$GIT_COMMITTER_EMAIL>" || return 1

            log INFO "Changes committed"
        fi
    fi

    log_stop
    return $error
}

function subcommand__backup__check() {
    local idx=1
    while [[ $idx -le $# ]]; do
        local opt=${!idx}
        case $opt in
            -w|--warning-age)
                ((idx++))
                LAST_CLUSTER_BACKUP_WARNING_AGE=${!idx}
                check_int "$LAST_CLUSTER_BACKUP_WARNING_AGE" 0 || \
                    usage "Invalid warning threshold specified for last cluster backup age. " \
                    "Must be a positive integer."
                ;;
            -c|--critical-age)
                ((idx++))
                LAST_CLUSTER_BACKUP_CRITICAL_AGE=${!idx}
                check_int "$LAST_CLUSTER_BACKUP_CRITICAL_AGE" 0 || \
                    usage "Invalid critical threshold specified for last cluster backup age. " \
                    "Must be a positive integer."
                ;;
            *)
                usage "Unknown parameter '$opt'"
                ;;
        esac
        ((idx++))
    done

    backup_telltale_path TELLTALE_PATH
    if [[ ! -e "$TELLTALE_PATH" ]]; then
        echo "UNKNOWN - Last backup telltale file not found ($TELLTALE_PATH)"
        exit 3
    fi

    local last_backup_time current_time last_backup_age last_backup_age_min status exit_code
    last_backup_time=$( date -r "$TELLTALE_PATH" +%s )
    log DEBUG "Last backup time: $( date -d "@$last_backup_time" )"
	current_time=$( date +%s )
    log DEBUG "Current time: $( date -d "@$current_time" )"
	((last_backup_age=current_time-last_backup_time))
    log DEBUG "Last backup age: $( format_duration "$last_backup_age" )"
	((last_backup_age_min=last_backup_age/60))
    log DEBUG "Last backup age (in minutes): $last_backup_age_min"

    if [[ "$last_backup_age_min" -ge "$LAST_CLUSTER_BACKUP_CRITICAL_AGE" ]]; then
        status=CRITICAL
        exit_code=2
    elif [[ "$last_backup_age_min" -ge "$LAST_CLUSTER_BACKUP_WARNING_AGE" ]]; then
        status=WARNING
        exit_code=1
    else
        status=OK
        exit_code=0
    fi
    echo "$status - Last cluster backup occurred $( format_duration "$last_backup_age" ) ago"
    exit "$exit_code"
}

run_subcommand "${COMMAND_ARGS[@]}"
