1 contributor
#!/bin/bash
# pgs
# Manages VM and CT suspend/shutdown for planned maintenance.
#
# Before maintenance (suspend mode):
# - Suspends all running VMs to disk
# - Gracefully shuts down all running CTs
# - Saves state to a list for restoration
# - VMs already suspended to disk: logged as warning, not auto-resumed
# - VMs suspended to RAM: suspended to disk but not auto-resumed (preserving user intent)
#
# After maintenance (resume mode):
# - Resumes VMs from the saved list
# - Starts CTs from the saved list
# - Logs warnings for VMs/CTs skipped
# - Logs errors for VMs/CTs that fail to resume/start
#
# Usage: pgs suspend|resume [--dry-run] [-v]
#
# Version: 1.4 - Standardized xdev state path with legacy state migration
#
# TODO: Implement critical VM/CT migration support.
# Critical guests (tagged or listed) should be live-migrated to another
# node before maintenance instead of suspended/stopped. Rules TBD:
# - Which guests are critical (tag? config flag? external list?)
# - Target node selection (least loaded? affinity rules?)
# - Fallback if migration fails (suspend locally?)
# - Post-maintenance: migrate back or leave on target node?
PROJECT_ID="pve-guests-state"
ORG_ID="xdev"
DEFAULT_STATE_DIR="/var/lib/${ORG_ID}/${PROJECT_ID}"
LEGACY_STATE_DIR="/var/lib/pve-manager"
LEGACY_STATE_FILE="${LEGACY_STATE_DIR}/pgs-state.json"
STATE_DIR="${PGS_STATE_DIR:-${DEFAULT_STATE_DIR}}"
STATE_FILE="${STATE_DIR}/pgs-state.json"
LOCK_FILE="/run/pgs.lock"
SCRIPT_NAME=$(basename "$0")
DRY_RUN=0
VERBOSE=0
QUORUM_RELAXED=0
# Associative arrays for VM data (populated once)
declare -A VM_STATUS
declare -A VM_NAME
declare -A VM_HAS_LOCK
declare -A VM_VMSTATE
declare -A VMSTATE_TO_VMID
# Associative arrays for CT data (populated once)
declare -A CT_STATUS
declare -A CT_NAME
# Logging functions.
# When running inside systemd (JOURNAL_STREAM is set), stdout goes directly to
# the journal - calling logger in addition causes duplicate entries. When running
# interactively, use both echo (terminal) and logger (journal archive).
_log() {
local level="$1" prefix="$2"; shift 2
echo "$prefix $*"
[[ -z "${JOURNAL_STREAM:-}" ]] && logger -t "$SCRIPT_NAME" -p "$level" "$*"
}
log_info() {
# When in systemd: always log regardless of VERBOSE (journal is the destination)
# When interactive: only log if -v is set
if [[ -n "${JOURNAL_STREAM:-}" ]] || [[ $VERBOSE -ge 1 ]]; then
_log user.info "[INFO]" "$@"
fi
}
log_debug() {
if [[ -n "${JOURNAL_STREAM:-}" ]] || [[ $VERBOSE -ge 2 ]]; then
_log user.debug "[DEBUG]" "$@"
fi
}
log_warning() {
_log user.warning "[WARNING]" "$@"
}
log_error() {
echo "[ERROR] $*" >&2
[[ -z "${JOURNAL_STREAM:-}" ]] && logger -t "$SCRIPT_NAME" -p user.err "$*"
}
log_success() {
_log user.notice "[SUCCESS]" "$@"
}
usage() {
cat <<EOF
Usage: $0 suspend|resume|cleanup [OPTIONS]
Manage VM and CT suspend/shutdown for planned maintenance.
Commands:
suspend Suspend running VMs to disk, shutdown running CTs
resume Resume VMs and start CTs from saved state
cleanup Remove stale suspend artifacts from config and storage
Options:
-n, --dry-run Show what would be done without making changes
-v, --verbose Print informational messages (-vv adds debug detail)
-h, --help Display this help and exit
Examples:
$0 suspend # Suspend VMs, shutdown CTs
$0 resume # Resume VMs, start CTs
$0 cleanup -v # Remove orphan/stale suspend artifacts
$0 cleanup -vv # Include real filesystem paths in cleanup output
$0 suspend --dry-run # Show what would happen
EOF
}
refresh_vm_artifact_metadata() {
VM_HAS_LOCK=()
VM_VMSTATE=()
VMSTATE_TO_VMID=()
for conf in /etc/pve/qemu-server/*.conf; do
[[ ! -f "$conf" ]] && continue
local vmid=$(basename "$conf" .conf)
if grep -q '^lock: suspended$' "$conf" 2>/dev/null; then
VM_HAS_LOCK[$vmid]=1
fi
local vmstate
vmstate=$(awk -F': ' '/^vmstate: / {print $2; exit}' "$conf" 2>/dev/null)
if [[ -n "$vmstate" ]]; then
VM_VMSTATE[$vmid]="$vmstate"
VMSTATE_TO_VMID[$vmstate]="$vmid"
fi
done
}
load_vm_config_metadata() {
VM_STATUS=()
VM_NAME=()
while read -r vmid name status _rest; do
[[ "$vmid" == "VMID" ]] && continue
VM_NAME[$vmid]="$name"
done < <(qm list 2>/dev/null)
refresh_vm_artifact_metadata
}
# Load all VM info in one pass - FAST
load_vm_info() {
load_vm_config_metadata
# Get status and name from qm list (single call)
while read -r vmid name status _rest; do
[[ "$vmid" == "VMID" ]] && continue # skip header
VM_STATUS[$vmid]="$status"
VM_NAME[$vmid]="$name"
done < <(qm list 2>/dev/null)
# For "running" VMs, get actual status (qm list shows "running" for paused/suspended VMs)
# This is only a few VMs so the overhead is acceptable
for vmid in "${!VM_STATUS[@]}"; do
if [[ "${VM_STATUS[$vmid]}" == "running" ]]; then
local real_status
real_status=$(qm status "$vmid" 2>/dev/null | awk '{print $2}')
[[ -n "$real_status" ]] && VM_STATUS[$vmid]="$real_status"
fi
done
}
array_contains() {
local needle="$1"
shift
local item
for item in "$@"; do
[[ "$item" == "$needle" ]] && return 0
done
return 1
}
append_unique() {
local -n target_ref=$1
local value="$2"
array_contains "$value" "${target_ref[@]}" || target_ref+=("$value")
}
remove_value() {
local -n target_ref=$1
local value="$2"
local filtered=()
local item
for item in "${target_ref[@]}"; do
[[ "$item" == "$value" ]] && continue
filtered+=("$item")
done
target_ref=("${filtered[@]}")
}
extract_suspend_file_date() {
local vmid="$1"
local volume="$2"
local volume_name="${volume##*/}"
if [[ "$volume_name" =~ ^vm-${vmid}-state-suspend-([0-9]{4}-[0-9]{2}-[0-9]{2})\.raw$ ]]; then
echo "${BASH_REMATCH[1]}"
fi
}
# Load all CT info in one pass - FAST
load_ct_info() {
# pct list columns: VMID Status Lock Name
# When Lock is empty, read shifts Name into the lock variable
while read -r vmid status lock name; do
[[ "$vmid" == "VMID" ]] && continue # skip header
if [[ -z "$name" ]]; then
# No lock present: lock actually holds the name
name="$lock"
lock=""
fi
CT_STATUS[$vmid]="$status"
CT_NAME[$vmid]="$name"
done < <(pct list 2>/dev/null)
}
# Get VM name (from cache)
get_vm_name() {
echo "${VM_NAME[$1]:-unknown}"
}
vm_has_suspend_lock() {
local vmid="$1"
grep -q '^lock: suspended$' "/etc/pve/qemu-server/${vmid}.conf" 2>/dev/null
}
vm_has_vmstate_reference() {
local vmid="$1"
grep -q '^vmstate:' "/etc/pve/qemu-server/${vmid}.conf" 2>/dev/null
}
get_vm_vmstate_volume() {
local vmid="$1"
echo "${VM_VMSTATE[$vmid]:-}"
}
is_strict_suspend_volume_name() {
local vmid="$1"
local name="$2"
[[ "$name" =~ ^vm-${vmid}-state-suspend-[0-9]{4}-[0-9]{2}-[0-9]{2}\.raw$ ]]
}
storage_cleanup_supports_path_scan() {
local storage_type="$1"
# Cleanup walks filesystem paths directly under <path>/images.
# Keep this limited to local directory-backed storages so a stale remote
# mount cannot block planned maintenance in kernel I/O wait.
[[ "$storage_type" == "dir" ]]
}
vmstate_volume_looks_like_suspend_artifact() {
local vmid="$1"
local volume="$2"
local volume_name="${volume##*/}"
[[ -n "$volume" ]] || return 1
is_strict_suspend_volume_name "$vmid" "$volume_name"
}
resolve_storage_volume_path() {
local volume="$1"
pvesm path "$volume" 2>/dev/null
}
vmstate_volume_exists() {
local volume="$1"
local resolved_path
[[ -z "$volume" ]] && return 1
resolved_path=$(resolve_storage_volume_path "$volume") || return 1
[[ -n "$resolved_path" && -e "$resolved_path" ]]
}
remove_suspend_volume_by_volid() {
local vmid="$1"
local volume="$2"
local name="${VM_NAME[$vmid]:-unknown}"
local free_output
if ! vmstate_volume_looks_like_suspend_artifact "$vmid" "$volume"; then
log_warning "VM $vmid ($name) suspend volume does not look like a suspend artifact, leaving it untouched: ${volume:-none}"
return 1
fi
if [[ $DRY_RUN -eq 1 ]]; then
echo "would remove stale vmstate volume for VM $vmid ($name): $volume"
return 0
fi
free_output=$(pvesm free "$volume" 2>&1)
if [[ $? -eq 0 ]]; then
log_info "Removed stale vmstate volume for VM $vmid ($name): $volume"
return 0
fi
if maybe_relax_quorum "$free_output"; then
free_output=$(pvesm free "$volume" 2>&1)
if [[ $? -eq 0 ]]; then
log_info "Removed stale vmstate volume for VM $vmid ($name) after quorum recovery: $volume"
return 0
fi
fi
if echo "$free_output" | grep -qiE 'does not exist|no such file|not found'; then
log_info "Stale vmstate volume for VM $vmid ($name) was already absent: $volume"
return 0
fi
log_warning "VM $vmid ($name) stale vmstate volume could not be removed: $volume ($free_output)"
return 1
}
clear_vmstate_metadata() {
local vmid="$1"
local name="${VM_NAME[$vmid]:-unknown}"
local set_output
if [[ $DRY_RUN -eq 1 ]]; then
echo "would remove stale vmstate metadata for VM $vmid ($name)"
return 0
fi
set_output=$(qm set "$vmid" --delete vmstate 2>&1)
if [[ $? -eq 0 ]]; then
log_info "Removed stale vmstate metadata for VM $vmid ($name)"
return 0
fi
if maybe_relax_quorum "$set_output"; then
set_output=$(qm set "$vmid" --delete vmstate 2>&1)
if [[ $? -eq 0 ]]; then
log_info "Removed stale vmstate metadata for VM $vmid ($name) after quorum recovery"
return 0
fi
fi
log_warning "VM $vmid ($name) stale vmstate metadata could not be removed: $set_output"
return 1
}
free_stale_vmstate_volume() {
local vmid="$1"
local volume="$2"
remove_suspend_volume_by_volid "$vmid" "$volume"
}
cleanup_stale_suspend_artifacts() {
local vmid="$1"
local context="${2:-}"
local name="${VM_NAME[$vmid]:-unknown}"
local volume
local had_issue=0
local cleanup_failed=0
volume=$(get_vm_vmstate_volume "$vmid")
if vm_has_suspend_lock "$vmid"; then
had_issue=1
if ! unlock_vm_suspend_lock "$vmid" "$context"; then
cleanup_failed=1
fi
fi
if [[ -n "$volume" ]]; then
had_issue=1
if vmstate_volume_exists "$volume"; then
if ! free_stale_vmstate_volume "$vmid" "$volume"; then
cleanup_failed=1
fi
else
log_info "VM $vmid ($name) has stale vmstate metadata pointing to missing volume: $volume"
fi
if ! clear_vmstate_metadata "$vmid"; then
cleanup_failed=1
fi
fi
if [[ $had_issue -eq 0 ]]; then
return 0
fi
[[ $cleanup_failed -eq 0 ]]
}
vm_has_valid_suspend_state() {
local vmid="$1"
local volume
vm_has_suspend_lock "$vmid" || return 1
vm_has_vmstate_reference "$vmid" || return 1
volume=$(get_vm_vmstate_volume "$vmid")
vmstate_volume_looks_like_suspend_artifact "$vmid" "$volume" || return 1
vmstate_volume_exists "$volume"
}
get_referencing_vmid_for_vmstate() {
local target_volume="$1"
local vmid="${VMSTATE_TO_VMID[$target_volume]:-}"
[[ -n "$vmid" ]] || return 1
echo "$vmid"
return 0
}
list_suspend_artifact_files() {
awk '
BEGIN {
RS = ""
FS = "\n"
}
{
type = ""
name = ""
path = ""
content = ""
split($1, header_parts, /:[[:space:]]+/)
if (length(header_parts) >= 2) {
type = header_parts[1]
name = header_parts[2]
}
for (i = 2; i <= NF; i++) {
line = $i
sub(/^\t/, "", line)
if (line ~ /^path /) {
path = substr(line, 6)
} else if (line ~ /^content /) {
content = substr(line, 9)
}
}
if (name != "" && path != "" && content ~ /(^|,)images(,|$)/) {
print type "\t" name "\t" path
}
}
' /etc/pve/storage.cfg 2>/dev/null | while IFS=$'\t' read -r storage_type storage path; do
[[ -z "$storage" || -z "$path" ]] && continue
if ! storage_cleanup_supports_path_scan "$storage_type"; then
continue
fi
[[ -d "${path}/images" ]] || continue
local file
for file in "${path}"/images/[0-9]*/vm-*-state-suspend-????-??-??.raw; do
[[ -e "$file" ]] || continue
local relative_path="${file#${path}/images/}"
[[ "$relative_path" == "$file" ]] && continue
local vm_dir="${relative_path%%/*}"
local file_name="${relative_path##*/}"
[[ "$vm_dir" =~ ^[0-9]+$ ]] || continue
is_strict_suspend_volume_name "$vm_dir" "$file_name" || continue
printf '%s\t%s:%s/%s\t%s\n' "$storage" "$storage" "$vm_dir" "$file_name" "$file"
done
done
}
cleanup_orphan_suspend_artifacts() {
local cleaned_count=0
local skipped_count=0
local fail_count=0
local storage
local volume
local file_path
local vmid
log_info "Scanning storages for orphan suspend-state volumes..."
while IFS=$'\t' read -r storage volume file_path; do
[[ -z "$volume" ]] && continue
if vmid=$(get_referencing_vmid_for_vmstate "$volume"); then
if vm_has_valid_suspend_state "$vmid"; then
log_info "Keeping active suspend-state volume for VM $vmid (${VM_NAME[$vmid]:-unknown}): $volume"
((skipped_count++))
else
log_warning "VM $vmid (${VM_NAME[$vmid]:-unknown}) references inconsistent suspend artifacts - cleaning up"
if cleanup_stale_suspend_artifacts "$vmid" "during cleanup"; then
((cleaned_count++))
else
((fail_count++))
fi
fi
continue
fi
if [[ $DRY_RUN -eq 1 ]]; then
echo "would remove orphan suspend-state volume: $volume"
log_debug "real path: $file_path"
((cleaned_count++))
continue
fi
if [[ "$volume" =~ ^([^:]+):([0-9]+)/vm-([0-9]+)-state-suspend-([0-9]{4}-[0-9]{2}-[0-9]{2})\.raw$ ]]; then
vmid="${BASH_REMATCH[3]}"
else
log_warning "Skipping suspicious suspend-state volume with unexpected name: $volume"
((skipped_count++))
continue
fi
VM_NAME[$vmid]="${VM_NAME[$vmid]:-unknown}"
if remove_suspend_volume_by_volid "$vmid" "$volume"; then
log_info "Removed orphan suspend-state volume from $storage: $volume"
((cleaned_count++))
else
((fail_count++))
fi
done < <(list_suspend_artifact_files)
log_success "Suspend artifact cleanup complete: $cleaned_count cleaned, $skipped_count retained, $fail_count failed"
return $fail_count
}
unlock_vm_suspend_lock() {
local vmid="$1"
local context="${2:-}"
local name="${VM_NAME[$vmid]:-unknown}"
local unlock_output
if ! vm_has_suspend_lock "$vmid"; then
return 0
fi
if [[ $DRY_RUN -eq 1 ]]; then
if [[ -n "$context" ]]; then
echo "would remove stale suspend lock for VM $vmid ($name) $context"
else
echo "would remove stale suspend lock for VM $vmid ($name)"
fi
return 0
fi
unlock_output=$(qm unlock "$vmid" 2>&1)
if [[ $? -eq 0 ]]; then
if [[ -n "$context" ]]; then
log_info "Removed stale suspend lock for VM $vmid ($name) $context"
else
log_info "Removed stale suspend lock for VM $vmid ($name)"
fi
return 0
fi
if maybe_relax_quorum "$unlock_output"; then
unlock_output=$(qm unlock "$vmid" 2>&1)
if [[ $? -eq 0 ]]; then
if [[ -n "$context" ]]; then
log_info "Removed stale suspend lock for VM $vmid ($name) $context after quorum recovery"
else
log_info "Removed stale suspend lock for VM $vmid ($name) after quorum recovery"
fi
return 0
fi
fi
if [[ -n "$context" ]]; then
log_warning "VM $vmid ($name) has a stale suspend lock $context but it could not be removed: $unlock_output"
else
log_warning "VM $vmid ($name) has a stale suspend lock but it could not be removed: $unlock_output"
fi
return 1
}
unlock_vm_if_needed() {
unlock_vm_suspend_lock "$1" "while VM is running"
}
# Quorum-sensitive operations (qm suspend/start/resume) may fail during
# cluster-wide maintenance when pmxcfs becomes read-only. In that case, relax
# expected votes once and retry the failed operation.
maybe_relax_quorum() {
local cmd_output="$1"
# Already attempted in this run.
if [[ $QUORUM_RELAXED -eq 1 ]]; then
return 1
fi
if echo "$cmd_output" | grep -qiE "cluster not ready - no quorum|/etc/pve/.+\\.conf\\.tmp.+(Permission denied|Device or resource busy)"; then
log_warning "Detected quorum-related write failure in /etc/pve - attempting temporary 'pvecm expected 1'"
if pvecm expected 1 >/dev/null 2>&1; then
QUORUM_RELAXED=1
log_warning "Applied 'pvecm expected 1' for this maintenance cycle; retrying operation"
return 0
fi
log_error "Failed to apply 'pvecm expected 1' after quorum-related error"
fi
return 1
}
# Suspend a VM to disk
suspend_vm_to_disk() {
local vmid="$1"
local name="${VM_NAME[$vmid]:-unknown}"
local qm_output
local stale_path
local retry_output
local stale_retry_path
if [[ $DRY_RUN -eq 1 ]]; then
echo "would suspend VM $vmid ($name) to disk"
return 0
fi
log_info "Suspending VM $vmid ($name) to disk..."
qm_output=$(qm suspend "$vmid" --todisk 1 2>&1)
if [[ $? -eq 0 ]]; then
log_success "VM $vmid ($name) suspended to disk"
return 0
fi
# Recover from stale suspend image left from a previous interrupted suspend.
# Proxmox can emit either:
# - "stale saved state disk image ('...raw' already exists)"
# - "disk image '...raw' already exists"
stale_path=$(
echo "$qm_output" | sed -n \
-e "s/.*stale saved state[[:space:]]*disk image ('\\([^']*\\)' already exists).*/\\1/p" \
-e "s/.*disk image '\\([^']*\\)' already exists.*/\\1/p" | head -n 1
)
if [[ -n "$stale_path" && "$stale_path" =~ /vm-${vmid}-state-suspend-[0-9]{4}-[0-9]{2}-[0-9]{2}\.raw$ && -f "$stale_path" ]]; then
log_warning "VM $vmid ($name) has stale suspend image: $stale_path - removing and retrying once"
if rm -f -- "$stale_path"; then
retry_output=$(qm suspend "$vmid" --todisk 1 2>&1)
if [[ $? -eq 0 ]]; then
log_success "VM $vmid ($name) suspended to disk (after stale image cleanup)"
return 0
fi
if maybe_relax_quorum "$retry_output"; then
retry_output=$(qm suspend "$vmid" --todisk 1 2>&1)
if [[ $? -eq 0 ]]; then
log_success "VM $vmid ($name) suspended to disk (after stale image cleanup + quorum recovery)"
return 0
fi
stale_retry_path=$(
echo "$retry_output" | sed -n \
-e "s/.*stale saved state[[:space:]]*disk image ('\\([^']*\\)' already exists).*/\\1/p" \
-e "s/.*disk image '\\([^']*\\)' already exists.*/\\1/p" | head -n 1
)
if [[ -n "$stale_retry_path" && "$stale_retry_path" =~ /vm-${vmid}-state-suspend-[0-9]{4}-[0-9]{2}-[0-9]{2}\.raw$ && -f "$stale_retry_path" ]]; then
log_warning "VM $vmid ($name) retry left stale suspend image: $stale_retry_path - removing and retrying once more"
if rm -f -- "$stale_retry_path"; then
retry_output=$(qm suspend "$vmid" --todisk 1 2>&1)
if [[ $? -eq 0 ]]; then
log_success "VM $vmid ($name) suspended to disk (after stale image cleanup + quorum recovery + retry)"
return 0
fi
fi
fi
fi
log_error "Failed to suspend VM $vmid ($name) after stale image cleanup: $retry_output"
return 1
fi
log_error "Failed to remove stale suspend image for VM $vmid ($name): $stale_path"
return 1
fi
if maybe_relax_quorum "$qm_output"; then
retry_output=$(qm suspend "$vmid" --todisk 1 2>&1)
if [[ $? -eq 0 ]]; then
log_success "VM $vmid ($name) suspended to disk (after quorum recovery)"
return 0
fi
stale_retry_path=$(
echo "$retry_output" | sed -n \
-e "s/.*stale saved state[[:space:]]*disk image ('\\([^']*\\)' already exists).*/\\1/p" \
-e "s/.*disk image '\\([^']*\\)' already exists.*/\\1/p" | head -n 1
)
if [[ -n "$stale_retry_path" && "$stale_retry_path" =~ /vm-${vmid}-state-suspend-[0-9]{4}-[0-9]{2}-[0-9]{2}\.raw$ && -f "$stale_retry_path" ]]; then
log_warning "VM $vmid ($name) quorum retry hit stale suspend image: $stale_retry_path - removing and retrying once more"
if rm -f -- "$stale_retry_path"; then
retry_output=$(qm suspend "$vmid" --todisk 1 2>&1)
if [[ $? -eq 0 ]]; then
log_success "VM $vmid ($name) suspended to disk (after quorum recovery + stale retry)"
return 0
fi
fi
fi
log_error "Failed to suspend VM $vmid ($name) after quorum recovery: $retry_output"
return 1
fi
log_error "Failed to suspend VM $vmid ($name) to disk: $qm_output"
return 1
}
# Resume a VM from disk suspend
resume_vm() {
local vmid="$1"
local name="${VM_NAME[$vmid]:-unknown}"
local qm_output
local current_status
if [[ $DRY_RUN -eq 1 ]]; then
echo "would resume VM $vmid ($name)"
return 0
fi
log_info "Resuming VM $vmid ($name)..."
qm_output=$(qm resume "$vmid" 2>&1)
if [[ $? -eq 0 ]]; then
unlock_vm_if_needed "$vmid"
log_success "VM $vmid ($name) resumed successfully"
return 0
fi
if maybe_relax_quorum "$qm_output"; then
qm_output=$(qm resume "$vmid" 2>&1)
if [[ $? -eq 0 ]]; then
unlock_vm_if_needed "$vmid"
log_success "VM $vmid ($name) resumed successfully (after quorum recovery)"
return 0
fi
current_status=$(qm status "$vmid" 2>/dev/null | awk '{print $2}')
if [[ "$current_status" == "running" ]]; then
unlock_vm_if_needed "$vmid"
log_warning "VM $vmid ($name) is running despite resume error after quorum recovery - treating as resumed"
return 2
fi
log_error "Failed to resume VM $vmid ($name) after quorum recovery: $qm_output"
return 1
fi
if echo "$qm_output" | grep -qi "already running"; then
unlock_vm_if_needed "$vmid"
log_warning "VM $vmid ($name) is already running - treating as resumed"
return 2
fi
current_status=$(qm status "$vmid" 2>/dev/null | awk '{print $2}')
if [[ "$current_status" == "running" ]]; then
unlock_vm_if_needed "$vmid"
log_warning "VM $vmid ($name) is running despite resume error - treating as resumed"
return 2
fi
log_error "Failed to resume VM $vmid ($name): $qm_output"
return 1
}
# Graceful shutdown a CT
shutdown_ct() {
local ctid="$1"
local name="${CT_NAME[$ctid]:-unknown}"
if [[ $DRY_RUN -eq 1 ]]; then
echo "would shutdown CT $ctid ($name)"
return 0
fi
log_info "Shutting down CT $ctid ($name)..."
if pct shutdown "$ctid" --timeout 120; then
log_success "CT $ctid ($name) shut down gracefully"
return 0
else
log_error "Failed to shutdown CT $ctid ($name)"
return 1
fi
}
# Start a CT
start_ct() {
local ctid="$1"
local name="${CT_NAME[$ctid]:-unknown}"
local pct_output
if [[ $DRY_RUN -eq 1 ]]; then
echo "would start CT $ctid ($name)"
return 0
fi
log_info "Starting CT $ctid ($name)..."
pct_output=$(pct start "$ctid" 2>&1)
if [[ $? -eq 0 ]]; then
log_success "CT $ctid ($name) started successfully"
return 0
fi
if maybe_relax_quorum "$pct_output"; then
pct_output=$(pct start "$ctid" 2>&1)
if [[ $? -eq 0 ]]; then
log_success "CT $ctid ($name) started successfully (after quorum recovery)"
return 0
fi
if [[ "$(pct status "$ctid" 2>/dev/null | awk '{print $2}')" == "running" ]]; then
log_warning "CT $ctid ($name) is running despite start error after quorum recovery - treating as started"
return 2
fi
log_error "Failed to start CT $ctid ($name) after quorum recovery: $pct_output"
return 1
fi
if echo "$pct_output" | grep -qi "already running"; then
log_warning "CT $ctid ($name) is already running - treating as started"
return 2
fi
if [[ "$(pct status "$ctid" 2>/dev/null | awk '{print $2}')" == "running" ]]; then
log_warning "CT $ctid ($name) is running despite start error - treating as started"
return 2
fi
log_error "Failed to start CT $ctid ($name): $pct_output"
return 1
}
# Save state to JSON file
# Usage: save_state vm_resume_array vm_suspended_array ct_start_array
save_state() {
local -n to_resume_ref=$1
local -n was_suspended_ref=$2
local -n ct_to_start_ref=$3
local existing_state_json=""
local existing_to_resume=()
local existing_was_suspended=()
local existing_ct_to_start=()
local final_to_resume=()
local final_was_suspended=()
local final_ct_to_start=()
local vmid
local volume
local suspend_date
local -A existing_vm_volume=()
local -A existing_vm_date=()
local -A current_vm_volume=()
local -A current_vm_date=()
if [[ $DRY_RUN -eq 1 ]]; then
echo "would save state to $STATE_FILE"
echo " to_resume (VMs): ${to_resume_ref[*]}"
echo " was_suspended (VMs): ${was_suspended_ref[*]}"
echo " ct_to_start (CTs): ${ct_to_start_ref[*]}"
return 0
fi
if existing_state_json=$(load_state 2>/dev/null); then
mapfile -t existing_to_resume < <(echo "$existing_state_json" | jq -r '.to_resume[]?' 2>/dev/null)
mapfile -t existing_was_suspended < <(echo "$existing_state_json" | jq -r '.was_suspended[]?' 2>/dev/null)
mapfile -t existing_ct_to_start < <(echo "$existing_state_json" | jq -r '.ct_to_start[]?' 2>/dev/null)
while IFS=$'\t' read -r vmid volume suspend_date; do
[[ -z "$vmid" ]] && continue
existing_vm_volume[$vmid]="$volume"
existing_vm_date[$vmid]="$suspend_date"
done < <(
echo "$existing_state_json" | jq -r '
(.vm_details // {})
| to_entries[]
| [.key, (.value.suspend_volume // ""), (.value.suspend_file_date // "")]
| @tsv
' 2>/dev/null
)
fi
refresh_vm_artifact_metadata
for vmid in "${to_resume_ref[@]}"; do
append_unique final_to_resume "$vmid"
volume="${VM_VMSTATE[$vmid]:-}"
suspend_date=$(extract_suspend_file_date "$vmid" "$volume")
current_vm_volume[$vmid]="$volume"
current_vm_date[$vmid]="$suspend_date"
done
for vmid in "${existing_to_resume[@]}"; do
append_unique final_to_resume "$vmid"
done
for vmid in "${existing_was_suspended[@]}"; do
if ! array_contains "$vmid" "${final_to_resume[@]}"; then
append_unique final_was_suspended "$vmid"
fi
done
for vmid in "${was_suspended_ref[@]}"; do
if array_contains "$vmid" "${final_to_resume[@]}"; then
volume="${VM_VMSTATE[$vmid]:-}"
if [[ -n "$volume" ]]; then
current_vm_volume[$vmid]="$volume"
current_vm_date[$vmid]="$(extract_suspend_file_date "$vmid" "$volume")"
fi
continue
fi
append_unique final_was_suspended "$vmid"
volume="${VM_VMSTATE[$vmid]:-}"
suspend_date=$(extract_suspend_file_date "$vmid" "$volume")
current_vm_volume[$vmid]="$volume"
current_vm_date[$vmid]="$suspend_date"
done
for vmid in "${final_to_resume[@]}"; do
remove_value final_was_suspended "$vmid"
done
for vmid in "${existing_ct_to_start[@]}"; do
append_unique final_ct_to_start "$vmid"
done
for vmid in "${ct_to_start_ref[@]}"; do
append_unique final_ct_to_start "$vmid"
done
# Create JSON arrays (handle empty arrays properly)
local to_resume_json="[]"
local was_suspended_json="[]"
local ct_to_start_json="[]"
local vm_details_json="{}"
if [[ ${#final_to_resume[@]} -gt 0 ]]; then
to_resume_json=$(printf '%s\n' "${final_to_resume[@]}" | jq -R . | jq -s .)
fi
if [[ ${#final_was_suspended[@]} -gt 0 ]]; then
was_suspended_json=$(printf '%s\n' "${final_was_suspended[@]}" | jq -R . | jq -s .)
fi
if [[ ${#final_ct_to_start[@]} -gt 0 ]]; then
ct_to_start_json=$(printf '%s\n' "${final_ct_to_start[@]}" | jq -R . | jq -s .)
fi
for vmid in "${final_to_resume[@]}"; do
volume="${current_vm_volume[$vmid]:-${existing_vm_volume[$vmid]:-}}"
suspend_date="${current_vm_date[$vmid]:-${existing_vm_date[$vmid]:-}}"
vm_details_json=$(
jq \
--arg vmid "$vmid" \
--arg mode "to_resume" \
--arg volume "$volume" \
--arg suspend_date "$suspend_date" \
'
.[$vmid] = {
mode: $mode,
suspend_volume: $volume,
suspend_file_date: $suspend_date
}
' <<<"$vm_details_json"
)
done
for vmid in "${final_was_suspended[@]}"; do
volume="${current_vm_volume[$vmid]:-${existing_vm_volume[$vmid]:-}}"
suspend_date="${current_vm_date[$vmid]:-${existing_vm_date[$vmid]:-}}"
vm_details_json=$(
jq \
--arg vmid "$vmid" \
--arg mode "was_suspended" \
--arg volume "$volume" \
--arg suspend_date "$suspend_date" \
'
.[$vmid] = {
mode: $mode,
suspend_volume: $volume,
suspend_file_date: $suspend_date
}
' <<<"$vm_details_json"
)
done
cat > "$STATE_FILE" <<EOF
{
"timestamp": "$(date -Iseconds)",
"hostname": "$(hostname)",
"to_resume": $to_resume_json,
"was_suspended": $was_suspended_json,
"ct_to_start": $ct_to_start_json,
"vm_details": $vm_details_json
}
EOF
log_info "State saved to $STATE_FILE"
}
# Load state from JSON file (outputs JSON only, no logging to avoid capture issues)
load_state() {
if [[ ! -f "$STATE_FILE" ]]; then
return 1
fi
cat "$STATE_FILE"
}
# Remove state file after resume is complete
clear_state() {
if [[ $DRY_RUN -eq 1 ]]; then
echo "would remove state file $STATE_FILE"
return 0
fi
if [[ -f "$STATE_FILE" ]]; then
rm -f "$STATE_FILE"
log_info "State file removed"
fi
}
migrate_legacy_state_if_needed() {
if [[ "${STATE_FILE}" == "${LEGACY_STATE_FILE}" ]]; then
return 0
fi
if [[ -f "${LEGACY_STATE_FILE}" && ! -f "${STATE_FILE}" ]]; then
mkdir -p "${STATE_DIR}"
mv "${LEGACY_STATE_FILE}" "${STATE_FILE}"
log_warning "Migrated legacy state file from ${LEGACY_STATE_FILE} to ${STATE_FILE}"
fi
}
# Main suspend operation
do_suspend() {
log_info "Starting suspend/shutdown operation on $(hostname)"
# Clean stale suspend artifacts before creating new suspend volumes.
load_vm_config_metadata
if ! cleanup_orphan_suspend_artifacts; then
log_warning "Suspend artifact preflight cleanup had failures; continuing with suspend operation"
fi
# Load all VM and CT info in one pass
load_vm_info
load_ct_info
local to_resume=()
local was_suspended=()
local ct_to_start=()
local suspend_count=0
local skip_count=0
local fail_count=0
# --- Process QEMU VMs ---
log_info "Processing QEMU VMs..."
for conf in /etc/pve/qemu-server/*.conf; do
[[ ! -f "$conf" ]] && continue
local vmid=$(basename "$conf" .conf)
local name="${VM_NAME[$vmid]:-unknown}"
local status="${VM_STATUS[$vmid]:-stopped}"
case "$status" in
running)
# Running VM: suspend to disk, add to resume list
if suspend_vm_to_disk "$vmid"; then
to_resume+=("$vmid")
((suspend_count++))
else
((fail_count++))
fi
;;
suspended)
# Suspended to RAM: save state to disk but DON'T add to resume list
log_warning "VM $vmid ($name) is suspended to RAM - saving to disk but will NOT auto-resume (was manually suspended)"
if suspend_vm_to_disk "$vmid"; then
was_suspended+=("$vmid")
((suspend_count++))
else
((fail_count++))
fi
;;
stopped)
# Could be stopped normally or suspended to disk
if vm_has_valid_suspend_state "$vmid"; then
log_warning "VM $vmid ($name) is already suspended to disk - will NOT auto-resume"
was_suspended+=("$vmid")
((skip_count++))
elif vm_has_suspend_lock "$vmid" || vm_has_vmstate_reference "$vmid"; then
log_warning "VM $vmid ($name) has inconsistent suspend artifacts - treating them as stale"
if cleanup_stale_suspend_artifacts "$vmid" "while VM is stopped"; then
((skip_count++))
else
((fail_count++))
fi
else
log_info "VM $vmid ($name) is stopped, skipping"
fi
;;
paused)
# Paused/suspended to RAM: save state to disk but DON'T auto-resume
log_warning "VM $vmid ($name) is paused/suspended to RAM - saving to disk but will NOT auto-resume (was manually paused)"
if suspend_vm_to_disk "$vmid"; then
was_suspended+=("$vmid")
((suspend_count++))
else
((fail_count++))
fi
;;
*)
log_info "VM $vmid ($name) status '$status', skipping"
;;
esac
done
# --- Process LXC Containers ---
log_info "Processing LXC containers..."
for conf in /etc/pve/lxc/*.conf; do
[[ ! -f "$conf" ]] && continue
local ctid=$(basename "$conf" .conf)
local name="${CT_NAME[$ctid]:-unknown}"
local status="${CT_STATUS[$ctid]:-stopped}"
case "$status" in
running)
# Running CT: graceful shutdown, add to start list
if shutdown_ct "$ctid"; then
ct_to_start+=("$ctid")
((suspend_count++))
else
((fail_count++))
fi
;;
stopped)
log_info "CT $ctid ($name) is stopped, skipping"
;;
*)
log_info "CT $ctid ($name) status '$status', skipping"
;;
esac
done
# Save state
save_state to_resume was_suspended ct_to_start
# Summary
log_success "Suspend/shutdown complete: $suspend_count processed, $skip_count skipped, $fail_count failed"
log_info "VMs to auto-resume: ${to_resume[*]:-none}"
log_info "VMs NOT to auto-resume (were suspended): ${was_suspended[*]:-none}"
log_info "CTs to auto-start: ${ct_to_start[*]:-none}"
return $fail_count
}
do_cleanup() {
log_info "Starting suspend artifact cleanup on $(hostname)"
load_vm_config_metadata
cleanup_orphan_suspend_artifacts
return $?
}
# Main resume operation
do_resume() {
log_info "Starting resume/start operation on $(hostname)"
# Load all VM and CT info in one pass
load_vm_info
load_ct_info
local state_json
state_json=$(load_state)
if [[ $? -ne 0 ]]; then
log_warning "No saved state - nothing to resume"
return 0
fi
# Parse state file
local to_resume=($(echo "$state_json" | jq -r '.to_resume[]' 2>/dev/null))
local was_suspended=($(echo "$state_json" | jq -r '.was_suspended[]' 2>/dev/null))
local ct_to_start=($(echo "$state_json" | jq -r '.ct_to_start[]' 2>/dev/null))
local saved_timestamp=$(echo "$state_json" | jq -r '.timestamp' 2>/dev/null)
local -A saved_vm_volume=()
local -A saved_vm_date=()
local saved_volume
local current_volume
while IFS=$'\t' read -r vmid saved_volume saved_date; do
[[ -z "$vmid" ]] && continue
saved_vm_volume[$vmid]="$saved_volume"
saved_vm_date[$vmid]="$saved_date"
done < <(
echo "$state_json" | jq -r '
(.vm_details // {})
| to_entries[]
| [.key, (.value.suspend_volume // ""), (.value.suspend_file_date // "")]
| @tsv
' 2>/dev/null
)
log_info "State file from: $saved_timestamp"
local resume_count=0
local skip_count=0
local fail_count=0
# --- Resume QEMU VMs ---
# Log warnings for VMs that won't be resumed
for vmid in "${was_suspended[@]}"; do
local name="${VM_NAME[$vmid]:-unknown}"
log_warning "VM $vmid ($name) was already suspended before maintenance - NOT auto-resuming"
((skip_count++))
done
# Resume VMs that should be resumed
for vmid in "${to_resume[@]}"; do
local name="${VM_NAME[$vmid]:-unknown}"
# Verify VM still exists and has suspend lock
if [[ ! -f "/etc/pve/qemu-server/${vmid}.conf" ]]; then
log_error "VM $vmid config not found - skipping"
((fail_count++))
continue
fi
if [[ -z "${VM_HAS_LOCK[$vmid]}" ]]; then
log_warning "VM $vmid ($name) no longer has suspend lock - may have been manually resumed"
((skip_count++))
continue
fi
saved_volume="${saved_vm_volume[$vmid]:-}"
current_volume="${VM_VMSTATE[$vmid]:-}"
if [[ -n "$saved_volume" && "$current_volume" != "$saved_volume" ]]; then
log_warning "VM $vmid ($name) suspend volume changed since state file (${saved_vm_date[$vmid]:-unknown date}): saved=$saved_volume current=${current_volume:-none} - skipping auto-resume"
((skip_count++))
continue
fi
resume_vm "$vmid"
case $? in
0) ((resume_count++)) ;;
2) ((skip_count++)) ;;
*) ((fail_count++)) ;;
esac
done
# --- Start LXC Containers ---
for ctid in "${ct_to_start[@]}"; do
local name="${CT_NAME[$ctid]:-unknown}"
# Verify CT still exists
if [[ ! -f "/etc/pve/lxc/${ctid}.conf" ]]; then
log_error "CT $ctid config not found - skipping"
((fail_count++))
continue
fi
# Check if already running (someone started it manually)
if [[ "${CT_STATUS[$ctid]}" == "running" ]]; then
log_warning "CT $ctid ($name) is already running - skipping"
((skip_count++))
continue
fi
start_ct "$ctid"
case $? in
0) ((resume_count++)) ;;
2) ((skip_count++)) ;;
*) ((fail_count++)) ;;
esac
done
# Clear state file only on full success; keep it for retry if any failures.
if [[ $fail_count -eq 0 ]]; then
clear_state
else
log_warning "Resume/start encountered failures - keeping state file for retry"
fi
# Summary
log_success "Resume/start complete: $resume_count restored, $skip_count skipped, $fail_count failed"
return $fail_count
}
# Acquire lock to prevent concurrent runs
acquire_lock() {
if [[ $DRY_RUN -eq 1 ]]; then
return 0
fi
if [[ -f "$LOCK_FILE" ]]; then
local pid=$(cat "$LOCK_FILE" 2>/dev/null)
if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then
log_error "Another instance is running (PID $pid)"
exit 1
fi
# Stale lock file
rm -f "$LOCK_FILE"
fi
echo $$ > "$LOCK_FILE"
trap "rm -f '$LOCK_FILE'" EXIT
}
# Parse command line
COMMAND=""
while [[ $# -gt 0 ]]; do
case "$1" in
suspend|resume|cleanup)
COMMAND="$1"
shift
;;
-n|--dry-run)
DRY_RUN=1
shift
;;
-v|--verbose)
((VERBOSE++))
shift
;;
-vv)
VERBOSE=2
shift
;;
-h|--help)
usage
exit 0
;;
*)
echo "Unknown option: $1" >&2
usage
exit 1
;;
esac
done
if [[ -z "$COMMAND" ]]; then
echo "Error: No command specified" >&2
usage
exit 1
fi
# Ensure state directory exists
mkdir -p "$STATE_DIR"
# Migrate state from the legacy location used by older installs.
migrate_legacy_state_if_needed
# Acquire lock
acquire_lock
# Execute command
case "$COMMAND" in
suspend)
do_suspend
exit $?
;;
resume)
do_resume
exit $?
;;
cleanup)
do_cleanup
exit $?
;;
esac