From 141a55c30d25a91ef5a5fceb83dd66b1d5f52622 Mon Sep 17 00:00:00 2001
From: poddm <8801231+poddm@users.noreply.github.com>
Date: Tue, 28 Jul 2026 07:31:07 -0700
Subject: [PATCH] Fix HA-enabled KVM VMs stuck in Running after out-of-band
 stop

---
 .../cloud/vm/VirtualMachineManagerImpl.java   | 36 ++++++++++++++++++-
 .../resource/LibvirtComputingResource.java    |  3 ++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
index b69491b64668..37b5c4dedf16 100755
--- a/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
+++ b/engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java
@@ -5455,7 +5455,41 @@ private void handlePowerOffReportWithNoPendingJobsOnVM(final VMInstanceVO vm) {
                     && vm.getHypervisorType() != HypervisorType.Hyperv) {
                 logger.info("Detected out-of-band stop of a HA enabled VM {}, will schedule restart.", vm);
                 if (!_haMgr.hasPendingHaWork(vm.getId())) {
-                    _haMgr.scheduleRestart(vm, true);
+                    // The power state report already confirmed the VM is off
+                    // (e.g. OOM-killed QEMU process). We cannot use _haMgr.scheduleRestart() because it
+                    // calls advanceStop(), which submits a VM work job and blocks waiting for completion.
+                    // This code runs on the AgentManager-Handler thread, and the VM job queue does not
+                    // dispatch jobs from this context — causing an indefinite block that leaves the VM
+                    // stuck in Running. Instead we release resources, transition to Stopped, and insert
+                    // an HA work item directly into op_ha_work (Step.Scheduled) for the HA worker to pick up.
+                    final VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm);
+                    releaseVmResources(profile, true);
+
+                    // Save lastHostId before transition (stateTransitTo sets hostId=null)
+                    final Long lastHostId = vm.getHostId();
+                    try {
+                        stateTransitTo(vm, VirtualMachine.Event.FollowAgentPowerOffReport, null);
+                    } catch (final NoTransitionException e) {
+                        logger.warn("Failed to transition VM {} to Stopped state: {}", vm, e.getMessage());
+                        return;
+                    }
+
+                    _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_SYNC, vm.getDataCenterId(), vm.getPodIdToDeployIn(),
+                            VM_SYNC_ALERT_SUBJECT, String.format("VM %s(%s) stopped out-of-band (OOM-killed or crashed). HA restart scheduled.",
+                                    vm.getHostName(), vm.getInstanceName()));
+
+                    // Insert HA work item directly — bypasses advanceStop/job queue entirely
+                    final VMInstanceVO refreshedVm = _vmDao.findByUuid(vm.getUuid());
+                    final Long haHostId = lastHostId != null ? lastHostId : refreshedVm.getLastHostId();
+                    if (haHostId == null || haHostId == 0L) {
+                        logger.warn("Cannot schedule HA for VM {} — no valid host_id available (would violate FK constraint).", vm);
+                    } else {
+                        final HaWorkVO work = new HaWorkVO(refreshedVm.getId(), refreshedVm.getType(),
+                                WorkType.HA, HighAvailabilityManager.Step.Scheduled, haHostId,
+                                refreshedVm.getState(), 0, refreshedVm.getUpdated(), null);
+                        _haDao.persist(work);
+                        logger.info("Scheduled VM for HA: {}", refreshedVm);
+                    }
                 } else {
                     logger.info("VM {} already has a pending HA task working on it.", vm);
                 }
diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
index 80d9a51cb855..2baaae6098ed 100644
--- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
+++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java
@@ -537,6 +537,9 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
         POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_BLOCKED, PowerState.PowerOn);
         POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_NOSTATE, PowerState.PowerUnknown);
         POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_SHUTDOWN, PowerState.PowerOff);
+        // Report crashed domains as PowerOff immediately instead of omitting them
+        // from the report (which delays detection via the "missing VM" threshold).
+        POWER_STATES_TABLE.put(DomainState.VIR_DOMAIN_CRASHED, PowerState.PowerOff);
     }
 
     public VirtualRoutingResource virtRouterResource;