From 636670fdcdb9db202d9e7c570ca37d3a7b125e74 Mon Sep 17 00:00:00 2001 From: Nikolaus Eppinger Date: Fri, 10 Jul 2026 14:52:35 +0200 Subject: [PATCH] server: stop NoTransitionException warning spam for fenced hosts in HA manager After the Host HA framework successfully fences a host, FenceTask puts it into maintenance mode, which makes the host ineligible for the HA provider. The background poll task then attempted an Ineligible transition on every poll round, but the HA state machine intentionally has no Fenced -> Ineligible transition (a fenced host must remain Fenced until a health check passes). The result was an endless stream of warnings: WARN [o.a.c.h.HAManagerImpl] Unable to find next HA state for current HA state=[Fenced] for event=[Ineligible] ... com.cloud.utils.fsm.NoTransitionException Skip the Ineligible transition attempt for hosts in Fenced state, since their ineligibility is expected and the intended exit from Fenced is Fenced -> HealthCheckPassed -> Ineligible. No behavioural change otherwise: the host is still skipped by the poll task while fenced and ineligible. --- .../apache/cloudstack/ha/HAManagerImpl.java | 7 +- .../cloudstack/ha/HAManagerImplTest.java | 107 ++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 server/src/test/java/org/apache/cloudstack/ha/HAManagerImplTest.java diff --git a/server/src/main/java/org/apache/cloudstack/ha/HAManagerImpl.java b/server/src/main/java/org/apache/cloudstack/ha/HAManagerImpl.java index 93b237ec20a9..3dbaab449b89 100644 --- a/server/src/main/java/org/apache/cloudstack/ha/HAManagerImpl.java +++ b/server/src/main/java/org/apache/cloudstack/ha/HAManagerImpl.java @@ -221,13 +221,16 @@ private HAResource validateAndFindHAResource(final HAConfig haConfig) { return resource; } - private HAProvider validateAndFindHAProvider(final HAConfig haConfig, final HAResource resource) { + HAProvider validateAndFindHAProvider(final HAConfig haConfig, final HAResource resource) { if (haConfig == null) { return null; } final HAProvider haProvider = haProviderMap.get(haConfig.getHaProvider()); if (haProvider != null && !haProvider.isEligible(resource)) { - if (haConfig.getState() != HAConfig.HAState.Ineligible) { + // A fenced host is expected to be ineligible as fencing puts it in maintenance mode; + // it must remain Fenced until a health check passes (Fenced -> HealthCheckPassed -> Ineligible), + // so there is no Fenced -> Ineligible transition to attempt here. + if (haConfig.getState() != HAConfig.HAState.Ineligible && haConfig.getState() != HAConfig.HAState.Fenced) { transitionHAState(HAConfig.Event.Ineligible, haConfig); } return null; diff --git a/server/src/test/java/org/apache/cloudstack/ha/HAManagerImplTest.java b/server/src/test/java/org/apache/cloudstack/ha/HAManagerImplTest.java new file mode 100644 index 000000000000..0a5c95d2472f --- /dev/null +++ b/server/src/test/java/org/apache/cloudstack/ha/HAManagerImplTest.java @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package org.apache.cloudstack.ha; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import org.apache.cloudstack.ha.provider.HAProvider; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class HAManagerImplTest { + + private static final String HA_PROVIDER_NAME = "kvmhaprovider"; + + @Mock + private HAConfig haConfig; + + @Mock + private HAResource resource; + + @Mock + private HAProvider haProvider; + + private HAManagerImpl haManager; + + @Before + public void setUp() throws Exception { + haManager = Mockito.spy(new HAManagerImpl()); + + final Map> haProviderMap = new HashMap<>(); + haProviderMap.put(HA_PROVIDER_NAME, haProvider); + final Field field = HAManagerImpl.class.getDeclaredField("haProviderMap"); + field.setAccessible(true); + field.set(haManager, haProviderMap); + + Mockito.when(haConfig.getHaProvider()).thenReturn(HA_PROVIDER_NAME); + } + + @Test + public void validateAndFindHAProviderFencedHostDoesNotAttemptIneligibleTransition() { + // A fenced host is put in maintenance mode by the fencing flow and is therefore + // ineligible; the FSM has no Fenced -> Ineligible transition, so none must be attempted + // (it would log a NoTransitionException warning on every poll round otherwise). + Mockito.when(haProvider.isEligible(resource)).thenReturn(false); + Mockito.when(haConfig.getState()).thenReturn(HAConfig.HAState.Fenced); + + assertNull(haManager.validateAndFindHAProvider(haConfig, resource)); + + Mockito.verify(haManager, Mockito.never()).transitionHAState(Mockito.any(HAConfig.Event.class), Mockito.any(HAConfig.class)); + } + + @Test + public void validateAndFindHAProviderIneligibleResourceTransitionsToIneligible() { + Mockito.when(haProvider.isEligible(resource)).thenReturn(false); + Mockito.when(haConfig.getState()).thenReturn(HAConfig.HAState.Available); + Mockito.doReturn(true).when(haManager).transitionHAState(Mockito.any(HAConfig.Event.class), Mockito.any(HAConfig.class)); + + assertNull(haManager.validateAndFindHAProvider(haConfig, resource)); + + Mockito.verify(haManager).transitionHAState(HAConfig.Event.Ineligible, haConfig); + } + + @Test + public void validateAndFindHAProviderAlreadyIneligibleDoesNotRetransition() { + Mockito.when(haProvider.isEligible(resource)).thenReturn(false); + Mockito.when(haConfig.getState()).thenReturn(HAConfig.HAState.Ineligible); + + assertNull(haManager.validateAndFindHAProvider(haConfig, resource)); + + Mockito.verify(haManager, Mockito.never()).transitionHAState(Mockito.any(HAConfig.Event.class), Mockito.any(HAConfig.class)); + } + + @Test + public void validateAndFindHAProviderEligibleResourceTransitionsBackFromIneligible() { + Mockito.when(haProvider.isEligible(resource)).thenReturn(true); + Mockito.when(haConfig.getState()).thenReturn(HAConfig.HAState.Ineligible); + Mockito.doReturn(true).when(haManager).transitionHAState(Mockito.any(HAConfig.Event.class), Mockito.any(HAConfig.class)); + + assertEquals(haProvider, haManager.validateAndFindHAProvider(haConfig, resource)); + + Mockito.verify(haManager).transitionHAState(HAConfig.Event.Eligible, haConfig); + } +}