public HostHAResponse(final HAConfig config) { this(); if (config == null) { this.enabled = false; this.haState = HAConfig.HAState.Disabled; return; } setProvider(config.getHaProvider()); setEnabled(config.isEnabled()); setHaState(config.getState()); }
private HAResource validateAndFindHAResource(final HAConfig haConfig) { HAResource resource = null; if (haConfig == null) { return null; } if (haConfig.getResourceType() == HAResource.ResourceType.Host) { final Host host = hostDao.findById(haConfig.getResourceId()); if (host != null && host.getRemoved() != null) { return null; } resource = host; if (haConfig.getState() == null || (resource == null && haConfig.getState() != HAConfig.HAState.Disabled)) { disableHA(haConfig.getResourceId(), haConfig.getResourceType()); return null; } } if (!haConfig.isEnabled() || !isHAEnabledForZone(resource) || !isHAEnabledForCluster(resource)) { if (haConfig.getState() != HAConfig.HAState.Disabled) { if (transitionHAState(HAConfig.Event.Disabled, haConfig) ) { purgeHACounter(haConfig.getResourceId(), haConfig.getResourceType()); } } return null; } else if (haConfig.getState() == HAConfig.HAState.Disabled) { transitionHAState(HAConfig.Event.Enabled, haConfig); } return resource; }
private boolean checkHAOwnership(final HAConfig haConfig) { // Skip for resources not owned by this mgmt server return !(haConfig.getManagementServerId() != null && haConfig.getManagementServerId() != ManagementServerNode.getManagementServerId()); }
@Override public boolean preStateTransitionEvent(final HAConfig.HAState oldState, final HAConfig.Event event, final HAConfig.HAState newState, final HAConfig haConfig, final boolean status, final Object opaque) { if (oldState != newState || newState == HAConfig.HAState.Suspect || newState == HAConfig.HAState.Checking) { return false; } if (LOG.isTraceEnabled()) { LOG.trace("HA state pre-transition:: new state=" + newState + ", old state=" + oldState + ", for resource id=" + haConfig.getResourceId() + ", status=" + status + ", ha config state=" + haConfig.getState()); } if (status && haConfig.getState() != newState) { LOG.warn("HA state pre-transition:: HA state is not equal to transition state, HA state=" + haConfig.getState() + ", new state=" + newState); } return processHAStateChange(haConfig, newState, status); }
private HAProvider<HAResource> validateAndFindHAProvider(final HAConfig haConfig, final HAResource resource) { if (haConfig == null) { return null; } final HAProvider<HAResource> haProvider = haProviderMap.get(haConfig.getHaProvider()); if (haProvider != null && !haProvider.isEligible(resource)) { if (haConfig.getState() != HAConfig.HAState.Ineligible) { transitionHAState(HAConfig.Event.Ineligible, haConfig); } return null; } else if (haConfig.getState() == HAConfig.HAState.Ineligible) { transitionHAState(HAConfig.Event.Eligible, haConfig); } return haProvider; }
private void setupResponse(final List<HAConfig> hostHAConfigList) { final ListResponse<HostHAResponse> response = new ListResponse<>(); final List<HostHAResponse> hostHAResponses = new ArrayList<>(); for (final HAConfig config : hostHAConfigList) { final Host host = _resourceService.getHost(config.getResourceId()); if (host == null) { continue; } final HostHAResponse hostHAResponse = new HostHAResponse(); hostHAResponse.setId(host.getUuid()); hostHAResponse.setEnabled(config.isEnabled()); hostHAResponse.setHaState(config.getState()); hostHAResponse.setProvider(config.getHaProvider()); hostHAResponses.add(hostHAResponse); } response.setResponses(hostHAResponses); response.setResponseName(getCommandName()); setResponseObject(response); }
public void processResult(boolean result, Throwable e) { final HAConfig haConfig = getHaConfig(); final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType()); if (result) { haManager.transitionHAState(HAConfig.Event.HealthCheckPassed, haConfig); if (haConfig.getState() == HAConfig.HAState.Fenced) { haManager.disableHA(haConfig.getResourceId(), haConfig.getResourceType()); } counter.resetSuspectTimestamp(); counter.resetActivityCounter(); counter.resetRecoveryCounter(); } else { haManager.transitionHAState(HAConfig.Event.HealthCheckFailed, haConfig); counter.markResourceSuspected(); } } }
public void processResult(boolean result, Throwable e) { final HAConfig haConfig = getHaConfig(); final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType()); counter.incrRecoveryCounter(); counter.resetActivityCounter(); if (result) { haManager.transitionHAState(HAConfig.Event.Recovered, haConfig); getHaProvider().fenceSubResources(getResource()); } getHaProvider().sendAlert(getResource(), HAConfig.HAState.Recovering); } }
private boolean isHAEligibleForResource(final HAResource resource) { if (resource == null || resource.getId() < 1L) { return false; } HAResource.ResourceType resourceType = null; if (resource instanceof Host) { resourceType = HAResource.ResourceType.Host; } if (resourceType == null) { return false; } final HAConfig haConfig = haConfigDao.findHAResource(resource.getId(), resourceType); return haConfig != null && haConfig.isEnabled() && haConfig.getState() != HAConfig.HAState.Disabled && haConfig.getState() != HAConfig.HAState.Ineligible; }
public Status getHostStatus(final Host host) { final HAConfig haConfig = haConfigDao.findHAResource(host.getId(), HAResource.ResourceType.Host); if (haConfig != null) { if (haConfig.getState() == HAConfig.HAState.Fenced) { if (LOG.isDebugEnabled()){ LOG.debug("HA: Agent is available/suspect/checking Up " + host.getId()); } return Status.Down; } else if (haConfig.getState() == HAConfig.HAState.Degraded || haConfig.getState() == HAConfig.HAState.Recovering || haConfig.getState() == HAConfig.HAState.Fencing) { if (LOG.isDebugEnabled()){ LOG.debug("HA: Agent is disconnected " + host.getId()); } return Status.Disconnected; } return Status.Up; } return Status.Unknown; }
switch (haConfig.getState()) { case Available: case Suspect: final HAResourceCounter counter = getHACounter(haConfig.getResourceId(), haConfig.getResourceType()); if (haConfig.getState() == HAConfig.HAState.Suspect) { if (counter.canPerformActivityCheck((Long)(haProvider.getConfigValue(HAProviderConfig.MaxActivityCheckInterval, resource)))) { transitionHAState(HAConfig.Event.PerformActivityCheck, haConfig); if (haConfig.getState() == HAConfig.HAState.Degraded) { if (counter.canRecheckActivity((Long)(haProvider.getConfigValue(HAProviderConfig.MaxDegradedWaitTimeout, resource)))) { transitionHAState(HAConfig.Event.PeriodicRecheckResourceActivity, haConfig); if (haConfig.getState() == HAConfig.HAState.Recovering) { if (counter.getRecoveryCounter() >= (Long) (haProvider.getConfigValue(HAProviderConfig.MaxRecoveryAttempts, resource))) { transitionHAState(HAConfig.Event.RecoveryOperationThresholdExceeded, haConfig); if (haConfig.getState() == HAConfig.HAState.Recovered) { counter.markRecoveryStarted(); if (counter.canExitRecovery((Long)(haProvider.getConfigValue(HAProviderConfig.RecoveryWaitTimeout, resource)))) { if (haConfig.getState() == HAConfig.HAState.Fencing && counter.canAttemptFencing()) { transitionHAState(HAConfig.Event.RetryFencing, haConfig);
@Override public boolean postStateTransitionEvent(final StateMachine2.Transition<HAConfig.HAState, HAConfig.Event> transition, final HAConfig haConfig, final boolean status, final Object opaque) { if (LOG.isTraceEnabled()) { LOG.trace("HA state post-transition:: new state=" + transition.getToState() + ", old state=" + transition.getCurrentState() + ", for resource id=" + haConfig.getResourceId() + ", status=" + status + ", ha config state=" + haConfig.getState()); } if (status && haConfig.getState() != transition.getToState()) { LOG.warn("HA state post-transition:: HA state is not equal to transition state, HA state=" + haConfig.getState() + ", new state=" + transition.getToState()); } return processHAStateChange(haConfig, transition.getToState(), status); }
public void processResult(boolean result, Throwable e) { final HAConfig haConfig = getHaConfig(); final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType()); if (result) { counter.resetRecoveryCounter(); haManager.transitionHAState(HAConfig.Event.Fenced, haConfig); getHaProvider().fenceSubResources(getResource()); getHaProvider().enableMaintenance(getResource()); } getHaProvider().sendAlert(getResource(), HAConfig.HAState.Fencing); } }
public Boolean isVMAliveOnHost(final Host host) throws Investigator.UnknownVM { final HAConfig haConfig = haConfigDao.findHAResource(host.getId(), HAResource.ResourceType.Host); if (haConfig != null) { if (haConfig.getState() == HAConfig.HAState.Fenced) { if (LOG.isDebugEnabled()){ LOG.debug("HA: Host is fenced " + host.getId()); } return false; } if (LOG.isDebugEnabled()){ LOG.debug("HA: HOST is alive " + host.getId()); } return true; } throw new Investigator.UnknownVM(); }
public boolean transitionHAState(final HAConfig.Event event, final HAConfig haConfig) { if (event == null || haConfig == null) { return false; } final HAConfig.HAState currentHAState = haConfig.getState(); try { final HAConfig.HAState nextState = HAConfig.HAState.getStateMachine().getNextState(currentHAState, event); boolean result = HAConfig.HAState.getStateMachine().transitTo(haConfig, event, null, haConfigDao); if (result) { final String message = String.format("Transitioned host HA state from:%s to:%s due to event:%s for the host id:%d", currentHAState, nextState, event, haConfig.getResourceId()); if (LOG.isTraceEnabled()) { LOG.trace(message); } if (nextState == HAConfig.HAState.Recovering || nextState == HAConfig.HAState.Fencing || nextState == HAConfig.HAState.Fenced) { ActionEventUtils.onActionEvent(CallContext.current().getCallingUserId(), CallContext.current().getCallingAccountId(), Domain.ROOT_DOMAIN, EventTypes.EVENT_HA_STATE_TRANSITION, message); } } return result; } catch (NoTransitionException e) { if (LOG.isTraceEnabled()) { LOG.trace("Unable to find next HA state for current HA state: " + currentHAState + " for event: " + event + " for host" + haConfig.getResourceId()); } } return false; }
final HAResourceCounter counter = getHACounter(haConfig.getResourceId(), haConfig.getResourceType());
public synchronized void processResult(boolean result, Throwable t) { final HAConfig haConfig = getHaConfig(); final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType()); if (t != null && t instanceof HACheckerException) { haManager.transitionHAState(HAConfig.Event.Ineligible, getHaConfig()); counter.resetActivityCounter(); return; } counter.incrActivityCounter(!result); if (counter.getActivityCheckCounter() < maxActivityChecks) { haManager.transitionHAState(HAConfig.Event.TooFewActivityCheckSamples, haConfig); return; } if (counter.hasActivityThresholdExceeded(activityCheckFailureRatio)) { haManager.transitionHAState(HAConfig.Event.ActivityCheckFailureOverThresholdRatio, haConfig); } else { if (haManager.transitionHAState(HAConfig.Event.ActivityCheckFailureUnderThresholdRatio, haConfig)) { counter.markResourceDegraded(); } } counter.resetActivityCounter(); } }