public ActivityCheckTask(final HAResource resource, final HAProvider<HAResource> haProvider, final HAConfig haConfig, final HAProvider.HAProviderConfig haProviderConfig, final ExecutorService executor, final long disconnectTime) { super(resource, haProvider, haConfig, haProviderConfig, executor); this.disconnectTime = disconnectTime; this.maxActivityChecks = (Long)haProvider.getConfigValue(HAProviderConfig.MaxActivityChecks, resource); this.activityCheckFailureRatio = (Double)haProvider.getConfigValue(HAProviderConfig.ActivityCheckFailureRatio, resource); }
@Override public boolean isActive(Host r, DateTime suspectTime) throws HACheckerException { try { return isVMActivtyOnHost(r, suspectTime); } catch (StorageUnavailableException e){ throw new HACheckerException("Storage is unavailable to do the check, mostly host is not reachable ", e); } catch (Exception e){ throw new HACheckerException("Operation timed out, mostly host is not reachable ", e); } }
public boolean performAction() throws HACheckerException, HAFenceException { return getHaProvider().fence(getResource()); }
public void processResult(boolean result, Throwable e) { final HAConfig haConfig = getHaConfig(); final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType()); if (result) { counter.resetRecoveryCounter(); haManager.transitionHAState(HAConfig.Event.Fenced, haConfig); getHaProvider().fenceSubResources(getResource()); getHaProvider().enableMaintenance(getResource()); } getHaProvider().sendAlert(getResource(), HAConfig.HAState.Fencing); } }
@Override public List<String> listHAProviders(final HAResource.ResourceType resourceType, final HAResource.ResourceSubType entityType) { final List<String> haProviderNames = new ArrayList<>(); for (final HAProvider<HAResource> haProvider : haProviders) { if (haProvider.resourceType().equals(resourceType) && haProvider.resourceSubType().equals(entityType)) { haProviderNames.add(haProvider.getClass().getSimpleName()); } } return haProviderNames; }
public void processResult(boolean result, Throwable e) { final HAConfig haConfig = getHaConfig(); final HAResourceCounter counter = haManager.getHACounter(haConfig.getResourceId(), haConfig.getResourceType()); counter.incrRecoveryCounter(); counter.resetActivityCounter(); if (result) { haManager.transitionHAState(HAConfig.Event.Recovered, haConfig); getHaProvider().fenceSubResources(getResource()); } getHaProvider().sendAlert(getResource(), HAConfig.HAState.Recovering); } }
public boolean performAction() throws HACheckerException, HARecoveryException { return getHaProvider().recover(getResource()); }
public boolean performAction() throws HACheckerException { return getHaProvider().isHealthy(getResource()); }
public boolean performAction() throws HACheckerException { return getHaProvider().hasActivity(getResource(), new DateTime(disconnectTime)); }
private HAProvider<HAResource> validateAndFindHAProvider(final HAConfig haConfig, final HAResource resource) { if (haConfig == null) { return null; } final HAProvider<HAResource> haProvider = haProviderMap.get(haConfig.getHaProvider()); if (haProvider != null && !haProvider.isEligible(resource)) { if (haConfig.getState() != HAConfig.HAState.Ineligible) { transitionHAState(HAConfig.Event.Ineligible, haConfig); } return null; } else if (haConfig.getState() == HAConfig.HAState.Ineligible) { transitionHAState(HAConfig.Event.Eligible, haConfig); } return haProvider; }
@Override public boolean recover(Host r) throws HARecoveryException { try { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.RESET, null); return resp.getSuccess(); } else { LOG.warn("OOBM recover operation failed for the host " + r.getName()); return false; } } catch (Exception e){ LOG.warn("OOBM service is not configured or enabled for this host " + r.getName() + " error is " + e.getMessage()); throw new HARecoveryException(" OOBM service is not configured or enabled for this host " + r.getName(), e); } }
@Override public boolean fence(Host r) throws HAFenceException { try { if (outOfBandManagementService.isOutOfBandManagementEnabled(r)){ final OutOfBandManagementResponse resp = outOfBandManagementService.executePowerOperation(r, PowerOperation.OFF, null); return resp.getSuccess(); } else { LOG.warn("OOBM fence operation failed for this host " + r.getName()); return false; } } catch (Exception e){ LOG.warn("OOBM service is not configured or enabled for this host " + r.getName() + " error is " + e.getMessage()); throw new HAFenceException("OOBM service is not configured or enabled for this host " + r.getName() , e); } }
public void validateHAProviderConfigForResource(final Long resourceId, final HAResource.ResourceType resourceType, final HAProvider<HAResource> haProvider) { if (HAResource.ResourceType.Host.equals(resourceType)) { final Host host = hostDao.findById(resourceId); if (host.getHypervisorType() == null || haProvider.resourceSubType() == null || !host.getHypervisorType().toString().equals(haProvider.resourceSubType().toString())) { throw new ServerApiException(ApiErrorCode.PARAM_ERROR, "Incompatible haprovider provided for the resource of hypervisor type:" + host.getHypervisorType()); } } }
@Override public Object getConfigValue(final HAProviderConfig name, final Host host) { final Long clusterId = host.getClusterId(); switch (name) { case HealthCheckTimeout: return KVMHAConfig.KvmHAHealthCheckTimeout.valueIn(clusterId); case ActivityCheckTimeout: return KVMHAConfig.KvmHAActivityCheckTimeout.valueIn(clusterId); case MaxActivityCheckInterval: return KVMHAConfig.KvmHAActivityCheckInterval.valueIn(clusterId); case MaxActivityChecks: return KVMHAConfig.KvmHAActivityCheckMaxAttempts.valueIn(clusterId); case ActivityCheckFailureRatio: return KVMHAConfig.KvmHAActivityCheckFailureThreshold.valueIn(clusterId); case RecoveryWaitTimeout: return KVMHAConfig.KvmHARecoverWaitPeriod.valueIn(clusterId); case RecoveryTimeout: return KVMHAConfig.KvmHARecoverTimeout.valueIn(clusterId); case FenceTimeout: return KVMHAConfig.KvmHAFenceTimeout.valueIn(clusterId); case MaxRecoveryAttempts: return KVMHAConfig.KvmHARecoverAttemptThreshold.valueIn(clusterId); case MaxDegradedWaitTimeout: return KVMHAConfig.KvmHADegradedMaxPeriod.valueIn(clusterId); default: throw new InvalidParameterException("Unknown HAProviderConfig " + name.toString()); } }
public BaseHATask(final HAResource resource, final HAProvider<HAResource> haProvider, final HAConfig haConfig, final HAProvider.HAProviderConfig haProviderConfig, final ExecutorService executor) { this.resource = resource; this.haProvider = haProvider; this.haConfig = haConfig; this.executor = executor; this.timeout = (Long)haProvider.getConfigValue(haProviderConfig, resource); this.created = new DateTime(); }
if (counter.canPerformActivityCheck((Long)(haProvider.getConfigValue(HAProviderConfig.MaxActivityCheckInterval, resource)))) { transitionHAState(HAConfig.Event.PerformActivityCheck, haConfig); if (counter.canRecheckActivity((Long)(haProvider.getConfigValue(HAProviderConfig.MaxDegradedWaitTimeout, resource)))) { transitionHAState(HAConfig.Event.PeriodicRecheckResourceActivity, haConfig); if (counter.getRecoveryCounter() >= (Long) (haProvider.getConfigValue(HAProviderConfig.MaxRecoveryAttempts, resource))) { transitionHAState(HAConfig.Event.RecoveryOperationThresholdExceeded, haConfig); } else { if (counter.canExitRecovery((Long)(haProvider.getConfigValue(HAProviderConfig.RecoveryWaitTimeout, resource)))) { if (transitionHAState(HAConfig.Event.RecoveryWaitPeriodTimeout, haConfig)) { counter.markRecoveryCompleted();
if (counter.getRecoveryCounter() >= (Long) (haProvider.getConfigValue(HAProviderConfig.MaxRecoveryAttempts, resource))) { return false;