/** {@inheritDoc} */ @Override public synchronized void resourceOffers(SchedulerDriver schedulerDriver, List<Protos.Offer> offers) { log.log(Level.FINE, "Offers resources: {0}", offers.size()); for (Protos.Offer offer : offers) { IgniteTask igniteTask = checkOffer(offer); // Decline offer which doesn't match by mem or cpu. if (igniteTask == null) { schedulerDriver.declineOffer(offer.getId()); continue; } // Generate a unique task ID. Protos.TaskID taskId = Protos.TaskID.newBuilder() .setValue(Integer.toString(taskIdGenerator.incrementAndGet())).build(); log.log(Level.INFO, "Launching task: {0}", igniteTask); // Create task to run. Protos.TaskInfo task = createTask(offer, igniteTask, taskId); try { schedulerDriver.launchTasks(Collections.singletonList(offer.getId()), Collections.singletonList(task), Protos.Filters.newBuilder().setRefuseSeconds(1).build()); } catch (RuntimeException e) { log.log(Level.SEVERE, "Failed launch task. Task id: {0}. Task info: {1}", new Object[]{taskId, task, e}); throw e; } tasks.put(taskId.getValue(), igniteTask); } }
@Override protected CompletableFuture<Void> clearStateAsync() { schedulerDriver.stop(true); workersInNew.clear(); workersInLaunch.clear(); workersBeingReturned.clear(); return stopSupportingActorsAsync(); }
@Override protected CompletableFuture<Void> prepareLeadershipAsync() { Preconditions.checkState(initializedMesosConfig != null); schedulerDriver = initializedMesosConfig.createDriver( new MesosResourceManagerSchedulerCallback(), false); // create supporting actors connectionMonitor = createConnectionMonitor(); launchCoordinator = createLaunchCoordinator(schedulerDriver, selfActor); reconciliationCoordinator = createReconciliationCoordinator(schedulerDriver); taskMonitor = createTaskMonitor(schedulerDriver); return getWorkersAsync().thenApplyAsync((tasksFromPreviousAttempts) -> { // recover state recoverWorkers(tasksFromPreviousAttempts); // begin scheduling connectionMonitor.tell(new ConnectionMonitor.Start(), selfActor); schedulerDriver.start(); LOG.info("Mesos resource manager started."); return null; }, getMainThreadExecutor()); }
@Override public void expect(ClusterState state, SchedulerDriver mockDriver) { ArgumentCaptor<Protos.OfferID> offerIdCaptor = ArgumentCaptor.forClass(Protos.OfferID.class); Set<String> lastCycleOfferIds = state.getLastOfferCycle().stream() .map(o -> o.getId().getValue()) .collect(Collectors.toSet()); Mockito.verify(mockDriver, Mockito.atLeast(lastCycleOfferIds.size())).declineOffer(offerIdCaptor.capture(), any()); // Check that the offer ids from the last cycle were all declined: Set<String> declinedOfferIds = offerIdCaptor.getAllValues().stream() .map(o -> o.getValue()) .collect(Collectors.toSet()); Assert.assertTrue( String.format("Expected all offers from last offer cycle to be declined: %s, got: %s", lastCycleOfferIds, declinedOfferIds), declinedOfferIds.containsAll(lastCycleOfferIds)); }
public void killTask(Protos.TaskID taskId) { driver.get().killTask(taskId); }
log.info("Declining all offers that are currently buffered because no topologies nor tasks need assignments. Declined offer ids: {}", offerMapKeySetToString(offers)); for (Protos.OfferID offerId : offers.keySet()) { driver.declineOffer(offerId); driver.suppressOffers(); offersSuppressed = true; driver.reviveOffers(); offersSuppressed = false;
_driver.launchTasks(offerIDList, taskInfoList); for (OfferID offerID: offerIDList) { _offers.remove(offerID);
@Test public void testEmptyOffers() throws Exception { UninstallScheduler uninstallScheduler = getUninstallScheduler(); Assert.assertEquals(ClientStatusResponse.launching(true), uninstallScheduler.getClientStatus()); Assert.assertEquals(OfferResponse.Result.PROCESSED, uninstallScheduler.offers(Collections.emptyList()).result); verify(mockSchedulerDriver, times(0)).acceptOffers(any(), any(), any()); verify(mockSchedulerDriver, times(0)).declineOffer(any(), any()); }
@Override public void expect(ClusterState state, SchedulerDriver mockDriver) { Mockito.verify(mockDriver, Mockito.atLeastOnce()).reconcileTasks(Collections.emptyList()); }
private void suppressOrRevive() { boolean hasOperations = !planManager.getPlan().isComplete() || recoveryScheduler.hasOperations(); LOGGER.debug(hasOperations ? "Scheduler has operations to perform." : "Scheduler has no operations to perform."); if (hasOperations) { // Revive offers only if they were previously suppressed. if (cassandraState.isSuppressed()) { LOGGER.info("Reviving offers."); driver.reviveOffers(); cassandraState.setSuppressed(false); } } else { LOGGER.info("Suppressing offers."); driver.suppressOffers(); cassandraState.setSuppressed(true); } }
Assert.assertEquals(Arrays.asList(insufficientOffer), OfferUtils.filterOutAcceptedOffers(offers, response.recommendations)); verify(mockSchedulerDriver, times(0)).killTask(any()); Assert.assertEquals(Status.PENDING, stepTaskA0.getStatus()); verify(mockSchedulerDriver, times(1)).reconcileTasks( Arrays.asList(getTaskStatus(launchedTaskId, Protos.TaskState.TASK_RUNNING))); statusUpdate(launchedTaskId, Protos.TaskState.TASK_RUNNING); Assert.assertEquals(Arrays.asList(insufficientOffer), OfferUtils.filterOutAcceptedOffers(offers, response.recommendations)); verify(mockSchedulerDriver, times(1)).killTask(launchedTaskId); Assert.assertEquals(Status.PREPARED, stepTaskA0.getStatus());
@Override public void expect(ClusterState state, SchedulerDriver mockDriver) { Mockito.verify(mockDriver, Mockito.times(totalTimes)).reviveOffers(); }
@Override public void expect(ClusterState state, SchedulerDriver mockDriver) { Mockito.verify(mockDriver, Mockito.times(totalTimes)).suppressOffers(); }
schedulerDriver.acceptOffers(msg.offerIds(), msg.operations(), msg.filters()); } catch (Exception ex) { onFatalError(new ResourceManagerException("unable to accept offers", ex));
/** * Decline unused {@link org.apache.mesos.Protos.Offer}s. * * @param unusedOffers The collection of Offers to decline * @param refuseSeconds The number of seconds for which the offers should be refused */ private static void declineOffers(Collection<Protos.Offer> unusedOffers, int refuseSeconds) { Collection<Protos.OfferID> offerIds = unusedOffers.stream() .map(offer -> offer.getId()) .collect(Collectors.toList()); LOGGER.info("Declining {} unused offer{} for {} seconds: {}", offerIds.size(), offerIds.size() == 1 ? "" : "s", refuseSeconds, offerIds.stream().map(Protos.OfferID::getValue).collect(Collectors.toList())); final Protos.Filters filters = Protos.Filters.newBuilder() .setRefuseSeconds(refuseSeconds) .build(); offerIds.forEach(offerId -> Driver.getInstance().declineOffer(offerId, filters)); }
public void killTracker(MesosTracker tracker) { if (metrics != null) { metrics.killMeter.mark(); } synchronized (this) { driver.killTask(tracker.taskId); } tracker.stop(); if (mesosTrackers.get(tracker.host) == tracker) { mesosTrackers.remove(tracker.host); } }
@Test public void testResourceOffers_launchTasks() { final Protos.Offer offer = newOffer("host3").build(); when(offerStrategy.evaluate(offer)).thenReturn(OfferStrategy.OfferResult.accept()); when(frameworkState.isRegistered()).thenReturn(true); Protos.TaskInfo taskInfo = ProtoTestUtil.getDefaultTaskInfo(); when(taskInfoFactory.createTask(any(), any(), any(), any())).thenReturn(taskInfo); scheduler.resourceOffers(driver, singletonList(offer)); verify(driver).launchTasks(singleton(offer.getId()), singleton(taskInfo)); }
@Test public void testAcceptedAndUnexpectedResources() throws InterruptedException { List<Protos.Offer> sentOffers = Arrays.asList(getOffer(), getOffer(), getOffer()); List<Protos.OfferID> sentOfferIds = sentOffers.stream().map(o -> o.getId()).collect(Collectors.toList()); Protos.OfferID offerToConsume = sentOfferIds.get(0); Protos.OfferID offerToUnreserve = sentOfferIds.get(2); when(mockMesosEventClient.offers(any())).thenAnswer(consumeOffer(offerToConsume)); when(mockMesosEventClient.getUnexpectedResources(any())).thenAnswer(unexpectedOffer(offerToUnreserve)); processor.setOfferQueueSize(0).start(); // unlimited queue size processor.enqueue(sentOffers); processor.awaitOffersProcessed(); // One declined offer, one reserved offer, one unreserved offer: verify(mockSchedulerDriver, times(1)).declineOffer(sentOfferIds.get(1), LONG_INTERVAL); verify(mockSchedulerDriver, times(1)).acceptOffers(offerIdCaptor.capture(), operationCaptor.capture(), any()); Assert.assertEquals(new HashSet<>(Arrays.asList(offerToConsume, offerToUnreserve)), offerIdCaptor.getValue()); List<Protos.Offer.Operation> operations = operationCaptor.getValue(); Assert.assertEquals(2, operations.size()); Assert.assertEquals(Protos.Offer.Operation.Type.RESERVE, operations.get(0).getType()); Assert.assertEquals(Protos.Offer.Operation.Type.UNRESERVE, operations.get(1).getType()); }
/** * 隐式协调. */ public void implicitReconcile() { lock.lock(); try { schedulerDriver.reconcileTasks(Collections.<Protos.TaskStatus>emptyList()); } finally { lock.unlock(); } }
@Test public void suppressRevive() { ReviveManager manager = new ReviveManager( TokenBucket.newBuilder().acquireInterval(Duration.ZERO).build(), mockSchedulerConfig); // Suppress: manager.suppressIfActive(); verify(driver, times(1)).suppressOffers(); // Revive: manager.requestReviveIfSuppressed(); manager.reviveIfRequested(); verify(driver, times(1)).reviveOffers(); // Revive again, because previous revive apparently didn't go through: manager.requestReviveIfSuppressed(); manager.reviveIfRequested(); verify(driver, times(2)).reviveOffers(); // Finally get an offer, un-suppress must've worked. manager.notifyOffersReceived(); // Now that we aren't suppressed, revive is not triggered by just needing offers: manager.requestReviveIfSuppressed(); manager.reviveIfRequested(); verify(driver, times(2)).reviveOffers(); // .. but still revives if specifically requested (due to new work): manager.requestRevive(); manager.reviveIfRequested(); verify(driver, times(3)).reviveOffers(); }