public void workerDisabled() { Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.SECONDS), "not started"); if (disabled.compareAndSet(false, true)) { changeHistory.addChangeRequest(new WorkerHistoryItem.Metadata(true)); } }
public void workerEnabled() { Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.SECONDS), "not started"); if (disabled.compareAndSet(true, false)) { changeHistory.addChangeRequest(new WorkerHistoryItem.Metadata(false)); } }
@Override public void handle() { synchronized (lock) { final TaskDetails details = runningTasks.get(taskId); if (details == null) { log.warn("Got location notice for task [%s] that isn't running...", taskId); return; } if (!Objects.equals(details.location, location)) { details.location = location; TaskAnnouncement latest = TaskAnnouncement.create( details.task, details.status, details.location ); changeHistory.addChangeRequest(new WorkerHistoryItem.TaskUpdate(latest)); taskAnnouncementChanged(latest); } } } }
public void assignTask(Task task) { Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.SECONDS), "not started"); synchronized (lock) { if (assignedTasks.containsKey(task.getId()) || runningTasks.containsKey(task.getId()) || completedTasks.containsKey(task.getId())) { log.info("Assign task[%s] request ignored because it exists already.", task.getId()); return; } try { jsonMapper.writeValue(new File(getAssignedTaskDir(), task.getId()), task); assignedTasks.put(task.getId(), task); } catch (IOException ex) { log.error(ex, "Error while trying to persist assigned task[%s]", task.getId()); throw new ISE("Assign Task[%s] Request failed because [%s].", task.getId(), ex.getMessage()); } changeHistory.addChangeRequest( new WorkerHistoryItem.TaskUpdate( TaskAnnouncement.create( task, TaskStatus.running(task.getId()), TaskLocation.unknown() ) ) ); } submitNoticeToExec(new RunNotice(task)); }
changes.addChangeRequest(new SegmentChangeRequestLoad(toAnnounce));
try { Files.deleteIfExists(taskFile.toPath()); changeHistory.addChangeRequest(new WorkerHistoryItem.TaskRemoval(taskId));
@Override public void unannounceSegment(DataSegment segment) { synchronized (lock) { final SegmentZNode segmentZNode = segmentLookup.remove(segment); if (segmentZNode == null) { log.warn("No path to unannounce segment[%s]", segment.getId()); return; } changes.addChangeRequest(new SegmentChangeRequestDrop(segment)); if (config.isSkipSegmentAnnouncementOnZk()) { return; } segmentZNode.removeSegment(segment); log.info("Unannouncing segment[%s] at path[%s]", segment.getId(), segmentZNode.getPath()); if (segmentZNode.getCount() == 0) { availableZNodes.remove(segmentZNode); announcer.unannounce(segmentZNode.getPath()); } else { announcer.update(segmentZNode.getPath(), segmentZNode.getBytes()); availableZNodes.add(segmentZNode); } } }
@Test public void testNonImmediateFuture() throws Exception { final ChangeRequestHistory history = new ChangeRequestHistory(); Future<ChangeRequestsSnapshot> future = history.getRequestsSince( ChangeRequestHistory.Counter.ZERO ); Assert.assertFalse(future.isDone()); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestsSnapshot snapshot = future.get(1, TimeUnit.MINUTES); Assert.assertEquals(1, snapshot.getCounter().getCounter()); Assert.assertEquals(1, snapshot.getRequests().size()); }
@Test public void testTruncatedHistory() throws Exception { ChangeRequestHistory<DataSegmentChangeRequest> history = new ChangeRequestHistory(2); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter one = history.getLastCounter(); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter two = history.getLastCounter(); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter three = history.getLastCounter(); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter four = history.getLastCounter(); Assert.assertTrue(history.getRequestsSince(ChangeRequestHistory.Counter.ZERO).get().isResetCounter()); Assert.assertTrue(history.getRequestsSince(one).get().isResetCounter()); Assert.assertTrue(history.getRequestsSince(two).get().isResetCounter()); ChangeRequestsSnapshot<DataSegmentChangeRequest> snapshot = history.getRequestsSince(three).get(); Assert.assertEquals(1, snapshot.getRequests().size()); Assert.assertEquals(4, snapshot.getCounter().getCounter()); }
changeHistory.addChangeRequest(new WorkerHistoryItem.TaskUpdate(latest)); taskAnnouncementChanged(latest);
@Override public void handle() { TaskAnnouncement announcement; synchronized (lock) { if (runningTasks.containsKey(task.getId()) || completedTasks.containsKey(task.getId())) { log.warn( "Got run notice for task [%s] that I am already running or completed...", task.getId() ); taskStarted(task.getId()); return; } final ListenableFuture<TaskStatus> future = taskRunner.run(task); addRunningTask(task, future); announcement = TaskAnnouncement.create( task, TaskStatus.running(task.getId()), TaskLocation.unknown() ); changeHistory.addChangeRequest(new WorkerHistoryItem.TaskUpdate(announcement)); cleanupAssignedTask(task); log.info("Task[%s] started.", task.getId()); } taskAnnouncementChanged(announcement); taskStarted(task.getId()); } }
@Test public void testCounterHashMismatch() throws Exception { ChangeRequestHistory<DataSegmentChangeRequest> history = new ChangeRequestHistory(3); Assert.assertTrue(history.getRequestsSince(new ChangeRequestHistory.Counter(0, 1234)).get().isResetCounter()); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter one = history.getLastCounter(); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter two = history.getLastCounter(); Assert.assertTrue(history.getRequestsSince(new ChangeRequestHistory.Counter(0, 1234)).get().isResetCounter()); ChangeRequestsSnapshot<DataSegmentChangeRequest> snapshot = history.getRequestsSince(one).get(); Assert.assertEquals(1, snapshot.getRequests().size()); Assert.assertEquals(2, snapshot.getCounter().getCounter()); Assert.assertTrue(history.getRequestsSince(new ChangeRequestHistory.Counter(1, 1234)).get().isResetCounter()); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter three = history.getLastCounter(); history.addChangeRequest(new SegmentChangeRequestNoop()); ChangeRequestHistory.Counter four = history.getLastCounter(); snapshot = history.getRequestsSince(two).get(); Assert.assertEquals(2, snapshot.getRequests().size()); Assert.assertEquals(4, snapshot.getCounter().getCounter()); Assert.assertTrue(history.getRequestsSince(new ChangeRequestHistory.Counter(2, 1234)).get().isResetCounter()); }
@Test public void testSimple() throws Exception { ChangeRequestHistory<DataSegmentChangeRequest> history = new ChangeRequestHistory(); Assert.assertEquals(0, history.getLastCounter().getCounter()); history.addChangeRequest(new SegmentChangeRequestNoop()); Assert.assertEquals(1, history.getLastCounter().getCounter()); ChangeRequestsSnapshot<DataSegmentChangeRequest> snapshot = history.getRequestsSince(ChangeRequestHistory.Counter.ZERO).get(); Assert.assertEquals(1, snapshot.getRequests().size()); Assert.assertEquals(1, snapshot.getCounter().getCounter()); history.addChangeRequest(new SegmentChangeRequestNoop()); Assert.assertEquals(2, history.getLastCounter().getCounter()); snapshot = history.getRequestsSince(snapshot.getCounter()).get(); Assert.assertEquals(1, snapshot.getRequests().size()); Assert.assertEquals(2, snapshot.getCounter().getCounter()); snapshot = history.getRequestsSince(ChangeRequestHistory.Counter.ZERO).get(); Assert.assertEquals(2, snapshot.getRequests().size()); Assert.assertEquals(2, snapshot.getCounter().getCounter()); }
public void workerEnabled() { Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.SECONDS), "not started"); if (disabled.compareAndSet(true, false)) { changeHistory.addChangeRequest(new WorkerHistoryItem.Metadata(false)); } }
public void workerDisabled() { Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.SECONDS), "not started"); if (disabled.compareAndSet(false, true)) { changeHistory.addChangeRequest(new WorkerHistoryItem.Metadata(true)); } }
@Override public void handle() { synchronized (lock) { final TaskDetails details = runningTasks.get(taskId); if (details == null) { log.warn("Got location notice for task [%s] that isn't running...", taskId); return; } if (!Objects.equals(details.location, location)) { details.location = location; TaskAnnouncement latest = TaskAnnouncement.create( details.task, details.status, details.location ); changeHistory.addChangeRequest(new WorkerHistoryItem.TaskUpdate(latest)); taskAnnouncementChanged(latest); } } } }
public void assignTask(Task task) { Preconditions.checkState(lifecycleLock.awaitStarted(1, TimeUnit.SECONDS), "not started"); synchronized (lock) { if (assignedTasks.containsKey(task.getId()) || runningTasks.containsKey(task.getId()) || completedTasks.containsKey(task.getId())) { log.info("Assign task[%s] request ignored because it exists already.", task.getId()); return; } try { jsonMapper.writeValue(new File(getAssignedTaskDir(), task.getId()), task); assignedTasks.put(task.getId(), task); } catch (IOException ex) { log.error(ex, "Error while trying to persist assigned task[%s]", task.getId()); throw new ISE("Assign Task[%s] Request failed because [%s].", task.getId(), ex.getMessage()); } changeHistory.addChangeRequest( new WorkerHistoryItem.TaskUpdate( TaskAnnouncement.create( task, TaskStatus.running(task.getId()), TaskLocation.unknown() ) ) ); } submitNoticeToExec(new RunNotice(task)); }
@Override public void unannounceSegment(DataSegment segment) { synchronized (lock) { final SegmentZNode segmentZNode = segmentLookup.remove(segment); if (segmentZNode == null) { log.warn("No path to unannounce segment[%s]", segment.getIdentifier()); return; } changes.addChangeRequest(new SegmentChangeRequestDrop(segment)); if (config.isSkipSegmentAnnouncementOnZk()) { return; } segmentZNode.removeSegment(segment); log.info("Unannouncing segment[%s] at path[%s]", segment.getIdentifier(), segmentZNode.getPath()); if (segmentZNode.getCount() == 0) { availableZNodes.remove(segmentZNode); announcer.unannounce(segmentZNode.getPath()); } else { announcer.update(segmentZNode.getPath(), segmentZNode.getBytes()); availableZNodes.add(segmentZNode); } } }
changeHistory.addChangeRequest(new WorkerHistoryItem.TaskUpdate(latest)); taskAnnouncementChanged(latest);
@Override public void handle() { TaskAnnouncement announcement = null; synchronized (lock) { if (runningTasks.containsKey(task.getId()) || completedTasks.containsKey(task.getId())) { log.warn( "Got run notice for task [%s] that I am already running or completed...", task.getId() ); taskStarted(task.getId()); return; } final ListenableFuture<TaskStatus> future = taskRunner.run(task); addRunningTask(task, future); announcement = TaskAnnouncement.create( task, TaskStatus.running(task.getId()), TaskLocation.unknown() ); changeHistory.addChangeRequest(new WorkerHistoryItem.TaskUpdate(announcement)); cleanupAssignedTask(task); log.info("Task[%s] started.", task.getId()); } taskAnnouncementChanged(announcement); taskStarted(task.getId()); } }