BPServiceActor(InetSocketAddress nnAddr, InetSocketAddress lifelineNnAddr, BPOfferService bpos) { this.bpos = bpos; this.dn = bpos.getDataNode(); this.nnAddr = nnAddr; this.lifelineSender = lifelineNnAddr != null ? new LifelineSender(lifelineNnAddr) : null; this.initialRegistrationComplete = lifelineNnAddr != null ? new CountDownLatch(1) : null; this.dnConf = dn.getDnConf(); this.ibrManager = new IncrementalBlockReportManager( dnConf.ibrInterval, dn.getMetrics()); prevBlockReportId = ThreadLocalRandom.current().nextLong(); scheduler = new Scheduler(dnConf.heartBeatInterval, dnConf.getLifelineIntervalMs(), dnConf.blockReportInterval, dnConf.outliersReportIntervalMs); // get the value of maxDataLength. this.maxDataLength = dnConf.getMaxDataLength(); }
try { DataNodeFaultInjector.get().startOfferService(); final long startTime = scheduler.monotonicNow(); final boolean sendHeartbeat = scheduler.isHeartbeatDue(startTime); HeartbeatResponse resp = null; if (sendHeartbeat) { scheduler.isBlockReportDue(startTime); if (!dn.areHeartbeatsDisabledForTests()) { resp = sendHeartBeat(requestBlockReportLease); dn.getMetrics().addHeartbeat(scheduler.monotonicNow() - startTime); scheduler.monotonicNow() - startTime); ibrManager.waitTillNextIBR(scheduler.getHeartbeatWaitTime()); } catch(RemoteException re) { String reClass = re.getClassName();
scheduler.scheduleBlockReport(dnConf.initialBlockReportDelayMs);
if (!scheduler.isBlockReportDue()) { return null; "."); scheduler.scheduleNextBlockReport(); return cmds.size() == 0 ? null : cmds;
if (!scheduler.isBlockReportDue()) { return null; "."); scheduler.scheduleNextBlockReport(); return cmds.size() == 0 ? null : cmds;
final long startTime = scheduler.monotonicNow(); final boolean sendHeartbeat = scheduler.isHeartbeatDue(startTime); if (sendHeartbeat) { HeartbeatResponse resp = sendHeartBeat(); assert resp != null; dn.getMetrics().addHeartbeat(scheduler.monotonicNow() - startTime); ibrManager.waitTillNextIBR(scheduler.getHeartbeatWaitTime()); } catch(RemoteException re) { String reClass = re.getClassName();
final long startTime = scheduler.monotonicNow(); final boolean sendHeartbeat = scheduler.isHeartbeatDue(startTime); if (sendHeartbeat) { HeartbeatResponse resp = sendHeartBeat(); assert resp != null; dn.getMetrics().addHeartbeat(scheduler.monotonicNow() - startTime); ibrManager.waitTillNextIBR(scheduler.getHeartbeatWaitTime()); } catch(RemoteException re) { String reClass = re.getClassName();
scheduler.scheduleBlockReport(dnConf.initialBlockReportDelay);
scheduler.scheduleBlockReport(dnConf.initialBlockReportDelay);
/** * Schedule the next block report after the block report interval. If the * current block report was delayed then the next block report is sent per * the original schedule. * Numerical overflow is possible here. */ void scheduleNextBlockReport() { // If we have sent the first set of block reports, then wait a random // time before we start the periodic block reports. if (resetBlockReportTime) { nextBlockReportTime = monotonicNow() + DFSUtil.getRandom().nextInt((int)(blockReportIntervalMs)); resetBlockReportTime = false; } else { /* say the last block report was at 8:20:14. The current report * should have started around 9:20:14 (default 1 hour interval). * If current time is : * 1) normal like 9:20:18, next report should be at 10:20:14 * 2) unexpected like 11:35:43, next report should be at 12:20:14 */ nextBlockReportTime += (((monotonicNow() - nextBlockReportTime + blockReportIntervalMs) / blockReportIntervalMs)) * blockReportIntervalMs; } }
/** * Schedule the next block report after the block report interval. If the * current block report was delayed then the next block report is sent per * the original schedule. * Numerical overflow is possible here. */ void scheduleNextBlockReport() { // If we have sent the first set of block reports, then wait a random // time before we start the periodic block reports. if (resetBlockReportTime) { nextBlockReportTime = monotonicNow() + DFSUtil.getRandom().nextInt((int)(blockReportIntervalMs)); resetBlockReportTime = false; } else { /* say the last block report was at 8:20:14. The current report * should have started around 9:20:14 (default 1 hour interval). * If current time is : * 1) normal like 9:20:18, next report should be at 10:20:14 * 2) unexpected like 11:35:43, next report should be at 12:20:14 */ nextBlockReportTime += (((monotonicNow() - nextBlockReportTime + blockReportIntervalMs) / blockReportIntervalMs)) * blockReportIntervalMs; } }
HeartbeatResponse sendHeartBeat() throws IOException { scheduler.scheduleNextHeartbeat(); StorageReport[] reports = dn.getFSDataset().getStorageReports(bpos.getBlockPoolId()); if (LOG.isDebugEnabled()) { LOG.debug("Sending heartbeat with " + reports.length + " storage reports from service actor: " + this); } VolumeFailureSummary volumeFailureSummary = dn.getFSDataset() .getVolumeFailureSummary(); int numFailedVolumes = volumeFailureSummary != null ? volumeFailureSummary.getFailedStorageLocations().length : 0; return bpNamenode.sendHeartbeat(bpRegistration, reports, dn.getFSDataset().getCacheCapacity(), dn.getFSDataset().getCacheUsed(), dn.getXmitsInProgress(), dn.getXceiverCount(), numFailedVolumes, volumeFailureSummary); }
/** * Tests the case when a block report was delayed past its scheduled time. * In that case the next block report should not be delayed for a full interval. */ @Test public void testScheduleNextBlockReport3() { for (final long now : getTimestamps()) { Scheduler scheduler = makeMockScheduler(now); scheduler.resetBlockReportTime = false; // Make it look like the block report was scheduled to be sent between 1-3 // intervals ago but sent just now. final long blockReportDelay = BLOCK_REPORT_INTERVAL_MS + random.nextInt(2 * (int) BLOCK_REPORT_INTERVAL_MS); final long origBlockReportTime = now - blockReportDelay; scheduler.nextBlockReportTime = origBlockReportTime; scheduler.scheduleNextBlockReport(); assertTrue(scheduler.nextBlockReportTime - now < BLOCK_REPORT_INTERVAL_MS); assertTrue(((scheduler.nextBlockReportTime - origBlockReportTime) % BLOCK_REPORT_INTERVAL_MS) == 0); } }
HeartbeatResponse sendHeartBeat() throws IOException { scheduler.scheduleNextHeartbeat(); StorageReport[] reports = dn.getFSDataset().getStorageReports(bpos.getBlockPoolId()); if (LOG.isDebugEnabled()) { LOG.debug("Sending heartbeat with " + reports.length + " storage reports from service actor: " + this); } VolumeFailureSummary volumeFailureSummary = dn.getFSDataset() .getVolumeFailureSummary(); int numFailedVolumes = volumeFailureSummary != null ? volumeFailureSummary.getFailedStorageLocations().length : 0; return bpNamenode.sendHeartbeat(bpRegistration, reports, dn.getFSDataset().getCacheCapacity(), dn.getFSDataset().getCacheUsed(), dn.getXmitsInProgress(), dn.getXceiverCount(), numFailedVolumes, volumeFailureSummary); }
/** * Run an immediate block report on this thread. Used by tests. */ @VisibleForTesting void triggerBlockReportForTests() { synchronized (ibrManager) { scheduler.scheduleHeartbeat(); long nextBlockReportTime = scheduler.scheduleBlockReport(0); ibrManager.notifyAll(); while (nextBlockReportTime - scheduler.nextBlockReportTime >= 0) { try { ibrManager.wait(100); } catch (InterruptedException e) { return; } } } }
/** * Run an immediate block report on this thread. Used by tests. */ @VisibleForTesting void triggerBlockReportForTests() { synchronized (ibrManager) { scheduler.scheduleHeartbeat(); long nextBlockReportTime = scheduler.scheduleBlockReport(0); ibrManager.notifyAll(); while (nextBlockReportTime - scheduler.nextBlockReportTime >= 0) { try { ibrManager.wait(100); } catch (InterruptedException e) { return; } } } }
/** * This methods arranges for the data node to send the block report at * the next heartbeat. */ long scheduleBlockReport(long delay) { if (delay > 0) { // send BR after random delay // Numerical overflow is possible here and is okay. nextBlockReportTime = monotonicNow() + DFSUtil.getRandom().nextInt((int) (delay)); } else { // send at next heartbeat nextBlockReportTime = monotonicNow(); } resetBlockReportTime = true; // reset future BRs for randomness return nextBlockReportTime; }
/** * Regression test for HDFS-9305. * Delayed processing of a heartbeat can cause a subsequent heartbeat * storm. */ @Test public void testScheduleDelayedHeartbeat() { for (final long now : getTimestamps()) { Scheduler scheduler = makeMockScheduler(now); scheduler.scheduleNextHeartbeat(); assertFalse(scheduler.isHeartbeatDue(now)); // Simulate a delayed heartbeat e.g. due to slow processing by NN. scheduler.nextHeartbeatTime = now - (HEARTBEAT_INTERVAL_MS * 10); scheduler.scheduleNextHeartbeat(); // Ensure that the next heartbeat is not due immediately. assertFalse(scheduler.isHeartbeatDue(now)); } }
/** * This methods arranges for the data node to send the block report at * the next heartbeat. */ long scheduleBlockReport(long delay) { if (delay > 0) { // send BR after random delay // Numerical overflow is possible here and is okay. nextBlockReportTime = monotonicNow() + DFSUtil.getRandom().nextInt((int) (delay)); } else { // send at next heartbeat nextBlockReportTime = monotonicNow(); } resetBlockReportTime = true; // reset future BRs for randomness return nextBlockReportTime; }
long getHeartbeatWaitTime() { return nextHeartbeatTime - monotonicNow(); }