/** * Issues a heartbeat request to the coordinator * @return the corresponding {@link ContainerHeartbeatResponse}. */ public ContainerHeartbeatResponse requestHeartbeat() { ObjectMapper mapper = new ObjectMapper(); ContainerHeartbeatResponse response; String reply = ""; try { reply = httpGet(new URL(heartbeatEndpoint)); LOG.debug("Container Heartbeat got response {}", reply); response = mapper.readValue(reply, ContainerHeartbeatResponse.class); return response; } catch (Exception e) { LOG.error("Error in container heartbeat to JobCoordinator.", e); } LOG.error("Container heartbeat expired"); response = new ContainerHeartbeatResponse(false); return response; }
public void start() { if (started) { LOG.warn("Skipping attempt to start an already started ContainerHeartbeatMonitor."); return; } LOG.info("Starting ContainerHeartbeatMonitor"); scheduler.scheduleAtFixedRate(() -> { ContainerHeartbeatResponse response = containerHeartbeatClient.requestHeartbeat(); if (!response.isAlive()) { scheduler.schedule(() -> { // On timeout of container shutting down, force exit. LOG.error("Graceful shutdown timeout expired. Force exiting."); ThreadUtil.logThreadDump("Thread dump at heartbeat monitor shutdown timeout."); System.exit(1); }, SHUTDOWN_TIMOUT_MS, TimeUnit.MILLISECONDS); onContainerExpired.run(); } }, 0, SCHEDULE_MS, TimeUnit.MILLISECONDS); started = true; }
/** * Issues a heartbeat request to the coordinator * @return the corresponding {@link ContainerHeartbeatResponse}. */ public ContainerHeartbeatResponse requestHeartbeat() { ObjectMapper mapper = new ObjectMapper(); ContainerHeartbeatResponse response; String reply = ""; try { reply = httpGet(new URL(heartbeatEndpoint)); LOG.debug("Container Heartbeat got response {}", reply); response = mapper.readValue(reply, ContainerHeartbeatResponse.class); return response; } catch (Exception e) { LOG.error("Error in container heartbeat to JobCoordinator.", e); } LOG.error("Container heartbeat expired"); response = new ContainerHeartbeatResponse(false); return response; }
public void start() { if (started) { LOG.warn("Skipping attempt to start an already started ContainerHeartbeatMonitor."); return; } LOG.info("Starting ContainerHeartbeatMonitor"); scheduler.scheduleAtFixedRate(() -> { ContainerHeartbeatResponse response = containerHeartbeatClient.requestHeartbeat(); if (!response.isAlive()) { scheduler.schedule(() -> { // On timeout of container shutting down, force exit. LOG.error("Graceful shutdown timeout expired. Force exiting."); Util.logThreadDump("Thread dump at heartbeat monitor shutdown timeout."); System.exit(1); }, SHUTDOWN_TIMOUT_MS, TimeUnit.MILLISECONDS); onContainerExpired.run(); } }, 0, SCHEDULE_MS, TimeUnit.MILLISECONDS); started = true; }
/** * Issues a heartbeat request to the coordinator * @return the corresponding {@link ContainerHeartbeatResponse}. */ public ContainerHeartbeatResponse requestHeartbeat() { ObjectMapper mapper = new ObjectMapper(); ContainerHeartbeatResponse response; String reply = ""; try { reply = httpGet(new URL(heartbeatEndpoint)); LOG.debug("Container Heartbeat got response {}", reply); response = mapper.readValue(reply, ContainerHeartbeatResponse.class); return response; } catch (Exception e) { LOG.error("Error in container heartbeat to JobCoordinator.", e); } LOG.error("Container heartbeat expired"); response = new ContainerHeartbeatResponse(false); return response; }
public void start() { if (started) { LOG.warn("Skipping attempt to start an already started ContainerHeartbeatMonitor."); return; } LOG.info("Starting ContainerHeartbeatMonitor"); scheduler.scheduleAtFixedRate(() -> { ContainerHeartbeatResponse response = containerHeartbeatClient.requestHeartbeat(); if (!response.isAlive()) { scheduler.schedule(() -> { // On timeout of container shutting down, force exit. LOG.error("Graceful shutdown timeout expired. Force exiting."); Util.logThreadDump("Thread dump at heartbeat monitor shutdown timeout."); System.exit(1); }, SHUTDOWN_TIMOUT_MS, TimeUnit.MILLISECONDS); onContainerExpired.run(); } }, 0, SCHEDULE_MS, TimeUnit.MILLISECONDS); started = true; }
/** * Issues a heartbeat request to the coordinator * @return the corresponding {@link ContainerHeartbeatResponse}. */ public ContainerHeartbeatResponse requestHeartbeat() { ObjectMapper mapper = new ObjectMapper(); ContainerHeartbeatResponse response; String reply = ""; try { reply = httpGet(new URL(heartbeatEndpoint)); LOG.debug("Container Heartbeat got response {}", reply); response = mapper.readValue(reply, ContainerHeartbeatResponse.class); return response; } catch (Exception e) { LOG.error("Error in container heartbeat to JobCoordinator.", e); } LOG.error("Container heartbeat expired"); response = new ContainerHeartbeatResponse(false); return response; }
public void start() { if (started) { LOG.warn("Skipping attempt to start an already started ContainerHeartbeatMonitor."); return; } LOG.info("Starting ContainerHeartbeatMonitor"); scheduler.scheduleAtFixedRate(() -> { ContainerHeartbeatResponse response = containerHeartbeatClient.requestHeartbeat(); if (!response.isAlive()) { scheduler.schedule(() -> { // On timeout of container shutting down, force exit. LOG.error("Graceful shutdown timeout expired. Force exiting."); Util.logThreadDump("Thread dump at heartbeat monitor shutdown timeout."); System.exit(1); }, SHUTDOWN_TIMOUT_MS, TimeUnit.MILLISECONDS); onContainerExpired.run(); } }, 0, SCHEDULE_MS, TimeUnit.MILLISECONDS); started = true; }
/** * Issues a heartbeat request to the coordinator * @return the corresponding {@link ContainerHeartbeatResponse}. */ public ContainerHeartbeatResponse requestHeartbeat() { ObjectMapper mapper = new ObjectMapper(); ContainerHeartbeatResponse response; String reply = ""; try { reply = httpGet(new URL(heartbeatEndpoint)); LOG.debug("Container Heartbeat got response {}", reply); response = mapper.readValue(reply, ContainerHeartbeatResponse.class); return response; } catch (Exception e) { LOG.error("Error in container heartbeat to JobCoordinator.", e); } LOG.error("Container heartbeat expired"); response = new ContainerHeartbeatResponse(false); return response; }
public void start() { if (started) { LOG.warn("Skipping attempt to start an already started ContainerHeartbeatMonitor."); return; } LOG.info("Starting ContainerHeartbeatMonitor"); scheduler.scheduleAtFixedRate(() -> { ContainerHeartbeatResponse response = containerHeartbeatClient.requestHeartbeat(); if (!response.isAlive()) { scheduler.schedule(() -> { // On timeout of container shutting down, force exit. LOG.error("Graceful shutdown timeout expired. Force exiting."); Util.logThreadDump("Thread dump at heartbeat monitor shutdown timeout."); System.exit(1); }, SHUTDOWN_TIMOUT_MS, TimeUnit.MILLISECONDS); onContainerExpired.run(); } }, 0, SCHEDULE_MS, TimeUnit.MILLISECONDS); started = true; }
@Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { ContainerId yarnContainerId; PrintWriter printWriter = resp.getWriter(); String containerIdParam = req.getParameter(YARN_CONTAINER_ID); ContainerHeartbeatResponse response; resp.setContentType(APPLICATION_JSON); boolean alive = false; try { yarnContainerId = ContainerId.fromString(containerIdParam); for (YarnContainer yarnContainer : yarnAppState.runningYarnContainers.values()) { if (yarnContainer.id().compareTo(yarnContainerId) == 0) { alive = true; break; } } if (!alive) { heartbeatsExpiredCount.inc(); } response = new ContainerHeartbeatResponse(alive); printWriter.write(mapper.writeValueAsString(response)); } catch (IllegalArgumentException e) { LOG.error("Container ID {} passed is invalid", containerIdParam); resp.sendError(HttpServletResponse.SC_BAD_REQUEST, e.getMessage()); } } }
@Test public void testClientResponseOnBadRequest() throws IOException { client.shouldThrowException(true); ContainerHeartbeatResponse response = client.requestHeartbeat(); Assert.assertFalse(response.isAlive()); }
@Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { ContainerId yarnContainerId; PrintWriter printWriter = resp.getWriter(); String containerIdParam = req.getParameter(YARN_CONTAINER_ID); ContainerHeartbeatResponse response; resp.setContentType(APPLICATION_JSON); boolean alive = false; try { yarnContainerId = ContainerId.fromString(containerIdParam); for (YarnContainer yarnContainer : yarnAppState.runningYarnContainers.values()) { if (yarnContainer.id().compareTo(yarnContainerId) == 0) { alive = true; break; } } if (!alive) { heartbeatsExpiredCount.inc(); } response = new ContainerHeartbeatResponse(alive); printWriter.write(mapper.writeValueAsString(response)); } catch (IllegalArgumentException e) { LOG.error("Container ID {} passed is invalid", containerIdParam); resp.sendError(HttpServletResponse.SC_BAD_REQUEST, e.getMessage()); } } }
@Test public void testClientResponseForHeartbeatAlive() throws IOException { client.setHttpOutput("{\"alive\": true}"); ContainerHeartbeatResponse response = client.requestHeartbeat(); Assert.assertTrue(response.isAlive()); }
@Test public void testDoesNotCallbackWhenHeartbeatAlive() throws InterruptedException { ContainerHeartbeatClient client = mock(ContainerHeartbeatClient.class); CountDownLatch countDownLatch = new CountDownLatch(1); Runnable onExpired = () -> { countDownLatch.countDown(); }; ContainerHeartbeatMonitor monitor = new ContainerHeartbeatMonitor(onExpired, client); ContainerHeartbeatResponse response = new ContainerHeartbeatResponse(true); when(client.requestHeartbeat()).thenReturn(response); monitor.start(); boolean success = countDownLatch.await(2, TimeUnit.SECONDS); Assert.assertFalse(success); Assert.assertEquals(1, countDownLatch.getCount()); } }
@Test public void testClientResponseForHeartbeatDead() throws IOException { client.setHttpOutput("{\"alive\": false}"); ContainerHeartbeatResponse response = client.requestHeartbeat(); Assert.assertFalse(response.isAlive()); }
@Test public void testCallbackWhenHeartbeatDead() throws InterruptedException { ContainerHeartbeatClient mockClient = mock(ContainerHeartbeatClient.class); CountDownLatch countDownLatch = new CountDownLatch(1); Runnable onExpired = () -> { countDownLatch.countDown(); }; ContainerHeartbeatMonitor monitor = new ContainerHeartbeatMonitor(onExpired, mockClient); ContainerHeartbeatResponse response = new ContainerHeartbeatResponse(false); when(mockClient.requestHeartbeat()).thenReturn(response); monitor.start(); boolean success = countDownLatch.await(2, TimeUnit.SECONDS); Assert.assertTrue(success); }