@GET @Path(SegmentCompletionProtocol.MSG_TYPE_STOPPED_CONSUMING) @Produces(MediaType.APPLICATION_JSON) public String segmentStoppedConsuming(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_REASON) String stopReason) { if (instanceId == null || segmentName == null || offset == -1) { LOGGER.error("Invalid call: offset={}, segmentName={}, instanceId={}", offset, segmentName, instanceId); return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset).withReason(stopReason); LOGGER.info("Processing segmentStoppedConsuming:{}", requestParams.toString()); SegmentCompletionProtocol.Response response = SegmentCompletionManager.getInstance().segmentStoppedConsuming(requestParams); final String responseStr = response.toJsonString(); LOGGER.info("Response to segmentStoppedConsuming:{}", responseStr); return responseStr; }
@GET @Path(SegmentCompletionProtocol.MSG_TYPE_CONSUMED) @Produces(MediaType.APPLICATION_JSON) public String segmentConsumed(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_REASON) String stopReason, @QueryParam(SegmentCompletionProtocol.PARAM_MEMORY_USED_BYTES) long memoryUsedBytes, @QueryParam(SegmentCompletionProtocol.PARAM_ROW_COUNT) int numRows) { if (instanceId == null || segmentName == null || offset == -1) { LOGGER.error("Invalid call: offset={}, segmentName={}, instanceId={}", offset, segmentName, instanceId); return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset).withReason(stopReason) .withMemoryUsedBytes(memoryUsedBytes).withNumRows(numRows); LOGGER.info("Processing segmentConsumed:{}", requestParams.toString()); SegmentCompletionProtocol.Response response = SegmentCompletionManager.getInstance().segmentConsumed(requestParams); final String responseStr = response.toJsonString(); LOGGER.info("Response to segmentConsumed:{}", responseStr); return responseStr; }
protected void postStopConsumedMsg(String reason) { do { SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withOffset(_currentOffset).withReason(reason).withSegmentName(_segmentNameStr).withInstanceId(_instanceId); SegmentCompletionProtocol.Response response = _protocolHandler.segmentStoppedConsuming(params); if (response.getStatus() == SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED) { segmentLogger.info("Got response {}", response.toJsonString()); break; } Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); segmentLogger.info("Retrying after response {}", response.toJsonString()); } while (!_shouldStop); }
new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr).withReason(reason); response = segmentCompletionMgr.segmentStoppedConsuming(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED);
new Request.Params().withInstanceId(s3).withOffset(s3Offset).withSegmentName(segmentNameStr).withReason(reason); response = segmentCompletionMgr.segmentStoppedConsuming(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED);
segmentCompletionMgr._secconds = 10L; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr) .withReason(SegmentCompletionProtocol.REASON_ROW_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.COMMIT); .withReason(SegmentCompletionProtocol.REASON_ROW_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.HOLD); .withReason(SegmentCompletionProtocol.REASON_TIME_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.HOLD); .withReason(SegmentCompletionProtocol.REASON_ROW_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.KEEP); .withReason(SegmentCompletionProtocol.REASON_TIME_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.DISCARD);
@Test public void testHappyPathAfterStoppedConsuming() throws Exception { SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 5; params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr) .withReason("some reason"); response = segmentCompletionMgr.segmentStoppedConsuming(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED); Assert.assertEquals(new LLCSegmentName(segmentNameStr), segmentManager._stoppedSegmentName); Assert.assertEquals(s2, segmentManager._stoppedInstance); segmentManager._stoppedSegmentName = null; segmentManager._stoppedInstance = null; testHappyPath(6L); }
@Test public void testControllerNotConnected() throws Exception { testCaseSetup(true, false); // Leader but not connected SegmentCompletionProtocol.Response response; Request.Params params; // s1 sends offset of 20, gets HOLD at t = 5s; segmentCompletionMgr._secconds = 5L; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr) .withReason("rowLimit"); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.NOT_LEADER); }
protected SegmentCompletionProtocol.Response postSegmentConsumedMsg() { // Post segmentConsumed to current leader. // Retry maybe once if leader is not found. SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withOffset(_currentOffset).withSegmentName(_segmentNameStr).withReason(_stopReason) .withNumRows(_numRowsConsumed).withInstanceId(_instanceId); if (_isOffHeap) { params.withMemoryUsedBytes(_memoryManager.getTotalAllocatedBytes()); } return _protocolHandler.segmentConsumed(params); }
@Test public void testWinnerOnTimeLimit() throws Exception { SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 10L; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr) .withReason(SegmentCompletionProtocol.REASON_TIME_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); }