@Override public void run() { int nAttempts = 0; SegmentCompletionProtocol.ControllerResponseStatus status = SegmentCompletionProtocol.ControllerResponseStatus.NOT_SENT; final String segmentId = _params.getSegmentName(); // Attempt to send a lease renewal message for MAX_NUM_ATTEMPTS number of times. If unsuccessful, // log a warning and let things take their course. At worst, the segment that is built will be rejected // in favor of another server. while (status != SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED && nAttempts < MAX_NUM_ATTEMPTS) { try { SegmentCompletionProtocol.Response response = _protocolHandler.extendBuildTime(_params); status = response.getStatus(); } catch (Exception e) { LOGGER.warn("Exception trying to send lease renewal for {}", segmentId); } if (status != SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED) { Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); LOGGER.warn("Retrying lease extension for {} because controller status {}", segmentId, status.toString()); nAttempts++; } } if (nAttempts >= MAX_NUM_ATTEMPTS) { LOGGER.error("Failed to send lease extension for {}", segmentId); } } }
@POST @Path(SegmentCompletionProtocol.MSG_TYPE_COMMIT) @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces(MediaType.APPLICATION_JSON) public String segmentCommit(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_MEMORY_USED_BYTES) long memoryUsedBytes, @QueryParam(SegmentCompletionProtocol.PARAM_BUILD_TIME_MILLIS) long buildTimeMillis, @QueryParam(SegmentCompletionProtocol.PARAM_WAIT_TIME_MILLIS) long waitTimeMillis, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_SIZE_BYTES) long segmentSizeBytes, @QueryParam(SegmentCompletionProtocol.PARAM_ROW_COUNT) int numRows, FormDataMultiPart multiPart) { SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset) .withSegmentSizeBytes(segmentSizeBytes).withBuildTimeMillis(buildTimeMillis).withWaitTimeMillis(waitTimeMillis) .withNumRows(numRows).withMemoryUsedBytes(memoryUsedBytes); LOGGER.info("Processing segmentCommit:{}", requestParams.toString()); final SegmentCompletionManager segmentCompletionManager = SegmentCompletionManager.getInstance(); SegmentCompletionProtocol.Response response = segmentCompletionManager.segmentCommitStart(requestParams); if (response.equals(SegmentCompletionProtocol.RESP_COMMIT_CONTINUE)) { // Get the segment and put it in the right place. boolean success = uploadSegment(multiPart, instanceId, segmentName, false) != null; response = segmentCompletionManager.segmentCommitEnd(requestParams, success, false); } LOGGER.info("Response to segmentCommit: instance={} segment={} status={} offset={}", requestParams.getInstanceId(), requestParams.getSegmentName(), response.getStatus(), response.getOffset()); return response.toJsonString(); }
protected void postStopConsumedMsg(String reason) { do { SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withOffset(_currentOffset).withReason(reason).withSegmentName(_segmentNameStr).withInstanceId(_instanceId); SegmentCompletionProtocol.Response response = _protocolHandler.segmentStoppedConsuming(params); if (response.getStatus() == SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED) { segmentLogger.info("Got response {}", response.toJsonString()); break; } Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); segmentLogger.info("Retrying after response {}", response.toJsonString()); } while (!_shouldStop); }
params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); params = new Request.Params().withInstanceId(s3).withOffset(s2Offset + 10).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_CONTINUE); segmentCompletionMgr._secconds += 5; params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr) .withSegmentLocation("location"); response = segmentCompletionMgr.segmentCommitEnd(params, true, isSplitCommit); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS); params = new Request.Params().withInstanceId(s3).withOffset(s2Offset + 10).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.DISCARD);
switch (response.getStatus()) { case NOT_SENT: _serverMetrics.addMeteredGlobalValue(ServerMeter.LLC_CONTROLLER_RESPONSE_NOT_SENT, 1);
params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP); Assert.assertEquals(response.getOffset(), s2Offset); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertFalse(fsmMap.containsKey(segmentNameStr)); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.HOLD); Assert.assertTrue(fsmMap.containsKey(segmentNameStr)); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.COMMIT);
.withReason(SegmentCompletionProtocol.REASON_ROW_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.COMMIT); .withReason(SegmentCompletionProtocol.REASON_ROW_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.HOLD); segmentCompletionMgr._secconds += 1; .withReason(SegmentCompletionProtocol.REASON_TIME_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_CONTINUE); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.KEEP); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.DISCARD);
params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.COMMIT); Assert.assertEquals(response.getOffset(), s1Offset); params = new Request.Params().withInstanceId(s3).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentCommitStart(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_CONTINUE); .withSegmentLocation("location"); response = segmentCompletionMgr.segmentCommitEnd(params, true, true); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.FAILED); params = new Request.Params().withInstanceId(s2).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
if (!segmentCommitStartResponse.getStatus() .equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_CONTINUE)) { segmentLogger.warn("CommitStart failed with response {}", segmentCommitStartResponse.toJsonString()); SegmentCompletionProtocol.Response segmentCommitUploadResponse = _protocolHandler.segmentCommitUpload(params, segmentTarFile, prevResponse.getControllerVipUrl()); if (!segmentCommitUploadResponse.getStatus() .equals(SegmentCompletionProtocol.ControllerResponseStatus.UPLOAD_SUCCESS)) { segmentLogger.warn("Segment upload failed with response {}", segmentCommitUploadResponse.toJsonString()); if (!commitEndResponse.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS)) { segmentLogger.warn("CommitEnd failed with response {}", commitEndResponse.toJsonString()); return SegmentCompletionProtocol.RESP_FAILED;
protected boolean commitSegment(SegmentCompletionProtocol.Response response) { final String segTarFileName = _segmentBuildDescriptor.getSegmentTarFilePath(); File segTarFile = new File(segTarFileName); if (!segTarFile.exists()) { throw new RuntimeException("Segment file does not exist:" + segTarFileName); } SegmentCompletionProtocol.Response returnedResponse; if (response.isSplitCommit() && _indexLoadingConfig.isEnableSplitCommit()) { // Send segmentStart, segmentUpload, & segmentCommitEnd to the controller // if that succeeds, swap in-memory segment with the one built. returnedResponse = doSplitCommit(response); } else { // Send segmentCommit() to the controller // if that succeeds, swap in-memory segment with the one built. returnedResponse = postSegmentCommitMsg(); } if (!returnedResponse.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS)) { return false; } _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig); removeSegmentFile(); return true; }
private SegmentCompletionProtocol.Response uploadSegment(String url, final String segmentName, final File segmentTarFile) { SegmentCompletionProtocol.Response response; try { String responseStr = _fileUploadDownloadClient .uploadSegment(new URI(url), segmentName, segmentTarFile, null, null, SEGMENT_UPLOAD_REQUEST_TIMEOUT_MS) .getResponse(); response = SegmentCompletionProtocol.Response.fromJsonString(responseStr); LOGGER.info("Controller response {} for {}", response.toJsonString(), url); if (response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)) { ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } } catch (Exception e) { // Catch all exceptions, we want the protocol to handle the case assuming the request was never sent. response = SegmentCompletionProtocol.RESP_NOT_SENT; LOGGER.error("Could not send request {}", url, e); // Invalidate controller leader cache, as exception could be because of leader being down (deployment/failure) and hence unable to send {@link SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER} // If cache is not invalidated, we will not recover from exceptions until the controller comes back up ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } raiseSegmentCompletionProtocolResponseMetric(response); return response; }
private SegmentCompletionProtocol.Response sendRequest(String url) { SegmentCompletionProtocol.Response response; try { String responseStr = _fileUploadDownloadClient.sendSegmentCompletionProtocolRequest(new URI(url), OTHER_REQUESTS_TIMEOUT) .getResponse(); response = SegmentCompletionProtocol.Response.fromJsonString(responseStr); LOGGER.info("Controller response {} for {}", response.toJsonString(), url); if (response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)) { ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } } catch (Exception e) { // Catch all exceptions, we want the protocol to handle the case assuming the request was never sent. response = SegmentCompletionProtocol.RESP_NOT_SENT; LOGGER.error("Could not send request {}", url, e); // Invalidate controller leader cache, as exception could be because of leader being down (deployment/failure) and hence unable to send {@link SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER} // If cache is not invalidated, we will not recover from exceptions until the controller comes back up ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } raiseSegmentCompletionProtocolResponseMetric(response); return response; }
protected SegmentCompletionProtocol.Response postSegmentCommitMsg() { final File segmentTarFile = new File(_segmentBuildDescriptor.getSegmentTarFilePath()); SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withInstanceId(_instanceId).withOffset(_currentOffset).withSegmentName(_segmentNameStr) .withNumRows(_numRowsConsumed).withInstanceId(_instanceId) .withBuildTimeMillis(_segmentBuildDescriptor.getBuildTimeMillis()) .withSegmentSizeBytes(_segmentBuildDescriptor.getSegmentSizeBytes()) .withWaitTimeMillis(_segmentBuildDescriptor.getWaitTimeMillis()); if (_isOffHeap) { params.withMemoryUsedBytes(_memoryManager.getTotalAllocatedBytes()); } SegmentCompletionProtocol.Response response = _protocolHandler.segmentCommit(params, segmentTarFile); if (!response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS)) { segmentLogger.warn("Commit failed with response {}", response.toJsonString()); } return response; }
@Test public void testHappyPathAfterStoppedConsuming() throws Exception { SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 5; params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr) .withReason("some reason"); response = segmentCompletionMgr.segmentStoppedConsuming(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED); Assert.assertEquals(new LLCSegmentName(segmentNameStr), segmentManager._stoppedSegmentName); Assert.assertEquals(s2, segmentManager._stoppedInstance); segmentManager._stoppedSegmentName = null; segmentManager._stoppedInstance = null; testHappyPath(6L); }
@Test public void testCompleteResponseParams() { // Test with all params SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSegmentLocation(SEGMENT_LOCATION).withSplitCommit(true) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); assertEquals(response.getBuildTimeSeconds(), BUILD_TIME_MILLIS); assertEquals(response.getOffset(), OFFSET); assertEquals(response.getSegmentLocation(), SEGMENT_LOCATION); assertTrue(response.isSplitCommit()); assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); }
@Test public void testIncompleteResponseParams() { // Test with reduced params SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); assertEquals(response.getBuildTimeSeconds(), BUILD_TIME_MILLIS); assertEquals(response.getOffset(), OFFSET); assertNull(response.getSegmentLocation()); assertFalse(response.isSplitCommit()); assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); }
@Test public void testNotLeader() throws Exception { testCaseSetup(false, true); SegmentCompletionProtocol.Response response; SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER); params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentCommitStart(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER); }
@Test public void testExceptionInConsumedMessage() throws Exception { segmentManager._segmentMetadata = null; SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 10; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.FAILED); }
@Test public void testControllerNotConnected() throws Exception { testCaseSetup(true, false); // Leader but not connected SegmentCompletionProtocol.Response response; Request.Params params; // s1 sends offset of 20, gets HOLD at t = 5s; segmentCompletionMgr._secconds = 5L; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr) .withReason("rowLimit"); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.NOT_LEADER); }
@Test public void testWinnerOnTimeLimit() throws Exception { SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 10L; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr) .withReason(SegmentCompletionProtocol.REASON_TIME_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); }