@Override public void run() { int nAttempts = 0; SegmentCompletionProtocol.ControllerResponseStatus status = SegmentCompletionProtocol.ControllerResponseStatus.NOT_SENT; final String segmentId = _params.getSegmentName(); // Attempt to send a lease renewal message for MAX_NUM_ATTEMPTS number of times. If unsuccessful, // log a warning and let things take their course. At worst, the segment that is built will be rejected // in favor of another server. while (status != SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED && nAttempts < MAX_NUM_ATTEMPTS) { try { SegmentCompletionProtocol.Response response = _protocolHandler.extendBuildTime(_params); status = response.getStatus(); } catch (Exception e) { LOGGER.warn("Exception trying to send lease renewal for {}", segmentId); } if (status != SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED) { Uninterruptibles.sleepUninterruptibly(2, TimeUnit.SECONDS); LOGGER.warn("Retrying lease extension for {} because controller status {}", segmentId, status.toString()); nAttempts++; } } if (nAttempts >= MAX_NUM_ATTEMPTS) { LOGGER.error("Failed to send lease extension for {}", segmentId); } } }
params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentCommitStart(params); Assert.assertTrue(response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.HOLD));
params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentCommitStart(params); Assert.assertTrue(response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.HOLD));
.equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_CONTINUE)) { segmentLogger.warn("CommitStart failed with response {}", segmentCommitStartResponse.toJsonString()); return SegmentCompletionProtocol.RESP_FAILED; _protocolHandler.segmentCommitUpload(params, segmentTarFile, prevResponse.getControllerVipUrl()); if (!segmentCommitUploadResponse.getStatus() .equals(SegmentCompletionProtocol.ControllerResponseStatus.UPLOAD_SUCCESS)) { segmentLogger.warn("Segment upload failed with response {}", segmentCommitUploadResponse.toJsonString()); return SegmentCompletionProtocol.RESP_FAILED; if (!commitEndResponse.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS)) { segmentLogger.warn("CommitEnd failed with response {}", commitEndResponse.toJsonString()); return SegmentCompletionProtocol.RESP_FAILED;
protected boolean commitSegment(SegmentCompletionProtocol.Response response) { final String segTarFileName = _segmentBuildDescriptor.getSegmentTarFilePath(); File segTarFile = new File(segTarFileName); if (!segTarFile.exists()) { throw new RuntimeException("Segment file does not exist:" + segTarFileName); } SegmentCompletionProtocol.Response returnedResponse; if (response.isSplitCommit() && _indexLoadingConfig.isEnableSplitCommit()) { // Send segmentStart, segmentUpload, & segmentCommitEnd to the controller // if that succeeds, swap in-memory segment with the one built. returnedResponse = doSplitCommit(response); } else { // Send segmentCommit() to the controller // if that succeeds, swap in-memory segment with the one built. returnedResponse = postSegmentCommitMsg(); } if (!returnedResponse.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS)) { return false; } _realtimeTableDataManager.replaceLLSegment(_segmentNameStr, _indexLoadingConfig); removeSegmentFile(); return true; }
private SegmentCompletionProtocol.Response uploadSegment(String url, final String segmentName, final File segmentTarFile) { SegmentCompletionProtocol.Response response; try { String responseStr = _fileUploadDownloadClient .uploadSegment(new URI(url), segmentName, segmentTarFile, null, null, SEGMENT_UPLOAD_REQUEST_TIMEOUT_MS) .getResponse(); response = SegmentCompletionProtocol.Response.fromJsonString(responseStr); LOGGER.info("Controller response {} for {}", response.toJsonString(), url); if (response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)) { ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } } catch (Exception e) { // Catch all exceptions, we want the protocol to handle the case assuming the request was never sent. response = SegmentCompletionProtocol.RESP_NOT_SENT; LOGGER.error("Could not send request {}", url, e); // Invalidate controller leader cache, as exception could be because of leader being down (deployment/failure) and hence unable to send {@link SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER} // If cache is not invalidated, we will not recover from exceptions until the controller comes back up ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } raiseSegmentCompletionProtocolResponseMetric(response); return response; }
private SegmentCompletionProtocol.Response sendRequest(String url) { SegmentCompletionProtocol.Response response; try { String responseStr = _fileUploadDownloadClient.sendSegmentCompletionProtocolRequest(new URI(url), OTHER_REQUESTS_TIMEOUT) .getResponse(); response = SegmentCompletionProtocol.Response.fromJsonString(responseStr); LOGGER.info("Controller response {} for {}", response.toJsonString(), url); if (response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER)) { ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } } catch (Exception e) { // Catch all exceptions, we want the protocol to handle the case assuming the request was never sent. response = SegmentCompletionProtocol.RESP_NOT_SENT; LOGGER.error("Could not send request {}", url, e); // Invalidate controller leader cache, as exception could be because of leader being down (deployment/failure) and hence unable to send {@link SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER} // If cache is not invalidated, we will not recover from exceptions until the controller comes back up ControllerLeaderLocator.getInstance().invalidateCachedControllerLeader(); } raiseSegmentCompletionProtocolResponseMetric(response); return response; }
protected SegmentCompletionProtocol.Response postSegmentCommitMsg() { final File segmentTarFile = new File(_segmentBuildDescriptor.getSegmentTarFilePath()); SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withInstanceId(_instanceId).withOffset(_currentOffset).withSegmentName(_segmentNameStr) .withNumRows(_numRowsConsumed).withInstanceId(_instanceId) .withBuildTimeMillis(_segmentBuildDescriptor.getBuildTimeMillis()) .withSegmentSizeBytes(_segmentBuildDescriptor.getSegmentSizeBytes()) .withWaitTimeMillis(_segmentBuildDescriptor.getWaitTimeMillis()); if (_isOffHeap) { params.withMemoryUsedBytes(_memoryManager.getTotalAllocatedBytes()); } SegmentCompletionProtocol.Response response = _protocolHandler.segmentCommit(params, segmentTarFile); if (!response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS)) { segmentLogger.warn("Commit failed with response {}", response.toJsonString()); } return response; }
@Test public void testJsonResponseWithAllParams() { // Test with all params SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSegmentLocation(SEGMENT_LOCATION).withSplitCommit(true).withControllerVipUrl(CONTROLLER_VIP_URL) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertEquals(jsonNode.get("segmentLocation").asText(), SEGMENT_LOCATION); assertTrue(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertEquals(jsonNode.get("controllerVipUrl").asText(), CONTROLLER_VIP_URL); }
@Test public void testJsonResponseWithSegmentLocationNullVip() { // Should never happen because if split commit, should have both location and VIP, but testing deserialization regardless SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSegmentLocation(SEGMENT_LOCATION).withSplitCommit(false) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertEquals(jsonNode.get("segmentLocation").asText(), SEGMENT_LOCATION); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertNull(jsonNode.get("controllerVipUrl")); }
@Test public void testJsonResponseWithVipAndNullSegmentLocation() { // Should never happen because if split commit, should have both location and VIP, but testing deserialization regardless SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withControllerVipUrl(CONTROLLER_VIP_URL).withSplitCommit(false) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertNull(jsonNode.get("segmentLocation")); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertEquals(jsonNode.get("controllerVipUrl").asText(), CONTROLLER_VIP_URL); } }
@Test public void testJsonResponseWithoutSplitCommit() { SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSplitCommit(false).withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertNull(jsonNode.get("segmentLocation")); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertNull(jsonNode.get("controllerVipUrl")); }
@Test public void testJsonNullSegmentLocationAndVip() { SegmentCompletionProtocol.Response.Params params = new SegmentCompletionProtocol.Response.Params().withBuildTimeSeconds(BUILD_TIME_MILLIS).withOffset(OFFSET) .withSplitCommit(false).withStatus(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); SegmentCompletionProtocol.Response response = new SegmentCompletionProtocol.Response(params); JsonNode jsonNode = JsonUtils.objectToJsonNode(response); assertEquals(jsonNode.get("offset").asInt(), OFFSET); assertNull(jsonNode.get("segmentLocation")); assertFalse(jsonNode.get("isSplitCommitType").asBoolean()); assertEquals(jsonNode.get("status").asText(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT.toString()); assertNull(jsonNode.get("controllerVipUrl")); }