@GET @Path(SegmentCompletionProtocol.MSG_TYPE_STOPPED_CONSUMING) @Produces(MediaType.APPLICATION_JSON) public String segmentStoppedConsuming(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_REASON) String stopReason) { if (instanceId == null || segmentName == null || offset == -1) { LOGGER.error("Invalid call: offset={}, segmentName={}, instanceId={}", offset, segmentName, instanceId); return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset).withReason(stopReason); LOGGER.info("Processing segmentStoppedConsuming:{}", requestParams.toString()); SegmentCompletionProtocol.Response response = SegmentCompletionManager.getInstance().segmentStoppedConsuming(requestParams); final String responseStr = response.toJsonString(); LOGGER.info("Response to segmentStoppedConsuming:{}", responseStr); return responseStr; }
@GET @Path(SegmentCompletionProtocol.MSG_TYPE_CONSUMED) @Produces(MediaType.APPLICATION_JSON) public String segmentConsumed(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_REASON) String stopReason, @QueryParam(SegmentCompletionProtocol.PARAM_MEMORY_USED_BYTES) long memoryUsedBytes, @QueryParam(SegmentCompletionProtocol.PARAM_ROW_COUNT) int numRows) { if (instanceId == null || segmentName == null || offset == -1) { LOGGER.error("Invalid call: offset={}, segmentName={}, instanceId={}", offset, segmentName, instanceId); return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset).withReason(stopReason) .withMemoryUsedBytes(memoryUsedBytes).withNumRows(numRows); LOGGER.info("Processing segmentConsumed:{}", requestParams.toString()); SegmentCompletionProtocol.Response response = SegmentCompletionManager.getInstance().segmentConsumed(requestParams); final String responseStr = response.toJsonString(); LOGGER.info("Response to segmentConsumed:{}", responseStr); return responseStr; }
@GET @Path(SegmentCompletionProtocol.MSG_TYPE_EXTEND_BUILD_TIME) @Produces(MediaType.APPLICATION_JSON) public String extendBuildTime(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_EXTRA_TIME_SEC) int extraTimeSec) { if (instanceId == null || segmentName == null || offset == -1) { LOGGER.error("Invalid call: offset={}, segmentName={}, instanceId={}", offset, segmentName, instanceId); return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } if (extraTimeSec <= 0) { LOGGER.warn("Invalid value {} for extra build time from instance {} for segment {}", extraTimeSec, instanceId, segmentName); extraTimeSec = SegmentCompletionProtocol.getDefaultMaxSegmentCommitTimeSeconds(); } SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset) .withExtraTimeSec(extraTimeSec); LOGGER.info("Processing extendBuildTime:{}", requestParams.toString()); SegmentCompletionProtocol.Response response = SegmentCompletionManager.getInstance().extendBuildTime(requestParams); final String responseStr = response.toJsonString(); LOGGER.info("Response to extendBuildTime:{}", responseStr); return responseStr; }
params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); params = new Request.Params().withInstanceId(s2).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); params = new Request.Params().withInstanceId(s3).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.COMMIT); params = new Request.Params().withInstanceId(s3).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentCommitStart(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_CONTINUE); params = new Request.Params().withInstanceId(s3).withOffset(s3Offset).withSegmentName(segmentNameStr) .withSegmentLocation("location"); response = segmentCompletionMgr.segmentCommitEnd(params, true, true); params = new Request.Params().withInstanceId(s2).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD);
SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withSegmentName(_segmentNameStr).withOffset(_currentOffset).withNumRows(_numRowsConsumed) .withInstanceId(_instanceId).withBuildTimeMillis(_segmentBuildDescriptor.getBuildTimeMillis()) .withSegmentSizeBytes(_segmentBuildDescriptor.getSegmentSizeBytes()) params.withOffset(_currentOffset).withSegmentName(_segmentNameStr).withInstanceId(_instanceId); SegmentCompletionProtocol.Response segmentCommitUploadResponse = _protocolHandler.segmentCommitUpload(params, segmentTarFile, prevResponse.getControllerVipUrl()); params.withInstanceId(_instanceId).withOffset(_currentOffset).withSegmentName(_segmentNameStr) .withSegmentLocation(segmentCommitUploadResponse.getSegmentLocation()).withNumRows(_numRowsConsumed) .withBuildTimeMillis(_segmentBuildDescriptor.getBuildTimeMillis())
params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); params = new Request.Params().withInstanceId(s3).withOffset(s3Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.CATCH_UP); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.COMMIT); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); Assert.assertFalse(fsmMap.containsKey(segmentNameStr)); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.HOLD); params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.COMMIT);
requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset) .withSegmentLocation(segmentLocation).withSegmentSizeBytes(segmentSizeBytes) .withBuildTimeMillis(buildTimeMillis).withWaitTimeMillis(waitTimeMillis).withNumRows(numRows)
@Test public void testCommitSegmentWhenControllerWentThroughGC() throws InvalidConfigException { FakePinotLLCRealtimeSegmentManager segmentManager1 = new FakePinotLLCRealtimeSegmentManager(null); FakePinotLLCRealtimeSegmentManager segmentManager2 = new FakePinotLLCRealtimeSegmentManagerII(null, FakePinotLLCRealtimeSegmentManagerII.SCENARIO_1_ZK_VERSION_NUM_HAS_CHANGE); FakePinotLLCRealtimeSegmentManager segmentManager3 = new FakePinotLLCRealtimeSegmentManagerII(null, FakePinotLLCRealtimeSegmentManagerII.SCENARIO_2_METADATA_STATUS_HAS_CHANGE); final String rtTableName = "table_REALTIME"; final String rawTableName = TableNameBuilder.extractRawTableName(rtTableName); setupSegmentManager(segmentManager1, rtTableName); setupSegmentManager(segmentManager2, rtTableName); setupSegmentManager(segmentManager3, rtTableName); // Now commit the first segment of partition 6. final int committingPartition = 6; final long nextOffset = 3425666L; SegmentCompletionProtocol.Request.Params reqParams = new SegmentCompletionProtocol.Request.Params(); LLCRealtimeSegmentZKMetadata committingSegmentMetadata = new LLCRealtimeSegmentZKMetadata(segmentManager2._records.get(committingPartition)); reqParams.withSegmentName(committingSegmentMetadata.getSegmentName()).withOffset(nextOffset); CommittingSegmentDescriptor committingSegmentDescriptor = CommittingSegmentDescriptor.fromSegmentCompletionReqParams(reqParams); boolean status = segmentManager1.commitSegmentMetadata(rawTableName, committingSegmentDescriptor); Assert.assertTrue(status); // Committing segment metadata succeeded. status = segmentManager2.commitSegmentMetadata(rawTableName, committingSegmentDescriptor); Assert.assertFalse(status); // Committing segment metadata failed. status = segmentManager3.commitSegmentMetadata(rawTableName, committingSegmentDescriptor); Assert.assertFalse(status); // Committing segment metadata failed. }
@POST @Path(SegmentCompletionProtocol.MSG_TYPE_COMMIT) @Consumes(MediaType.MULTIPART_FORM_DATA) @Produces(MediaType.APPLICATION_JSON) public String segmentCommit(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_MEMORY_USED_BYTES) long memoryUsedBytes, @QueryParam(SegmentCompletionProtocol.PARAM_BUILD_TIME_MILLIS) long buildTimeMillis, @QueryParam(SegmentCompletionProtocol.PARAM_WAIT_TIME_MILLIS) long waitTimeMillis, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_SIZE_BYTES) long segmentSizeBytes, @QueryParam(SegmentCompletionProtocol.PARAM_ROW_COUNT) int numRows, FormDataMultiPart multiPart) { SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset) .withSegmentSizeBytes(segmentSizeBytes).withBuildTimeMillis(buildTimeMillis).withWaitTimeMillis(waitTimeMillis) .withNumRows(numRows).withMemoryUsedBytes(memoryUsedBytes); LOGGER.info("Processing segmentCommit:{}", requestParams.toString()); final SegmentCompletionManager segmentCompletionManager = SegmentCompletionManager.getInstance(); SegmentCompletionProtocol.Response response = segmentCompletionManager.segmentCommitStart(requestParams); if (response.equals(SegmentCompletionProtocol.RESP_COMMIT_CONTINUE)) { // Get the segment and put it in the right place. boolean success = uploadSegment(multiPart, instanceId, segmentName, false) != null; response = segmentCompletionManager.segmentCommitEnd(requestParams, success, false); } LOGGER.info("Response to segmentCommit: instance={} segment={} status={} offset={}", requestParams.getInstanceId(), requestParams.getSegmentName(), response.getStatus(), response.getOffset()); return response.toJsonString(); }
@GET @Path(SegmentCompletionProtocol.MSG_TYPE_COMMIT_START) @Produces(MediaType.APPLICATION_JSON) public String segmentCommitStart(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_MEMORY_USED_BYTES) long memoryUsedBytes, @QueryParam(SegmentCompletionProtocol.PARAM_BUILD_TIME_MILLIS) long buildTimeMillis, @QueryParam(SegmentCompletionProtocol.PARAM_WAIT_TIME_MILLIS) long waitTimeMillis, @QueryParam(SegmentCompletionProtocol.PARAM_ROW_COUNT) int numRows, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_SIZE_BYTES) long segmentSizeBytes) { if (instanceId == null || segmentName == null || offset == -1) { LOGGER.error("Invalid call: offset={}, segmentName={}, instanceId={}", offset, segmentName, instanceId); return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset) .withMemoryUsedBytes(memoryUsedBytes).withBuildTimeMillis(buildTimeMillis).withWaitTimeMillis(waitTimeMillis) .withNumRows(numRows).withSegmentSizeBytes(segmentSizeBytes); LOGGER.info("Processing segmentCommitStart:{}", requestParams.toString()); SegmentCompletionProtocol.Response response = SegmentCompletionManager.getInstance().segmentCommitStart(requestParams); final String responseStr = response.toJsonString(); LOGGER.info("Response to segmentCommitStart:{}", responseStr); return responseStr; }
@POST @Path(SegmentCompletionProtocol.MSG_TYPE_SEGMENT_UPLOAD) @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.MULTIPART_FORM_DATA) public String segmentUpload(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, FormDataMultiPart multiPart) { SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset); LOGGER.info("Processing segmentUpload:{}", requestParams.toString()); final String segmentLocation = uploadSegment(multiPart, instanceId, segmentName, true); if (segmentLocation == null) { return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } SegmentCompletionProtocol.Response.Params responseParams = new SegmentCompletionProtocol.Response.Params().withOffset(requestParams.getOffset()) .withSegmentLocation(segmentLocation) .withStatus(SegmentCompletionProtocol.ControllerResponseStatus.UPLOAD_SUCCESS); String response = new SegmentCompletionProtocol.Response(responseParams).toJsonString(); LOGGER.info("Response to segmentUpload:{}", response); return response; }
/** * Adds a segment for periodic lease request. * The first lease request is sent before {@param initialBuildTimeMs} exipres. Subsequent lease requests are sent * within two minutes. * @param segmentId is the name of he segment that is being built * @param initialBuildTimeMs is the initial time budget that SegmentCompletionManager has allocated. * @param offset The offset at which this segment is being built. */ public void addSegment(String segmentId, long initialBuildTimeMs, long offset) { final long initialDelayMs = initialBuildTimeMs * 9 / 10; final SegmentCompletionProtocol.Request.Params reqParams = new SegmentCompletionProtocol.Request.Params(); reqParams.withOffset(offset).withSegmentName(segmentId).withExtraTimeSec(EXTRA_TIME_SECONDS) .withInstanceId(_instanceId); Future future = _executor .scheduleWithFixedDelay(new LeaseExtender(reqParams), initialDelayMs, REPEAT_REQUEST_PERIOD_SEC * 1000L, TimeUnit.MILLISECONDS); _segmentToFutureMap.put(segmentId, future); }
protected SegmentCompletionProtocol.Response postSegmentCommitMsg() { final File segmentTarFile = new File(_segmentBuildDescriptor.getSegmentTarFilePath()); SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withInstanceId(_instanceId).withOffset(_currentOffset).withSegmentName(_segmentNameStr) .withNumRows(_numRowsConsumed).withInstanceId(_instanceId) .withBuildTimeMillis(_segmentBuildDescriptor.getBuildTimeMillis()) .withSegmentSizeBytes(_segmentBuildDescriptor.getSegmentSizeBytes()) .withWaitTimeMillis(_segmentBuildDescriptor.getWaitTimeMillis()); if (_isOffHeap) { params.withMemoryUsedBytes(_memoryManager.getTotalAllocatedBytes()); } SegmentCompletionProtocol.Response response = _protocolHandler.segmentCommit(params, segmentTarFile); if (!response.getStatus().equals(SegmentCompletionProtocol.ControllerResponseStatus.COMMIT_SUCCESS)) { segmentLogger.warn("Commit failed with response {}", response.toJsonString()); } return response; }
@Test public void testHappyPathAfterStoppedConsuming() throws Exception { SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 5; params = new Request.Params().withInstanceId(s2).withOffset(s2Offset).withSegmentName(segmentNameStr) .withReason("some reason"); response = segmentCompletionMgr.segmentStoppedConsuming(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED); Assert.assertEquals(new LLCSegmentName(segmentNameStr), segmentManager._stoppedSegmentName); Assert.assertEquals(s2, segmentManager._stoppedInstance); segmentManager._stoppedSegmentName = null; segmentManager._stoppedInstance = null; testHappyPath(6L); }
protected void postStopConsumedMsg(String reason) { do { SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withOffset(_currentOffset).withReason(reason).withSegmentName(_segmentNameStr).withInstanceId(_instanceId); SegmentCompletionProtocol.Response response = _protocolHandler.segmentStoppedConsuming(params); if (response.getStatus() == SegmentCompletionProtocol.ControllerResponseStatus.PROCESSED) { segmentLogger.info("Got response {}", response.toJsonString()); break; } Uninterruptibles.sleepUninterruptibly(10, TimeUnit.SECONDS); segmentLogger.info("Retrying after response {}", response.toJsonString()); } while (!_shouldStop); }
@Test public void testControllerNotConnected() throws Exception { testCaseSetup(true, false); // Leader but not connected SegmentCompletionProtocol.Response response; Request.Params params; // s1 sends offset of 20, gets HOLD at t = 5s; segmentCompletionMgr._secconds = 5L; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr) .withReason("rowLimit"); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.NOT_LEADER); }
@Test public void testNotLeader() throws Exception { testCaseSetup(false, true); SegmentCompletionProtocol.Response response; SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER); params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentCommitStart(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.NOT_LEADER); }
protected SegmentCompletionProtocol.Response postSegmentConsumedMsg() { // Post segmentConsumed to current leader. // Retry maybe once if leader is not found. SegmentCompletionProtocol.Request.Params params = new SegmentCompletionProtocol.Request.Params(); params.withOffset(_currentOffset).withSegmentName(_segmentNameStr).withReason(_stopReason) .withNumRows(_numRowsConsumed).withInstanceId(_instanceId); if (_isOffHeap) { params.withMemoryUsedBytes(_memoryManager.getTotalAllocatedBytes()); } return _protocolHandler.segmentConsumed(params); }
@Test public void testWinnerOnTimeLimit() throws Exception { SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 10L; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr) .withReason(SegmentCompletionProtocol.REASON_TIME_LIMIT); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), SegmentCompletionProtocol.ControllerResponseStatus.HOLD); }
@Test public void testExceptionInConsumedMessage() throws Exception { segmentManager._segmentMetadata = null; SegmentCompletionProtocol.Response response; Request.Params params; segmentCompletionMgr._secconds = 10; params = new Request.Params().withInstanceId(s1).withOffset(s1Offset).withSegmentName(segmentNameStr); response = segmentCompletionMgr.segmentConsumed(params); Assert.assertEquals(response.getStatus(), ControllerResponseStatus.FAILED); }