public int getSegmentCommitTimeoutSeconds() { if (containsKey(SEGMENT_COMMIT_TIMEOUT_SECONDS)) { return Integer.parseInt((String) getProperty(SEGMENT_COMMIT_TIMEOUT_SECONDS)); } return SegmentCompletionProtocol.getDefaultMaxSegmentCommitTimeSeconds(); }
public long getCommitTimeoutMS(String tableName) { long commitTimeoutMS = SegmentCompletionProtocol.getMaxSegmentCommitTimeMs(); if (_propertyStore == null) { return commitTimeoutMS; } TableConfig tableConfig = getRealtimeTableConfig(tableName); final Map<String, String> streamConfigs = tableConfig.getIndexingConfig().getStreamConfigs(); if (streamConfigs != null && streamConfigs.containsKey(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS)) { final String commitTimeoutSecondsStr = streamConfigs.get(StreamConfigProperties.SEGMENT_COMMIT_TIMEOUT_SECONDS); try { return TimeUnit.MILLISECONDS.convert(Integer.parseInt(commitTimeoutSecondsStr), TimeUnit.SECONDS); } catch (Exception e) { LOGGER.warn("Failed to parse flush size of {}", commitTimeoutSecondsStr, e); return commitTimeoutMS; } } return commitTimeoutMS; }
public static SegmentCompletionManager create(HelixManager helixManager, PinotLLCRealtimeSegmentManager segmentManager, ControllerConf controllerConf, ControllerMetrics controllerMetrics) { if (_instance != null) { throw new RuntimeException("Cannot create multiple instances"); } _instance = new SegmentCompletionManager(helixManager, segmentManager, controllerMetrics); SegmentCompletionProtocol.setMaxSegmentCommitTimeMs( TimeUnit.MILLISECONDS.convert(controllerConf.getSegmentCommitTimeoutSeconds(), TimeUnit.SECONDS)); return _instance; }
@GET @Path(SegmentCompletionProtocol.MSG_TYPE_EXTEND_BUILD_TIME) @Produces(MediaType.APPLICATION_JSON) public String extendBuildTime(@QueryParam(SegmentCompletionProtocol.PARAM_INSTANCE_ID) String instanceId, @QueryParam(SegmentCompletionProtocol.PARAM_SEGMENT_NAME) String segmentName, @QueryParam(SegmentCompletionProtocol.PARAM_OFFSET) long offset, @QueryParam(SegmentCompletionProtocol.PARAM_EXTRA_TIME_SEC) int extraTimeSec) { if (instanceId == null || segmentName == null || offset == -1) { LOGGER.error("Invalid call: offset={}, segmentName={}, instanceId={}", offset, segmentName, instanceId); return SegmentCompletionProtocol.RESP_FAILED.toJsonString(); } if (extraTimeSec <= 0) { LOGGER.warn("Invalid value {} for extra build time from instance {} for segment {}", extraTimeSec, instanceId, segmentName); extraTimeSec = SegmentCompletionProtocol.getDefaultMaxSegmentCommitTimeSeconds(); } SegmentCompletionProtocol.Request.Params requestParams = new SegmentCompletionProtocol.Request.Params(); requestParams.withInstanceId(instanceId).withSegmentName(segmentName).withOffset(offset) .withExtraTimeSec(extraTimeSec); LOGGER.info("Processing extendBuildTime:{}", requestParams.toString()); SegmentCompletionProtocol.Response response = SegmentCompletionManager.getInstance().extendBuildTime(requestParams); final String responseStr = response.toJsonString(); LOGGER.info("Response to extendBuildTime:{}", responseStr); return responseStr; }
SegmentCompletionProtocol.getMaxSegmentCommitTimeMs() * SegmentCompletionProtocol.MAX_HOLD_TIME_MS / 1000;
protected void buildSegmentForCommit(long buildTimeLeaseMs) { try { if (_segmentBuildDescriptor != null && _segmentBuildDescriptor.getOffset() == _currentOffset) { // Double-check that we have the file, just in case. String segTarFile = _segmentBuildDescriptor.getSegmentTarFilePath(); if (new File(segTarFile).exists()) { return; } } removeSegmentFile(); if (buildTimeLeaseMs <= 0) { if (_segBuildSemaphore == null) { buildTimeLeaseMs = SegmentCompletionProtocol.getDefaultMaxSegmentCommitTimeSeconds() * 1000L; } else { // We know we are going to use a semaphore to limit number of segment builds, and could be // blocked for a long time. The controller has not provided a lease time, so set one to // some reasonable guess here. buildTimeLeaseMs = BUILD_TIME_LEASE_SECONDS * 1000; } } _leaseExtender.addSegment(_segmentNameStr, buildTimeLeaseMs, _currentOffset); _segmentBuildDescriptor = buildSegmentInternal(true); } finally { _leaseExtender.removeSegment(_segmentNameStr); } }