private void setFetchThresholds() { // get the thresholds, and set defaults if not defined. KafkaConfig kafkaConfig = new KafkaConfig(config); Option<String> fetchThresholdOption = kafkaConfig.getConsumerFetchThreshold(systemName); long fetchThreshold = FETCH_THRESHOLD; if (fetchThresholdOption.isDefined()) { fetchThreshold = Long.valueOf(fetchThresholdOption.get()); } Option<String> fetchThresholdBytesOption = kafkaConfig.getConsumerFetchThresholdBytes(systemName); long fetchThresholdBytes = FETCH_THRESHOLD_BYTES; if (fetchThresholdBytesOption.isDefined()) { fetchThresholdBytes = Long.valueOf(fetchThresholdBytesOption.get()); } int numPartitions = topicPartitionsToSSP.size(); if (numPartitions != topicPartitionsToOffset.size()) { throw new SamzaException("topicPartitionsToSSP.size() doesn't match topicPartitionsToOffset.size()"); } if (numPartitions > 0) { perPartitionFetchThreshold = fetchThreshold / numPartitions; if (fetchThresholdBytesEnabled) { // currently this feature cannot be enabled, because we do not have the size of the messages available. // messages get double buffered, hence divide by 2 perPartitionFetchThresholdBytes = (fetchThresholdBytes / 2) / numPartitions; } } LOG.info("{}: fetchThresholdBytes = {}; fetchThreshold={}; numPartitions={}, perPartitionFetchThreshold={}, perPartitionFetchThresholdBytes(0 if disabled)={}", this, fetchThresholdBytes, fetchThreshold, numPartitions, perPartitionFetchThreshold, perPartitionFetchThresholdBytes); }
private void setFetchThresholds() { // get the thresholds, and set defaults if not defined. KafkaConfig kafkaConfig = new KafkaConfig(config); Option<String> fetchThresholdOption = kafkaConfig.getConsumerFetchThreshold(systemName); long fetchThreshold = FETCH_THRESHOLD; if (fetchThresholdOption.isDefined()) { fetchThreshold = Long.valueOf(fetchThresholdOption.get()); } Option<String> fetchThresholdBytesOption = kafkaConfig.getConsumerFetchThresholdBytes(systemName); long fetchThresholdBytes = FETCH_THRESHOLD_BYTES; if (fetchThresholdBytesOption.isDefined()) { fetchThresholdBytes = Long.valueOf(fetchThresholdBytesOption.get()); } int numPartitions = topicPartitionsToSSP.size(); if (numPartitions != topicPartitionsToOffset.size()) { throw new SamzaException("topicPartitionsToSSP.size() doesn't match topicPartitionsToOffset.size()"); } if (numPartitions > 0) { perPartitionFetchThreshold = fetchThreshold / numPartitions; if (fetchThresholdBytesEnabled) { // currently this feature cannot be enabled, because we do not have the size of the messages available. // messages get double buffered, hence divide by 2 perPartitionFetchThresholdBytes = (fetchThresholdBytes / 2) / numPartitions; } } LOG.info("{}: fetchThresholdBytes = {}; fetchThreshold={}; numPartitions={}, perPartitionFetchThreshold={}, perPartitionFetchThresholdBytes(0 if disabled)={}", this, fetchThresholdBytes, fetchThreshold, numPartitions, perPartitionFetchThreshold, perPartitionFetchThresholdBytes); }
private void setFetchThresholds() { // get the thresholds, and set defaults if not defined. KafkaConfig kafkaConfig = new KafkaConfig(config); Option<String> fetchThresholdOption = kafkaConfig.getConsumerFetchThreshold(systemName); long fetchThreshold = FETCH_THRESHOLD; if (fetchThresholdOption.isDefined()) { fetchThreshold = Long.valueOf(fetchThresholdOption.get()); } Option<String> fetchThresholdBytesOption = kafkaConfig.getConsumerFetchThresholdBytes(systemName); long fetchThresholdBytes = FETCH_THRESHOLD_BYTES; if (fetchThresholdBytesOption.isDefined()) { fetchThresholdBytes = Long.valueOf(fetchThresholdBytesOption.get()); } int numPartitions = topicPartitionsToSSP.size(); if (numPartitions != topicPartitionsToOffset.size()) { throw new SamzaException("topicPartitionsToSSP.size() doesn't match topicPartitionsToOffset.size()"); } if (numPartitions > 0) { perPartitionFetchThreshold = fetchThreshold / numPartitions; if (fetchThresholdBytesEnabled) { // currently this feature cannot be enabled, because we do not have the size of the messages available. // messages get double buffered, hence divide by 2 perPartitionFetchThresholdBytes = (fetchThresholdBytes / 2) / numPartitions; } } LOG.info("{}: fetchThresholdBytes = {}; fetchThreshold={}; numPartitions={}, perPartitionFetchThreshold={}, perPartitionFetchThresholdBytes(0 if disabled)={}", this, fetchThresholdBytes, fetchThreshold, numPartitions, perPartitionFetchThreshold, perPartitionFetchThresholdBytes); }