private boolean currentPartitionFinished() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { return true; } else if (this.nextWatermark.get(this.currentPartitionIdx) >= this.highWatermark.get(this.currentPartitionIdx)) { LOG.info("Finished pulling partition " + this.getCurrentPartition()); return true; } else { return false; } }
/** * Record the avg time per record for the current partition, then increment this.currentPartitionIdx, * and switch metric context to the new partition. */ private void moveToNextPartition() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { LOG.info("Pulling topic " + this.topicName); this.currentPartitionIdx = 0; } else { updateStatisticsForCurrentPartition(); this.currentPartitionIdx++; this.currentPartitionRecordCount = 0; this.currentPartitionTotalSize = 0; this.currentPartitionDecodeRecordTime = 0; this.currentPartitionFetchMessageBufferTime = 0; this.currentPartitionReadRecordTime = 0; this.currentPartitionLastSuccessfulRecord = null; } this.messageIterator = null; if (this.currentPartitionIdx < this.partitions.size()) { LOG.info(String.format("Pulling partition %s from offset %d to %d, range=%d", this.getCurrentPartition(), this.nextWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx) - this.nextWatermark.get(this.currentPartitionIdx))); switchMetricContextToCurrentPartition(); } if (!allPartitionsFinished()) { this.startFetchEpochTime.put(this.getCurrentPartition(), System.currentTimeMillis()); } }
KafkaPartition partition = partitions.get(i); if (watermark.get(i) != ConfigurationKeys.DEFAULT_WATERMARK_VALUE) { this.previousOffsets.put(partition, watermark.get(i)); if (previousLowWatermark.get(i) != ConfigurationKeys.DEFAULT_WATERMARK_VALUE) { this.previousLowWatermarks.put(partition, previousLowWatermark.get(i)); if (previousExpectedHighWatermark.get(i) != ConfigurationKeys.DEFAULT_WATERMARK_VALUE) { this.previousExpectedHighWatermarks.put(partition, previousExpectedHighWatermark.get(i));
@Override public void close() throws IOException { if (currentPartitionIdx != INITIAL_PARTITION_IDX) { updateStatisticsForCurrentPartition(); } Map<KafkaPartition, Map<String, String>> tagsForPartitionsMap = Maps.newHashMap(); // Add error partition count and error message count to workUnitState this.workUnitState.setProp(ConfigurationKeys.ERROR_PARTITION_COUNT, this.errorPartitions.size()); this.workUnitState.setProp(ConfigurationKeys.ERROR_MESSAGE_UNDECODABLE_COUNT, this.undecodableMessageCount); for (int i = 0; i < this.partitions.size(); i++) { LOG.info(String.format("Actual high watermark for partition %s=%d, expected=%d", this.partitions.get(i), this.nextWatermark.get(i), this.highWatermark.get(i))); tagsForPartitionsMap.put(this.partitions.get(i), createTagsForPartition(i)); } this.workUnitState.setActualHighWatermark(this.nextWatermark); if (isInstrumentationEnabled()) { for (Map.Entry<KafkaPartition, Map<String, String>> eventTags : tagsForPartitionsMap.entrySet()) { new EventSubmitter.Builder(getMetricContext(), GOBBLIN_KAFKA_NAMESPACE).build() .submit(KAFKA_EXTRACTOR_TOPIC_METADATA_EVENT_NAME, eventTags.getValue()); } } this.closer.close(); }
tagsForPartition.put(LOW_WATERMARK, Long.toString(this.lowWatermark.get(partitionId))); tagsForPartition.put(ACTUAL_HIGH_WATERMARK, Long.toString(this.nextWatermark.get(partitionId))); tagsForPartition.put(EXPECTED_HIGH_WATERMARK, Long.toString(this.highWatermark.get(partitionId))); tagsForPartition.put(KafkaSource.PREVIOUS_OFFSET_FETCH_EPOCH_TIME, Long.toString(KafkaUtils.getPropAsLongFromSingleOrMultiWorkUnitState(this.workUnitState,
if (nextValidMessage.getOffset() < this.nextWatermark.get(this.currentPartitionIdx)) { continue;
private boolean currentPartitionFinished() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { return true; } else if (this.nextWatermark.get(this.currentPartitionIdx) >= this.highWatermark.get(this.currentPartitionIdx)) { LOG.info("Finished pulling partition " + this.getCurrentPartition()); return true; } else { return false; } }
/** * Record the avg time per record for the current partition, then increment this.currentPartitionIdx, * and switch metric context to the new partition. */ private void moveToNextPartition() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { LOG.info("Pulling topic " + this.topicName); this.currentPartitionIdx = 0; } else { updateStatisticsForCurrentPartition(); this.currentPartitionIdx++; this.currentPartitionRecordCount = 0; this.currentPartitionTotalSize = 0; this.currentPartitionDecodeRecordTime = 0; this.currentPartitionFetchMessageBufferTime = 0; this.currentPartitionReadRecordTime = 0; this.currentPartitionLastSuccessfulRecord = null; } this.messageIterator = null; if (this.currentPartitionIdx < this.partitions.size()) { LOG.info(String.format("Pulling partition %s from offset %d to %d, range=%d", this.getCurrentPartition(), this.nextWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx) - this.nextWatermark.get(this.currentPartitionIdx))); switchMetricContextToCurrentPartition(); } if (!allPartitionsFinished()) { this.startFetchEpochTime.put(this.getCurrentPartition(), System.currentTimeMillis()); } }
KafkaPartition partition = partitions.get(i); if (watermark.get(i) != ConfigurationKeys.DEFAULT_WATERMARK_VALUE) { this.previousOffsets.put(partition, watermark.get(i)); if (previousLowWatermark.get(i) != ConfigurationKeys.DEFAULT_WATERMARK_VALUE) { this.previousLowWatermarks.put(partition, previousLowWatermark.get(i)); if (previousExpectedHighWatermark.get(i) != ConfigurationKeys.DEFAULT_WATERMARK_VALUE) { this.previousExpectedHighWatermarks.put(partition, previousExpectedHighWatermark.get(i));
tagsForPartition.put(LOW_WATERMARK, Long.toString(this.lowWatermark.get(partitionId))); tagsForPartition.put(ACTUAL_HIGH_WATERMARK, Long.toString(this.nextWatermark.get(partitionId))); tagsForPartition.put(EXPECTED_HIGH_WATERMARK, Long.toString(this.highWatermark.get(partitionId))); tagsForPartition.put(KafkaSource.PREVIOUS_OFFSET_FETCH_EPOCH_TIME, Long.toString(KafkaUtils.getPropAsLongFromSingleOrMultiWorkUnitState(this.workUnitState,
@Override public void close() throws IOException { if (currentPartitionIdx != INITIAL_PARTITION_IDX) { updateStatisticsForCurrentPartition(); } Map<KafkaPartition, Map<String, String>> tagsForPartitionsMap = Maps.newHashMap(); // Add error partition count and error message count to workUnitState this.workUnitState.setProp(ConfigurationKeys.ERROR_PARTITION_COUNT, this.errorPartitions.size()); this.workUnitState.setProp(ConfigurationKeys.ERROR_MESSAGE_UNDECODABLE_COUNT, this.undecodableMessageCount); for (int i = 0; i < this.partitions.size(); i++) { LOG.info(String.format("Actual high watermark for partition %s=%d, expected=%d", this.partitions.get(i), this.nextWatermark.get(i), this.highWatermark.get(i))); tagsForPartitionsMap.put(this.partitions.get(i), createTagsForPartition(i)); } this.workUnitState.setActualHighWatermark(this.nextWatermark); if (isInstrumentationEnabled()) { for (Map.Entry<KafkaPartition, Map<String, String>> eventTags : tagsForPartitionsMap.entrySet()) { new EventSubmitter.Builder(getMetricContext(), GOBBLIN_KAFKA_NAMESPACE).build() .submit(KAFKA_EXTRACTOR_TOPIC_METADATA_EVENT_NAME, eventTags.getValue()); } } this.closer.close(); }
if (nextValidMessage.getOffset() < this.nextWatermark.get(this.currentPartitionIdx)) { continue;