while (!allPartitionsFinished()) { if (currentPartitionFinished()) { moveToNextPartition(); continue; try { long fetchStartTime = System.nanoTime(); this.messageIterator = fetchNextMessageBuffer(); this.currentPartitionFetchMessageBufferTime += System.nanoTime() - fetchStartTime; } catch (Exception e) { LOG.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e); moveToNextPartition(); continue; moveToNextPartition(); continue; while (!currentPartitionFinished()) { if (!this.messageIterator.hasNext()) { break; record = decodeRecord((ByteArrayBasedKafkaRecord)nextValidMessage); } else if (nextValidMessage instanceof DecodeableKafkaRecord){ record = convertRecord(((DecodeableKafkaRecord<?, D>) nextValidMessage).getValue()); } else { throw new IllegalStateException(
@Override public void close() throws IOException { if (currentPartitionIdx != INITIAL_PARTITION_IDX) { updateStatisticsForCurrentPartition(); } Map<KafkaPartition, Map<String, String>> tagsForPartitionsMap = Maps.newHashMap(); // Add error partition count and error message count to workUnitState this.workUnitState.setProp(ConfigurationKeys.ERROR_PARTITION_COUNT, this.errorPartitions.size()); this.workUnitState.setProp(ConfigurationKeys.ERROR_MESSAGE_UNDECODABLE_COUNT, this.undecodableMessageCount); for (int i = 0; i < this.partitions.size(); i++) { LOG.info(String.format("Actual high watermark for partition %s=%d, expected=%d", this.partitions.get(i), this.nextWatermark.get(i), this.highWatermark.get(i))); tagsForPartitionsMap.put(this.partitions.get(i), createTagsForPartition(i)); } this.workUnitState.setActualHighWatermark(this.nextWatermark); if (isInstrumentationEnabled()) { for (Map.Entry<KafkaPartition, Map<String, String>> eventTags : tagsForPartitionsMap.entrySet()) { new EventSubmitter.Builder(getMetricContext(), GOBBLIN_KAFKA_NAMESPACE).build() .submit(KAFKA_EXTRACTOR_TOPIC_METADATA_EVENT_NAME, eventTags.getValue()); } } this.closer.close(); }
protected void updateStatisticsForCurrentPartition() { long stopFetchEpochTime = System.currentTimeMillis(); if (!allPartitionsFinished()) { this.stopFetchEpochTime.put(this.getCurrentPartition(), stopFetchEpochTime); } if (this.currentPartitionRecordCount != 0) { long currentPartitionFetchDuration = stopFetchEpochTime - this.startFetchEpochTime.get(this.getCurrentPartition()); double avgMillisForCurrentPartition = (double) currentPartitionFetchDuration / (double) this.currentPartitionRecordCount; this.avgMillisPerRecord.put(this.getCurrentPartition(), avgMillisForCurrentPartition); long avgRecordSize = this.currentPartitionTotalSize / this.currentPartitionRecordCount; this.avgRecordSizes.put(this.getCurrentPartition(), avgRecordSize); this.elapsedTime.put(this.getCurrentPartition(), currentPartitionFetchDuration); this.processedRecordCount.put(this.getCurrentPartition(), this.currentPartitionRecordCount); this.partitionTotalSize.put(this.getCurrentPartition(), this.currentPartitionTotalSize); this.decodeRecordTime.put(this.getCurrentPartition(), this.currentPartitionDecodeRecordTime); this.fetchMessageBufferTime.put(this.getCurrentPartition(), this.currentPartitionFetchMessageBufferTime); this.readRecordTime.put(this.getCurrentPartition(), this.currentPartitionReadRecordTime); } }
private void switchMetricContextToCurrentPartition() { if (this.currentPartitionIdx >= this.partitions.size()) { return; } int currentPartitionId = this.getCurrentPartition().getId(); switchMetricContext(Lists.<Tag<?>> newArrayList(new Tag<>("kafka_partition", currentPartitionId))); }
/** * Record the avg time per record for the current partition, then increment this.currentPartitionIdx, * and switch metric context to the new partition. */ private void moveToNextPartition() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { LOG.info("Pulling topic " + this.topicName); this.currentPartitionIdx = 0; } else { updateStatisticsForCurrentPartition(); this.currentPartitionIdx++; this.currentPartitionRecordCount = 0; this.currentPartitionTotalSize = 0; this.currentPartitionDecodeRecordTime = 0; this.currentPartitionFetchMessageBufferTime = 0; this.currentPartitionReadRecordTime = 0; this.currentPartitionLastSuccessfulRecord = null; } this.messageIterator = null; if (this.currentPartitionIdx < this.partitions.size()) { LOG.info(String.format("Pulling partition %s from offset %d to %d, range=%d", this.getCurrentPartition(), this.nextWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx) - this.nextWatermark.get(this.currentPartitionIdx))); switchMetricContextToCurrentPartition(); } if (!allPartitionsFinished()) { this.startFetchEpochTime.put(this.getCurrentPartition(), System.currentTimeMillis()); } }
private void incrementErrorCount() { if (this.decodingErrorCount.containsKey(getCurrentPartition())) { this.decodingErrorCount.put(getCurrentPartition(), this.decodingErrorCount.get(getCurrentPartition()) + 1); } else { this.decodingErrorCount.put(getCurrentPartition(), 1L); } }
/** * Record the avg time per record for the current partition, then increment this.currentPartitionIdx, * and switch metric context to the new partition. */ private void moveToNextPartition() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { LOG.info("Pulling topic " + this.topicName); this.currentPartitionIdx = 0; } else { updateStatisticsForCurrentPartition(); this.currentPartitionIdx++; this.currentPartitionRecordCount = 0; this.currentPartitionTotalSize = 0; this.currentPartitionDecodeRecordTime = 0; this.currentPartitionFetchMessageBufferTime = 0; this.currentPartitionReadRecordTime = 0; this.currentPartitionLastSuccessfulRecord = null; } this.messageIterator = null; if (this.currentPartitionIdx < this.partitions.size()) { LOG.info(String.format("Pulling partition %s from offset %d to %d, range=%d", this.getCurrentPartition(), this.nextWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx) - this.nextWatermark.get(this.currentPartitionIdx))); switchMetricContextToCurrentPartition(); } if (!allPartitionsFinished()) { this.startFetchEpochTime.put(this.getCurrentPartition(), System.currentTimeMillis()); } }
private void switchMetricContextToCurrentPartition() { if (this.currentPartitionIdx >= this.partitions.size()) { return; } int currentPartitionId = this.getCurrentPartition().getId(); switchMetricContext(Lists.<Tag<?>> newArrayList(new Tag<>("kafka_partition", currentPartitionId))); }
protected void updateStatisticsForCurrentPartition() { long stopFetchEpochTime = System.currentTimeMillis(); if (!allPartitionsFinished()) { this.stopFetchEpochTime.put(this.getCurrentPartition(), stopFetchEpochTime); } if (this.currentPartitionRecordCount != 0) { long currentPartitionFetchDuration = stopFetchEpochTime - this.startFetchEpochTime.get(this.getCurrentPartition()); double avgMillisForCurrentPartition = (double) currentPartitionFetchDuration / (double) this.currentPartitionRecordCount; this.avgMillisPerRecord.put(this.getCurrentPartition(), avgMillisForCurrentPartition); long avgRecordSize = this.currentPartitionTotalSize / this.currentPartitionRecordCount; this.avgRecordSizes.put(this.getCurrentPartition(), avgRecordSize); this.elapsedTime.put(this.getCurrentPartition(), currentPartitionFetchDuration); this.processedRecordCount.put(this.getCurrentPartition(), this.currentPartitionRecordCount); this.partitionTotalSize.put(this.getCurrentPartition(), this.currentPartitionTotalSize); this.decodeRecordTime.put(this.getCurrentPartition(), this.currentPartitionDecodeRecordTime); this.fetchMessageBufferTime.put(this.getCurrentPartition(), this.currentPartitionFetchMessageBufferTime); this.readRecordTime.put(this.getCurrentPartition(), this.currentPartitionReadRecordTime); } }
private boolean shouldLogError() { return !this.decodingErrorCount.containsKey(getCurrentPartition()) || this.decodingErrorCount.get(getCurrentPartition()) <= MAX_LOG_DECODING_ERRORS; }
while (!allPartitionsFinished()) { if (currentPartitionFinished()) { moveToNextPartition(); continue; try { long fetchStartTime = System.nanoTime(); this.messageIterator = fetchNextMessageBuffer(); this.currentPartitionFetchMessageBufferTime += System.nanoTime() - fetchStartTime; } catch (Exception e) { LOG.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e); moveToNextPartition(); continue; moveToNextPartition(); continue; while (!currentPartitionFinished()) { if (!this.messageIterator.hasNext()) { break; record = decodeRecord((ByteArrayBasedKafkaRecord)nextValidMessage); } else if (nextValidMessage instanceof DecodeableKafkaRecord){ record = convertRecord(((DecodeableKafkaRecord<?, D>) nextValidMessage).getValue()); } else { throw new IllegalStateException(
@Override public void close() throws IOException { if (currentPartitionIdx != INITIAL_PARTITION_IDX) { updateStatisticsForCurrentPartition(); } Map<KafkaPartition, Map<String, String>> tagsForPartitionsMap = Maps.newHashMap(); // Add error partition count and error message count to workUnitState this.workUnitState.setProp(ConfigurationKeys.ERROR_PARTITION_COUNT, this.errorPartitions.size()); this.workUnitState.setProp(ConfigurationKeys.ERROR_MESSAGE_UNDECODABLE_COUNT, this.undecodableMessageCount); for (int i = 0; i < this.partitions.size(); i++) { LOG.info(String.format("Actual high watermark for partition %s=%d, expected=%d", this.partitions.get(i), this.nextWatermark.get(i), this.highWatermark.get(i))); tagsForPartitionsMap.put(this.partitions.get(i), createTagsForPartition(i)); } this.workUnitState.setActualHighWatermark(this.nextWatermark); if (isInstrumentationEnabled()) { for (Map.Entry<KafkaPartition, Map<String, String>> eventTags : tagsForPartitionsMap.entrySet()) { new EventSubmitter.Builder(getMetricContext(), GOBBLIN_KAFKA_NAMESPACE).build() .submit(KAFKA_EXTRACTOR_TOPIC_METADATA_EVENT_NAME, eventTags.getValue()); } } this.closer.close(); }
private boolean currentPartitionFinished() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { return true; } else if (this.nextWatermark.get(this.currentPartitionIdx) >= this.highWatermark.get(this.currentPartitionIdx)) { LOG.info("Finished pulling partition " + this.getCurrentPartition()); return true; } else { return false; } }
private void incrementErrorCount() { if (this.decodingErrorCount.containsKey(getCurrentPartition())) { this.decodingErrorCount.put(getCurrentPartition(), this.decodingErrorCount.get(getCurrentPartition()) + 1); } else { this.decodingErrorCount.put(getCurrentPartition(), 1L); } }
private boolean shouldLogError() { return !this.decodingErrorCount.containsKey(getCurrentPartition()) || this.decodingErrorCount.get(getCurrentPartition()) <= MAX_LOG_DECODING_ERRORS; }
private boolean currentPartitionFinished() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { return true; } else if (this.nextWatermark.get(this.currentPartitionIdx) >= this.highWatermark.get(this.currentPartitionIdx)) { LOG.info("Finished pulling partition " + this.getCurrentPartition()); return true; } else { return false; } }