private void incrementErrorCount() { if (this.decodingErrorCount.containsKey(getCurrentPartition())) { this.decodingErrorCount.put(getCurrentPartition(), this.decodingErrorCount.get(getCurrentPartition()) + 1); } else { this.decodingErrorCount.put(getCurrentPartition(), 1L); } }
private boolean shouldLogError() { return !this.decodingErrorCount.containsKey(getCurrentPartition()) || this.decodingErrorCount.get(getCurrentPartition()) <= MAX_LOG_DECODING_ERRORS; }
protected void updateStatisticsForCurrentPartition() { long stopFetchEpochTime = System.currentTimeMillis(); if (!allPartitionsFinished()) { this.stopFetchEpochTime.put(this.getCurrentPartition(), stopFetchEpochTime); } if (this.currentPartitionRecordCount != 0) { long currentPartitionFetchDuration = stopFetchEpochTime - this.startFetchEpochTime.get(this.getCurrentPartition()); double avgMillisForCurrentPartition = (double) currentPartitionFetchDuration / (double) this.currentPartitionRecordCount; this.avgMillisPerRecord.put(this.getCurrentPartition(), avgMillisForCurrentPartition); long avgRecordSize = this.currentPartitionTotalSize / this.currentPartitionRecordCount; this.avgRecordSizes.put(this.getCurrentPartition(), avgRecordSize); this.elapsedTime.put(this.getCurrentPartition(), currentPartitionFetchDuration); this.processedRecordCount.put(this.getCurrentPartition(), this.currentPartitionRecordCount); this.partitionTotalSize.put(this.getCurrentPartition(), this.currentPartitionTotalSize); this.decodeRecordTime.put(this.getCurrentPartition(), this.currentPartitionDecodeRecordTime); this.fetchMessageBufferTime.put(this.getCurrentPartition(), this.currentPartitionFetchMessageBufferTime); this.readRecordTime.put(this.getCurrentPartition(), this.currentPartitionReadRecordTime); } }
private boolean currentPartitionFinished() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { return true; } else if (this.nextWatermark.get(this.currentPartitionIdx) >= this.highWatermark.get(this.currentPartitionIdx)) { LOG.info("Finished pulling partition " + this.getCurrentPartition()); return true; } else { return false; } }
private void switchMetricContextToCurrentPartition() { if (this.currentPartitionIdx >= this.partitions.size()) { return; } int currentPartitionId = this.getCurrentPartition().getId(); switchMetricContext(Lists.<Tag<?>> newArrayList(new Tag<>("kafka_partition", currentPartitionId))); }
/** * Record the avg time per record for the current partition, then increment this.currentPartitionIdx, * and switch metric context to the new partition. */ private void moveToNextPartition() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { LOG.info("Pulling topic " + this.topicName); this.currentPartitionIdx = 0; } else { updateStatisticsForCurrentPartition(); this.currentPartitionIdx++; this.currentPartitionRecordCount = 0; this.currentPartitionTotalSize = 0; this.currentPartitionDecodeRecordTime = 0; this.currentPartitionFetchMessageBufferTime = 0; this.currentPartitionReadRecordTime = 0; this.currentPartitionLastSuccessfulRecord = null; } this.messageIterator = null; if (this.currentPartitionIdx < this.partitions.size()) { LOG.info(String.format("Pulling partition %s from offset %d to %d, range=%d", this.getCurrentPartition(), this.nextWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx) - this.nextWatermark.get(this.currentPartitionIdx))); switchMetricContextToCurrentPartition(); } if (!allPartitionsFinished()) { this.startFetchEpochTime.put(this.getCurrentPartition(), System.currentTimeMillis()); } }
} catch (Exception e) { LOG.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e); moveToNextPartition(); continue; this.undecodableMessageCount++; if (shouldLogError()) { LOG.error(String.format("A record from partition %s cannot be decoded.", getCurrentPartition()), t);
private void incrementErrorCount() { if (this.decodingErrorCount.containsKey(getCurrentPartition())) { this.decodingErrorCount.put(getCurrentPartition(), this.decodingErrorCount.get(getCurrentPartition()) + 1); } else { this.decodingErrorCount.put(getCurrentPartition(), 1L); } }
private boolean shouldLogError() { return !this.decodingErrorCount.containsKey(getCurrentPartition()) || this.decodingErrorCount.get(getCurrentPartition()) <= MAX_LOG_DECODING_ERRORS; }
protected void updateStatisticsForCurrentPartition() { long stopFetchEpochTime = System.currentTimeMillis(); if (!allPartitionsFinished()) { this.stopFetchEpochTime.put(this.getCurrentPartition(), stopFetchEpochTime); } if (this.currentPartitionRecordCount != 0) { long currentPartitionFetchDuration = stopFetchEpochTime - this.startFetchEpochTime.get(this.getCurrentPartition()); double avgMillisForCurrentPartition = (double) currentPartitionFetchDuration / (double) this.currentPartitionRecordCount; this.avgMillisPerRecord.put(this.getCurrentPartition(), avgMillisForCurrentPartition); long avgRecordSize = this.currentPartitionTotalSize / this.currentPartitionRecordCount; this.avgRecordSizes.put(this.getCurrentPartition(), avgRecordSize); this.elapsedTime.put(this.getCurrentPartition(), currentPartitionFetchDuration); this.processedRecordCount.put(this.getCurrentPartition(), this.currentPartitionRecordCount); this.partitionTotalSize.put(this.getCurrentPartition(), this.currentPartitionTotalSize); this.decodeRecordTime.put(this.getCurrentPartition(), this.currentPartitionDecodeRecordTime); this.fetchMessageBufferTime.put(this.getCurrentPartition(), this.currentPartitionFetchMessageBufferTime); this.readRecordTime.put(this.getCurrentPartition(), this.currentPartitionReadRecordTime); } }
private boolean currentPartitionFinished() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { return true; } else if (this.nextWatermark.get(this.currentPartitionIdx) >= this.highWatermark.get(this.currentPartitionIdx)) { LOG.info("Finished pulling partition " + this.getCurrentPartition()); return true; } else { return false; } }
private void switchMetricContextToCurrentPartition() { if (this.currentPartitionIdx >= this.partitions.size()) { return; } int currentPartitionId = this.getCurrentPartition().getId(); switchMetricContext(Lists.<Tag<?>> newArrayList(new Tag<>("kafka_partition", currentPartitionId))); }
/** * Record the avg time per record for the current partition, then increment this.currentPartitionIdx, * and switch metric context to the new partition. */ private void moveToNextPartition() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { LOG.info("Pulling topic " + this.topicName); this.currentPartitionIdx = 0; } else { updateStatisticsForCurrentPartition(); this.currentPartitionIdx++; this.currentPartitionRecordCount = 0; this.currentPartitionTotalSize = 0; this.currentPartitionDecodeRecordTime = 0; this.currentPartitionFetchMessageBufferTime = 0; this.currentPartitionReadRecordTime = 0; this.currentPartitionLastSuccessfulRecord = null; } this.messageIterator = null; if (this.currentPartitionIdx < this.partitions.size()) { LOG.info(String.format("Pulling partition %s from offset %d to %d, range=%d", this.getCurrentPartition(), this.nextWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx), this.highWatermark.get(this.currentPartitionIdx) - this.nextWatermark.get(this.currentPartitionIdx))); switchMetricContextToCurrentPartition(); } if (!allPartitionsFinished()) { this.startFetchEpochTime.put(this.getCurrentPartition(), System.currentTimeMillis()); } }
} catch (Exception e) { LOG.error(String.format("Failed to fetch next message buffer for partition %s. Will skip this partition.", getCurrentPartition()), e); moveToNextPartition(); continue; this.undecodableMessageCount++; if (shouldLogError()) { LOG.error(String.format("A record from partition %s cannot be decoded.", getCurrentPartition()), t);