private KafkaAPI getKafkaAPI(Config config) { if (this.useNewKafkaAPI) { return new KafkaNewAPI(config); } return new KafkaOldAPI(config); }
private WorkUnit createEmptyWorkUnit(KafkaPartition partition, long previousOffset, long previousFetchEpochTime, Optional<State> topicSpecificState) { Offsets offsets = new Offsets(); offsets.setEarliestOffset(previousOffset); offsets.setLatestOffset(previousOffset); offsets.startAtEarliestOffset(); offsets.setOffsetFetchEpochTime(previousFetchEpochTime); return getWorkUnitForTopicPartition(partition, offsets, topicSpecificState); }
private KafkaPartition(Builder builder) { this.id = builder.id; this.topicName = builder.topicName; this.leader = new KafkaLeader(builder.leaderId, builder.leaderHostAndPort); }
/** * Set the average time in milliseconds to pull a record of a partition, which will be stored in property * "[topicname].[partitionid].avg.record.millis". */ public static void setPartitionAvgRecordMillis(State state, KafkaPartition partition, double millis) { state.setProp( getPartitionPropName(partition.getTopicName(), partition.getId()) + "." + KafkaSource.AVG_RECORD_MILLIS, millis); }
private long getPreviousExpectedHighWatermark(KafkaPartition partition, SourceState state) throws PreviousOffsetNotFoundException { getAllPreviousOffsetState(state); if (this.previousExpectedHighWatermarks.containsKey(partition)) { return this.previousExpectedHighWatermarks.get(partition); } throw new PreviousOffsetNotFoundException(String .format("Previous expected high watermark for topic %s, partition %s not found.", partition.getTopicName(), partition.getId())); }
private KafkaDeserializerExtractor(WorkUnitState state, Optional<Deserializers> deserializerType) throws ReflectiveOperationException { this(state, deserializerType, getDeserializer(getProps(state), deserializerType), getKafkaSchemaRegistry(getProps(state))); }
@Override public long getExpectedRecordCount() { return this.lowWatermark.getGap(this.highWatermark); }
private boolean currentPartitionFinished() { if (this.currentPartitionIdx == INITIAL_PARTITION_IDX) { return true; } else if (this.nextWatermark.get(this.currentPartitionIdx) >= this.highWatermark.get(this.currentPartitionIdx)) { LOG.info("Finished pulling partition " + this.getCurrentPartition()); return true; } else { return false; } }
@Override public String apply(KafkaTopic topic) { return topic.getName(); } }), state);
@Override public Extractor getExtractor(WorkUnitState state) throws IOException { return new KafkaSimpleStreamingExtractor<S, D>(state); } }
private void addTopicSpecificPropsToWorkUnits(List<WorkUnit> workUnits, Map<String, State> topicSpecificStateMap) { for (WorkUnit workUnit : workUnits) { addTopicSpecificPropsToWorkUnit(workUnit, topicSpecificStateMap); } }
private KafkaWrapper(Builder builder) { this.useNewKafkaAPI = builder.useNewKafkaAPI; this.brokers = builder.brokers; this.kafkaAPI = getKafkaAPI(builder.config); }
@Override public void close() throws IOException { this.kafkaAPI.close(); }
public KafkaPartition build() { return new KafkaPartition(this); } }
@Override public Extractor<JsonArray, JsonObject> getExtractor(WorkUnitState state) throws IOException { return new JsonExtractor(state); }
@Override public boolean equals(Object obj) { if (obj == null) { return false; } if (!(obj instanceof KafkaWatermark)) { return false; } return this.compareTo((CheckpointableWatermark) obj) == 0; }
/** * Determines whether the given {@link State} contains "[topicname].[partitionid].avg.record.size". */ public static boolean containsPartitionAvgRecordSize(State state, KafkaPartition partition) { return state.contains( getPartitionPropName(partition.getTopicName(), partition.getId()) + "." + KafkaSource.AVG_RECORD_SIZE); }
private long getPreviousLowWatermark(KafkaPartition partition, SourceState state) throws PreviousOffsetNotFoundException { getAllPreviousOffsetState(state); if (this.previousLowWatermarks.containsKey(partition)) { return this.previousLowWatermarks.get(partition); } throw new PreviousOffsetNotFoundException(String .format("Previous low watermark for topic %s, partition %s not found.", partition.getTopicName(), partition.getId())); }
public KafkaPartition(KafkaPartition other) { this.topicName = other.topicName; this.id = other.id; this.leader = new KafkaLeader(other.leader.id, other.leader.hostAndPort); }
/** * Get the average record size of a partition, which is stored in property "[topicname].[partitionid].avg.record.size". * If state doesn't contain this property, it returns defaultSize. */ public static long getPartitionAvgRecordSize(State state, KafkaPartition partition) { return state.getPropAsLong( getPartitionPropName(partition.getTopicName(), partition.getId()) + "." + KafkaSource.AVG_RECORD_SIZE); }