public static SourcePartition getCopyOf(SourcePartition origin) { SourcePartition copy = new SourcePartition(); copy.setTSRange(origin.getTSRange()); copy.setSegRange(origin.getSegRange()); if (origin.getSourcePartitionOffsetStart() != null) { copy.setSourcePartitionOffsetStart(new HashMap<>(origin.getSourcePartitionOffsetStart())); } if (origin.getSourcePartitionOffsetEnd() != null) { copy.setSourcePartitionOffsetEnd(new HashMap<>(origin.getSourcePartitionOffsetEnd())); } return copy; } }
@Override public SourcePartition enrichSourcePartitionBeforeBuild(IBuildable buildable, SourcePartition srcPartition) { SourcePartition result = SourcePartition.getCopyOf(srcPartition); if (srcPartition.getTSRange() != null) { result.setSegRange(null); } return result; }
@Override public SourcePartition enrichSourcePartitionBeforeBuild(IBuildable buildable, SourcePartition srcPartition) { SourcePartition result = SourcePartition.getCopyOf(srcPartition); result.setSegRange(null); return result; }
@Override public SourcePartition enrichSourcePartitionBeforeBuild(IBuildable buildable, SourcePartition srcPartition) { checkSourceOffsets(srcPartition); final SourcePartition result = SourcePartition.getCopyOf(srcPartition); final SegmentRange range = result.getSegRange(); final CubeInstance cube = (CubeInstance) buildable; if (range == null || range.start.v.equals(0L)) { result.setSourcePartitionOffsetStart(last.getSourcePartitionOffsetEnd()); } else if (cube.getDescriptor().getPartitionOffsetStart() != null && cube.getDescriptor().getPartitionOffsetStart().size() > 0) { logger.debug("Last segment doesn't exist, use the start offset that be initiated previously: " + cube.getDescriptor().getPartitionOffsetStart()); result.setSourcePartitionOffsetStart(cube.getDescriptor().getPartitionOffsetStart()); } else { result.setSourcePartitionOffsetStart(KafkaClient.getEarliestOffsets(cube)); logger.info("Get {} partitions for topic {} ", partitionInfos.size(), topic); for (PartitionInfo partitionInfo : partitionInfos) { if (result.getSourcePartitionOffsetStart().containsKey(partitionInfo.partition()) == false) { long earliest = KafkaClient.getEarliestOffset(consumer, topic, partitionInfo.partition()); logger.debug("New partition {} added, with start offset {}", partitionInfo.partition(), earliest); result.getSourcePartitionOffsetStart().put(partitionInfo.partition(), earliest); if (result.getSourcePartitionOffsetStart().containsKey(partitionId)) { if (result.getSourcePartitionOffsetStart().get(partitionId) > latestOffsets.get(partitionId)) { throw new IllegalArgumentException("Partition " + partitionId + " end offset (" + latestOffsets.get(partitionId) + ") is smaller than start offset ( " + result.getSourcePartitionOffsetStart().get(partitionId) + ")");
public CubeSegment appendSegment(CubeInstance cube, SourcePartition src) throws IOException { return appendSegment(cube, src.getTSRange(), src.getSegRange(), src.getSourcePartitionOffsetStart(), src.getSourcePartitionOffsetEnd()); }
private void checkSourceOffsets(SourcePartition src) { if (src.getSegRange() == null) return; long startOffset = (Long) src.getSegRange().start.v; long endOffset = (Long) src.getSegRange().end.v; final Map<Integer, Long> sourcePartitionOffsetStart = src.getSourcePartitionOffsetStart(); final Map<Integer, Long> sourcePartitionOffsetEnd = src.getSourcePartitionOffsetEnd(); if (endOffset <= 0 || startOffset >= endOffset) { throw new IllegalArgumentException("'startOffset' need be smaller than 'endOffset'");
if (buildType == CubeBuildTypeEnum.BUILD) { ISource source = SourceManager.getSource(cube); SourcePartition src = new SourcePartition(tsRange, segRange, sourcePartitionOffsetStart, sourcePartitionOffsetEnd); src = source.enrichSourcePartitionBeforeBuild(cube, src);
@Override public SourcePartition enrichSourcePartitionBeforeBuild(IBuildable buildable, SourcePartition srcPartition) { checkSourceOffsets(srcPartition); final SourcePartition result = SourcePartition.getCopyOf(srcPartition); final SegmentRange range = result.getSegRange(); final CubeInstance cube = (CubeInstance) buildable; if (range == null || range.start.v.equals(0L)) { result.setSourcePartitionOffsetStart(last.getSourcePartitionOffsetEnd()); } else if (cube.getDescriptor().getPartitionOffsetStart() != null && cube.getDescriptor().getPartitionOffsetStart().size() > 0) { logger.debug("Last segment doesn't exist, use the start offset that be initiated previously: " + cube.getDescriptor().getPartitionOffsetStart()); result.setSourcePartitionOffsetStart(cube.getDescriptor().getPartitionOffsetStart()); } else { result.setSourcePartitionOffsetStart(KafkaClient.getEarliestOffsets(cube)); logger.info("Get {} partitions for topic {} ", partitionInfos.size(), topic); for (PartitionInfo partitionInfo : partitionInfos) { if (result.getSourcePartitionOffsetStart().containsKey(partitionInfo.partition()) == false) { long earliest = KafkaClient.getEarliestOffset(consumer, topic, partitionInfo.partition()); logger.debug("New partition {} added, with start offset {}", partitionInfo.partition(), earliest); result.getSourcePartitionOffsetStart().put(partitionInfo.partition(), earliest); if (result.getSourcePartitionOffsetStart().containsKey(partitionId)) { if (result.getSourcePartitionOffsetStart().get(partitionId) > latestOffsets.get(partitionId)) { throw new IllegalArgumentException("Partition " + partitionId + " end offset (" + latestOffsets.get(partitionId) + ") is smaller than start offset ( " + result.getSourcePartitionOffsetStart().get(partitionId) + ")");
public CubeSegment appendSegment(CubeInstance cube, SourcePartition src) throws IOException { return appendSegment(cube, src.getTSRange(), src.getSegRange(), src.getSourcePartitionOffsetStart(), src.getSourcePartitionOffsetEnd()); }
private void checkSourceOffsets(SourcePartition src) { if (src.getSegRange() == null) return; long startOffset = (Long) src.getSegRange().start.v; long endOffset = (Long) src.getSegRange().end.v; final Map<Integer, Long> sourcePartitionOffsetStart = src.getSourcePartitionOffsetStart(); final Map<Integer, Long> sourcePartitionOffsetEnd = src.getSourcePartitionOffsetEnd(); if (endOffset <= 0 || startOffset >= endOffset) { throw new IllegalArgumentException("'startOffset' need be smaller than 'endOffset'");
@Override public SourcePartition enrichSourcePartitionBeforeBuild(IBuildable buildable, SourcePartition srcPartition) { SourcePartition result = SourcePartition.getCopyOf(srcPartition); result.setSegRange(null); return result; }
public static SourcePartition getCopyOf(SourcePartition origin) { SourcePartition copy = new SourcePartition(); copy.setTSRange(origin.getTSRange()); copy.setSegRange(origin.getSegRange()); if (origin.getSourcePartitionOffsetStart() != null) { copy.setSourcePartitionOffsetStart(new HashMap<>(origin.getSourcePartitionOffsetStart())); } if (origin.getSourcePartitionOffsetEnd() != null) { copy.setSourcePartitionOffsetEnd(new HashMap<>(origin.getSourcePartitionOffsetEnd())); } return copy; } }
@Override public SourcePartition enrichSourcePartitionBeforeBuild(IBuildable buildable, SourcePartition srcPartition) { SourcePartition result = SourcePartition.getCopyOf(srcPartition); if (srcPartition.getTSRange() != null) { result.setSegRange(null); } return result; }