void setAppendingSegment(SegmentWithState appendingSegment) { Preconditions.checkArgument( appendingSegment.getState() == SegmentState.APPENDING, "segment[%s] is not in the APPENDING state", appendingSegment.getSegmentIdentifier() ); // There should be only one appending segment at any time Preconditions.checkState( this.appendingSegment == null, "WTF?! Current appendingSegment[%s] is not null. " + "Its state must be changed before setting a new appendingSegment[%s]", this.appendingSegment, appendingSegment ); this.appendingSegment = appendingSegment; }
SegmentsOfInterval( Interval interval, @Nullable SegmentWithState appendingSegment, List<SegmentWithState> appendFinishedSegments ) { this.interval = interval; this.appendingSegment = appendingSegment; this.appendFinishedSegments.addAll(appendFinishedSegments); if (appendingSegment != null) { Preconditions.checkArgument( appendingSegment.getState() == SegmentState.APPENDING, "appendingSegment[%s] is not in the APPENDING state", appendingSegment.getSegmentIdentifier() ); } if (appendFinishedSegments .stream() .anyMatch(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING)) { throw new ISE("Some appendFinishedSegments[%s] is in the APPENDING state", appendFinishedSegments); } }
/** * Find a segment in the {@link SegmentState#APPENDING} state for the given timestamp and sequenceName. */ private SegmentIdWithShardSpec getAppendableSegment(final DateTime timestamp, final String sequenceName) { synchronized (segments) { final SegmentsForSequence segmentsForSequence = segments.get(sequenceName); if (segmentsForSequence == null) { return null; } final Map.Entry<Long, SegmentsOfInterval> candidateEntry = segmentsForSequence.floor( timestamp.getMillis() ); if (candidateEntry != null) { final SegmentsOfInterval segmentsOfInterval = candidateEntry.getValue(); if (segmentsOfInterval.interval.contains(timestamp)) { return segmentsOfInterval.appendingSegment == null ? null : segmentsOfInterval.appendingSegment.getSegmentIdentifier(); } else { return null; } } else { return null; } } }
segmentWithState.getDataSegment(), "dataSegment for segmentId[%s]", segmentWithState.getSegmentIdentifier()
void add(SegmentWithState segmentWithState) { final SegmentIdWithShardSpec identifier = segmentWithState.getSegmentIdentifier(); final Pair<SegmentWithState, List<SegmentWithState>> pair = intervalToSegments.get(identifier); final List<SegmentWithState> appendFinishedSegments = pair == null || pair.rhs == null ? new ArrayList<>() : pair.rhs; // always keep APPENDING segments for an interval start millis in the front if (segmentWithState.getState() == SegmentState.APPENDING) { if (pair != null && pair.lhs != null) { throw new ISE( "WTF?! there was already an appendingSegment[%s] before adding an appendingSegment[%s]", pair.lhs, segmentWithState ); } intervalToSegments.put(identifier, Pair.of(segmentWithState, appendFinishedSegments)); } else { final SegmentWithState appendingSegment = pair == null ? null : pair.lhs; appendFinishedSegments.add(segmentWithState); intervalToSegments.put(identifier, Pair.of(appendingSegment, appendFinishedSegments)); } }
final SegmentWithState appendingSegment = segmentsOfInterval.getAppendingSegment(); if (appendingSegment != null) { final DataSegment pushedSegment = pushedSegmentIdToSegmentMap.get(appendingSegment.getSegmentIdentifier()); if (pushedSegment == null) { throw new ISE("Can't find pushedSegments for segment[%s]", appendingSegment.getSegmentIdentifier());
/** * Move a set of identifiers out from "active", making way for newer segments. * This method is to support KafkaIndexTask's legacy mode and will be removed in the future. * See KakfaIndexTask.runLegacy(). */ public void moveSegmentOut(final String sequenceName, final List<SegmentIdWithShardSpec> identifiers) { synchronized (segments) { final SegmentsForSequence activeSegmentsForSequence = segments.get(sequenceName); if (activeSegmentsForSequence == null) { throw new ISE("WTF?! Asked to remove segments for sequenceName[%s] which doesn't exist...", sequenceName); } for (final SegmentIdWithShardSpec identifier : identifiers) { log.info("Moving segment[%s] out of active list.", identifier); final long key = identifier.getInterval().getStartMillis(); final SegmentsOfInterval segmentsOfInterval = activeSegmentsForSequence.get(key); if (segmentsOfInterval == null || segmentsOfInterval.getAppendingSegment() == null || !segmentsOfInterval.getAppendingSegment().getSegmentIdentifier().equals(identifier)) { throw new ISE("WTF?! Asked to remove segment[%s] that didn't exist...", identifier); } segmentsOfInterval.finishAppendingToCurrentActiveSegment(SegmentWithState::finishAppending); } } }
void setAppendingSegment(SegmentWithState appendingSegment) { Preconditions.checkArgument( appendingSegment.getState() == SegmentState.APPENDING, "segment[%s] is not in the APPENDING state", appendingSegment.getSegmentIdentifier() ); // There should be only one appending segment at any time Preconditions.checkState( this.appendingSegment == null, "WTF?! Current appendingSegment[%s] is not null. " + "Its state must be changed before setting a new appendingSegment[%s]", this.appendingSegment, appendingSegment ); this.appendingSegment = appendingSegment; }
SegmentsOfInterval( Interval interval, @Nullable SegmentWithState appendingSegment, List<SegmentWithState> appendFinishedSegments ) { this.interval = interval; this.appendingSegment = appendingSegment; this.appendFinishedSegments.addAll(appendFinishedSegments); if (appendingSegment != null) { Preconditions.checkArgument( appendingSegment.getState() == SegmentState.APPENDING, "appendingSegment[%s] is not in the APPENDING state", appendingSegment.getSegmentIdentifier() ); } if (appendFinishedSegments .stream() .anyMatch(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING)) { throw new ISE("Some appendFinishedSegments[%s] is in the APPENDING state", appendFinishedSegments); } }
/** * Find a segment in the {@link SegmentState#APPENDING} state for the given timestamp and sequenceName. */ private SegmentIdentifier getAppendableSegment(final DateTime timestamp, final String sequenceName) { synchronized (segments) { final SegmentsForSequence segmentsForSequence = segments.get(sequenceName); if (segmentsForSequence == null) { return null; } final Map.Entry<Long, SegmentsOfInterval> candidateEntry = segmentsForSequence.floor( timestamp.getMillis() ); if (candidateEntry != null) { final SegmentsOfInterval segmentsOfInterval = candidateEntry.getValue(); if (segmentsOfInterval.interval.contains(timestamp)) { return segmentsOfInterval.appendingSegment == null ? null : segmentsOfInterval.appendingSegment.getSegmentIdentifier(); } else { return null; } } else { return null; } } }
segmentWithState.getDataSegment(), "dataSegment for segmentId[%s]", segmentWithState.getSegmentIdentifier()
void add(SegmentWithState segmentWithState) { final SegmentIdentifier identifier = segmentWithState.getSegmentIdentifier(); final Pair<SegmentWithState, List<SegmentWithState>> pair = intervalToSegments.get(identifier); final List<SegmentWithState> appendFinishedSegments = pair == null || pair.rhs == null ? new ArrayList<>() : pair.rhs; // always keep APPENDING segments for an interval start millis in the front if (segmentWithState.getState() == SegmentState.APPENDING) { if (pair != null && pair.lhs != null) { throw new ISE( "WTF?! there was already an appendingSegment[%s] before adding an appendingSegment[%s]", pair.lhs, segmentWithState ); } intervalToSegments.put(identifier, Pair.of(segmentWithState, appendFinishedSegments)); } else { final SegmentWithState appendingSegment = pair == null ? null : pair.lhs; appendFinishedSegments.add(segmentWithState); intervalToSegments.put(identifier, Pair.of(appendingSegment, appendFinishedSegments)); } }
final SegmentWithState appendingSegment = segmentsOfInterval.getAppendingSegment(); if (appendingSegment != null) { final DataSegment pushedSegment = pushedSegmentIdToSegmentMap.get(appendingSegment.getSegmentIdentifier()); if (pushedSegment == null) { throw new ISE("Can't find pushedSegments for segment[%s]", appendingSegment.getSegmentIdentifier());
/** * Move a set of identifiers out from "active", making way for newer segments. * This method is to support KafkaIndexTask's legacy mode and will be removed in the future. * See KakfaIndexTask.runLegacy(). */ public void moveSegmentOut(final String sequenceName, final List<SegmentIdentifier> identifiers) { synchronized (segments) { final SegmentsForSequence activeSegmentsForSequence = segments.get(sequenceName); if (activeSegmentsForSequence == null) { throw new ISE("WTF?! Asked to remove segments for sequenceName[%s] which doesn't exist...", sequenceName); } for (final SegmentIdentifier identifier : identifiers) { log.info("Moving segment[%s] out of active list.", identifier); final long key = identifier.getInterval().getStartMillis(); final SegmentsOfInterval segmentsOfInterval = activeSegmentsForSequence.get(key); if (segmentsOfInterval == null || segmentsOfInterval.getAppendingSegment() == null || !segmentsOfInterval.getAppendingSegment().getSegmentIdentifier().equals(identifier)) { throw new ISE("WTF?! Asked to remove segment[%s] that didn't exist...", identifier); } segmentsOfInterval.finishAppendingToCurrentActiveSegment(SegmentWithState::finishAppending); } } }