void setAppendingSegment(SegmentWithState appendingSegment) { Preconditions.checkArgument( appendingSegment.getState() == SegmentState.APPENDING, "segment[%s] is not in the APPENDING state", appendingSegment.getSegmentIdentifier() ); // There should be only one appending segment at any time Preconditions.checkState( this.appendingSegment == null, "WTF?! Current appendingSegment[%s] is not null. " + "Its state must be changed before setting a new appendingSegment[%s]", this.appendingSegment, appendingSegment ); this.appendingSegment = appendingSegment; }
static SegmentWithState newSegment(SegmentIdWithShardSpec segmentIdentifier) { return new SegmentWithState(segmentIdentifier, SegmentState.APPENDING, null); }
/** * Change the segment state to {@link SegmentState#APPEND_FINISHED}. The current state should be * {@link SegmentState#APPENDING}. */ public void finishAppending() { checkStateTransition(this.state, SegmentState.APPENDING, SegmentState.APPEND_FINISHED); this.state = SegmentState.APPEND_FINISHED; }
.map(segmentWithState -> Preconditions .checkNotNull( segmentWithState.getDataSegment(), "dataSegment for segmentId[%s]", segmentWithState.getSegmentIdentifier()
final SegmentWithState appendingSegment = segmentsOfInterval.getAppendingSegment(); if (appendingSegment != null) { final DataSegment pushedSegment = pushedSegmentIdToSegmentMap.get(appendingSegment.getSegmentIdentifier()); if (pushedSegment == null) { throw new ISE("Can't find pushedSegments for segment[%s]", appendingSegment.getSegmentIdentifier()); segmentWithState -> segmentWithState.pushAndDrop(pushedSegment) );
private void checkSegmentStates(int expectedNumSegmentsInState, SegmentState expectedState) { final SegmentsForSequence segmentsForSequence = driver.getSegments().get("dummy"); Assert.assertNotNull(segmentsForSequence); final List<SegmentWithState> segmentWithStates = segmentsForSequence .allSegmentStateStream() .filter(segmentWithState -> segmentWithState.getState() == expectedState) .collect(Collectors.toList()); Assert.assertEquals(expectedNumSegmentsInState, segmentWithStates.size()); }
.map(segmentIdentifier -> { activeSegmentsAlreadySeen.add(segmentIdentifier.asSegmentId()); return SegmentWithState.newSegment(segmentIdentifier); }) .collect(Collectors.toList()) .stream() .filter(segmentIdentifier -> !activeSegmentsAlreadySeen.contains(segmentIdentifier.asSegmentId())) .map(segmentIdentifier -> SegmentWithState.newSegment(segmentIdentifier, SegmentState.APPEND_FINISHED)) .forEach(segmentWithStates::add); });
/** * Find a segment in the {@link SegmentState#APPENDING} state for the given timestamp and sequenceName. */ private SegmentIdWithShardSpec getAppendableSegment(final DateTime timestamp, final String sequenceName) { synchronized (segments) { final SegmentsForSequence segmentsForSequence = segments.get(sequenceName); if (segmentsForSequence == null) { return null; } final Map.Entry<Long, SegmentsOfInterval> candidateEntry = segmentsForSequence.floor( timestamp.getMillis() ); if (candidateEntry != null) { final SegmentsOfInterval segmentsOfInterval = candidateEntry.getValue(); if (segmentsOfInterval.interval.contains(timestamp)) { return segmentsOfInterval.appendingSegment == null ? null : segmentsOfInterval.appendingSegment.getSegmentIdentifier(); } else { return null; } } else { return null; } } }
.map(segmentWithState -> Preconditions .checkNotNull( segmentWithState.getDataSegment(), "dataSegment for segmentId[%s]", segmentWithState.getSegmentIdentifier()
final SegmentWithState appendingSegment = segmentsOfInterval.getAppendingSegment(); if (appendingSegment != null) { final DataSegment pushedSegment = pushedSegmentIdToSegmentMap.get(appendingSegment.getSegmentIdentifier()); if (pushedSegment == null) { throw new ISE("Can't find pushedSegments for segment[%s]", appendingSegment.getSegmentIdentifier()); segmentWithState -> segmentWithState.pushAndDrop(pushedSegment) );
void add(SegmentIdWithShardSpec identifier) { intervalToSegmentStates.computeIfAbsent( identifier.getInterval().getStartMillis(), k -> new SegmentsOfInterval(identifier.getInterval()) ).setAppendingSegment(SegmentWithState.newSegment(identifier)); lastSegmentId = identifier.toString(); }
/** * Move a set of identifiers out from "active", making way for newer segments. * This method is to support KafkaIndexTask's legacy mode and will be removed in the future. * See KakfaIndexTask.runLegacy(). */ public void moveSegmentOut(final String sequenceName, final List<SegmentIdWithShardSpec> identifiers) { synchronized (segments) { final SegmentsForSequence activeSegmentsForSequence = segments.get(sequenceName); if (activeSegmentsForSequence == null) { throw new ISE("WTF?! Asked to remove segments for sequenceName[%s] which doesn't exist...", sequenceName); } for (final SegmentIdWithShardSpec identifier : identifiers) { log.info("Moving segment[%s] out of active list.", identifier); final long key = identifier.getInterval().getStartMillis(); final SegmentsOfInterval segmentsOfInterval = activeSegmentsForSequence.get(key); if (segmentsOfInterval == null || segmentsOfInterval.getAppendingSegment() == null || !segmentsOfInterval.getAppendingSegment().getSegmentIdentifier().equals(identifier)) { throw new ISE("WTF?! Asked to remove segment[%s] that didn't exist...", identifier); } segmentsOfInterval.finishAppendingToCurrentActiveSegment(SegmentWithState::finishAppending); } } }
SegmentsOfInterval( Interval interval, @Nullable SegmentWithState appendingSegment, List<SegmentWithState> appendFinishedSegments ) { this.interval = interval; this.appendingSegment = appendingSegment; this.appendFinishedSegments.addAll(appendFinishedSegments); if (appendingSegment != null) { Preconditions.checkArgument( appendingSegment.getState() == SegmentState.APPENDING, "appendingSegment[%s] is not in the APPENDING state", appendingSegment.getSegmentIdentifier() ); } if (appendFinishedSegments .stream() .anyMatch(segmentWithState -> segmentWithState.getState() == SegmentState.APPENDING)) { throw new ISE("Some appendFinishedSegments[%s] is in the APPENDING state", appendFinishedSegments); } }
.map(segmentIdentifier -> { activeSegmentsAlreadySeen.add(segmentIdentifier.toString()); return SegmentWithState.newSegment(segmentIdentifier); }) .collect(Collectors.toList()) .filter(segmentIdentifier -> !activeSegmentsAlreadySeen.contains( segmentIdentifier.toString())) .map(segmentIdentifier -> SegmentWithState.newSegment( segmentIdentifier, SegmentState.APPEND_FINISHED
/** * Change the segment state to {@link SegmentState#PUSHED_AND_DROPPED}. The current state should be * {@link SegmentState#APPENDING}. This method should be called after the segment of {@link #segmentIdentifier} is * completely pushed and dropped. * * @param dataSegment pushed {@link DataSegment} */ public void pushAndDrop(DataSegment dataSegment) { checkStateTransition(this.state, SegmentState.APPENDING, SegmentState.PUSHED_AND_DROPPED); this.state = SegmentState.PUSHED_AND_DROPPED; this.dataSegment = dataSegment; }
static SegmentWithState newSegment(SegmentIdWithShardSpec segmentIdentifier, SegmentState state) { return new SegmentWithState(segmentIdentifier, state, null); }
/** * Find a segment in the {@link SegmentState#APPENDING} state for the given timestamp and sequenceName. */ private SegmentIdentifier getAppendableSegment(final DateTime timestamp, final String sequenceName) { synchronized (segments) { final SegmentsForSequence segmentsForSequence = segments.get(sequenceName); if (segmentsForSequence == null) { return null; } final Map.Entry<Long, SegmentsOfInterval> candidateEntry = segmentsForSequence.floor( timestamp.getMillis() ); if (candidateEntry != null) { final SegmentsOfInterval segmentsOfInterval = candidateEntry.getValue(); if (segmentsOfInterval.interval.contains(timestamp)) { return segmentsOfInterval.appendingSegment == null ? null : segmentsOfInterval.appendingSegment.getSegmentIdentifier(); } else { return null; } } else { return null; } } }
void add(SegmentWithState segmentWithState) { final SegmentIdWithShardSpec identifier = segmentWithState.getSegmentIdentifier(); final Pair<SegmentWithState, List<SegmentWithState>> pair = intervalToSegments.get(identifier); final List<SegmentWithState> appendFinishedSegments = pair == null || pair.rhs == null ? new ArrayList<>() : pair.rhs; // always keep APPENDING segments for an interval start millis in the front if (segmentWithState.getState() == SegmentState.APPENDING) { if (pair != null && pair.lhs != null) { throw new ISE( "WTF?! there was already an appendingSegment[%s] before adding an appendingSegment[%s]", pair.lhs, segmentWithState ); } intervalToSegments.put(identifier, Pair.of(segmentWithState, appendFinishedSegments)); } else { final SegmentWithState appendingSegment = pair == null ? null : pair.lhs; appendFinishedSegments.add(segmentWithState); intervalToSegments.put(identifier, Pair.of(appendingSegment, appendFinishedSegments)); } }
void add(SegmentIdentifier identifier) { intervalToSegmentStates.computeIfAbsent( identifier.getInterval().getStartMillis(), k -> new SegmentsOfInterval(identifier.getInterval()) ).setAppendingSegment(SegmentWithState.newSegment(identifier)); lastSegmentId = identifier.getIdentifierAsString(); }
/** * Change the segment state to {@link SegmentState#APPEND_FINISHED}. The current state should be * {@link SegmentState#APPENDING}. */ public void finishAppending() { checkStateTransition(this.state, SegmentState.APPENDING, SegmentState.APPEND_FINISHED); this.state = SegmentState.APPEND_FINISHED; }