SegmentsForSequence build() { final NavigableMap<Long, SegmentsOfInterval> map = new TreeMap<>(); for (Entry<SegmentIdWithShardSpec, Pair<SegmentWithState, List<SegmentWithState>>> entry : intervalToSegments.entrySet()) { map.put( entry.getKey().getInterval().getStartMillis(), new SegmentsOfInterval(entry.getKey().getInterval(), entry.getValue().lhs, entry.getValue().rhs) ); } return new SegmentsForSequence(map, lastSegmentId); } }
Iterables.transform(entry.getValue(), input -> input.getInterval()) );
public void dropSegment(final SegmentIdWithShardSpec identifier) { log.info("Dropping segment: %s", identifier); segments.remove(identifier.getInterval().getStartMillis()); Futures.addCallback( appenderator.drop(identifier), new FutureCallback<Object>() { @Override public void onSuccess(Object result) { log.info("Dropped segment: %s", identifier); } @Override public void onFailure(Throwable e) { // TODO: Retry? log.warn(e, "Failed to drop segment: %s", identifier); } } ); }
void add(SegmentIdWithShardSpec identifier) { intervalToSegmentStates.computeIfAbsent( identifier.getInterval().getStartMillis(), k -> new SegmentsOfInterval(identifier.getInterval()) ).setAppendingSegment(SegmentWithState.newSegment(identifier)); lastSegmentId = identifier.toString(); }
private void addSegment(final SegmentIdWithShardSpec identifier) { segments.put(identifier.getInterval().getStartMillis(), identifier); try { segmentAnnouncer.announceSegment( new DataSegment( identifier.getDataSource(), identifier.getInterval(), identifier.getVersion(), ImmutableMap.of(), ImmutableList.of(), ImmutableList.of(), identifier.getShardSpec(), null, 0 ) ); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", identifier.getDataSource()) .addData("interval", identifier.getInterval()) .emit(); } }
handoffNotifier.registerSegmentHandoffCallback( new SegmentDescriptor( segmentIdentifier.getInterval(), segmentIdentifier.getVersion(), segmentIdentifier.getShardSpec().getPartitionNum()
identifier.getInterval(), schema, identifier.getShardSpec(),
private List<SegmentIdWithShardSpec> getPendingSegmentsForIntervalWithHandle( final Handle handle, final String dataSource, final Interval interval ) throws IOException { final List<SegmentIdWithShardSpec> identifiers = new ArrayList<>(); final ResultIterator<byte[]> dbSegments = handle.createQuery( StringUtils.format( "SELECT payload FROM %1$s WHERE dataSource = :dataSource AND start <= :end and %2$send%2$s >= :start", dbTables.getPendingSegmentsTable(), connector.getQuoteString() ) ) .bind("dataSource", dataSource) .bind("start", interval.getStart().toString()) .bind("end", interval.getEnd().toString()) .map(ByteArrayMapper.FIRST) .iterator(); while (dbSegments.hasNext()) { final byte[] payload = dbSegments.next(); final SegmentIdWithShardSpec identifier = jsonMapper.readValue(payload, SegmentIdWithShardSpec.class); if (interval.overlaps(identifier.getInterval())) { identifiers.add(identifier); } } dbSegments.close(); return identifiers; }
); if (existingIdentifier.getInterval().getStartMillis() == interval.getStartMillis() && existingIdentifier.getInterval().getEndMillis() == interval.getEndMillis()) { if (previousSegmentId == null) { log.info("Found existing pending segment [%s] for sequence[%s] in DB", existingIdentifier, sequenceName);
final Long intervalStart = segment.getInterval().getStartMillis(); if (intervalStart < minTimestamp) { log.info("Adding entry [%s] for merge and push.", segment);
/** * Move a set of identifiers out from "active", making way for newer segments. * This method is to support KafkaIndexTask's legacy mode and will be removed in the future. * See KakfaIndexTask.runLegacy(). */ public void moveSegmentOut(final String sequenceName, final List<SegmentIdWithShardSpec> identifiers) { synchronized (segments) { final SegmentsForSequence activeSegmentsForSequence = segments.get(sequenceName); if (activeSegmentsForSequence == null) { throw new ISE("WTF?! Asked to remove segments for sequenceName[%s] which doesn't exist...", sequenceName); } for (final SegmentIdWithShardSpec identifier : identifiers) { log.info("Moving segment[%s] out of active list.", identifier); final long key = identifier.getInterval().getStartMillis(); final SegmentsOfInterval segmentsOfInterval = activeSegmentsForSequence.get(key); if (segmentsOfInterval == null || segmentsOfInterval.getAppendingSegment() == null || !segmentsOfInterval.getAppendingSegment().getSegmentIdentifier().equals(identifier)) { throw new ISE("WTF?! Asked to remove segment[%s] that didn't exist...", identifier); } segmentsOfInterval.finishAppendingToCurrentActiveSegment(SegmentWithState::finishAppending); } } }
id -> new DataSegment( id.getDataSource(), id.getInterval(), id.getVersion(), ImmutableMap.of(),
private Sink getOrCreateSink(final SegmentIdWithShardSpec identifier) { Sink retVal = sinks.get(identifier); if (retVal == null) { retVal = new Sink( identifier.getInterval(), schema, identifier.getShardSpec(), identifier.getVersion(), tuningConfig.getMaxRowsInMemory(), maxBytesTuningConfig, tuningConfig.isReportParseExceptions(), null ); try { segmentAnnouncer.announceSegment(retVal.getSegment()); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", retVal.getInterval()) .emit(); } sinks.put(identifier, retVal); metrics.setSinkCount(sinks.size()); sinkTimeline.add(retVal.getInterval(), retVal.getVersion(), identifier.getShardSpec().createChunk(retVal)); } return retVal; }
new NumberedShardSpec(0, 0) ); } else if (!maxId.getInterval().equals(interval) || maxId.getVersion().compareTo(maxVersion) > 0) { log.warn( "Cannot allocate new segment for dataSource[%s], interval[%s], maxVersion[%s]: conflicting segment[%s].", return new SegmentIdWithShardSpec( dataSource, maxId.getInterval(), maxId.getVersion(), new LinearShardSpec(maxId.getShardSpec().getPartitionNum() + 1) return new SegmentIdWithShardSpec( dataSource, maxId.getInterval(), maxId.getVersion(), new NumberedShardSpec(
@Override public Set<DataSegment> findUsedSegments(Set<SegmentIdWithShardSpec> identifiers) { final VersionedIntervalTimeline<String, DataSegment> timeline = new VersionedIntervalTimeline<>(Ordering.natural()); for (DataSegment dataSegment : appenderatorTester.getPushedSegments()) { timeline.add( dataSegment.getInterval(), dataSegment.getVersion(), dataSegment.getShardSpec().createChunk(dataSegment) ); } final Set<DataSegment> retVal = new HashSet<>(); for (SegmentIdWithShardSpec identifier : identifiers) { for (TimelineObjectHolder<String, DataSegment> holder : timeline.lookup(identifier.getInterval())) { for (PartitionChunk<DataSegment> chunk : holder.getObject()) { if (identifiers.contains(SegmentIdWithShardSpec.fromDataSegment(chunk.getObject()))) { retVal.add(chunk.getObject()); } } } } return retVal; } }
persistedFile = indexMerger.persist( indexToPersist.getIndex(), identifier.getInterval(), new File(persistDir, String.valueOf(indexToPersist.getCount())), indexSpec,
@Test public void testSerde() throws Exception { final ObjectMapper objectMapper = new DefaultObjectMapper(); objectMapper.registerSubtypes(NumberedShardSpec.class); final SegmentIdWithShardSpec id2 = objectMapper.readValue( objectMapper.writeValueAsBytes(ID_1), SegmentIdWithShardSpec.class ); Assert.assertEquals(ID_1, id2); Assert.assertEquals(DATA_SOURCE, id2.getDataSource()); Assert.assertEquals(INTERVAL, id2.getInterval()); Assert.assertEquals(VERSION, id2.getVersion()); Assert.assertEquals(SHARD_SPEC_1.getPartitionNum(), id2.getShardSpec().getPartitionNum()); Assert.assertEquals(SHARD_SPEC_1.getPartitions(), ((NumberedShardSpec) id2.getShardSpec()).getPartitions()); }
ImmutableList.of( new SegmentDescriptor( IDENTIFIERS.get(2).getInterval(), IDENTIFIERS.get(2).getVersion(), IDENTIFIERS.get(2).getShardSpec().getPartitionNum()