@Override public String apply(Sink input) { return input.getSegment().getIdentifier(); } }
@Override public String apply(Sink input) { return input.getSegment().getIdentifier(); } }
@Override public ScheduledExecutors.Signal call() throws Exception { log.info("Abandoning segment %s", sink.getSegment().getIdentifier()); abandonSegment(truncatedTime, sink); return ScheduledExecutors.Signal.STOP; } }
@Override public ScheduledExecutors.Signal call() throws Exception { log.info("Abandoning segment %s", sink.getSegment().getIdentifier()); abandonSegment(truncatedTime, sink); return ScheduledExecutors.Signal.STOP; } }
protected void flushAfterDuration(final long truncatedTime, final Sink sink) { log.info( "Abandoning segment %s at %s", sink.getSegment().getIdentifier(), new DateTime().plusMillis((int) flushDuration.getMillis()) ); ScheduledExecutors.scheduleWithFixedDelay( flushScheduledExec, flushDuration, new Callable<ScheduledExecutors.Signal>() { @Override public ScheduledExecutors.Signal call() throws Exception { log.info("Abandoning segment %s", sink.getSegment().getIdentifier()); abandonSegment(truncatedTime, sink); return ScheduledExecutors.Signal.STOP; } } ); }
protected void flushAfterDuration(final long truncatedTime, final Sink sink) { log.info( "Abandoning segment %s at %s", sink.getSegment().getIdentifier(), DateTimes.nowUtc().plusMillis((int) flushDuration.getMillis()) ); ScheduledExecutors.scheduleWithFixedDelay( flushScheduledExec, flushDuration, new Callable<ScheduledExecutors.Signal>() { @Override public ScheduledExecutors.Signal call() throws Exception { log.info("Abandoning segment %s", sink.getSegment().getIdentifier()); abandonSegment(truncatedTime, sink); return ScheduledExecutors.Signal.STOP; } } ); }
for (Map.Entry<SegmentIdentifier, Sink> entry : sinks.entrySet()) { try { segmentAnnouncer.unannounceSegment(entry.getValue().getSegment());
final DataSegment segmentToUpload = theSink.getSegment() .withDimensions(ImmutableList.copyOf(mappedSegment.getAvailableDimensions())) .withBinaryVersion(SegmentUtils.getVersionFromDir(fileToUpload));
/** * Unannounces a given sink and removes all local references to it. */ protected void abandonSegment(final long truncatedTime, final Sink sink) { try { segmentAnnouncer.unannounceSegment(sink.getSegment()); FileUtils.deleteDirectory(computePersistDir(schema, sink.getInterval())); log.info("Removing sinkKey %d for segment %s", truncatedTime, sink.getSegment().getIdentifier()); sinks.remove(truncatedTime); sinkTimeline.remove( sink.getInterval(), sink.getVersion(), new SingleElementPartitionChunk<>(sink) ); synchronized (handoffCondition) { handoffCondition.notifyAll(); } } catch (IOException e) { log.makeAlert(e, "Unable to abandon old segment for dataSource[%s]", schema.getDataSource()) .addData("interval", sink.getInterval()) .emit(); } }
private FireHydrant makeNewCurrIndex(long minTimestamp, Schema schema) { IncrementalIndex newIndex = new IncrementalIndex( new IncrementalIndexSchema.Builder() .withMinTimestamp(minTimestamp) .withQueryGranularity(schema.getIndexGranularity()) .withSpatialDimensions(schema.getSpatialDimensions()) .withMetrics(schema.getAggregators()) .build() ); FireHydrant old; if (currIndex == null) { // Only happens on initialization, cannot synchronize on null old = currIndex; currIndex = new FireHydrant(newIndex, hydrants.size(), getSegment().getIdentifier()); hydrants.add(currIndex); } else { synchronized (currIndex) { old = currIndex; currIndex = new FireHydrant(newIndex, hydrants.size(), getSegment().getIdentifier()); hydrants.add(currIndex); } } return old; }
@Override public Sink getSink(long timestamp) { if (!rejectionPolicy.accept(timestamp)) { return null; } final long truncatedTime = segmentGranularity.truncate(timestamp); Sink retVal = sinks.get(truncatedTime); if (retVal == null) { final Interval sinkInterval = new Interval( new DateTime(truncatedTime), segmentGranularity.increment(new DateTime(truncatedTime)) ); retVal = new Sink(sinkInterval, schema, versioningPolicy.getVersion(sinkInterval)); try { segmentAnnouncer.announceSegment(retVal.getSegment()); sinks.put(truncatedTime, retVal); sinkTimeline.add(retVal.getInterval(), retVal.getVersion(), new SingleElementPartitionChunk<Sink>(retVal)); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", retVal.getInterval()) .emit(); } } return retVal; }
final DataSegment segmentToUpload = theSink.getSegment() .withDimensions(ImmutableList.copyOf(mappedSegment.getAvailableDimensions())) .withBinaryVersion(SegmentUtils.getVersionFromDir(fileToUpload));
final String sinkVersion = sink.getSegment().getVersion(); if (segmentVersion.compareTo(sinkVersion) >= 0) { log.info("Segment version[%s] >= sink version[%s]", segmentVersion, sinkVersion);
segmentAnnouncer.unannounceSegment(sink.getSegment()); removeSegment(sink, computePersistDir(schema, sink.getInterval())); log.info("Removing sinkKey %d for segment %s", truncatedTime, sink.getSegment().getIdentifier()); sinks.remove(truncatedTime); metrics.setSinkCount(sinks.size());
private void addSink(final Sink sink) { sinks.put(sink.getInterval().getStartMillis(), sink); metrics.setSinkCount(sinks.size()); sinkTimeline.add( sink.getInterval(), sink.getVersion(), new SingleElementPartitionChunk<Sink>(sink) ); try { segmentAnnouncer.announceSegment(sink.getSegment()); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", sink.getInterval()) .emit(); } }
segmentAnnouncer.unannounceSegment(sink.getSegment());
private Sink getOrCreateSink(final SegmentIdentifier identifier) { Sink retVal = sinks.get(identifier); if (retVal == null) { retVal = new Sink( identifier.getInterval(), schema, identifier.getShardSpec(), identifier.getVersion(), tuningConfig.getMaxRowsInMemory(), tuningConfig.isReportParseExceptions() ); try { segmentAnnouncer.announceSegment(retVal.getSegment()); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", retVal.getInterval()) .emit(); } sinks.put(identifier, retVal); metrics.setSinkCount(sinks.size()); sinkTimeline.add(retVal.getInterval(), retVal.getVersion(), identifier.getShardSpec().createChunk(retVal)); } return retVal; }
new SegmentDescriptor( holder.getInterval(), theSink.getSegment().getVersion(), theSink.getSegment().getShardSpec().getPartitionNum()
sink.getSegment().withDimensions(Lists.newArrayList(index.getAvailableDimensions())) );
currHydrant = new FireHydrant(newIndex, newCount, getSegment().getIdentifier()); if (old != null) { numRowsExcludingCurrIndex.addAndGet(old.getIndex().size());