@Override public void announceSegment(final DataSegment segment) throws IOException { // Side effect: Calling announceSegment causes a lock to be acquired Preconditions.checkNotNull( toolbox.getTaskActionClient().submit( new LockAcquireAction(TaskLockType.EXCLUSIVE, segment.getInterval(), lockTimeoutMs) ), "Cannot acquire a lock for interval[%s]", segment.getInterval() ); toolbox.getSegmentAnnouncer().announceSegment(segment); }
@Override public void run() { synchronized (lock) { try { if (!(finished && queue.isEmpty())) { final List<DataSegment> segments = new ArrayList<>(); queue.drainTo(segments); try { announcer.announceSegments(segments); nextAnnoucement = exec.schedule(this, intervalMillis, TimeUnit.MILLISECONDS); } catch (IOException e) { doneAnnouncing.setException( new SegmentLoadingException(e, "Failed to announce segments[%s]", segments) ); } } else { doneAnnouncing.set(true); } } catch (Exception e) { doneAnnouncing.setException(e); } } } },
@Override public void unannounceSegment(final DataSegment segment) throws IOException { try { toolbox.getSegmentAnnouncer().unannounceSegment(segment); } finally { toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval())); } }
@Override public void unannounceSegments(Iterable<DataSegment> segments) throws IOException { try { toolbox.getSegmentAnnouncer().unannounceSegments(segments); } finally { for (DataSegment segment : segments) { toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval())); } } } };
@Override public void unannounceSegments(Iterable<DataSegment> segments) throws IOException { try { toolbox.getSegmentAnnouncer().unannounceSegments(segments); } finally { for (DataSegment segment : segments) { toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval())); } } } };
private void addSegment(final SegmentIdWithShardSpec identifier) { segments.put(identifier.getInterval().getStartMillis(), identifier); try { segmentAnnouncer.announceSegment( new DataSegment( identifier.getDataSource(), identifier.getInterval(), identifier.getVersion(), ImmutableMap.of(), ImmutableList.of(), ImmutableList.of(), identifier.getShardSpec(), null, 0 ) ); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", identifier.getDataSource()) .addData("interval", identifier.getInterval()) .emit(); } }
for (Map.Entry<SegmentIdWithShardSpec, Sink> entry : sinks.entrySet()) { try { segmentAnnouncer.unannounceSegment(entry.getValue().getSegment());
final List<DataSegment> segments = new ArrayList<>(); queue.drainTo(segments); announcer.announceSegments(segments);
); segmentAnnouncer.announceSegment(currSink.getSegment());
segmentAnnouncer.unannounceSegment(sink.getSegment());
@Override public void announceSegments(Iterable<DataSegment> segments) throws IOException { // Side effect: Calling announceSegments causes locks to be acquired for (DataSegment segment : segments) { Preconditions.checkNotNull( toolbox.getTaskActionClient().submit( new LockAcquireAction(TaskLockType.EXCLUSIVE, segment.getInterval(), lockTimeoutMs) ), "Cannot acquire a lock for interval[%s]", segment.getInterval() ); } toolbox.getSegmentAnnouncer().announceSegments(segments); }
private void addSink(final Sink sink) { sinks.put(sink.getInterval().getStartMillis(), sink); metrics.setSinkCount(sinks.size()); sinkTimeline.add( sink.getInterval(), sink.getVersion(), new SingleElementPartitionChunk<Sink>(sink) ); try { segmentAnnouncer.announceSegment(sink.getSegment()); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", sink.getInterval()) .emit(); } clearDedupCache(); }
announcer.unannounceSegment(segment); segmentsToDelete.add(segment);
@Override public void run() { synchronized (lock) { try { if (!(finished && queue.isEmpty())) { final List<DataSegment> segments = new ArrayList<>(); queue.drainTo(segments); try { announcer.announceSegments(segments); nextAnnoucement = exec.schedule(this, intervalMillis, TimeUnit.MILLISECONDS); } catch (IOException e) { doneAnnouncing.setException( new SegmentLoadingException(e, "Failed to announce segments[%s]", segments) ); } } else { doneAnnouncing.set(true); } } catch (Exception e) { doneAnnouncing.setException(e); } } } },
private Sink getOrCreateSink(final SegmentIdWithShardSpec identifier) { Sink retVal = sinks.get(identifier); if (retVal == null) { retVal = new Sink( identifier.getInterval(), schema, identifier.getShardSpec(), identifier.getVersion(), tuningConfig.getMaxRowsInMemory(), maxBytesTuningConfig, tuningConfig.isReportParseExceptions(), null ); try { segmentAnnouncer.announceSegment(retVal.getSegment()); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", retVal.getInterval()) .emit(); } sinks.put(identifier, retVal); metrics.setSinkCount(sinks.size()); sinkTimeline.add(retVal.getInterval(), retVal.getVersion(), identifier.getShardSpec().createChunk(retVal)); } return retVal; }
segmentAnnouncer.unannounceSegment(sink.getSegment()); removeSegment(sink, computePersistDir(schema, sink.getInterval())); log.info("Removing sinkKey %d for segment %s", truncatedTime, sink.getSegment().getId());
@Override public void announceSegments(Iterable<DataSegment> segments) throws IOException { // Side effect: Calling announceSegments causes locks to be acquired for (DataSegment segment : segments) { Preconditions.checkNotNull( toolbox.getTaskActionClient().submit( new LockAcquireAction(TaskLockType.EXCLUSIVE, segment.getInterval(), lockTimeoutMs) ), "Cannot acquire a lock for interval[%s]", segment.getInterval() ); } toolbox.getSegmentAnnouncer().announceSegments(segments); }
announcer.announceSegment(segment);
@Override public void unannounceSegment(final DataSegment segment) throws IOException { try { toolbox.getSegmentAnnouncer().unannounceSegment(segment); } finally { toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval())); } }
final List<DataSegment> segments = new ArrayList<>(); queue.drainTo(segments); announcer.announceSegments(segments);