@LifecycleStart public void start() throws IOException { serverAnnouncer.announce(); fireChiefExecutor = Execs.multiThreaded(fireDepartments.size(), "chief-%d"); for (final FireDepartment fireDepartment : fireDepartments) { final DataSchema schema = fireDepartment.getDataSchema(); final FireChief chief = new FireChief(fireDepartment, conglomerate); chiefs.computeIfAbsent(schema.getDataSource(), k -> new HashMap<>()) .put(fireDepartment.getTuningConfig().getShardSpec().getPartitionNum(), chief); fireChiefExecutor.submit(chief); } }
private static String makeTaskId(FireDepartment fireDepartment) { return makeTaskId( fireDepartment.getDataSchema().getDataSource(), fireDepartment.getTuningConfig().getShardSpec().getPartitionNum(), DateTimes.nowUtc(), random.nextInt() ); }
interval, schema, config.getShardSpec(), version, config.getMaxRowsInMemory(),
"%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum() ); ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable =
"%s-overseer-%d", schema.getDataSource(), config.getShardSpec().getPartitionNum() ); ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable =
@Override public Appenderator build( final DataSchema schema, final RealtimeTuningConfig config, final FireDepartmentMetrics metrics ) { return Appenderators.createRealtime( schema, config.withBasePersistDirectory( makeBasePersistSubdirectory( config.getBasePersistDirectory(), schema.getDataSource(), config.getShardSpec() ) ), metrics, dataSegmentPusher, objectMapper, indexIO, indexMerger, conglomerate, segmentAnnouncer, emitter, queryExecutorService, cache, cacheConfig ); }
sinkInterval.getEnd(), versioningPolicy.getVersion(sinkInterval), config.getShardSpec() ), queryableIndex sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getMaxRowsInMemory(),
"%s-flusher-%d", getSchema().getDataSource(), getConfig().getShardSpec().getPartitionNum() ); ThreadRenamingCallable<ScheduledExecutors.Signal> threadRenamingCallable =
private SegmentIdentifier getSegmentIdentifier(long timestamp) { if (!rejectionPolicy.accept(timestamp)) { return null; } final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity(); final VersioningPolicy versioningPolicy = config.getVersioningPolicy(); DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp)); final long truncatedTime = truncatedDateTime.getMillis(); SegmentIdentifier retVal = segments.get(truncatedTime); if (retVal == null) { final Interval interval = new Interval( truncatedDateTime, segmentGranularity.increment(truncatedDateTime) ); retVal = new SegmentIdentifier( schema.getDataSource(), interval, versioningPolicy.getVersion(interval), config.getShardSpec() ); addSegment(retVal); } return retVal; }
private Sink getSink(long timestamp) { if (!rejectionPolicy.accept(timestamp)) { return null; } final Granularity segmentGranularity = schema.getGranularitySpec().getSegmentGranularity(); final VersioningPolicy versioningPolicy = config.getVersioningPolicy(); DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp)); final long truncatedTime = truncatedDateTime.getMillis(); Sink retVal = sinks.get(truncatedTime); if (retVal == null) { final Interval sinkInterval = new Interval( truncatedDateTime, segmentGranularity.increment(truncatedDateTime) ); retVal = new Sink( sinkInterval, schema, config.getShardSpec(), versioningPolicy.getVersion(sinkInterval), config.getMaxRowsInMemory(), config.isReportParseExceptions() ); addSink(retVal); } return retVal; }
new SegmentDescriptor(sink.getInterval(), sink.getVersion(), config.getShardSpec().getPartitionNum()), mergeExecutor, new Runnable()