@Override public Response processStats(BatchedOperatorStats batchedOperatorStats) Response response = new Response(); int operatorId = batchedOperatorStats.getOperatorId();
@Override public Response processStats(BatchedOperatorStats stats) { Response response = new Response(); response.repartitionRequired = false; if (!partitionedInstanceStatus.containsKey(stats.getOperatorId())) { return response; } partitionedInstanceStatus.put(stats.getOperatorId(), stats); if (getLoad(stats) != 0) { if (repartition && System.currentTimeMillis() > nextMillis) { repartition = false; response.repartitionRequired = true; logger.debug("setting repartition to true"); } else if (!repartition) { repartition = true; nextMillis = System.currentTimeMillis() + cooldownMillis; } } else { repartition = false; } return response; }
@Override public Response processStats(BatchedOperatorStats batchedOperatorStats) { long t = System.currentTimeMillis(); if (repartitionInterval < 0 || repartitionCheckInterval < 0 || t - lastCheckTime < repartitionCheckInterval || t - lastRepartitionTime < repartitionInterval) { // return false if it's within repartitionCheckInterval since last time it check the stats Response response = new Response(); response.repartitionRequired = false; return response; } try { logger.debug("Process stats"); initPartitioner(); return partitioner.processStats(batchedOperatorStats); } finally { lastCheckTime = System.currentTimeMillis(); } }
@Override @SuppressWarnings("unchecked") public Response processStats(BatchedOperatorStats batchedOperatorStats) { BasicCounters<MutableLong> fileCounters = null; for (OperatorStats operatorStats : batchedOperatorStats.getLastWindowedStats()) { if (operatorStats.counters != null) { fileCounters = (BasicCounters<MutableLong>)operatorStats.counters; } } Response response = new Response(); if (fileCounters != null && fileCounters.getCounter(FileCounters.PENDING_FILES).longValue() > 0L || System.currentTimeMillis() - repartitionInterval <= lastRepartition) { response.repartitionRequired = false; return response; } response.repartitionRequired = true; return response; } }
@Override public Response processStats(BatchedOperatorStats batchedOperatorStats) { final long emittedCount = batchedOperatorStats.getTuplesEmittedPSMA(); // we only perform a single dynamic repartition Response res = new Response(); res.repartitionRequired = false; if (emittedCount > 500 && partitions < MAX_PARTITIONS) { LOG.info("processStats: trying repartition of input operator current {} required {}", partitions, MAX_PARTITIONS); LOG.info("**** operator id = {}, window id = {}, tuplesProcessedPSMA = {}, tuplesEmittedPSMA = {}", batchedOperatorStats.getOperatorId(), batchedOperatorStats.getCurrentWindowId(), batchedOperatorStats.getTuplesProcessedPSMA(), emittedCount); partitions = MAX_PARTITIONS; res.repartitionRequired = true; } return res; } // processStats
@Override public Response processStats(BatchedOperatorStats batchedOperatorStats) { long t = System.currentTimeMillis(); if (repartitionInterval < 0 || repartitionCheckInterval < 0 || t - lastCheckTime < repartitionCheckInterval || t - lastRepartitionTime < repartitionInterval) { // return false if it's within repartitionCheckInterval since last time it check the stats Response response = new Response(); response.repartitionRequired = false; return response; } try { logger.debug("Process stats"); initPartitioner(); return partitioner.processStats(batchedOperatorStats); } finally { lastCheckTime = System.currentTimeMillis(); } }
/** * Repartition is required when number of partitions are not equal to required * partitions. * @param batchedOperatorStats the stats to use when repartitioning. * @return Returns the stats listener response. */ @Override public Response processStats(BatchedOperatorStats batchedOperatorStats) { Response res = new Response(); res.repartitionRequired = false; if (currentPartitions != partitionCount) { LOG.info("processStats: trying repartition of input operator current {} required {}", currentPartitions, partitionCount); res.repartitionRequired = true; } return res; }
@Override public Response processStats(BatchedOperatorStats stats) { Response response = new Response(); if ( ( partitionPieAssignment == null) || (partitionPieAssignment.size() == 0) ) { response.repartitionRequired = true; } else { response.repartitionRequired = false; } return response; }
@Override public StatsListener.Response processStats(StatsListener.BatchedOperatorStats stats) { StatsListener.Response resp = new StatsListener.Response(); List<KafkaConsumer.KafkaMeterStats> kstats = extractKafkaStats(stats); resp.repartitionRequired = isPartitionRequired(stats.getOperatorId(), kstats); return resp; }
@Override public Response processStats(BatchedOperatorStats batchedOperatorStats) { Response response = new Response(); response.repartitionRequired = true; return response; }
@Override public StatsListener.Response processStats(StatsListener.BatchedOperatorStats stats) { StatsListener.Response resp = new StatsListener.Response(); List<KafkaConsumer.KafkaMeterStats> kstats = extractKafkaStats(stats); resp.repartitionRequired = isPartitionRequired(stats.getOperatorId(), kstats); return resp; }
@Override public Response processStats(BatchedOperatorStats stats) { Response resp = new Response(); List<KinesisConsumer.KinesisShardStats> kstats = extractkinesisStats(stats); resp.repartitionRequired = isPartitionRequired(kstats); return resp; }
@Override public Response processStats(BatchedOperatorStats stats) { Response resp = new Response(); List<KinesisConsumer.KinesisShardStats> kstats = extractkinesisStats(stats); resp.repartitionRequired = isPartitionRequired(kstats); return resp; }
@Override public Response processStats(BatchedOperatorStats batchedOperatorStats) { Response response = new Response(); response.repartitionRequired = true; return response; }
public AbstractBlockReader() { maxReaders = 16; minReaders = 1; intervalMillis = 2 * 60 * 1000L; response = new StatsListener.Response(); backlogPerOperator = Maps.newHashMap(); partitionCount = 1; counters = new BasicCounters<>(MutableLong.class); collectStats = true; lastBlockOpenTime = -1; }