/** * getStatMapFromExecutorSummary. * @param executorSummary executorSummary * @return getStatMapFromExecutorSummary */ public static Map<String, Object> getStatMapFromExecutorSummary(ExecutorSummary executorSummary) { Map<String, Object> result = new HashMap(); result.put(":host", executorSummary.get_host()); result.put(":port", executorSummary.get_port()); result.put(":uptime_secs", executorSummary.get_uptime_secs()); result.put(":transferred", null); if (executorSummary.is_set_stats()) { result.put(":transferred", sanitizeTransferredStats(executorSummary.get_stats().get_transferred())); } return result; }
public java.lang.Object getFieldValue(_Fields field) { switch (field) { case EXECUTOR_INFO: return get_executor_info(); case COMPONENT_ID: return get_component_id(); case HOST: return get_host(); case PORT: return get_port(); case UPTIME_SECS: return get_uptime_secs(); case STATS: return get_stats(); } throw new java.lang.IllegalStateException(); }
private static ExecutorAggregateStats thriftifyExecAggStats(String compId, String compType, Map m) { ExecutorSummary executorSummary = new ExecutorSummary(); List executor = (List) m.get(EXECUTOR_ID); executorSummary.set_executor_info(new ExecutorInfo(((Number) executor.get(0)).intValue(), ((Number) executor.get(1)).intValue())); executorSummary.set_component_id(compId); executorSummary.set_host((String) m.get(HOST)); executorSummary.set_port(getByKeyOr0(m, PORT).intValue()); int uptime = getByKeyOr0(m, ClientStatsUtil.UPTIME).intValue(); executorSummary.set_uptime_secs(uptime); ExecutorAggregateStats stats = new ExecutorAggregateStats(); stats.set_exec_summary(executorSummary); if (compType.equals(ClientStatsUtil.SPOUT)) { stats.set_stats(thriftifySpoutAggStats(m)); } else { stats.set_stats(thriftifyBoltAggStats(m)); } return stats; }
/** Returns true if field corresponding to fieldID is set (has been assigned a value) and false otherwise */ public boolean isSet(_Fields field) { if (field == null) { throw new java.lang.IllegalArgumentException(); } switch (field) { case EXECUTOR_INFO: return is_set_executor_info(); case COMPONENT_ID: return is_set_component_id(); case HOST: return is_set_host(); case PORT: return is_set_port(); case UPTIME_SECS: return is_set_uptime_secs(); case STATS: return is_set_stats(); } throw new java.lang.IllegalStateException(); }
@Override public int hashCode() { int hashCode = 1; hashCode = hashCode * 8191 + ((is_set_executor_info()) ? 131071 : 524287); if (is_set_executor_info()) hashCode = hashCode * 8191 + executor_info.hashCode(); hashCode = hashCode * 8191 + ((is_set_component_id()) ? 131071 : 524287); if (is_set_component_id()) hashCode = hashCode * 8191 + component_id.hashCode(); hashCode = hashCode * 8191 + ((is_set_host()) ? 131071 : 524287); if (is_set_host()) hashCode = hashCode * 8191 + host.hashCode(); hashCode = hashCode * 8191 + port; hashCode = hashCode * 8191 + uptime_secs; hashCode = hashCode * 8191 + ((is_set_stats()) ? 131071 : 524287); if (is_set_stats()) hashCode = hashCode * 8191 + stats.hashCode(); return hashCode; }
case EXECUTOR_INFO: if (value == null) { unset_executor_info(); } else { set_executor_info((ExecutorInfo)value); unset_component_id(); } else { set_component_id((java.lang.String)value); unset_host(); } else { set_host((java.lang.String)value); unset_port(); } else { set_port((java.lang.Integer)value); unset_uptime_secs(); } else { set_uptime_secs((java.lang.Integer)value); unset_stats(); } else { set_stats((ExecutorStats)value);
double weightedAvgTotal = 0.0; for (ExecutorSummary exec : info.get_executors()) { if ("spout".equals(exec.get_component_id())) { SpoutStats stats = exec.get_stats().get_specific().get_spout(); Map<String, Long> failedMap = stats.get_failed().get(":all-time"); Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
Map<String, Object> result = new HashMap(); ExecutorSummary executorSummary = executorAggregateStats.get_exec_summary(); ExecutorInfo executorInfo = executorSummary.get_executor_info(); ComponentAggregateStats componentAggregateStats = executorAggregateStats.get_stats(); SpecificAggregateStats specificAggregateStats = componentAggregateStats.get_specific_stats(); result.put("id", executorId); result.put("encodedId", URLEncoder.encode(executorId)); result.put("uptime", prettyUptimeSec(executorSummary.get_uptime_secs())); result.put("uptimeSeconds", executorSummary.get_uptime_secs()); String host = executorSummary.get_host(); result.put("host", host); int port = executorSummary.get_port(); result.put("port", port); result.put("emitted", nullToZero(commonAggregateStats.get_emitted()));
/** * filter ExecutorSummary whose stats is null. * * @param summs a list of ExecutorSummary * @return filtered summs */ public static List<ExecutorSummary> getFilledStats(List<ExecutorSummary> summs) { List<ExecutorSummary> ret = new ArrayList<>(); for (ExecutorSummary summ : summs) { if (summ.get_stats() != null) { ret.add(summ); } } return ret; }
uptime = Math.max(uptime, info.get_uptime_secs()); for (ExecutorSummary exec : info.get_executors()) { hosts.add(exec.get_host()); workers.add(exec.get_host() + exec.get_port()); executors++; if (exec.get_stats() != null && exec.get_stats().get_specific() != null && exec.get_stats().get_specific().is_set_spout()) { SpoutStats stats = exec.get_stats().get_specific().get_spout(); Map<String, Long> failedMap = stats.get_failed().get(":all-time"); Map<String, Long> ackedMap = stats.get_acked().get(":all-time");
private static List<Double> extractBoltValues(List<ExecutorSummary> summaries, GlobalStreamId id, Function<BoltStats, Map<String, Map<GlobalStreamId, Double>>> func) { List<Double> ret = new ArrayList<>(); if (summaries != null) { for (ExecutorSummary summ : summaries) { if (summ != null && summ.is_set_stats()) { Map<String, Map<GlobalStreamId, Double>> data = func.apply(summ.get_stats().get_specific().get_bolt()); if (data != null) { List<Double> subvalues = data.values().stream() .map((subMap) -> subMap.get(id)) .filter((value) -> value != null) .collect(Collectors.toList()); ret.addAll(subvalues); } } } } return ret; }
/** * Compute the capacity of a executor. approximation of the % of time spent doing real work. * @param summary the stats for the executor. * @return the capacity of the executor. */ public static double computeExecutorCapacity(ExecutorSummary summary) { ExecutorStats stats = summary.get_stats(); if (stats == null) { return 0.0; } else { // actual value of m is: Map<String, Map<String/GlobalStreamId, Long/Double>> ({win -> stream -> value}) Map<String, Map> m = aggregateBoltStats(Lists.newArrayList(summary), true); // {metric -> win -> value} ==> {win -> metric -> value} m = swapMapOrder(aggregateBoltStreams(m)); // {metric -> value} Map data = ClientStatsUtil.getMapByKey(m, TEN_MIN_IN_SECONDS_STR); int uptime = summary.get_uptime_secs(); int win = Math.min(uptime, TEN_MIN_IN_SECONDS); long executed = getByKeyOr0(data, EXECUTED).longValue(); double latency = getByKeyOr0(data, EXEC_LATENCIES).doubleValue(); if (win > 0) { return executed * latency / (1000 * win); } return 0.0; } }
@Override public void read(org.apache.storm.thrift.protocol.TProtocol prot, ExecutorAggregateStats struct) throws org.apache.storm.thrift.TException { org.apache.storm.thrift.protocol.TTupleProtocol iprot = (org.apache.storm.thrift.protocol.TTupleProtocol) prot; java.util.BitSet incoming = iprot.readBitSet(2); if (incoming.get(0)) { struct.exec_summary = new ExecutorSummary(); struct.exec_summary.read(iprot); struct.set_exec_summary_isSet(true); } if (incoming.get(1)) { struct.stats = new ComponentAggregateStats(); struct.stats.read(iprot); struct.set_stats_isSet(true); } } }
/** * getSpoutExecutors. * @param executorSummaries executorSummaries * @param stormTopology stormTopology * @return getSpoutExecutors. */ public static Map<String, List<ExecutorSummary>> getSpoutExecutors(List<ExecutorSummary> executorSummaries, StormTopology stormTopology) { Map<String, List<ExecutorSummary>> result = new HashMap(); for (ExecutorSummary executorSummary : executorSummaries) { if (StatsUtil.componentType(stormTopology, executorSummary.get_component_id()).equals("spout")) { List<ExecutorSummary> executorSummaryList = result.getOrDefault(executorSummary.get_component_id(), new ArrayList()); executorSummaryList.add(executorSummary); result.put(executorSummary.get_component_id(), executorSummaryList); } } return result; }
/** * getTopologyWorkers. * @param topologyInfo topologyInfo * @param config config * @return getTopologyWorkers. */ public static Map<String, Object> getTopologyWorkers(TopologyInfo topologyInfo, Map config) { List<Map> executorSummaries = new ArrayList(); for (ExecutorSummary executorSummary : topologyInfo.get_executors()) { Map<String, Object> executorSummaryMap = new HashMap(); executorSummaryMap.put("host", executorSummary.get_host()); executorSummaryMap.put("port", executorSummary.get_port()); executorSummaries.add(executorSummaryMap); } HashSet hashSet = new HashSet(); hashSet.addAll(executorSummaries); executorSummaries.clear(); executorSummaries.addAll(hashSet); Map<String, Object> result = new HashMap(); result.put("hostPortList", executorSummaries); addLogviewerInfo(config, result); return result; }
public void write(org.apache.storm.thrift.protocol.TProtocol oprot, ExecutorSummary struct) throws org.apache.storm.thrift.TException { struct.validate(); oprot.writeFieldEnd(); if (struct.stats != null) { if (struct.is_set_stats()) { oprot.writeFieldBegin(STATS_FIELD_DESC); struct.stats.write(oprot);
sb.append(this.uptime_secs); first = false; if (is_set_stats()) { if (!first) sb.append(", "); sb.append("stats:");
public ExecutorSummary deepCopy() { return new ExecutorSummary(this); }
heartbeat = Collections.emptyMap(); ExecutorSummary summ = new ExecutorSummary(execInfo, common.taskToComponent.get(execInfo.get_task_start()), nodeToHost.get(ni.get_node()), ni.get_port_iterator().next().intValue(), if (ex != null) { ExecutorStats stats = StatsUtil.thriftifyExecutorStats(ex); summ.set_stats(stats);
case EXECUTOR_INFO: if (value == null) { unset_executor_info(); } else { set_executor_info((ExecutorInfo)value); unset_component_id(); } else { set_component_id((String)value); unset_host(); } else { set_host((String)value); unset_port(); } else { set_port((Integer)value); unset_uptime_secs(); } else { set_uptime_secs((Integer)value); unset_stats(); } else { set_stats((ExecutorStats)value);