public SparkStatisticsBuilder add(String groupName, String name, String value) { List<SparkStatistic> statisticList = statisticMap.get(groupName); if (statisticList == null) { statisticList = new LinkedList<SparkStatistic>(); statisticMap.put(groupName, statisticList); } statisticList.add(new SparkStatistic(name, value)); return this; } }
private void logSparkStatistic(SparkStatistics sparkStatistic) { Iterator<SparkStatisticGroup> groupIterator = sparkStatistic.getStatisticGroups(); while (groupIterator.hasNext()) { SparkStatisticGroup group = groupIterator.next(); LOG.info(group.getGroupName()); Iterator<SparkStatistic> statisticIterator = group.getStatistics(); while (statisticIterator.hasNext()) { SparkStatistic statistic = statisticIterator.next(); LOG.info("\t" + statistic.getName() + ": " + statistic.getValue()); } } }
SparkStatisticGroup(String groupName, List<SparkStatistic> statisticList) { this.groupName = groupName; for (SparkStatistic sparkStatistic : statisticList) { this.statistics.put(sparkStatistic.getName(), sparkStatistic); } }
public static long getSparkStatisticAsLong(SparkStatisticGroup group, String name) { return Long.parseLong(group.getSparkStatistic(name).getValue()); } }
@VisibleForTesting static String sparkStatisticsToString(SparkStatistics sparkStatistic, int sparkJobID) { StringBuilder sparkStatsString = new StringBuilder(); sparkStatsString.append("\n\n"); sparkStatsString.append(String.format("=====Spark Job[%d] Statistics=====", sparkJobID)); sparkStatsString.append("\n\n"); Iterator<SparkStatisticGroup> groupIterator = sparkStatistic.getStatisticGroups(); while (groupIterator.hasNext()) { SparkStatisticGroup group = groupIterator.next(); sparkStatsString.append(group.getGroupName()).append("\n"); Iterator<SparkStatistic> statisticIterator = group.getStatistics(); while (statisticIterator.hasNext()) { SparkStatistic statistic = statisticIterator.next(); sparkStatsString.append("\t").append(statistic.getName()).append(": ").append( statistic.getValue()).append("\n"); } } return sparkStatsString.toString(); }
public SparkStatisticsBuilder add(String groupName, String name, String value) { List<SparkStatistic> statisticList = statisticMap.get(groupName); if (statisticList == null) { statisticList = new LinkedList<SparkStatistic>(); statisticMap.put(groupName, statisticList); } statisticList.add(new SparkStatistic(name, value)); return this; } }
private void logSparkStatistic(SparkStatistics sparkStatistic) { Iterator<SparkStatisticGroup> groupIterator = sparkStatistic.getStatisticGroups(); while (groupIterator.hasNext()) { SparkStatisticGroup group = groupIterator.next(); LOG.info(group.getGroupName()); Iterator<SparkStatistic> statisticIterator = group.getStatistics(); while (statisticIterator.hasNext()) { SparkStatistic statistic = statisticIterator.next(); LOG.info("\t" + statistic.getName() + ": " + statistic.getValue()); } } }
public SparkStatisticsBuilder add(SparkCounters sparkCounters) { for (SparkCounterGroup counterGroup : sparkCounters.getSparkCounterGroups().values()) { String groupDisplayName = counterGroup.getGroupDisplayName(); List<SparkStatistic> statisticList = statisticMap.get(groupDisplayName); if (statisticList == null) { statisticList = new LinkedList<SparkStatistic>(); statisticMap.put(groupDisplayName, statisticList); } for (SparkCounter counter : counterGroup.getSparkCounters().values()) { String displayName = counter.getDisplayName(); statisticList.add(new SparkStatistic(displayName, Long.toString(counter.getValue()))); } } return this; }
public SparkStatisticsBuilder add(SparkCounters sparkCounters) { for (SparkCounterGroup counterGroup : sparkCounters.getSparkCounterGroups().values()) { String groupDisplayName = counterGroup.getGroupDisplayName(); List<SparkStatistic> statisticList = statisticMap.get(groupDisplayName); if (statisticList == null) { statisticList = new LinkedList<SparkStatistic>(); statisticMap.put(groupDisplayName, statisticList); } for (SparkCounter counter : counterGroup.getSparkCounters().values()) { String displayName = counter.getDisplayName(); statisticList.add(new SparkStatistic(displayName, Long.toString(counter.getValue()))); } } return this; }
public SparkStatisticsBuilder add(String groupName, String name, String value) { List<SparkStatistic> statisticList = statisticMap.get(groupName); if (statisticList == null) { statisticList = new LinkedList<SparkStatistic>(); statisticMap.put(groupName, statisticList); } statisticList.add(new SparkStatistic(name, value)); return this; } }
public SparkStatisticsBuilder add(SparkCounters sparkCounters) { for (SparkCounterGroup counterGroup : sparkCounters.getSparkCounterGroups().values()) { String groupDisplayName = counterGroup.getGroupDisplayName(); List<SparkStatistic> statisticList = statisticMap.get(groupDisplayName); if (statisticList == null) { statisticList = new LinkedList<SparkStatistic>(); statisticMap.put(groupDisplayName, statisticList); } for (SparkCounter counter : counterGroup.getSparkCounters().values()) { String displayName = counter.getDisplayName(); statisticList.add(new SparkStatistic(displayName, Long.toString(counter.getValue()))); } } return this; }