/** * Get the Map of Statistics object indexed by URI Scheme. * @return a Map having a key as URI scheme and value as Statistics object * @deprecated use {@link #getGlobalStorageStatistics()} */ @Deprecated public static synchronized Map<String, Statistics> getStatistics() { Map<String, Statistics> result = new HashMap<>(); for(Statistics stat: statisticsTable.values()) { result.put(stat.getScheme(), stat); } return result; }
public static Map<String, FileSystem.Statistics> getCombinedFileSystemStatistics() { final List<FileSystem.Statistics> allStats = FileSystem.getAllStatistics(); final Map<String, FileSystem.Statistics> result = new HashMap<>(); for (FileSystem.Statistics statistics : allStats) { final String scheme = statistics.getScheme(); if (result.containsKey(scheme)) { FileSystem.Statistics existing = result.get(scheme); FileSystem.Statistics combined = combineFileSystemStatistics(existing, statistics); result.put(scheme, combined); } else { result.put(scheme, statistics); } } return result; }
public static List<StatisticsData> cloneThreadLocalFileSystemStatistics() { List<StatisticsData> result = new ArrayList<>(); // thread local filesystem stats is private and cannot be cloned. So make a copy to new class for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { result.add(new StatisticsData(statistics.getScheme(), statistics.getThreadStatistics())); } return result; }
/** * Get the Map of Statistics object indexed by URI Scheme. * @return a Map having a key as URI scheme and value as Statistics object * @deprecated use {@link #getAllStatistics} instead */ @Deprecated public static synchronized Map<String, Statistics> getStatistics() { Map<String, Statistics> result = new HashMap<String, Statistics>(); for(Statistics stat: statisticsTable.values()) { result.put(stat.getScheme(), stat); } return result; }
@Test(timeout = 1000) public void testStatistics() throws Exception { int fileSchemeCount = 0; for (Statistics stats : FileSystem.getAllStatistics()) { if (stats.getScheme().equals("file")) { fileSchemeCount++; } } assertEquals(1, fileSchemeCount); }
if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); int readOpsDelta = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; ((MockFileSystem) fs1).touch(mockFile); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); assertEquals(2, splits.length); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; ((MockFileSystem) fs1).touch(mockFile); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); assertEquals(2, splits.length); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; if (statistics.getScheme().equalsIgnoreCase("mock")) {
if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); int readOpsDelta = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); assertEquals(2, splits.length); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); assertEquals(2, splits.length); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); int readOpsDelta = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); assertEquals(2, splits.length); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; conf.set(HiveConf.ConfVars.HIVE_ORC_SPLIT_STRATEGY.varname, "HYBRID"); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); assertEquals(2, splits.length); for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore; if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps();
int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
int readOpsBefore = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsBefore = statistics.getReadOps(); int readOpsDelta = -1; for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { if (statistics.getScheme().equalsIgnoreCase("mock")) { readOpsDelta = statistics.getReadOps() - readOpsBefore;
@Override public void print(SessionState.LogHelper console) { console.printInfo("FileSystem Counters Summary"); SortedSet<String> keys = new TreeSet<>(progressMap.keySet()); Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); // Assuming FileSystem.getAllStatistics() returns all schemes that are accessed on task side // as well. If not, we need a way to get all the schemes that are accessed by the tez task/llap. for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { final String scheme = statistics.getScheme().toUpperCase(); console.printInfo(""); console.printInfo("Scheme: " + scheme); console.printInfo(SEPARATOR); console.printInfo(HEADER); console.printInfo(SEPARATOR); for (String vertexName : keys) { TezCounters vertexCounters = vertexCounters(statusOptions, vertexName); if (vertexCounters != null) { console.printInfo(summary(scheme, vertexName, vertexCounters)); } } console.printInfo(SEPARATOR); } }
@Override public void print(SessionState.LogHelper console) { console.printInfo("FileSystem Counters Summary"); SortedSet<String> keys = new TreeSet<>(progressMap.keySet()); Set<StatusGetOpts> statusOptions = new HashSet<>(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); // Assuming FileSystem.getAllStatistics() returns all schemes that are accessed on task side // as well. If not, we need a way to get all the schemes that are accessed by the tez task/llap. for (FileSystem.Statistics statistics : FileSystem.getAllStatistics()) { final String scheme = statistics.getScheme().toUpperCase(); console.printInfo(""); console.printInfo("Scheme: " + scheme); console.printInfo(SEPARATOR); console.printInfo(HEADER); console.printInfo(SEPARATOR); for (String vertexName : keys) { TezCounters vertexCounters = vertexCounters(statusOptions, vertexName); if (vertexCounters != null) { console.printInfo(summary(scheme, vertexName, vertexCounters)); } } console.printInfo(SEPARATOR); } }
@Override public String getScheme() { return stats.getScheme(); }