protected static synchronized Map<URI, Statistics> getAllStatistics() { Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>( STATISTICS_TABLE.size()); for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) { URI key = pair.getKey(); Statistics value = pair.getValue(); Statistics newStatsObj = new Statistics(value); statsMap.put(URI.create(key.toString()), newStatsObj); } return statsMap; }
/** * @param file File. * @param append Append flag. * @return Output stream. * @throws IOException If failed. */ private FSDataOutputStream out(File file, boolean append, int bufSize) throws IOException { return new FSDataOutputStream(new BufferedOutputStream(new FileOutputStream(file, append), bufSize < 32 * 1024 ? 32 * 1024 : bufSize), new Statistics(getUri().getScheme())); }
/** * Get the statistics for a particular file system. * * @param uri * used as key to lookup STATISTICS_TABLE. Only scheme and authority * part of the uri are used. * @return a statistics object */ protected static synchronized Statistics getStatistics(URI uri) { String scheme = uri.getScheme(); if (scheme == null) { throw new IllegalArgumentException("Scheme not defined in the uri: " + uri); } URI baseUri = getBaseUri(uri); Statistics result = STATISTICS_TABLE.get(baseUri); if (result == null) { result = new Statistics(scheme); STATISTICS_TABLE.put(baseUri, result); } return result; }
/** * {@inheritDoc} */ @Override protected void prepareWrite(WritableByteChannel channel) throws Exception { LOG.debug("Opening new writer"); Configuration configuration = new Configuration(false); configuration.setStrings("io.serializations", writeOperation.serializationNames); FSDataOutputStream outputStream = new FSDataOutputStream(new OutputStreamWrapper(channel), new Statistics("dataflow")); sequenceFile = SequenceFile.createWriter(configuration, SequenceFile.Writer.stream(outputStream), SequenceFile.Writer.keyClass(writeOperation.keyClass), SequenceFile.Writer.valueClass(writeOperation.valueClass), SequenceFile.Writer.compression(SequenceFile.CompressionType.BLOCK) ); }
/** * Get the statistics for a particular file system * @param cls the class to lookup * @return a statistics object */ public static synchronized Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { Statistics result = statisticsTable.get(cls); if (result == null) { result = new Statistics(scheme); statisticsTable.put(cls, result); } return result; }
/** * Get the statistics for a particular file system * @param cls the class to lookup * @return a statistics object */ public static synchronized Statistics getStatistics(Class<? extends FileSystem> cls) { Statistics result = statisticsTable.get(cls); if (result == null) { result = new Statistics(); statisticsTable.put(cls, result); } return result; }
/** * Get the statistics for a particular file system * @param cls the class to lookup * @return a statistics object */ public static synchronized Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { Statistics result = statisticsTable.get(cls); if (result == null) { result = new Statistics(scheme); statisticsTable.put(cls, result); } return result; }
/** * Get the statistics for a particular file system * @param cls the class to lookup * @return a statistics object */ public static synchronized Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { Statistics result = statisticsTable.get(cls); if (result == null) { result = new Statistics(scheme); statisticsTable.put(cls, result); } return result; }
/** * Get the statistics for a particular file system * @param cls the class to lookup * @return a statistics object */ public static synchronized Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { Statistics result = statisticsTable.get(cls); if (result == null) { result = new Statistics(scheme); statisticsTable.put(cls, result); } return result; }
/** * Get the statistics for a particular file system * @param cls the class to lookup * @return a statistics object */ public static synchronized Statistics getStatistics(String scheme, Class<? extends FileSystem> cls) { Statistics result = statisticsTable.get(cls); if (result == null) { result = new Statistics(scheme); statisticsTable.put(cls, result); } return result; }
/** * Instantiates a new Git file system. */ public GitFileSystem() { statistics = new Statistics(""); }
protected static synchronized Map<URI, Statistics> getAllStatistics() { Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>( STATISTICS_TABLE.size()); for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) { URI key = pair.getKey(); Statistics value = pair.getValue(); Statistics newStatsObj = new Statistics(value); statsMap.put(URI.create(key.toString()), newStatsObj); } return statsMap; }
protected static synchronized Map<URI, Statistics> getAllStatistics() { Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>( STATISTICS_TABLE.size()); for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) { URI key = pair.getKey(); Statistics value = pair.getValue(); Statistics newStatsObj = new Statistics(value); statsMap.put(URI.create(key.toString()), newStatsObj); } return statsMap; }
protected static synchronized Map<URI, Statistics> getAllStatistics() { Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>( STATISTICS_TABLE.size()); for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) { URI key = pair.getKey(); Statistics value = pair.getValue(); Statistics newStatsObj = new Statistics(value); statsMap.put(URI.create(key.toString()), newStatsObj); } return statsMap; }
MyDistributedFileSystem() { statistics = new FileSystem.Statistics("myhdfs"); // can't mock finals dfs = mock(DFSClient.class); } @Override
protected static synchronized Map<URI, Statistics> getAllStatistics() { Map<URI, Statistics> statsMap = new HashMap<URI, Statistics>( STATISTICS_TABLE.size()); for (Map.Entry<URI, Statistics> pair : STATISTICS_TABLE.entrySet()) { URI key = pair.getKey(); Statistics value = pair.getValue(); Statistics newStatsObj = new Statistics(value); statsMap.put(URI.create(key.toString()), newStatsObj); } return statsMap; }
@Override public RFileWriter build() throws IOException { FileOperations fileops = FileOperations.getInstance(); AccumuloConfiguration acuconf = DefaultConfiguration.getInstance(); HashMap<String,String> userProps = new HashMap<>(); userProps.putAll(tableConfig); userProps.putAll(summarizerProps); userProps.putAll(samplerProps); if (userProps.size() > 0) { acuconf = new ConfigurationCopy(Iterables.concat(acuconf, userProps.entrySet())); } CryptoService cs = CryptoServiceFactory.newInstance(acuconf, ClassloaderType.JAVA); if (out.getOutputStream() != null) { FSDataOutputStream fsdo; if (out.getOutputStream() instanceof FSDataOutputStream) { fsdo = (FSDataOutputStream) out.getOutputStream(); } else { fsdo = new FSDataOutputStream(out.getOutputStream(), new FileSystem.Statistics("foo")); } return new RFileWriter( fileops.newWriterBuilder().forOutputStream(".rf", fsdo, out.getConf(), cs) .withTableConfiguration(acuconf).withStartDisabled().build(), visCacheSize); } else { return new RFileWriter(fileops.newWriterBuilder() .forFile(out.path.toString(), out.getFileSystem(), out.getConf(), cs) .withTableConfiguration(acuconf).withStartDisabled().build(), visCacheSize); } }
@Override public void process(InputStream in, OutputStream out) throws IOException { // Use a FilterableOutputStream to change 'InputStreamWritable' to 'BytesWritable' - see comment // above for an explanation of why we want to do this. final ByteFilteringOutputStream bwos = new ByteFilteringOutputStream(out); // TODO: Adding this filter could be dangerous... A Sequence File's header contains 3 bytes: "SEQ", // followed by 1 byte that is the Sequence File version, followed by 2 "entries." These "entries" // contain the size of the Key/Value type and the Key/Value type. So, we will be writing the // value type as InputStreamWritable -- which we need to change to BytesWritable. This means that // we must also change the "size" that is written, but replacing this single byte could be // dangerous. However, we know exactly what will be written to the header, and we limit this at one // replacement, so we should be just fine. bwos.addFilter(toReplace, replaceWith, 1); bwos.addFilter((byte) InputStreamWritable.class.getCanonicalName().length(), (byte) BytesWritable.class.getCanonicalName().length(), 1); try (final FSDataOutputStream fsDataOutputStream = new FSDataOutputStream(bwos, new Statistics("")); final SequenceFile.Writer writer = SequenceFile.createWriter(configuration, SequenceFile.Writer.stream(fsDataOutputStream), SequenceFile.Writer.keyClass(Text.class), SequenceFile.Writer.valueClass(InputStreamWritable.class), SequenceFile.Writer.compression(compressionType, compressionCodec))) { processInputStream(in, flowFile, writer); } finally { watch.stop(); } } });
/** * Get the statistics for a particular file system. * @param cls the class to lookup * @return a statistics object * @deprecated use {@link #getGlobalStorageStatistics()} */ @Deprecated public static synchronized Statistics getStatistics(final String scheme, Class<? extends FileSystem> cls) { checkArgument(scheme != null, "No statistics is allowed for a file system with null scheme!"); Statistics result = statisticsTable.get(cls); if (result == null) { final Statistics newStats = new Statistics(scheme); statisticsTable.put(cls, newStats); result = newStats; GlobalStorageStatistics.INSTANCE.put(scheme, new StorageStatisticsProvider() { @Override public StorageStatistics provide() { return new FileSystemStorageStatistics(scheme, newStats); } }); } return result; }
private static FileSystem.Statistics combineFileSystemStatistics(final FileSystem.Statistics s1, final FileSystem.Statistics s2) { FileSystem.Statistics result = new FileSystem.Statistics(s1); result.incrementReadOps(s2.getReadOps()); result.incrementLargeReadOps(s2.getLargeReadOps()); result.incrementWriteOps(s2.getWriteOps()); result.incrementBytesRead(s2.getBytesRead()); result.incrementBytesWritten(s2.getBytesWritten()); return result; }