public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) throws IOException { this(fs, file, fs.getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT)); }
private int getSumBufferSize(int bytesPerSum, int bufferSize) { int defaultBufferSize = getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT); int proportionalBufferSize = bufferSize / bytesPerSum; return Math.max(bytesPerSum, Math.max(proportionalBufferSize, defaultBufferSize)); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), src, this, dst, delSrc, conf); }
/** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. */ @Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf); }
if (!fs.isDirectory(src)) { // source is a file fs.copyToLocalFile(src, dst); FileSystem localFs = getLocal(getConf()).getRawFileSystem(); if (localFs.isDirectory(dst)) { dst = new Path(dst, src.getName());
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) throws IOException { this(fs, file, fs.getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT)); }
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) throws IOException { this(fs, file, fs.getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT)); }
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) throws IOException { this(fs, file, fs.getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT)); }
public ChecksumFSInputChecker(ChecksumFileSystem fs, Path file) throws IOException { this(fs, file, fs.getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT)); }
private int getSumBufferSize(int bytesPerSum, int bufferSize) { int defaultBufferSize = getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT); int proportionalBufferSize = bufferSize / bytesPerSum; return Math.max(bytesPerSum, Math.max(proportionalBufferSize, defaultBufferSize)); }
private int getSumBufferSize(int bytesPerSum, int bufferSize) { int defaultBufferSize = getConf().getInt("io.file.buffer.size", 4096); int proportionalBufferSize = bufferSize / bytesPerSum; return Math.max(bytesPerSum, Math.max(proportionalBufferSize, defaultBufferSize)); }
private int getSumBufferSize(int bytesPerSum, int bufferSize) { int defaultBufferSize = getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT); int proportionalBufferSize = bufferSize / bytesPerSum; return Math.max(bytesPerSum, Math.max(proportionalBufferSize, defaultBufferSize)); }
private int getSumBufferSize(int bytesPerSum, int bufferSize) { int defaultBufferSize = getConf().getInt( LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_KEY, LocalFileSystemConfigKeys.LOCAL_FS_STREAM_BUFFER_SIZE_DEFAULT); int proportionalBufferSize = bufferSize / bytesPerSum; return Math.max(bytesPerSum, Math.max(proportionalBufferSize, defaultBufferSize)); }
/** {@inheritDoc} */ @Override public FSDataOutputStream createNonRecursive(Path f, FsPermission permission, boolean overwrite, int bufferSize, short replication, long blockSize, Progressable progress) throws IOException { return create(f, permission, overwrite, false, bufferSize, replication, blockSize, getConf().getInt("io.bytes.per.checksum", 512), progress); }
/** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. */ @Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), src, this, dst, delSrc, conf); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), src, this, dst, delSrc, conf); }
/** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. */ @Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf); }
@Override public void copyFromLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(getLocal(conf), src, this, dst, delSrc, conf); }
/** * The src file is under FS, and the dst is on the local disk. * Copy it from FS control to the local dst name. */ @Override public void copyToLocalFile(boolean delSrc, Path src, Path dst) throws IOException { Configuration conf = getConf(); FileUtil.copy(this, src, getLocal(conf), dst, delSrc, conf); }