private FSDataInputStream getInputStream() throws IOException { if (this.inputStream == null) { this.inputStream = this.fs.open(this.checkpointFile, this.buf.length); } return this.inputStream; }
private FSDataInputStream getInputStream() throws IOException { if (this.inputStream == null) { this.inputStream = this.fs.open(this.checkpointFile, this.buf.length); } return this.inputStream; }
final FSDataInputStream inStream = fs.open(this.userJars.get(i)); final byte[] buf = new byte[BUFFERSIZE]; int read = inStream.read(buf, 0, buf.length);
final FSDataInputStream inStream = fs.open(this.userJars.get(i)); final byte[] buf = new byte[BUFFERSIZE]; int read = inStream.read(buf, 0, buf.length);
public Path call() { Path tmp = getTempDir(jobID, name); try { if (!lfs.exists(tmp)) { FSDataOutputStream lfsOutput = lfs.create(tmp, false); Path distributedPath = new Path(filePath); FileSystem fs = distributedPath.getFileSystem(); FSDataInputStream fsInput = fs.open(distributedPath); IOUtils.copyBytes(fsInput, lfsOutput); } } catch (IOException e1) { throw new RuntimeException("Error copying a file from hdfs to the local fs", e1); } return tmp; } }
out.writeLong(status.getLen()); final FSDataInputStream inStream = fs.open(storePath); final byte[] buf = new byte[8192]; // 8K Buffer*/ int read = inStream.read(buf, 0, buf.length);
out.writeLong(status.getLen()); final FSDataInputStream inStream = fs.open(storePath); final byte[] buf = new byte[8192]; // 8K Buffer*/ int read = inStream.read(buf, 0, buf.length);
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
@Override public void run() { try { final FileSystem fs = FileSystem.get(this.split.getPath().toUri()); this.fdis = fs.open(this.split.getPath()); // check for canceling and close the stream in that case, because no one will obtain it if (this.aborted) { final FSDataInputStream f = this.fdis; this.fdis = null; f.close(); } } catch (Throwable t) { this.error = t; } }
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException { FileSystem sFS = sourcePath.getFileSystem(); FileSystem tFS = targetPath.getFileSystem(); if (!tFS.exists(targetPath)) { if (sFS.getFileStatus(sourcePath).isDir()) { tFS.mkdirs(targetPath); FileStatus[] contents = sFS.listStatus(sourcePath); for (FileStatus content : contents) { String distPath = content.getPath().toString(); if (content.isDir()) { if (distPath.endsWith("/")) { distPath = distPath.substring(0, distPath.length() - 1); } } String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/")); copy(content.getPath(), new Path(localPath), executable); } } else { try { FSDataOutputStream lfsOutput = tFS.create(targetPath, false); FSDataInputStream fsInput = sFS.open(sourcePath); IOUtils.copyBytes(fsInput, lfsOutput); new File(targetPath.toString()).setExecutable(executable); } catch (IOException ioe) { } } } }
/** * Fill in the statistics. The last modification time and the total input size are prefilled. * * @param files * The files that are associated with this block input format. * @param stats * The pre-filled statistics. */ protected SequentialStatistics createStatistics(final List<FileStatus> files, final FileBaseStatistics stats) throws IOException { if (files.isEmpty()) return null; final BlockInfo blockInfo = this.createBlockInfo(); long totalCount = 0; for (final FileStatus file : files) { // invalid file if (file.getLen() < blockInfo.getInfoSize()) continue; final FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize()); fdis.seek(file.getLen() - blockInfo.getInfoSize()); final DataInputStream input = new DataInputStream(fdis); blockInfo.read(input); totalCount += blockInfo.getAccumulatedRecordCount(); } final float avgWidth = totalCount == 0 ? 0 : (float) stats.getTotalInputSize() / totalCount; return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth, totalCount); }
/** * Fill in the statistics. The last modification time and the total input size are prefilled. * * @param files * The files that are associated with this block input format. * @param stats * The pre-filled statistics. */ protected SequentialStatistics createStatistics(List<FileStatus> files, FileBaseStatistics stats) throws IOException { if (files.isEmpty()) { return null; } BlockInfo blockInfo = this.createBlockInfo(); long totalCount = 0; for (FileStatus file : files) { // invalid file if (file.getLen() < blockInfo.getInfoSize()) { continue; } FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize()); fdis.seek(file.getLen() - blockInfo.getInfoSize()); DataInputStream input = new DataInputStream(fdis); blockInfo.read(input); totalCount += blockInfo.getAccumulatedRecordCount(); } final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount); return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth, totalCount); }
/** * Fill in the statistics. The last modification time and the total input size are prefilled. * * @param files * The files that are associated with this block input format. * @param stats * The pre-filled statistics. */ protected SequentialStatistics createStatistics(List<FileStatus> files, FileBaseStatistics stats) throws IOException { if (files.isEmpty()) { return null; } BlockInfo blockInfo = this.createBlockInfo(); long totalCount = 0; for (FileStatus file : files) { // invalid file if (file.getLen() < blockInfo.getInfoSize()) { continue; } FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize()); fdis.seek(file.getLen() - blockInfo.getInfoSize()); DataInputStream input = new DataInputStream(fdis); blockInfo.read(input); totalCount += blockInfo.getAccumulatedRecordCount(); } final float avgWidth = totalCount == 0 ? 0 : ((float) stats.getTotalInputSize() / totalCount); return new SequentialStatistics(stats.getLastModificationTime(), stats.getTotalInputSize(), avgWidth, totalCount); }