/** * Read from non-checksum stream failed, fall back to FS checksum. Thread-safe. * @param offCount For how many checksumOk calls to turn off the HBase checksum. */ public FSDataInputStream fallbackToFsChecksum(int offCount) throws IOException { // checksumOffCount is speculative, but let's try to reset it less. boolean partOfConvoy = false; if (this.stream == null) { synchronized (streamNoFsChecksumFirstCreateLock) { partOfConvoy = (this.stream != null); if (!partOfConvoy) { this.stream = (link != null) ? link.open(hfs) : hfs.open(path); } } } if (!partOfConvoy) { this.useHBaseChecksum = false; this.hbaseChecksumOffCount.set(offCount); } return this.stream; }
private FSDataInputStreamWrapper(FileSystem fs, FileLink link, Path path, boolean dropBehind, long readahead) throws IOException { assert (path == null) != (link == null); this.path = path; this.link = link; this.doCloseStreams = true; this.dropBehind = dropBehind; this.readahead = readahead; // If the fs is not an instance of HFileSystem, then create an instance of HFileSystem // that wraps over the specified fs. In this case, we will not be able to avoid // checksumming inside the filesystem. this.hfs = (fs instanceof HFileSystem) ? (HFileSystem) fs : new HFileSystem(fs); // Initially we are going to read the tail block. Open the reader w/FS checksum. this.useHBaseChecksumConfigured = this.useHBaseChecksum = false; this.stream = (link != null) ? link.open(hfs) : hfs.open(path); setStreamOptions(stream); }
/** * Prepares the streams for block reader. NOT THREAD SAFE. Must be called once, after any * reads finish and before any other reads start (what happens in reality is we read the * tail, then call this based on what's in the tail, then read blocks). * @param forceNoHBaseChecksum Force not using HBase checksum. */ public void prepareForBlockReader(boolean forceNoHBaseChecksum) throws IOException { if (hfs == null) return; assert this.stream != null && !this.useHBaseChecksumConfigured; boolean useHBaseChecksum = !forceNoHBaseChecksum && hfs.useHBaseChecksum() && (hfs.getNoChecksumFs() != hfs); if (useHBaseChecksum) { FileSystem fsNc = hfs.getNoChecksumFs(); this.streamNoFsChecksum = (link != null) ? link.open(fsNc) : fsNc.open(path); setStreamOptions(streamNoFsChecksum); this.useHBaseChecksumConfigured = this.useHBaseChecksum = useHBaseChecksum; // Close the checksum stream; we will reopen it if we get an HBase checksum failure. this.stream.close(); this.stream = null; } }
@Test(expected = FileNotFoundException.class) public void testLinkReadWithMissingFile() throws Exception { HBaseTestingUtility testUtil = new HBaseTestingUtility(); FileSystem fs = new MyDistributedFileSystem(); Path originalPath = new Path(testUtil.getDefaultRootDirPath(), "test.file"); Path archivedPath = new Path(testUtil.getDefaultRootDirPath(), "archived.file"); List<Path> files = new ArrayList<Path>(); files.add(originalPath); files.add(archivedPath); FileLink link = new FileLink(files); link.open(fs); }
/** * Try to open the "source" file. * Throws an IOException if the communication with the inputFs fail or * if the file is not found. */ private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo) throws IOException { try { Configuration conf = context.getConfiguration(); FileLink link = null; switch (fileInfo.getType()) { case HFILE: Path inputPath = new Path(fileInfo.getHfile()); link = getFileLink(inputPath, conf); break; case WAL: String serverName = fileInfo.getWalServer(); String logName = fileInfo.getWalName(); link = new WALLink(inputRoot, serverName, logName); break; default: throw new IOException("Invalid File Type: " + fileInfo.getType().toString()); } return link.open(inputFs); } catch (IOException e) { context.getCounter(Counter.MISSING_FILES).increment(1); LOG.error("Unable to open source file=" + fileInfo.toString(), e); throw e; } }
FSDataInputStream in = link.open(fs); try { byte[] data = new byte[8192];
/** * Read from non-checksum stream failed, fall back to FS checksum. Thread-safe. * @param offCount For how many checksumOk calls to turn off the HBase checksum. */ public FSDataInputStream fallbackToFsChecksum(int offCount) throws IOException { // checksumOffCount is speculative, but let's try to reset it less. boolean partOfConvoy = false; if (this.stream == null) { synchronized (streamNoFsChecksumFirstCreateLock) { partOfConvoy = (this.stream != null); if (!partOfConvoy) { this.stream = (link != null) ? link.open(hfs) : hfs.open(path); } } } if (!partOfConvoy) { this.useHBaseChecksum = false; this.hbaseChecksumOffCount = offCount; } return this.stream; }
/** * Prepares the streams for block reader. NOT THREAD SAFE. Must be called once, after any * reads finish and before any other reads start (what happens in reality is we read the * tail, then call this based on what's in the tail, then read blocks). * @param forceNoHBaseChecksum Force not using HBase checksum. */ public void prepareForBlockReader(boolean forceNoHBaseChecksum) throws IOException { if (hfs == null) return; assert this.stream != null && !this.useHBaseChecksumConfigured; boolean useHBaseChecksum = !forceNoHBaseChecksum && hfs.useHBaseChecksum() && (hfs.getNoChecksumFs() != hfs); if (useHBaseChecksum) { FileSystem fsNc = hfs.getNoChecksumFs(); this.streamNoFsChecksum = (link != null) ? link.open(fsNc) : fsNc.open(path); this.useHBaseChecksumConfigured = this.useHBaseChecksum = useHBaseChecksum; // Close the checksum stream; we will reopen it if we get an HBase checksum failure. this.stream.close(); this.stream = null; } }
private FSDataInputStreamWrapper(FileSystem fs, FileLink link, Path path, boolean dropBehind) throws IOException { assert (path == null) != (link == null); this.path = path; this.link = link; this.doCloseStreams = true; // If the fs is not an instance of HFileSystem, then create an instance of HFileSystem // that wraps over the specified fs. In this case, we will not be able to avoid // checksumming inside the filesystem. this.hfs = (fs instanceof HFileSystem) ? (HFileSystem)fs : new HFileSystem(fs); // Initially we are going to read the tail block. Open the reader w/FS checksum. this.useHBaseChecksumConfigured = this.useHBaseChecksum = false; this.stream = (link != null) ? link.open(hfs) : hfs.open(path); try { this.stream.setDropBehind(dropBehind); } catch (Exception e) { // Skipped. } }
fsdis = fLink.open(fs); lenToRead = fLink.getFileStatus(fs).getLen(); } else {
@Test(expected = FileNotFoundException.class) public void testLinkReadWithMissingFile() throws Exception { HBaseTestingUtility testUtil = new HBaseTestingUtility(); FileSystem fs = new MyDistributedFileSystem(); Path originalPath = new Path(testUtil.getDefaultRootDirPath(), "test.file"); Path archivedPath = new Path(testUtil.getDefaultRootDirPath(), "archived.file"); List<Path> files = new ArrayList<Path>(); files.add(originalPath); files.add(archivedPath); FileLink link = new FileLink(files); link.open(fs); }
/** * Try to open the "source" file. * Throws an IOException if the communication with the inputFs fail or * if the file is not found. */ private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo) throws IOException { try { Configuration conf = context.getConfiguration(); FileLink link = null; switch (fileInfo.getType()) { case HFILE: Path inputPath = new Path(fileInfo.getHfile()); link = getFileLink(inputPath, conf); break; case WAL: String serverName = fileInfo.getWalServer(); String logName = fileInfo.getWalName(); link = new WALLink(inputRoot, serverName, logName); break; default: throw new IOException("Invalid File Type: " + fileInfo.getType().toString()); } return link.open(inputFs); } catch (IOException e) { context.getCounter(Counter.MISSING_FILES).increment(1); LOG.error("Unable to open source file=" + fileInfo.toString(), e); throw e; } }
/** * Try to open the "source" file. * Throws an IOException if the communication with the inputFs fail or * if the file is not found. */ private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo) throws IOException { try { Configuration conf = context.getConfiguration(); FileLink link = null; switch (fileInfo.getType()) { case HFILE: Path inputPath = new Path(fileInfo.getHfile()); link = getFileLink(inputPath, conf); break; case WAL: String serverName = fileInfo.getWalServer(); String logName = fileInfo.getWalName(); link = new WALLink(inputRoot, serverName, logName); break; default: throw new IOException("Invalid File Type: " + fileInfo.getType().toString()); } return link.open(inputFs); } catch (IOException e) { context.getCounter(Counter.MISSING_FILES).increment(1); LOG.error("Unable to open source file=" + fileInfo.toString(), e); throw e; } }
/** * Try to open the "source" file. * Throws an IOException if the communication with the inputFs fail or * if the file is not found. */ private FSDataInputStream openSourceFile(Context context, final SnapshotFileInfo fileInfo) throws IOException { try { Configuration conf = context.getConfiguration(); FileLink link = null; switch (fileInfo.getType()) { case HFILE: Path inputPath = new Path(fileInfo.getHfile()); link = HFileLink.buildFromHFileLinkPattern(conf, inputPath); break; case WAL: String serverName = fileInfo.getWalServer(); String logName = fileInfo.getWalName(); link = new WALLink(inputRoot, serverName, logName); break; default: throw new IOException("Invalid File Type: " + fileInfo.getType().toString()); } return link.open(inputFs); } catch (IOException e) { context.getCounter(Counter.MISSING_FILES).increment(1); LOG.error("Unable to open source file=" + fileInfo.toString(), e); throw e; } }
FSDataInputStream in = link.open(fs); try { byte[] data = new byte[8192];