@Test(expected = FileNotFoundException.class) public void testLinkReadWithMissingFile() throws Exception { HBaseTestingUtility testUtil = new HBaseTestingUtility(); FileSystem fs = new MyDistributedFileSystem(); Path originalPath = new Path(testUtil.getDefaultRootDirPath(), "test.file"); Path archivedPath = new Path(testUtil.getDefaultRootDirPath(), "archived.file"); List<Path> files = new ArrayList<Path>(); files.add(originalPath); files.add(archivedPath); FileLink link = new FileLink(files); link.open(fs); }
private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo) throws IOException { try { Configuration conf = context.getConfiguration(); FileLink link = null; switch (fileInfo.getType()) { case HFILE: Path inputPath = new Path(fileInfo.getHfile()); link = getFileLink(inputPath, conf); break; case WAL: link = new WALLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName()); break; default: throw new IOException("Invalid File Type: " + fileInfo.getType().toString()); } return link.getFileStatus(inputFs); } catch (FileNotFoundException e) { context.getCounter(Counter.MISSING_FILES).increment(1); LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e); throw e; } catch (IOException e) { LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e); throw e; } }
for (Path path: fileLink.getLocations()) { if (path.equals(currentPath)) continue; try {
@Test public void testHashCode() { Path p1 = new Path("/p1"); Path p2 = new Path("/p2"); Path p3 = new Path("/p3"); assertEquals(new FileLink().hashCode(), new FileLink().hashCode()); assertEquals(new FileLink(p1).hashCode(), new FileLink(p1).hashCode()); assertEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1, p2).hashCode()); assertEquals(new FileLink(p1, p2, p3).hashCode(), new FileLink(p1, p2, p3).hashCode()); assertNotEquals(new FileLink(p1).hashCode(), new FileLink(p3).hashCode()); assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1).hashCode()); assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2).hashCode()); assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2, p1).hashCode()); // ordering }
/** * Read from non-checksum stream failed, fall back to FS checksum. Thread-safe. * @param offCount For how many checksumOk calls to turn off the HBase checksum. */ public FSDataInputStream fallbackToFsChecksum(int offCount) throws IOException { // checksumOffCount is speculative, but let's try to reset it less. boolean partOfConvoy = false; if (this.stream == null) { synchronized (streamNoFsChecksumFirstCreateLock) { partOfConvoy = (this.stream != null); if (!partOfConvoy) { this.stream = (link != null) ? link.open(hfs) : hfs.open(path); } } } if (!partOfConvoy) { this.useHBaseChecksum = false; this.hbaseChecksumOffCount.set(offCount); } return this.stream; }
fsdis = fLink.open(fs); lenToRead = fLink.getFileStatus(fs).getLen(); } else {
/** * @param originPath Original location of the file to link * @param alternativePaths Alternative locations to look for the linked file */ public FileLink(Path originPath, Path... alternativePaths) { setLocations(originPath, alternativePaths); }
@Test public void testEquals() { Path p1 = new Path("/p1"); Path p2 = new Path("/p2"); Path p3 = new Path("/p3"); assertEquals(new FileLink(), new FileLink()); assertEquals(new FileLink(p1), new FileLink(p1)); assertEquals(new FileLink(p1, p2), new FileLink(p1, p2)); assertEquals(new FileLink(p1, p2, p3), new FileLink(p1, p2, p3)); assertNotEquals(new FileLink(p1), new FileLink(p3)); assertNotEquals(new FileLink(p1, p2), new FileLink(p1)); assertNotEquals(new FileLink(p1, p2), new FileLink(p2)); assertNotEquals(new FileLink(p1, p2), new FileLink(p2, p1)); // ordering important! }
Path backRefPath = FileLink.getBackReferencesDir(HFileArchiveUtil .getStoreArchivePath(conf, HFileLink.getReferencedTableName(path.getName().toString()), HFileLink.getReferencedRegionName(path.getName().toString()),
private FSDataInputStreamWrapper(FileSystem fs, FileLink link, Path path, boolean dropBehind, long readahead) throws IOException { assert (path == null) != (link == null); this.path = path; this.link = link; this.doCloseStreams = true; this.dropBehind = dropBehind; this.readahead = readahead; // If the fs is not an instance of HFileSystem, then create an instance of HFileSystem // that wraps over the specified fs. In this case, we will not be able to avoid // checksumming inside the filesystem. this.hfs = (fs instanceof HFileSystem) ? (HFileSystem) fs : new HFileSystem(fs); // Initially we are going to read the tail block. Open the reader w/FS checksum. this.useHBaseChecksumConfigured = this.useHBaseChecksum = false; this.stream = (link != null) ? link.open(hfs) : hfs.open(path); setStreamOptions(stream); }
/** * @param originPath Original location of the file to link * @param alternativePaths Alternative locations to look for the linked file */ public FileLink(Path originPath, Path... alternativePaths) { setLocations(originPath, alternativePaths); }
@Test public void testHashCode() { Path p1 = new Path("/p1"); Path p2 = new Path("/p2"); Path p3 = new Path("/p3"); assertEquals(new FileLink().hashCode(), new FileLink().hashCode()); assertEquals(new FileLink(p1).hashCode(), new FileLink(p1).hashCode()); assertEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1, p2).hashCode()); assertEquals(new FileLink(p1, p2, p3).hashCode(), new FileLink(p1, p2, p3).hashCode()); assertNotEquals(new FileLink(p1).hashCode(), new FileLink(p3).hashCode()); assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p1).hashCode()); assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2).hashCode()); assertNotEquals(new FileLink(p1, p2).hashCode(), new FileLink(p2, p1).hashCode()); // ordering }
@Test public void testEquals() { Path p1 = new Path("/p1"); Path p2 = new Path("/p2"); Path p3 = new Path("/p3"); assertEquals(new FileLink(), new FileLink()); assertEquals(new FileLink(p1), new FileLink(p1)); assertEquals(new FileLink(p1, p2), new FileLink(p1, p2)); assertEquals(new FileLink(p1, p2, p3), new FileLink(p1, p2, p3)); assertNotEquals(new FileLink(p1), new FileLink(p3)); assertNotEquals(new FileLink(p1, p2), new FileLink(p1)); assertNotEquals(new FileLink(p1, p2), new FileLink(p2)); assertNotEquals(new FileLink(p1, p2), new FileLink(p2, p1)); // ordering important! }
/** * Prepares the streams for block reader. NOT THREAD SAFE. Must be called once, after any * reads finish and before any other reads start (what happens in reality is we read the * tail, then call this based on what's in the tail, then read blocks). * @param forceNoHBaseChecksum Force not using HBase checksum. */ public void prepareForBlockReader(boolean forceNoHBaseChecksum) throws IOException { if (hfs == null) return; assert this.stream != null && !this.useHBaseChecksumConfigured; boolean useHBaseChecksum = !forceNoHBaseChecksum && hfs.useHBaseChecksum() && (hfs.getNoChecksumFs() != hfs); if (useHBaseChecksum) { FileSystem fsNc = hfs.getNoChecksumFs(); this.streamNoFsChecksum = (link != null) ? link.open(fsNc) : fsNc.open(path); setStreamOptions(streamNoFsChecksum); this.useHBaseChecksumConfigured = this.useHBaseChecksum = useHBaseChecksum; // Close the checksum stream; we will reopen it if we get an HBase checksum failure. this.stream.close(); this.stream = null; } }
private FileStatus getSourceFileStatus(Context context, final SnapshotFileInfo fileInfo) throws IOException { try { Configuration conf = context.getConfiguration(); FileLink link = null; switch (fileInfo.getType()) { case HFILE: Path inputPath = new Path(fileInfo.getHfile()); link = getFileLink(inputPath, conf); break; case WAL: link = new WALLink(inputRoot, fileInfo.getWalServer(), fileInfo.getWalName()); break; default: throw new IOException("Invalid File Type: " + fileInfo.getType().toString()); } return link.getFileStatus(inputFs); } catch (FileNotFoundException e) { context.getCounter(Counter.MISSING_FILES).increment(1); LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e); throw e; } catch (IOException e) { LOG.error("Unable to get the status for source file=" + fileInfo.toString(), e); throw e; } }
/** * @param originPath Original location of the file to link * @param alternativePaths Alternative locations to look for the linked file */ public FileLink(Path originPath, Path... alternativePaths) { setLocations(originPath, alternativePaths); }
/** * Try to open the file from one of the available locations. * * @return FSDataInputStream stream of the opened file link * @throws IOException on unexpected error, or file not found. */ private FSDataInputStream tryOpen() throws IOException { for (Path path: fileLink.getLocations()) { if (path.equals(currentPath)) continue; try { in = fs.open(path, bufferSize); in.seek(pos); assert(in.getPos() == pos) : "Link unable to seek to the right position=" + pos; if (LOG.isTraceEnabled()) { if (currentPath != null) { LOG.debug("link open path=" + path); } else { LOG.trace("link switch from path=" + currentPath + " to path=" + path); } } currentPath = path; return(in); } catch (FileNotFoundException e) { // Try another file location } } throw new FileNotFoundException("Unable to open link: " + fileLink); } }
/** * Creates a FileLink which adds pre-namespace paths in its list of available paths. This is used * when reading a snapshot file in a pre-namespace file layout, for example, while upgrading. * @param storeFilePath * @return a FileLink which could read from pre-namespace paths. * @throws IOException */ public FileLink getFileLinkWithPreNSPath(Path storeFilePath) throws IOException { HFileLink link = HFileLink.buildFromHFileLinkPattern(getConf(), storeFilePath); List<Path> pathsToProcess = getPreNSPathsForHFileLink(link); pathsToProcess.addAll(Arrays.asList(link.getLocations())); return new FileLink(pathsToProcess); }