/** * Create a {@link RegionInfo} from the serialized version on-disk. * @param fs {@link FileSystem} that contains the Region Info file * @param regionDir {@link Path} to the Region Directory that contains the Info file * @return An {@link RegionInfo} instance gotten from the Region Info file. * @throws IOException if an error occurred during file open/read operation. */ public static RegionInfo loadRegionInfoFileContent(final FileSystem fs, final Path regionDir) throws IOException { FSDataInputStream in = fs.open(new Path(regionDir, REGION_INFO_FILE)); try { return RegionInfo.parseFrom(in); } finally { in.close(); } }
@Override public InputStream openStream() throws IOException { log.info("Reading task log from: %s", path); final long seekPos; if (offset < 0) { final FileStatus stat = fs.getFileStatus(path); seekPos = Math.max(0, stat.getLen() + offset); } else { seekPos = offset; } final FSDataInputStream inputStream = fs.open(path); inputStream.seek(seekPos); log.info("Read task log from: %s (seek = %,d)", path, seekPos); return inputStream; } }
public static String readAsString(final FileSystem fs, final Path fromMetadataPath) throws IOException { try (FSDataInputStream stream = fs.open(fromMetadataPath)) { byte[] buffer = new byte[1024]; ByteArrayOutputStream sb = new ByteArrayOutputStream(); int read = stream.read(buffer); while (read != -1) { sb.write(buffer, 0, read); read = stream.read(buffer); } return new String(sb.toByteArray(), "UTF-8"); } }
public void addSampleFile(Path inputPath, JobConf job) throws IOException { FileSystem fs = inputPath.getFileSystem(job); FSDataInputStream input = fs.open(inputPath); try { int count = input.readInt(); for (int i = 0; i < count; i++) { byte[] key = new byte[input.readInt()]; input.readFully(key); sampled.add(key); } } finally { IOUtils.closeStream(input); } }
private void corruptDataFile(final String file, final Configuration conf, final int addRemoveBytes) throws Exception { Path bPath = new Path(file); Path cPath = new Path(bPath.getParent(), bPath.getName() + ".corrupt"); FileSystem fs = bPath.getFileSystem(conf); FileStatus fileStatus = fs.getFileStatus(bPath); int len = addRemoveBytes == Integer.MIN_VALUE ? 0 : (int) fileStatus.getLen() + addRemoveBytes; byte[] buffer = new byte[len]; FSDataInputStream fdis = fs.open(bPath); fdis.readFully(0, buffer, 0, (int) Math.min(fileStatus.getLen(), buffer.length)); fdis.close(); FSDataOutputStream fdos = fs.create(cPath, true); fdos.write(buffer, 0, buffer.length); fdos.close(); fs.delete(bPath, false); fs.rename(cPath, bPath); }
private void testSecureWALInternal(boolean offheap) throws IOException, FileNotFoundException { Configuration conf = TEST_UTIL.getConfiguration(); conf.setClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class, WAL.Reader.class); conf.setClass("hbase.regionserver.hlog.writer.impl", SecureProtobufLogWriter.class, WALProvider.Writer.class); conf.setClass("hbase.regionserver.hlog.async.writer.impl", SecureAsyncProtobufLogWriter.class, WALProvider.AsyncWriter.class); conf.setBoolean(WAL_ENCRYPTION, true); long length = fs.getFileStatus(walPath).getLen(); FSDataInputStream in = fs.open(walPath); byte[] fileData = new byte[(int)length]; IOUtils.readFully(in, fileData); in.close(); assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value)); FileStatus[] listStatus = fs.listStatus(walPath.getParent()); Path rootdir = FSUtils.getRootDir(conf); try { WALSplitter s = new WALSplitter(wals, conf, rootdir, fs, null, null); s.splitLogFile(listStatus[0], null); Path file = new Path(ZKSplitLog.getSplitLogDir(rootdir, listStatus[0].getPath().getName()), "corrupt"); assertTrue(fs.exists(file));
private Schema loadFromUrl(String schemaUrl) throws IOException { Configuration conf = new Configuration(); Schema.Parser parser = new Schema.Parser(); if (schemaUrl.toLowerCase(Locale.ENGLISH).startsWith("hdfs://")) { FileSystem fs = FileSystem.get(conf); FSDataInputStream input = null; try { input = fs.open(new Path(schemaUrl)); return parser.parse(input); } finally { if (input != null) { input.close(); } } } else { InputStream is = null; try { is = new URL(schemaUrl).openStream(); return parser.parse(is); } finally { if (is != null) { is.close(); } } } }
@Test public void testMovePath() throws IOException, URISyntaxException { String expected = "test"; ByteArrayOutputStream actual = new ByteArrayOutputStream(); Path src = new Path("/src/file.txt"); Path dst = new Path("/dst/file.txt"); FileSystem fs1 = Mockito.mock(FileSystem.class); Mockito.when(fs1.exists(src)).thenReturn(true); Mockito.when(fs1.isFile(src)).thenReturn(true); Mockito.when(fs1.getUri()).thenReturn(new URI("fs1:////")); Mockito.when(fs1.getFileStatus(src)).thenReturn(new FileStatus(1, false, 1, 1, 1, src)); Mockito.when(fs1.open(src)) .thenReturn(new FSDataInputStream(new SeekableFSInputStream(new ByteArrayInputStream(expected.getBytes())))); Mockito.when(fs1.delete(src, true)).thenReturn(true); FileSystem fs2 = Mockito.mock(FileSystem.class); Mockito.when(fs2.exists(dst)).thenReturn(false); Mockito.when(fs2.getUri()).thenReturn(new URI("fs2:////")); Mockito.when(fs2.getConf()).thenReturn(new Configuration()); Mockito.when(fs2.create(dst, false)).thenReturn(new FSDataOutputStream(actual, null)); try (ParallelRunner parallelRunner = new ParallelRunner(1, fs1)) { parallelRunner.movePath(src, fs2, dst, Optional.<String>absent()); } Assert.assertEquals(actual.toString(), expected); }
private void metablocks(final String compress) throws Exception { Path mFile = new Path(ROOT_DIR, "meta.hfile"); FSDataOutputStream fout = createFSOutput(mFile); HFileContext meta = new HFileContextBuilder() .withCompression(HFileWriterImpl.compressionByName(compress)) .withBlockSize(minBlockSize).build(); Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta) .create(); someTestingWithMetaBlock(writer); writer.close(); fout.close(); FSDataInputStream fin = fs.open(mFile); Reader reader = HFile.createReaderFromStream(mFile, fs.open(mFile), this.fs.getFileStatus(mFile).getLen(), cacheConf, conf); reader.loadFileInfo(); // No data -- this should return false. assertFalse(reader.getScanner(false, false).seekTo()); someReadingWithMetaBlock(reader); fs.delete(mFile, true); reader.close(); fin.close(); }
private void corruptFile(Path p) throws IOException { String manifestName = p.getName(); // Rename the original region-manifest file Path newP = new Path(p.getParent(), manifestName + "1"); fs.rename(p, newP); // Create a new region-manifest file FSDataOutputStream out = fs.create(p); //Copy the first 25 bytes of the original region-manifest into the new one, //make it a corrupted region-manifest file. FSDataInputStream input = fs.open(newP); byte[] buffer = new byte[25]; int len = input.read(0, buffer, 0, 25); if (len > 1) { out.write(buffer, 0, len - 1); } out.close(); // Delete the original region-manifest fs.delete(newP); }
Configuration conf = new Configuration(); Path p = new Path(file); FileSystem fs = p.getFileSystem(conf); if(fs.exists(p)) FSDataInputStream in = fs.open(p); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String s1 = null; in.close();
private void buildContainerRequest(String[] hostLocals) { if (conf.getBoolean(XLearningConfiguration.XLEARNING_HOST_LOCAL_ENABLE, XLearningConfiguration.DEFAULT_XLEARNING_HOST_LOCAL_ENABLE)) { XLearningConfiguration xlConf = new XLearningConfiguration(); String hostLocaldir = xlConf.get("fs.defaultFS") + conf.get(XLearningConfiguration.XLEARNING_HISTORY_LOG_DIR, XLearningConfiguration.DEFAULT_XLEARNING_HISTORY_LOG_DIR) + "/" + conf.get("hadoop.job.ugi").split(",")[0] + "/" + envs.get(XLearningConstants.Environment.XLEARNING_APP_NAME.toString()); Path hostLocalPath = new Path(hostLocaldir); String line; try { if (hostLocalPath.getFileSystem(xlConf).exists(hostLocalPath)) { FSDataInputStream in = hostLocalPath.getFileSystem(xlConf).open(hostLocalPath); BufferedReader br = new BufferedReader(new InputStreamReader(in)); line = br.readLine(); hostLocals = line.split(","); LOG.info("now in buildContainerRequest, host local is: " + Arrays.toString(hostLocals)); in.close();
/** @throws Exception If failed. */ @Test public void testDeleteSuccessfulIfPathIsOpenedToRead() throws Exception { Path fsHome = new Path(primaryFsUri); final Path file = new Path(fsHome, "myFile"); FSDataOutputStream os = fs.create(file, false, 128); final int cnt = 5 * FileSystemConfiguration.DFLT_BLOCK_SIZE; // Write 5 blocks. for (int i = 0; i < cnt; i++) os.writeInt(i); os.close(); final FSDataInputStream is = fs.open(file, -1); for (int i = 0; i < cnt / 2; i++) assertEquals(i, is.readInt()); assert fs.delete(file, false); assert !fs.exists(file); is.close(); }
/** @throws Exception If failed. */ @Test public void testOpenIfPathIsAlreadyOpened() throws Exception { Path fsHome = new Path(primaryFsUri); Path file = new Path(fsHome, "someFile"); FSDataOutputStream os = fs.create(file); os.close(); FSDataInputStream is1 = fs.open(file); FSDataInputStream is2 = fs.open(file); is1.close(); is2.close(); }
@Test public void testReadRequestRangeNotSatisfiable() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_REQUESTED_RANGE_NOT_SATISFIABLE); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { assertEquals(inputStream.read(), -1); } } }
@SuppressWarnings({"ResultOfMethodCallIgnored", "OverlyStrongTypeCast", "ConstantConditions"}) @Test public void testReadRetryCounters() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { int maxRetries = 2; MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectHttpErrorCode(SC_INTERNAL_SERVER_ERROR); Configuration configuration = new Configuration(); configuration.set(S3_MAX_BACKOFF_TIME, "1ms"); configuration.set(S3_MAX_RETRY_TIME, "5s"); configuration.setInt(S3_MAX_CLIENT_RETRIES, maxRetries); fs.initialize(new URI("s3n://test-bucket/"), configuration); fs.setS3Client(s3); try (FSDataInputStream inputStream = fs.open(new Path("s3n://test-bucket/test"))) { inputStream.read(); } catch (Throwable expected) { assertInstanceOf(expected, AmazonS3Exception.class); assertEquals(((AmazonS3Exception) expected).getStatusCode(), SC_INTERNAL_SERVER_ERROR); assertEquals(PrestoS3FileSystem.getFileSystemStats().getReadRetries().getTotalCount(), maxRetries); assertEquals(PrestoS3FileSystem.getFileSystemStats().getGetObjectRetries().getTotalCount(), (maxRetries + 1L) * maxRetries); } } }
Path dir = PerformanceEvaluation.writeInputFile(HTU.getConfiguration(), opts, HTU.getDataTestDir()); FileSystem fs = FileSystem.get(HTU.getConfiguration()); Path p = new Path(dir, PerformanceEvaluation.JOB_INPUT_FILENAME); long len = fs.getFileStatus(p).getLen(); assertTrue(len > 0); byte [] content = new byte[(int)len]; FSDataInputStream dis = fs.open(p); try { dis.readFully(content); BufferedReader br = new BufferedReader(new InputStreamReader(new ByteArrayInputStream(content))); dis.close();
/** @throws Exception If failed. */ @Test public void testGetFileBlockLocations() throws Exception { Path igfsHome = new Path(PRIMARY_URI); Path file = new Path(igfsHome, "someFile"); try (OutputStream out = new BufferedOutputStream(fs.create(file, true, 1024 * 1024))) { byte[] data = new byte[128 * 1024]; for (int i = 0; i < 100; i++) out.write(data); out.flush(); } try (FSDataInputStream in = fs.open(file, 1024 * 1024)) { byte[] data = new byte[128 * 1024]; int read; do { read = in.read(data); } while (read > 0); } FileStatus status = fs.getFileStatus(file); int grpLen = 128 * 512 * 1024; int grpCnt = (int)((status.getLen() + grpLen - 1) / grpLen); BlockLocation[] locations = fs.getFileBlockLocations(status, 0, status.getLen()); assertEquals(grpCnt, locations.length); }