public void mkdirs(String folder) throws IOException { Path path = new Path(folder); FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.setVerifyChecksum(false); FileSystem.getLocal(conf).setVerifyChecksum(false); if (!fs.exists(path)) { fs.mkdirs(path); System.out.println("Create: " + folder); } fs.close(); }
public void ls(String folder) throws IOException { Path path = new Path(folder); FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.setVerifyChecksum(false); FileSystem.getLocal(conf).setVerifyChecksum(false); FileStatus[] list = fs.listStatus(path); System.out.println("ls: " + folder); for (FileStatus f : list) { System.out.printf("name: %s, folder: %s, size: %d\n", f.getPath(), f.isDir(), f.getLen()); } fs.close(); }
public void download(String remote, String local) throws IOException { Path path = new Path(remote); FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.setVerifyChecksum(false); FileSystem.getLocal(conf).setVerifyChecksum(false); fs.copyToLocalFile(path, new Path(local)); System.out.println("download: from " + remote + " to " + local); fs.close(); }
public void copyFile(String local, String remote) throws IOException { FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.setVerifyChecksum(false); FileSystem.getLocal(conf).setVerifyChecksum(false); fs.copyFromLocalFile(new Path(local), new Path(remote)); System.out.println("copy from: " + local + " to " + remote); fs.close(); }
public void createFile(String file, String content) throws IOException { FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.setVerifyChecksum(false); FileSystem.getLocal(conf).setVerifyChecksum(false); byte[] buff = content.getBytes(); FSDataOutputStream os = null; try { os = fs.create(new Path(file)); os.write(buff, 0, buff.length); System.out.println("Create: " + file); } finally { if (os != null) os.close(); } fs.close(); }
public void rmr(String folder) throws IOException { Path path = new Path(folder); FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.setVerifyChecksum(false); FileSystem.getLocal(conf).setVerifyChecksum(false); fs.deleteOnExit(path); System.out.println("Delete: " + folder); fs.close(); }
public void cat(String remoteFile) throws IOException { Path path = new Path(remoteFile); FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf); fs.setVerifyChecksum(false); FileSystem.getLocal(conf).setVerifyChecksum(false); FSDataInputStream fsdis = null; System.out.println("cat: " + remoteFile); try { fsdis = fs.open(path); IOUtils.copyBytes(fsdis, System.out, 4096, false); } finally { IOUtils.closeStream(fsdis); fs.close(); } } }
@Before public void resetLocalFs() throws Exception { localFs = FileSystem.getLocal(new Configuration()); localFs.setVerifyChecksum(true); }
@Before public void resetLocalFs() throws Exception { localFs = FileSystem.getLocal(new Configuration()); localFs.setVerifyChecksum(true); }
@Test public void testStreamType() throws Exception { Path testPath = new Path(TEST_ROOT_DIR, "testStreamType"); localFs.create(testPath).close(); FSDataInputStream in = null; localFs.setVerifyChecksum(true); in = localFs.open(testPath); assertTrue("stream is input checker", in.getWrappedStream() instanceof FSInputChecker); localFs.setVerifyChecksum(false); in = localFs.open(testPath); assertFalse("stream is not input checker", in.getWrappedStream() instanceof FSInputChecker); }
@Test public void testStreamType() throws Exception { Path testPath = new Path(TEST_ROOT_DIR, "testStreamType"); localFs.create(testPath).close(); FSDataInputStream in = null; localFs.setVerifyChecksum(true); in = localFs.open(testPath); assertTrue("stream is input checker", in.getWrappedStream() instanceof FSInputChecker); localFs.setVerifyChecksum(false); in = localFs.open(testPath); assertFalse("stream is not input checker", in.getWrappedStream() instanceof FSInputChecker); }
localFs.setVerifyChecksum(false); String str = readFile(localFs, testPath, 1024).toString(); assertTrue("read", "testing truncation".equals(str));
localFs.setVerifyChecksum(false); String str = readFile(localFs, testPath, 1024).toString(); assertTrue("read", "testing truncation".equals(str));
localFs.setVerifyChecksum(true); readFile(localFs, testPath, 1024); } catch (ChecksumException ce) { localFs.setVerifyChecksum(false); String str = readFile(localFs, testPath, 1024); assertEquals("testing stale checksum", str);
localFs.setVerifyChecksum(true); readFile(localFs, testPath, 1024); } catch (ChecksumException ce) { localFs.setVerifyChecksum(false); String str = readFile(localFs, testPath, 1024); assertEquals("testing stale checksum", str);
@Before public void prepFiles() throws Exception { lfs.setVerifyChecksum(true); lfs.setWriteChecksum(true); lfs.delete(srcPath, true); lfs.delete(dstPath, true); FSDataOutputStream out = lfs.create(srcPath); out.writeChars("hi"); out.close(); assertTrue(lfs.exists(lfs.getChecksumFile(srcPath))); }
@Before public void prepFiles() throws Exception { lfs.setVerifyChecksum(true); lfs.setWriteChecksum(true); lfs.delete(srcPath, true); lfs.delete(dstPath, true); FSDataOutputStream out = lfs.create(srcPath); out.writeChars("hi"); out.close(); assertTrue(lfs.exists(lfs.getChecksumFile(srcPath))); }
localFs.setVerifyChecksum(false); String str = readFile(localFs, testPath, 1024).toString(); assertTrue("read", "testing".equals(str));
localFs.setVerifyChecksum(false); String str = readFile(localFs, testPath, 1024).toString(); assertTrue("read", "testing".equals(str));
@Before public void init() throws IOException { // Initialize a local spark env jsc = new JavaSparkContext(HoodieClientTestUtils.getSparkConfForTest("TestHoodieClient")); jsc.setLogLevel("ERROR"); //SQLContext stuff sqlContext = new SQLContext(jsc); folder = new TemporaryFolder(); folder.create(); basePath = folder.getRoot().getAbsolutePath(); fs = FSUtils.getFs(basePath, jsc.hadoopConfiguration()); if (fs instanceof LocalFileSystem) { LocalFileSystem lfs = (LocalFileSystem) fs; // With LocalFileSystem, with checksum disabled, fs.open() returns an inputStream which is FSInputStream // This causes ClassCastExceptions in LogRecordScanner (and potentially other places) calling fs.open // So, for the tests, we enforce checksum verification to circumvent the problem lfs.setVerifyChecksum(true); } HoodieTestUtils.initTableType(jsc.hadoopConfiguration(), basePath, getTableType()); dataGen = new HoodieTestDataGenerator(); }