/** * Load the properties from the specified file into a {@link Properties} object. * * @param fileName the name of the file to load properties from * @param conf configuration object to determine the file system to be used * @return a new {@link Properties} instance */ public static Properties fileToProperties(String fileName, Configuration conf) throws IOException, ConfigurationException { PropertiesConfiguration propsConfig = new PropertiesConfiguration(); Path filePath = new Path(fileName); URI fileURI = filePath.toUri(); if (fileURI.getScheme() == null && fileURI.getAuthority() == null) { propsConfig.load(FileSystem.getLocal(conf).open(filePath)); } else { propsConfig.load(filePath.getFileSystem(conf).open(filePath)); } return ConfigurationConverter.getProperties(propsConfig); }
public static byte[] getLocalSha(Path path, Configuration conf) throws IOException { InputStream is = null; try { is = FileSystem.getLocal(conf).open(path); return DigestUtils.sha256(is); } finally { if (is != null) { is.close(); } } }
/** * Copy local file to HDFS. This is used to set as classpath by distributed cache. */ private static Path shipToHDFS(Configuration conf, String fileName) throws IOException { Path dst = new Path(File.separator + "tmp" + File.separator + System.getProperty("user.name") + "_" + System.currentTimeMillis(), fileName.substring(fileName.lastIndexOf(File.separator) + 1)); FileSystem fs = dst.getFileSystem(conf); OutputStream os = null; InputStream is = null; try { is = FileSystem.getLocal(conf).open(new Path(fileName)); os = fs.create(dst); IOUtils.copyBytes(is, os, 4096, true); } finally { org.apache.commons.io.IOUtils.closeQuietly(is); // IOUtils should not close stream to HDFS quietly if(os != null) { os.close(); } } return dst; }
/** * Copy local file to HDFS. This is used to set as classpath by distributed cache. */ private static Path shipToHDFS(Configuration conf, String fileName) throws IOException { Path dst = new Path(File.separator + "tmp" + File.separator + System.getProperty("user.name") + "_" + System.currentTimeMillis(), fileName.substring(fileName.lastIndexOf(File.separator) + 1)); FileSystem fs = dst.getFileSystem(conf); OutputStream os = null; InputStream is = null; try { is = FileSystem.getLocal(conf).open(new Path(fileName)); os = fs.create(dst); IOUtils.copyBytes(is, os, 4096, true); } finally { org.apache.commons.io.IOUtils.closeQuietly(is); // IOUtils should not close stream to HDFS quietly if(os != null) { os.close(); } } return dst; }
/** * Copy local file to HDFS. This is used to set as classpath by distributed cache. */ private static Path shipToHDFS(Configuration conf, String fileName) throws IOException { Path dst = new Path(File.separator + "tmp" + File.separator + System.getProperty("user.name") + "_" + System.currentTimeMillis(), fileName.substring(fileName.lastIndexOf(File.separator) + 1)); FileSystem fs = dst.getFileSystem(conf); OutputStream os = null; InputStream is = null; try { is = FileSystem.getLocal(conf).open(new Path(fileName)); os = fs.create(dst); IOUtils.copyBytes(is, os, 4096, true); } finally { org.apache.commons.io.IOUtils.closeQuietly(is); // IOUtils should not close stream to HDFS quietly if(os != null) { os.close(); } } return dst; }
/** * Copy local file to HDFS. This is used to set as classpath by distributed cache. */ private static Path shipToHDFS(Configuration conf, String fileName) throws IOException { Path dst = new Path(File.separator + "tmp" + File.separator + System.getProperty("user.name") + "_" + System.currentTimeMillis(), fileName.substring(fileName.lastIndexOf(File.separator) + 1)); FileSystem fs = dst.getFileSystem(conf); OutputStream os = null; InputStream is = null; try { is = FileSystem.getLocal(conf).open(new Path(fileName)); os = fs.create(dst); IOUtils.copyBytes(is, os, 4096, true); } finally { org.apache.commons.io.IOUtils.closeQuietly(is); // IOUtils should not close stream to HDFS quietly if(os != null) { os.close(); } } return dst; }
/** * Load the properties from the specified file into a {@link Properties} object. * * @param fileName the name of the file to load properties from * @param conf configuration object to determine the file system to be used * @return a new {@link Properties} instance */ public static Properties fileToProperties(String fileName, Configuration conf) throws IOException, ConfigurationException { PropertiesConfiguration propsConfig = new PropertiesConfiguration(); Path filePath = new Path(fileName); URI fileURI = filePath.toUri(); if (fileURI.getScheme() == null && fileURI.getAuthority() == null) { propsConfig.load(FileSystem.getLocal(conf).open(filePath)); } else { propsConfig.load(filePath.getFileSystem(conf).open(filePath)); } return ConfigurationConverter.getProperties(propsConfig); }
/** * Load the properties from the specified file into a {@link Properties} object. * * @param fileName the name of the file to load properties from * @param conf configuration object to determine the file system to be used * @return a new {@link Properties} instance */ public static Properties fileToProperties(String fileName, Configuration conf) throws IOException, ConfigurationException { PropertiesConfiguration propsConfig = new PropertiesConfiguration(); Path filePath = new Path(fileName); URI fileURI = filePath.toUri(); if (fileURI.getScheme() == null && fileURI.getAuthority() == null) { propsConfig.load(FileSystem.getLocal(conf).open(filePath)); } else { propsConfig.load(filePath.getFileSystem(conf).open(filePath)); } return ConfigurationConverter.getProperties(propsConfig); }
public static void main(String... args) throws Exception { Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(config); Class<?> codecClass = Class.forName(args[0]); CompressionCodec codec = (CompressionCodec) ReflectionUtils.newInstance(codecClass, config); InputStream is = FileSystem.getLocal(config).open(new Path(args[1])); OutputStream os = hdfs.create( new Path(args[2] + codec.getDefaultExtension())); OutputStream cos = codec.createOutputStream(os); IOUtils.copyBytes(is, cos, config, true); IOUtils.closeStream(os); IOUtils.closeStream(is); } }
private void checkFileCorruption(LocalFileSystem fileSys, Path file, Path fileToCorrupt) throws IOException { // corrupt the file RandomAccessFile out = new RandomAccessFile(new File(fileToCorrupt.toString()), "rw"); byte[] buf = new byte[(int)fileSys.getFileStatus(file).getLen()]; int corruptFileLen = (int)fileSys.getFileStatus(fileToCorrupt).getLen(); assertTrue(buf.length >= corruptFileLen); rand.nextBytes(buf); out.seek(corruptFileLen/2); out.write(buf, 0, corruptFileLen/4); out.close(); boolean gotException = false; InputStream in = fileSys.open(file); try { IOUtils.readFully(in, buf, 0, buf.length); } catch (ChecksumException e) { gotException = true; } assertTrue(gotException); in.close(); }
@Override protected InputStream getInputStream(int bufferSize, byte[] key, byte[] iv) throws IOException { return new CryptoInputStream(fileSys.open(file), codec, bufferSize, key, iv); }
@Override protected InputStream getInputStream(int bufferSize, byte[] key, byte[] iv) throws IOException { return new CryptoInputStream(fileSys.open(file), codec, bufferSize, key, iv); }
private void testFileCorruption(LocalFileSystem fileSys) throws IOException { // create a file and verify that checksum corruption results in // a checksum exception on LocalFS String dir = System.getProperty("test.build.data", "."); Path file = new Path(dir + "/corruption-test.dat"); Path crcFile = new Path(dir + "/.corruption-test.dat.crc"); writeFile(fileSys, file); int fileLen = (int)fileSys.getFileStatus(file).getLen(); byte [] buf = new byte[fileLen]; InputStream in = fileSys.open(file); IOUtils.readFully(in, buf, 0, buf.length); in.close(); // check .crc corruption checkFileCorruption(fileSys, file, crcFile); fileSys.delete(file, true); writeFile(fileSys, file); // check data corrutpion checkFileCorruption(fileSys, file, file); fileSys.delete(file, true); }
private String readFile(String out) throws IOException { Path path = new Path(out); FileStatus stat = lfs.getFileStatus(path); FSDataInputStream in = lfs.open(path); byte[] buffer = new byte[(int)stat.getLen()]; in.readFully(buffer); in.close(); lfs.delete(path, false); return new String(buffer); }
private String readFile(String out) throws IOException { Path path = new Path(out); FileStatus stat = lfs.getFileStatus(path); FSDataInputStream in = lfs.open(path); byte[] buffer = new byte[(int)stat.getLen()]; in.readFully(buffer); in.close(); lfs.delete(path, false); return new String(buffer); }
private void checkFileCorruption(LocalFileSystem fileSys, Path file, Path fileToCorrupt) throws IOException { // corrupt the file RandomAccessFile out = new RandomAccessFile(new File(fileToCorrupt.toString()), "rw"); byte[] buf = new byte[(int)fileSys.getFileStatus(file).getLen()]; int corruptFileLen = (int)fileSys.getFileStatus(fileToCorrupt).getLen(); assertTrue(buf.length >= corruptFileLen); rand.nextBytes(buf); out.seek(corruptFileLen/2); out.write(buf, 0, corruptFileLen/4); out.close(); boolean gotException = false; InputStream in = fileSys.open(file); try { IOUtils.readFully(in, buf, 0, buf.length); } catch (ChecksumException e) { gotException = true; } assertTrue(gotException); in.close(); }
@Test public void testStreamType() throws Exception { Path testPath = new Path(TEST_ROOT_DIR, "testStreamType"); localFs.create(testPath).close(); FSDataInputStream in = null; localFs.setVerifyChecksum(true); in = localFs.open(testPath); assertTrue("stream is input checker", in.getWrappedStream() instanceof FSInputChecker); localFs.setVerifyChecksum(false); in = localFs.open(testPath); assertFalse("stream is not input checker", in.getWrappedStream() instanceof FSInputChecker); }
@Test public void testStreamType() throws Exception { Path testPath = new Path(TEST_ROOT_DIR, "testStreamType"); localFs.create(testPath).close(); FSDataInputStream in = null; localFs.setVerifyChecksum(true); in = localFs.open(testPath); assertTrue("stream is input checker", in.getWrappedStream() instanceof FSInputChecker); localFs.setVerifyChecksum(false); in = localFs.open(testPath); assertFalse("stream is not input checker", in.getWrappedStream() instanceof FSInputChecker); }
private void testFileCorruption(LocalFileSystem fileSys) throws IOException { // create a file and verify that checksum corruption results in // a checksum exception on LocalFS String dir = PathUtils.getTestDirName(getClass()); Path file = new Path(dir + "/corruption-test.dat"); Path crcFile = new Path(dir + "/.corruption-test.dat.crc"); writeFile(fileSys, file); int fileLen = (int)fileSys.getFileStatus(file).getLen(); byte [] buf = new byte[fileLen]; InputStream in = fileSys.open(file); IOUtils.readFully(in, buf, 0, buf.length); in.close(); // check .crc corruption checkFileCorruption(fileSys, file, crcFile); fileSys.delete(file, true); writeFile(fileSys, file); // check data corrutpion checkFileCorruption(fileSys, file, file); fileSys.delete(file, true); }
public static void uploadFile(HomeFileConf homeFileStore, Path inputFile, String name, String extension ,FileFormat fileFormat, FolderPath parent) throws Exception { FilePath filePath; if (parent == null) { filePath = new FilePath(ImmutableList.of(HomeName.getUserHomePath(DEFAULT_USER_NAME).getName(), name)); } else { List<String> path = Lists.newArrayList(parent.toPathList()); path.add(name); filePath = new FilePath(path); } FSDataInputStream inputStream = FileSystem.getLocal(new Configuration()).open(inputFile); FileSystem fs = homeFileStore.getFilesystemAndCreatePaths(null); Path stagingLocation = new HomeFileTool(homeFileStore, fs, "localhost").stageFile(filePath, extension, inputStream); Path finalLocation = new HomeFileTool(homeFileStore, fs, "localhost").saveFile(stagingLocation, filePath, extension); inputStream.close(); // create file in namespace fileFormat.setFullPath(filePath.toPathList()); fileFormat.setName(name); fileFormat.setLocation(finalLocation.toString()); DatasetConfig datasetConfig = DatasetsUtil.toDatasetConfig(fileFormat.asFileConfig(), DatasetType.PHYSICAL_DATASET_HOME_FILE, null, null); newCatalogService().getCatalog(SchemaConfig.newBuilder(SystemUser.SYSTEM_USERNAME).build()).createOrUpdateDataset(newNamespaceService(), new NamespaceKey(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME), filePath.toNamespaceKey(), datasetConfig); }