/** * Returns the Path where the YARN application files should be uploaded to. * * @param appId YARN application id */ private Path getYarnFilesDir(final ApplicationId appId) throws IOException { final FileSystem fileSystem = FileSystem.get(yarnConfiguration); final Path homeDir = fileSystem.getHomeDirectory(); return new Path(homeDir, ".flink/" + appId + '/'); }
/** * Create FileSystem object */ public FileSystem createFs() throws IOException { fs = FileSystem.get(new Configuration()); return fs; }
/** * Creates the HDFS filesystem to store output files. * * @param conf Hadoop configuration */ private void createHdfsFilesystem(Configuration conf) throws Exception { // Inits HDFS file system object mFileSystem = FileSystem.get(URI.create(conf.get("fs.defaultFS")), conf); mOutputFilePath = new Path("./MapReduceOutputFile"); if (mFileSystem.exists(mOutputFilePath)) { mFileSystem.delete(mOutputFilePath, true); } }
private PasswordManager(CachedInstanceKey cacheKey) { this.useStrongEncryptor = cacheKey.useStrongEncryptor; try { this.fs = cacheKey.fsURI != null ? FileSystem.get(URI.create(cacheKey.fsURI), new Configuration()) : (cacheKey.masterPasswordFile != null ? new Path(cacheKey.masterPasswordFile).getFileSystem(new Configuration()) : null); } catch (IOException e) { LOG.warn("Failed to instantiate FileSystem.", e); } this.encryptors = getEncryptors(cacheKey); }
public static List<String> listPaths(JavaSparkContext sc, String path, boolean recursive) throws IOException { if (path.endsWith(".blob.core.windows.net/") || path.endsWith(".blob.core.windows.net")) { //Azure library bug: seems that we get an infinite loop if we try to list paths on the // root directory, for some versions of the Azure Hadoop library - deadlocks on fileIter.hasNext() throw new IllegalStateException("Cannot list paths from root directory due to Azure library bug"); } List<String> paths = new ArrayList<>(); Configuration config = new Configuration(); FileSystem hdfs = FileSystem.get(URI.create(path), config); RemoteIterator fileIter = hdfs.listFiles(new Path(path), recursive); while (fileIter.hasNext()) { String filePath = ((LocatedFileStatus) fileIter.next()).getPath().toString(); paths.add(filePath); } return paths; }
public void start() throws IOException { random = new SecureRandom(); userProvider = UserProvider.instantiate(conf); ugiReferenceCounter = new ConcurrentHashMap<>(); fs = FileSystem.get(conf); baseStagingDir = new Path(FSUtils.getRootDir(conf), HConstants.BULKLOAD_STAGING_DIR_NAME); if (conf.get("hbase.bulkload.staging.dir") != null) { LOG.warn("hbase.bulkload.staging.dir " + " is deprecated. Bulkload staging directory is " + baseStagingDir); } if (!fs.exists(baseStagingDir)) { fs.mkdirs(baseStagingDir, PERM_HIDDEN); } }
@Test public void testCorrectComputationOfTargetPathsWhenUsingGlob() throws Exception { Path source = new Path("/source/directory"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1")); List<FileStatus> targetFiles = Lists.newArrayList(); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); FileSystem sourceUnderlyingFS = FileSystem.getLocal(new Configuration()); FileSystem sourceFs = Mockito.spy(sourceUnderlyingFS); Mockito.doReturn(new FileStatus(0, true, 0, 0, 0, source)).when(sourceFs).getFileStatus(source); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, new Path(target, "directory"), sourceFiles, targetFiles, properties, new Path("/source/*"), sourceFs); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.get(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 1); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "directory/file1")); }
private ResolverUrlClassLoader createClassLoader(ClassLoader parent, org.apache.hadoop.fs.Path path) throws IOException { Path tempPath = Files.createTempFile("hbase-coprocessor-", ".jar"); org.apache.hadoop.fs.Path destination = new org.apache.hadoop.fs.Path(tempPath.toString()); LOG.debug("Copying coprocessor jar '{}' to '{}'.", path, tempPath); FileSystem fileSystem = FileSystem.get(getConf()); fileSystem.copyToLocalFile(path, destination); URL url = tempPath.toUri().toURL(); return createClassLoader(new URL[] { url }, parent); }
private List<Path> getFilesRecursively(String fileBackupDir) throws IllegalArgumentException, IOException { FileSystem fs = FileSystem.get((new Path(fileBackupDir)).toUri(), new Configuration()); List<Path> list = new ArrayList<>(); RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true); while (it.hasNext()) { Path p = it.next().getPath(); if (HFile.isHFileFormat(fs, p)) { list.add(p); } } return list; }
@Test public void TestGetJobWorkingDir() throws IOException { FileSystem fileSystem = FileSystem.get(new Configuration()); Path jobWorkDirPath = null; KylinConfig kylinConfig = mock(KylinConfig.class); try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig)) { when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true); when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/"); DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class); defaultChainedExecutable.setId(RandomUtil.randomUUID().toString()); String jobWorkingDir = HiveInputBase.getJobWorkingDir(defaultChainedExecutable, KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); jobWorkDirPath = new Path(jobWorkingDir); Assert.assertTrue(fileSystem.exists(jobWorkDirPath)); } finally { if (jobWorkDirPath != null) fileSystem.deleteOnExit(jobWorkDirPath); } }
public PerformTestRCFileAndSeqFile(boolean local, String file) throws IOException { if (local) { fs = FileSystem.getLocal(conf); } else { fs = FileSystem.get(conf); } conf.setInt(RCFile.Writer.COLUMNS_BUFFER_SIZE_CONF_STR, 1 * 1024 * 1024); if (file == null) { Path dir = new Path(System.getProperty("test.tmp.dir", ".") + "/mapred"); testRCFile = new Path(dir, "test_rcfile"); testSeqFile = new Path(dir, "test_seqfile"); } else { testRCFile = new Path(file + "-rcfile"); testSeqFile = new Path(file + "-seqfile"); } fs.delete(testRCFile, true); fs.delete(testSeqFile, true); System.out.println("RCFile:" + testRCFile.toString()); System.out.println("SequenceFile:" + testSeqFile.toString()); }
@BeforeClass public static void beforeClass() throws Throwable { // Up the handlers; this test needs more than usual. UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); UTIL.startMiniCluster(); FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString(); }
/** * Uploads a local file to HDFS * This method is not thread safe * * @param source * @param conf * @return * @throws IOException */ public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException { Path localFile = new Path(source.getPath()); Path remoteFile = new Path(SessionState.get().getSparkSession().getHDFSSessionDir(), getFileName(source)); FileSystem fileSystem = FileSystem.get(remoteFile.toUri(), conf); // Overwrite if the remote file already exists. Whether the file can be added // on executor is up to spark, i.e. spark.files.overwrite fileSystem.copyFromLocalFile(false, true, localFile, remoteFile); Path fullPath = fileSystem.getFileStatus(remoteFile).getPath(); return fullPath.toUri(); }
private static void removeExportDir(final Path path) throws IOException { FileSystem fs = FileSystem.get(path.toUri(), new Configuration()); fs.delete(path, true); } }
public int processFile(String fileName) throws IOException { Path path = new Path(fileName); FileSystem fs; if (!path.toUri().isAbsolute()) { fs = FileSystem.getLocal(conf); path = fs.makeQualified(path); } else { fs = FileSystem.get(path.toUri(), conf); } BufferedReader bufferReader = null; int rc = 0; try { bufferReader = new BufferedReader(new InputStreamReader(fs.open(path))); rc = processReader(bufferReader); } finally { IOUtils.closeStream(bufferReader); } return rc; }
@Before public void setup() throws IOException { // Set it to a non-standard port. TEST_UTIL.getConfiguration().setInt(HConstants.ZOOKEEPER_CLIENT_PORT, PORT_NO); this.dataDir = TEST_UTIL.getDataTestDir(this.getClass().getName()); FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration()); if (fs.exists(this.dataDir)) { if (!fs.delete(this.dataDir, true)) { throw new IOException("Failed cleanup of " + this.dataDir); } } if (!fs.mkdirs(this.dataDir)) { throw new IOException("Failed create of " + this.dataDir); } }
private void moveTmpPathToOutputPath() throws IOException { Retryer<Void> retryer = RetryerFactory.newInstance(this.retrierConfig); LOG.info(String.format("Moving %s to %s", this.dataset.outputTmpPath(), this.dataset.outputPath())); this.fs.delete(this.dataset.outputPath(), true); if (this.isRetryEnabled) { try { retryer.call(() -> { if (fs.exists(this.dataset.outputPath())) { throw new IOException("Path " + this.dataset.outputPath() + " exists however it should not. Will wait more."); } return null; }); } catch (Exception e) { throw new IOException(e); } } WriterUtils.mkdirsWithRecursivePermissionWithRetry(MRCompactorJobRunner.this.fs, this.dataset.outputPath().getParent(), this.perm, this.retrierConfig); Log.info("Moving from fs: ("+MRCompactorJobRunner.this.tmpFs.getUri()+") path: "+ this.dataset.outputTmpPath() + " to "+ "fs: ("+ FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()).getUri()+") output path: " + this.dataset.outputPath()); HadoopUtils.movePath (MRCompactorJobRunner.this.tmpFs, this.dataset.outputTmpPath(), FileSystem.get(this.dataset.outputPath().getParent().toUri(), this.fs.getConf()), this.dataset.outputPath(), false, this.fs.getConf()) ; }
private String setRootDirAndCleanIt(final HBaseTestingUtility htu, final String subdir) throws IOException { Path testdir = htu.getDataTestDir(subdir); FileSystem fs = FileSystem.get(htu.getConfiguration()); if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true)); FSUtils.setRootDir(htu.getConfiguration(), testdir); return FSUtils.getRootDir(htu.getConfiguration()).toString(); }