private String pathToObjectName(final Path path) { org.apache.hadoop.fs.Path hadoopPath = HadoopFileSystem.toHadoopPath(path); if (!hadoopPath.isAbsolute()) { hadoopPath = new org.apache.hadoop.fs.Path(fs.getWorkingDirectory(), hadoopPath); } return hadoopPath.toUri().getScheme() != null && hadoopPath.toUri().getPath().isEmpty() ? "" : hadoopPath.toUri().getPath().substring(1); }
@Override public void abortJob(JobContext context, int status) throws IOException { JobConf conf = ShimLoader.getHadoopShims().getJobConf(context); Path tmpLocation = new Path(conf.get(TMP_LOCATION)); FileSystem fs = tmpLocation.getFileSystem(conf); LOG.debug("Removing " + tmpLocation.toString()); fs.delete(tmpLocation, true); } }
@Test(enabled = false) public void testCreate7() throws IOException, URISyntaxException { HDFSRoot hdfsRoot = new HDFSRoot("/tmp/create"); MetricsFileSystemInstrumentation fs = (MetricsFileSystemInstrumentation) FileSystem.get(new URI(instrumentedURI), new Configuration()); FsPermission permission = new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.READ); Path newFile = new Path("/tmp/create/newFile"); FSDataOutputStream fstream = fs.create(newFile, permission, true, 100, (short)2, 1048576, null); Assert.assertEquals(fs.createTimer.getCount(), 1); fstream.close(); hdfsRoot.cleanupRoot(); }
@Test public void testGetMetadataNotFound() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectMetadataHttpCode(SC_NOT_FOUND); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); assertEquals(fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")), null); } }
@Test public void testFromConfigurationOverride() throws Exception { Configuration configuration = new Configuration(); configuration.set("fs.file.impl", InstrumentedLocalFileSystem.class.getName()); FileSystem fs = FileSystem.newInstance(new URI("file:///"), configuration); Assert.assertTrue(fs instanceof InstrumentedLocalFileSystem); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); Assert.assertEquals(fs.getFileStatus(new Path("/tmp")).getPath(), new Path("file:///tmp")); Assert.assertEquals(fs.getUri().getScheme(), "file"); }
@Test(expectedExceptions = IOException.class, expectedExceptionsMessageRegExp = "Configured staging path is not a directory: .*") public void testCreateWithStagingDirectoryFile() throws Exception { java.nio.file.Path staging = createTempFile("staging", null); // staging = /tmp/stagingXXX.tmp try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); Configuration conf = new Configuration(); conf.set(S3_STAGING_DIRECTORY, staging.toString()); fs.initialize(new URI("s3n://test-bucket/"), conf); fs.setS3Client(s3); fs.create(new Path("s3n://test-bucket/test")); } finally { Files.deleteIfExists(staging); } }
/** * Creates the HDFS filesystem to store output files. * * @param conf Hadoop configuration */ private void createHdfsFilesystem(Configuration conf) throws Exception { // Inits HDFS file system object mFileSystem = FileSystem.get(URI.create(conf.get("fs.defaultFS")), conf); mOutputFilePath = new Path("./MapReduceOutputFile"); if (mFileSystem.exists(mOutputFilePath)) { mFileSystem.delete(mOutputFilePath, true); } }
@Test public void testUnderscoreBucket() throws Exception { Configuration config = new Configuration(); config.setBoolean(S3_PATH_STYLE_ACCESS, true); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); String expectedBucketName = "test-bucket_underscore"; fs.initialize(new URI("s3n://" + expectedBucketName + "/"), config); fs.setS3Client(s3); fs.getS3ObjectMetadata(new Path("/test/path")); assertEquals(expectedBucketName, s3.getGetObjectMetadataRequest().getBucketName()); } }
@Override public void killAll() throws IOException { log.info("Deleting all segment files from hdfs dir [%s].", storageDirectory.toUri().toString()); final FileSystem fs = storageDirectory.getFileSystem(config); fs.delete(storageDirectory, true); }
@Override public void killCluster(ApplicationId applicationId) throws FlinkException { try { yarnClient.killApplication(applicationId); Utils.deleteApplicationFiles(Collections.singletonMap( YarnConfigKeys.FLINK_YARN_FILES, getYarnFilesDir(applicationId).toUri().toString())); } catch (YarnException | IOException e) { throw new FlinkException("Could not kill the Yarn Flink cluster with id " + applicationId + '.', e); } }
/** @throws Exception If failed. */ @Test public void testSetWorkingDirectory() throws Exception { Path dir = new Path("/tmp/nested/dir"); Path file = new Path("file"); fs.mkdirs(dir); fs.setWorkingDirectory(dir); FSDataOutputStream os = fs.create(file); os.close(); String filePath = fs.getFileStatus(new Path(dir, file)).getPath().toString(); assertTrue(filePath.contains("/tmp/nested/dir/file")); }
public static String keyFromPath(Path path) { checkArgument(path.isAbsolute(), "Path is not absolute: %s", path); String key = nullToEmpty(path.toUri().getPath()); if (key.startsWith(PATH_SEPARATOR)) { key = key.substring(PATH_SEPARATOR.length()); } if (key.endsWith(PATH_SEPARATOR)) { key = key.substring(0, key.length() - PATH_SEPARATOR.length()); } return key; }
public static Path prependFSIfNullScheme(FileSystem fs, Path path) { if (path.toUri().getScheme() == null) { path = fs.makeQualified(path); } return path; }
public static String runTask(String[] args) throws Exception { String workingPath = args[0]; log.info("Deleting indexing hadoop working path [%s].", workingPath); Path p = new Path(workingPath); FileSystem fs = p.getFileSystem(new Configuration()); fs.delete(p, true); return null; } }
@SuppressWarnings("unchecked") public static HadoopDruidIndexerConfig fromDistributedFileSystem(String path) { try { Path pt = new Path(path); FileSystem fs = pt.getFileSystem(new Configuration()); Reader reader = new InputStreamReader(fs.open(pt), StandardCharsets.UTF_8); return fromMap( HadoopDruidIndexerConfig.JSON_MAPPER.readValue(reader, JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT) ); } catch (Exception e) { throw Throwables.propagate(e); } }
private void copyLocal(String var, Configuration conf) throws IOException { String[] filenames = TempletonUtils.decodeArray(conf.get(var)); if (filenames != null) { for (String filename : filenames) { Path src = new Path(filename); Path dst = new Path(src.getName()); FileSystem fs = src.getFileSystem(conf); LOG.info("templeton: copy " + src + " => " + dst); fs.copyToLocalFile(src, dst); } } }
/** * @param c configuration * @return {@link Path} to hbase root directory from * configuration as a qualified Path. * @throws IOException e */ public static Path getRootDir(final Configuration c) throws IOException { Path p = new Path(c.get(HConstants.HBASE_DIR)); FileSystem fs = p.getFileSystem(c); return p.makeQualified(fs.getUri(), fs.getWorkingDirectory()); }