public static String runTask(String[] args) throws Exception { String workingPath = args[0]; log.info("Deleting indexing hadoop working path [%s].", workingPath); Path p = new Path(workingPath); FileSystem fs = p.getFileSystem(new Configuration()); fs.delete(p, true); return null; } }
@VisibleForTesting static void serializeJobState(FileSystem fs, Path mrJobDir, Configuration conf, JobState jobState, Job job) throws IOException { Path jobStateFilePath = new Path(mrJobDir, JOB_STATE_FILE_NAME); // Write the job state with an empty task set (work units are read by the mapper from a different file) try (DataOutputStream dataOutputStream = new DataOutputStream(fs.create(jobStateFilePath))) { jobState.write(dataOutputStream, false, conf.getBoolean(SERIALIZE_PREVIOUS_WORKUNIT_STATES_KEY, DEFAULT_SERIALIZE_PREVIOUS_WORKUNIT_STATES)); } job.getConfiguration().set(ConfigurationKeys.JOB_STATE_FILE_PATH_KEY, jobStateFilePath.toString()); DistributedCache.addCacheFile(jobStateFilePath.toUri(), job.getConfiguration()); job.getConfiguration().set(ConfigurationKeys.JOB_STATE_DISTRIBUTED_CACHE_NAME, jobStateFilePath.getName()); }
/** * Creates the HDFS filesystem to store output files. * * @param conf Hadoop configuration */ private void createHdfsFilesystem(Configuration conf) throws Exception { // Inits HDFS file system object mFileSystem = FileSystem.get(URI.create(conf.get("fs.defaultFS")), conf); mOutputFilePath = new Path("./MapReduceOutputFile"); if (mFileSystem.exists(mOutputFilePath)) { mFileSystem.delete(mOutputFilePath, true); } }
public static void setReadColumns(Configuration configuration, List<Integer> readHiveColumnIndexes) { configuration.set(READ_COLUMN_IDS_CONF_STR, Joiner.on(',').join(readHiveColumnIndexes)); configuration.setBoolean(READ_ALL_COLUMNS, false); }
@Test public void testGetMetadataNotFound() throws Exception { try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); s3.setGetObjectMetadataHttpCode(SC_NOT_FOUND); fs.initialize(new URI("s3n://test-bucket/"), new Configuration()); fs.setS3Client(s3); assertEquals(fs.getS3ObjectMetadata(new Path("s3n://test-bucket/test")), null); } }
@Test public void testFromConfigurationOverride() throws Exception { Configuration configuration = new Configuration(); configuration.set("fs.file.impl", InstrumentedLocalFileSystem.class.getName()); FileSystem fs = FileSystem.newInstance(new URI("file:///"), configuration); Assert.assertTrue(fs instanceof InstrumentedLocalFileSystem); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); Assert.assertEquals(fs.getFileStatus(new Path("/tmp")).getPath(), new Path("file:///tmp")); Assert.assertEquals(fs.getUri().getScheme(), "file"); }
private static void setUpKeyProvider(Configuration conf) throws Exception { URI keyProviderUri = new URI("jceks://file" + TEST_UTIL.getDataTestDir("test.jks").toUri().toString()); conf.set("dfs.encryption.key.provider.uri", keyProviderUri.toString()); KeyProvider keyProvider = KeyProviderFactory.get(keyProviderUri, conf); keyProvider.createKey(TEST_KEY_NAME, KeyProvider.options(conf)); keyProvider.flush(); keyProvider.close(); }
@BeforeClass public static void createHDFS() throws Exception { final File baseDir = TMP.newFolder(); Configuration hdConf = new Configuration(); hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath()); MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf); hdfsCluster = builder.build(); org.apache.hadoop.fs.FileSystem hdfs = hdfsCluster.getFileSystem(); fs = new HadoopFileSystem(hdfs); basePath = new Path(hdfs.getUri().toString() + "/tests"); }
@Test(enabled = false) public void testCreate6() throws IOException, URISyntaxException { HDFSRoot hdfsRoot = new HDFSRoot("/tmp/create"); MetricsFileSystemInstrumentation fs = (MetricsFileSystemInstrumentation) FileSystem.get(new URI(instrumentedURI), new Configuration()); Path newFile = new Path("/tmp/create/newFile"); FSDataOutputStream fstream = fs.create(newFile, true, 300, (short)1, 1048576, null); Assert.assertEquals(fs.createTimer.getCount(), 1); fstream.close(); hdfsRoot.cleanupRoot(); }
@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "Error creating an instance of .*") public void testCustomCredentialsClassCannotBeFound() throws Exception { Configuration config = new Configuration(); config.set(S3_USE_INSTANCE_CREDENTIALS, "false"); config.set(S3_CREDENTIALS_PROVIDER, "com.example.DoesNotExist"); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); } }
@Test(expectedExceptions = RuntimeException.class, expectedExceptionsMessageRegExp = "S3 credentials not configured") public void testInstanceCredentialsDisabled() throws Exception { Configuration config = new Configuration(); config.setBoolean(S3_USE_INSTANCE_CREDENTIALS, false); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { fs.initialize(new URI("s3n://test-bucket/"), config); } }
@Test public void testUnderscoreBucket() throws Exception { Configuration config = new Configuration(); config.setBoolean(S3_PATH_STYLE_ACCESS, true); try (PrestoS3FileSystem fs = new PrestoS3FileSystem()) { MockAmazonS3 s3 = new MockAmazonS3(); String expectedBucketName = "test-bucket_underscore"; fs.initialize(new URI("s3n://" + expectedBucketName + "/"), config); fs.setS3Client(s3); fs.getS3ObjectMetadata(new Path("/test/path")); assertEquals(expectedBucketName, s3.getGetObjectMetadataRequest().getBucketName()); } }
/** * Gets the file system we work upon. * @return The file system. * @throws Exception */ FileSystem getFileSystem() throws Exception{ return FileSystem.get(new URI(getFsBase()), new Configuration()); }
@BeforeClass public void setUp() throws Exception { this.localFs = FileSystem.getLocal(new Configuration()); this.localFs.mkdirs(this.outputTaskStateDir); this.taskStateStore = new FsStateStore<>(this.localFs, this.outputTaskStateDir.toUri().getPath(), TaskState.class); this.taskStateCollectorService = new TaskStateCollectorService(new Properties(), this.jobState, this.eventBus, this.taskStateStore, new Path(this.outputTaskStateDir, JOB_ID)); this.eventBus.register(this); }
@Before public void setUp() throws Exception { Configuration c = TEST_UTIL.getConfiguration(); c.setBoolean("dfs.support.append", true); TEST_UTIL.startMiniCluster(1); table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, FAMILY); TEST_UTIL.loadTable(table, FAMILY); // setup the hdfssnapshots client = new DFSClient(TEST_UTIL.getDFSCluster().getURI(), TEST_UTIL.getConfiguration()); String fullUrIPath = TEST_UTIL.getDefaultRootDirPath().toString(); String uriString = TEST_UTIL.getTestFileSystem().getUri().toString(); baseDir = StringUtils.removeStart(fullUrIPath, uriString); client.allowSnapshot(baseDir); }
public void run() throws Exception { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); Path path = new Path(_segmentPath); FileStatus[] fileStatusArr = fs.globStatus(path); for (FileStatus fileStatus : fileStatusArr) { if (fileStatus.isDirectory()) { pushDir(fs, fileStatus.getPath()); } else { pushOneTarFile(fs, fileStatus.getPath()); } } }
@BeforeClass public static void beforeClass() throws Throwable { // Up the handlers; this test needs more than usual. UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10); UTIL.startMiniCluster(); FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString(); }
@BeforeClass public static void setupBeforeClass() throws Exception { Configuration conf = TEST_UTIL.getConfiguration(); conf.set(RSRpcServices.MASTER_RPC_SCHEDULER_FACTORY_CLASS, "org.apache.hadoop.hbase.regionserver.MasterFifoRpcSchedulerFactory"); conf.setInt(HConstants.REGION_SERVER_HANDLER_COUNT, 5); conf.setInt(MasterFifoRpcScheduler.MASTER_SERVER_REPORT_HANDLER_COUNT, 2); TEST_UTIL.startMiniCluster(); }