/** * Returns the Path where the YARN application files should be uploaded to. * * @param appId YARN application id */ private Path getYarnFilesDir(final ApplicationId appId) throws IOException { final FileSystem fileSystem = FileSystem.get(yarnConfiguration); final Path homeDir = fileSystem.getHomeDirectory(); return new Path(homeDir, ".flink/" + appId + '/'); }
@Override public Void run() throws IOException { FileSystem fs = renamedScriptPath.getFileSystem(conf); fs.rename(new Path(jstormMasterContext.scriptPath), renamedScriptPath); return null; } });
/** * Test for the {@link HadoopUtils#getPathWithoutScheme(Path)} method. */ @Test public void testGetPathWithoutSchema() { final Path path = new Path("/foo/bar/baz"); final String output = HadoopUtils.getPathWithoutScheme(path); assertEquals("/foo/bar/baz", output); }
@Test public void testAddPartitionNullLocationInTableToo() throws Exception { createTable(DB_NAME, TABLE_NAME, null); Partition partition = buildPartition(DB_NAME, TABLE_NAME, DEFAULT_YEAR_VALUE, null); client.add_partition(partition); Partition part = client.getPartition(DB_NAME, TABLE_NAME, "year=2017"); Assert.assertEquals( metaStore.getWarehouseRoot() + "/test_partition_db.db/test_partition_table/year=2017", part.getSd().getLocation()); Assert.assertTrue(metaStore.isPathExists(new Path(part.getSd().getLocation()))); }
@Test public void testSetWALRootDir() throws Exception { Path p = new Path("file:///hbase/root"); CommonFSUtils.setWALRootDir(conf, p); assertEquals(p.toString(), conf.get(CommonFSUtils.HBASE_WAL_DIR)); }
@Test public void testMovePathsThatCanBeMerged() { final Path condInputPath = new Path("s3a://bucket/scratch/-ext-10000"); final Path condOutputPath = new Path("s3a://bucket/scratch/-ext-10002"); final Path targetMoveWorkPath = new Path("s3a://bucket/scratch/-ext-10003"); final MoveWork mockWork = mock(MoveWork.class); when(mockWork.getLoadFileWork()).thenReturn(new LoadFileDesc( condOutputPath, targetMoveWorkPath, false, "", "", false)); assertTrue("Merging BlobStore paths should be allowed.", GenMapRedUtils.shouldMergeMovePaths(hiveConf, condInputPath, condOutputPath, mockWork)); }
@Test public void testMiniDFSCluster() throws Exception { HBaseTestingUtility hbt = new HBaseTestingUtility(); MiniDFSCluster cluster = hbt.startMiniDFSCluster(null); FileSystem dfs = cluster.getFileSystem(); Path dir = new Path("dir"); Path qualifiedDir = dfs.makeQualified(dir); LOG.info("dir=" + dir + ", qualifiedDir=" + qualifiedDir); assertFalse(dfs.exists(qualifiedDir)); assertTrue(dfs.mkdirs(qualifiedDir)); assertTrue(dfs.delete(qualifiedDir, true)); hbt.shutdownMiniCluster(); }
@Test public void testSkipEmptyColumns() throws Exception { Path bulkOutputPath = new Path(util.getDataTestDirOnTestFS(tn.getNameAsString()), "hfiles"); args.put(ImportTsv.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString()); args.put(ImportTsv.COLUMNS_CONF_KEY, "HBASE_ROW_KEY,HBASE_TS_KEY,FAM:A,FAM:B"); args.put(ImportTsv.SEPARATOR_CONF_KEY, ","); args.put(ImportTsv.SKIP_EMPTY_COLUMNS, "true"); // 2 Rows of data as input. Both rows are valid and only 3 columns are no-empty among 4 String data = "KEY,1234,VALUE1,VALUE2\nKEY,1235,,VALUE2\n"; doMROnTableTest(util, tn, FAMILY, data, args, 1, 3); util.deleteTable(tn); }
@Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable"))); }
@Test(expected = MetastoreException.class) public void testInvalidPartitionKeyName() throws HiveException, AlreadyExistsException, IOException, MetastoreException { Table table = createTestTable(); List<Partition> partitions = hive.getPartitions(table); assertEquals(2, partitions.size()); // add a fake partition dir on fs fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf()); Path fakePart = new Path(table.getDataLocation().toString(), "fakedate=2009-01-01/fakecity=sanjose"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); checker.checkMetastore(catName, dbName, tableName, null, new CheckResult()); }
@Test public void testShouldCreateNewTableDescriptorIfForcefulCreationIsFalse() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); Path rootdir = new Path(UTIL.getDataTestDir(), name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); assertTrue("Should create new table descriptor", fstd.createTableDescriptor(TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(), false)); }
private Path generateUniqTempDir(boolean withDirCreated) throws IOException { FileSystem fs = FSUtils.getCurrentFileSystem(getConf()); Path dir = new Path(fs.getWorkingDirectory(), NAME); if (!fs.exists(dir)) { fs.mkdirs(dir); } Path newDir = new Path(dir, UUID.randomUUID().toString()); if (withDirCreated) { fs.mkdirs(newDir); } return newDir; }
/** @throws Exception If failed. */ @Test public void testSetWorkingDirectory() throws Exception { Path dir = new Path("/tmp/nested/dir"); Path file = new Path("file"); fs.mkdirs(dir); fs.setWorkingDirectory(dir); FSDataOutputStream os = fs.create(file); os.close(); String filePath = fs.getFileStatus(new Path(dir, file)).getPath().toString(); assertTrue(filePath.contains("/tmp/nested/dir/file")); }
@Test(expected = IOException.class) public void testBackupPathIsNotAccessible() throws Exception { Path path = new Path(PERMISSION_TEST_PATH); FileSystem rootFs = FileSystem.get(TEST_UTIL.getConnection().getConfiguration()); rootFs.mkdirs(path.getParent()); rootFs.setPermission(path.getParent(), FsPermission.createImmutable((short) 000)); FileSystem fs = DFSTestUtil.getFileSystemAs(DIANA, TEST_UTIL.getConnection().getConfiguration()); fs.mkdirs(path); } }
private List<Path> getFilesRecursively(String fileBackupDir) throws IllegalArgumentException, IOException { FileSystem fs = FileSystem.get((new Path(fileBackupDir)).toUri(), new Configuration()); List<Path> list = new ArrayList<>(); RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(fileBackupDir), true); while (it.hasNext()) { Path p = it.next().getPath(); if (HFile.isHFileFormat(fs, p)) { list.add(p); } } return list; }
@Override public boolean connect(StatsCollectionContext context) { conf = context.getHiveConf(); List<String> statsDirs = context.getStatsTmpDirs(); assert statsDirs.size() == 1 : "Found multiple stats dirs: " + statsDirs; Path statsDir = new Path(statsDirs.get(0)); LOG.debug("Connecting to : " + statsDir); statsMap = new HashMap<String, Map<String,String>>(); try { return statsDir.getFileSystem(conf).exists(statsDir); } catch (IOException e) { LOG.error("Failed to check if dir exists", e); return false; } }