FileSetProperties.builder() .setBasePath(absolutePath) .setDataExternal(true) .build());
static void performAdmin(RuntimeContext context) { Admin admin = context.getAdmin(); Map<String, String> args = context.getRuntimeArguments(); try { // if invoked with dropAll=true, clean up all datasets (a, b, c, d) if ("true".equals(args.get("dropAll"))) { for (String name : new String[]{"a", "b", "c", "d"}) { if (admin.datasetExists(name)) { admin.dropDataset(name); } } } else { // create a, update b with /extra in base path, truncate c, drop d admin.createDataset("a", Table.class.getName(), DatasetProperties.EMPTY); String type = admin.getDatasetType("b"); Assert.assertEquals(FileSet.class.getName(), type); DatasetProperties bProps = admin.getDatasetProperties("b"); String base = bProps.getProperties().get("base.path"); Assert.assertNotNull(base); String newBase = args.get("new.base.path"); DatasetProperties newBProps = ((FileSetProperties.Builder) FileSetProperties.builder() .addAll(bProps.getProperties())).setDataExternal(true).setBasePath(newBase).build(); admin.updateDataset("b", newBProps); admin.truncateDataset("c"); admin.dropDataset("d"); } } catch (DatasetManagementException e) { Throwables.propagate(e); } }
@Test public void testNonExternalExistentPath() throws Exception { // Create an instance at a location String absolutePath = tmpFolder.newFolder() + "/some/existing/location"; File file = new File(absolutePath); Assert.assertTrue(file.mkdirs()); // Try to add another instance of non external fileset at the same location try { dsFrameworkUtil.createInstance("fileSet", testFileSetInstance6, FileSetProperties.builder() .setBasePath(absolutePath) .setDataExternal(false) .build()); Assert.fail("Expected IOException from createInstance()"); } catch (IOException e) { // expected } }
framework.addInstance(FileSet.class.getName(), MY_DS, FileSetProperties.builder() .setBasePath(baseDir.getPath()) .setDataExternal(true) .build()); .setDataExternal(true) .build()); spec = framework.getDatasetSpec(MY_DS);
.setPartitioning(PARTITIONING_1) .setBasePath(absolutePath.getPath()) .setDataExternal(true) .build()); final PartitionedFileSet pfs = dsFrameworkUtil.getInstance(pfsExternalInstance);
@Test public void testFixProperties() throws DatasetManagementException, UnsupportedTypeException { testFix("fileSet", FileSetProperties.builder().setBasePath("/tmp/nn").setDataExternal(true).build()); testFix(FileSet.class.getName(), FileSetProperties.builder().setEnableExploreOnCreate(true).setExploreFormat("csv").build()); testFix("timePartitionedFileSet", FileSetProperties.builder().setBasePath("relative").build()); testFix(TimePartitionedFileSet.class.getName(), FileSetProperties.builder().setBasePath("relative").add("custom", "value").build()); testFix("objectMappedTable", ObjectMappedTableProperties.builder().setType(TestObject.class) .setRowKeyExploreName("x").setRowKeyExploreType(Schema.Type.STRING) .setConflictDetection(ConflictDetection.NONE).build()); testFix(ObjectMappedTable.class.getName(), ObjectMappedTableProperties.builder().setType(TestObject.class) .setRowKeyExploreName("x").setRowKeyExploreType(Schema.Type.STRING) .setConflictDetection(ConflictDetection.NONE).build()); testFix("lineageDataset", DatasetProperties.EMPTY); testFix(LineageDataset.class.getName(), TableProperties.builder().setTTL(1000).build()); testFix(UsageDataset.class.getSimpleName(), DatasetProperties.EMPTY); testFix("table", TableProperties.builder().setColumnFamily("fam").build()); testFix("indexedTable", DatasetProperties.builder().add(IndexedTable.INDEX_COLUMNS_CONF_KEY, "a,c").build()); }
@Test(expected = IOException.class) public void testExternalNonExistentPath() throws IOException, DatasetManagementException { // create an external dir and create a file in it String absolutePath = tmpFolder.newFolder() + "/not/there"; // attempt to create an external dataset - should fail dsFrameworkUtil.createInstance("fileSet", testFileSetInstance5, FileSetProperties.builder() .setBasePath(absolutePath) .setDataExternal(true) .build()); }
@Test(expected = IllegalArgumentException.class) public void testPossessAndExternal() throws IOException, DatasetManagementException { dsFrameworkUtil.createInstance("fileSet", DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("badFileSet"), FileSetProperties.builder() .setDataExternal(true) .setPossessExisting(true) .build()); }
private DatasetProperties buildFileSetProps(DatasetProperties extraProps, Boolean external, String path) { FileSetProperties.Builder builder = FileSetProperties.builder(); builder.addAll(extraProps.getProperties()); if (external != null) { builder.setDataExternal(external); } if (path != null) { builder.setBasePath(path); } return builder.build(); }
@Test(expected = IllegalArgumentException.class) public void testReuseAndExternal() throws IOException, DatasetManagementException { dsFrameworkUtil.createInstance("fileSet", DatasetFrameworkTestUtil.NAMESPACE_ID.dataset("badFileSet"), FileSetProperties.builder() .setDataExternal(true) .setUseExisting(true) .build()); }