public static PartitionedFileSetProperties.Builder getBaseProperties(SnapshotFileSetConfig config) { PartitionedFileSetProperties.Builder propertiesBuilder = PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder().addLongField(SNAPSHOT_FIELD).build()); if (!Strings.isNullOrEmpty(config.getBasePath())) { propertiesBuilder.setBasePath(config.getBasePath()); } try { Map<String, String> properties = GSON.fromJson(config.getFileProperties(), MAP_TYPE); if (properties != null) { propertiesBuilder.addAll(properties); } } catch (Exception e) { throw new IllegalArgumentException("Could not decode the 'properties' setting. Please check that it " + "is a JSON Object of string to string. Failed with error: " + e.getMessage(), e); } return propertiesBuilder; }
.setBasePath(absolutePath.getPath()) .setDataExternal(true) .build());
@Override public void configure() { setName("AppWithMapReduceUsingFile"); setDescription("Application with MapReduce job using file as dataset"); createDataset(INPUT, "table"); createDataset(OUTPUT, "table"); Class<? extends InputFormat> inputFormatClass = getConfig().isUseCombineFileInputFormat() ? CombineTextInputFormat.class : TextInputFormat.class; createDataset(PARTITIONED, "partitionedFileSet", PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder() .addStringField("type") .addLongField("time") .build()) // properties for file set .setBasePath("partitioned") .setInputFormat(inputFormatClass) .setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, SEPARATOR) // don't configure properties for the Hive table - this is used in a context where explore is disabled .build()); addMapReduce(new PartitionWriter()); addMapReduce(new PartitionReader()); }
@Before public void before() throws Exception { txClient = new InMemoryTxSystemClient(dsFrameworkUtil.getTxManager()); dsFrameworkUtil.createInstance("partitionedFileSet", pfsInstance, PartitionedFileSetProperties.builder() .setPartitioning(PARTITIONING_1) .setTablePermissions(tablePermissions) .setBasePath("testDir") .setFilePermissions(fsPermissions) .setFileGroup(group) .build()); pfsBaseLocation = ((PartitionedFileSet) dsFrameworkUtil.getInstance(pfsInstance)) .getEmbeddedFileSet().getBaseLocation(); Assert.assertTrue(pfsBaseLocation.exists()); }
@Before public void before() throws Exception { dsFrameworkUtil.createInstance("partitionedFileSet", pfsInstance, PartitionedFileSetProperties.builder() .setPartitioning(PARTITIONING_1) .setBasePath("testDir") .build()); pfsBaseLocation = ((PartitionedFileSet) dsFrameworkUtil.getInstance(pfsInstance)) .getEmbeddedFileSet().getBaseLocation(); Assert.assertTrue(pfsBaseLocation.exists()); }