public static PartitionedFileSetProperties.Builder getBaseProperties(SnapshotFileSetConfig config) { PartitionedFileSetProperties.Builder propertiesBuilder = PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder().addLongField(SNAPSHOT_FIELD).build()); if (!Strings.isNullOrEmpty(config.getBasePath())) { propertiesBuilder.setBasePath(config.getBasePath()); } try { Map<String, String> properties = GSON.fromJson(config.getFileProperties(), MAP_TYPE); if (properties != null) { propertiesBuilder.addAll(properties); } } catch (Exception e) { throw new IllegalArgumentException("Could not decode the 'properties' setting. Please check that it " + "is a JSON Object of string to string. Failed with error: " + e.getMessage(), e); } return propertiesBuilder; }
@Test public void testFieldOrder() { Partitioning partitioning = Partitioning.builder() .addIntField("1") .addLongField("2") .addStringField("3") .build(); Iterator<Map.Entry<String, FieldType>> iterator = partitioning.getFields().entrySet().iterator(); Assert.assertEquals("1", iterator.next().getKey()); Assert.assertEquals("2", iterator.next().getKey()); Assert.assertEquals("3", iterator.next().getKey()); Assert.assertFalse(iterator.hasNext()); // the previous order may have been preserved by chance. Now try the reverse order partitioning = Partitioning.builder() .addIntField("3") .addLongField("2") .addStringField("1") .build(); iterator = partitioning.getFields().entrySet().iterator(); Assert.assertEquals("3", iterator.next().getKey()); Assert.assertEquals("2", iterator.next().getKey()); Assert.assertEquals("1", iterator.next().getKey()); Assert.assertFalse(iterator.hasNext()); }
private void testIllegalFieldValue(Function<PartitionKey.Builder, PartitionKey.Builder> function) { PartitionKey.Builder builder = PartitionKey.builder( Partitioning.builder().addIntField("x").addLongField("y").addStringField("z").build()); try { function.apply(builder); Assert.fail("builder should have thrown exception for invalid field type"); } catch (IllegalArgumentException e) { //expected } }
Partitioning.builder().addLongField("i").addStringField("s").build()).build(); try { ((Reconfigurable) pfsDef).reconfigure("pfs", longIprops, spec);
addDatasetInstance(PartitionedFileSet.class.getName(), orcPFS, PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder().addLongField("time").build())
.setPartitioning(Partitioning.builder().addLongField("time").build()) .setPartitioning(Partitioning.builder().addLongField("time").build())
@Test public void testBuilderGetters() { Partitioning partitioning = Partitioning.builder() .addField("a", FieldType.STRING) .addField("b", FieldType.INT) .addField("c", FieldType.LONG) .addStringField("d") .addIntField("e") .addLongField("f") .build(); Assert.assertEquals(FieldType.STRING, partitioning.getFieldType("a")); Assert.assertEquals(FieldType.INT, partitioning.getFieldType("b")); Assert.assertEquals(FieldType.LONG, partitioning.getFieldType("c")); Assert.assertEquals(FieldType.STRING, partitioning.getFieldType("d")); Assert.assertEquals(FieldType.INT, partitioning.getFieldType("e")); Assert.assertEquals(FieldType.LONG, partitioning.getFieldType("f")); Assert.assertNull(partitioning.getFieldType("x")); Assert.assertEquals(partitioning.getFields().keySet(), ImmutableSet.of("a", "b", "c", "d", "e", "f")); }
@Override public void configure() { setName("AppWithMapReduceUsingFile"); setDescription("Application with MapReduce job using file as dataset"); createDataset(INPUT, "table"); createDataset(OUTPUT, "table"); Class<? extends InputFormat> inputFormatClass = getConfig().isUseCombineFileInputFormat() ? CombineTextInputFormat.class : TextInputFormat.class; createDataset(PARTITIONED, "partitionedFileSet", PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder() .addStringField("type") .addLongField("time") .build()) // properties for file set .setBasePath("partitioned") .setInputFormat(inputFormatClass) .setOutputFormat(TextOutputFormat.class) .setOutputProperty(TextOutputFormat.SEPERATOR, SEPARATOR) // don't configure properties for the Hive table - this is used in a context where explore is disabled .build()); addMapReduce(new PartitionWriter()); addMapReduce(new PartitionReader()); }
@Test(expected = IllegalArgumentException.class) public void testBuilderUnknownField() { PartitionKey.builder( Partitioning.builder().addIntField("x").addLongField("y").build()) .addField("x", 10).addField("y", 10L).addField("z", 15).build(); }
@Override public void configure() { // A PFS for storing uploaded file createDataset(PFS_NAME, PartitionedFileSet.class, PartitionedFileSetProperties.builder() .setPartitioning(Partitioning.builder().addLongField("time").build()) .setInputFormat(TextInputFormat.class) .build() ); // A KV table for tracking chunks sizes createDataset(KV_TABLE_NAME, KeyValueTable.class); addService(SERVICE_NAME, new FileHandler()); }
@Test(expected = IllegalStateException.class) public void testBuilderMissingField() { PartitionKey.builder( Partitioning.builder().addIntField("x").addLongField("y").addStringField("z").build()) .addField("x", 10).addField("y", 10L).build(); }