/** * Provided a PartitionSpec this method attempts to retrieve each field from the deserialized * event object. * * @param devent deserialized event to extract partitions from */ protected LinkedHashMap<String, String> getPartitions(DeserializedEvent devent) { int numPartSpecs = partitionSpecs.size(); /* * Loop through each partition spec fetching the associated field from the event. Set to null if * field does not exist. */ LinkedHashMap<String, String> partitions = new LinkedHashMap<String, String>(numPartSpecs); for (PartitionSpec spec : partitionSpecs) { String key = null; for (String source : spec.getSources()) { try { key = devent.getFieldAsString(source); if (key != null) { break; } } catch (FieldNotFoundException e) { continue; } } partitions.put(spec.getName(), spec.interpret(key)); } return partitions; }
/** * Provided a PartitionSpec this method attempts to retrieve each field from the deserialized * event object. * * @param devent deserialized event to extract partitions from */ protected LinkedHashMap<String, String> getPartitions(DeserializedEvent devent) { int numPartSpecs = partitionSpecs.size(); /* * Loop through each partition spec fetching the associated field from the event. Set to null if * field does not exist. */ LinkedHashMap<String, String> partitions = new LinkedHashMap<String, String>(numPartSpecs); for (PartitionSpec spec : partitionSpecs) { String key = null; for (String source : spec.getSources()) { try { key = devent.getFieldAsString(source); if (key != null) { break; } } catch (FieldNotFoundException e) { continue; } } partitions.put(spec.getName(), spec.interpret(key)); } return partitions; }
@Test public void testLoadConfg() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); assertEquals("type", spec.getName()); assertEquals(Interpreter.STRING, spec.getInterpreter()); List<String> expected = Arrays.asList("one", "two", "three"); assertEquals(expected, spec.getSources()); }
@Test public void testLoadConfgTime() { BenderConfig config = BenderConfig.load("/com/nextdoor/bender/partition/partition_config_time.json"); PartitionOperationConfig op = (PartitionOperationConfig) config.getSources().get(0).getOperationConfigs().get(0); PartitionSpec spec = op.getPartitionSpecs().get(0); assertEquals("dt", spec.getName()); assertEquals(Interpreter.SECONDS, spec.getInterpreter()); List<String> expected = Arrays.asList("epoch"); assertEquals(expected, spec.getSources()); assertEquals("2017-01-19 05:05:59", spec.interpret("1484802359")); }