@Override public InputRowParser withParseSpec(ParseSpec parseSpec) { return new MapInputRowParser(parseSpec); } }
@JsonCreator public StringInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, @JsonProperty("encoding") String encoding ) { this.parseSpec = Preconditions.checkNotNull(parseSpec, "parseSpec"); this.mapParser = new MapInputRowParser(parseSpec); if (encoding != null) { this.charset = Charset.forName(encoding); } else { this.charset = DEFAULT_CHARSET; } }
@JsonCreator public AvroHadoopInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, @JsonProperty("fromPigAvroStorage") Boolean fromPigAvroStorage ) { this.parseSpec = parseSpec; this.fromPigAvroStorage = fromPigAvroStorage == null ? false : fromPigAvroStorage; this.avroFlattener = AvroParsers.makeFlattener(parseSpec, this.fromPigAvroStorage, false); this.mapParser = new MapInputRowParser(parseSpec); }
@JsonCreator public AvroStreamInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, @JsonProperty("avroBytesDecoder") AvroBytesDecoder avroBytesDecoder ) { this.parseSpec = Preconditions.checkNotNull(parseSpec, "parseSpec"); this.avroBytesDecoder = Preconditions.checkNotNull(avroBytesDecoder, "avroBytesDecoder"); this.avroFlattener = AvroParsers.makeFlattener(parseSpec, false, false); this.mapParser = new MapInputRowParser(parseSpec); }
@JsonCreator public ParquetHadoopInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, @JsonProperty("binaryAsString") Boolean binaryAsString ) { this.parseSpec = parseSpec; this.binaryAsString = binaryAsString == null ? false : binaryAsString; final JSONPathSpec flattenSpec; if ((parseSpec instanceof ParquetParseSpec)) { flattenSpec = ((ParquetParseSpec) parseSpec).getFlattenSpec(); } else { flattenSpec = JSONPathSpec.DEFAULT; } this.groupFlattener = ObjectFlatteners.create(flattenSpec, new ParquetGroupFlattenerMaker(this.binaryAsString)); this.parser = new MapInputRowParser(parseSpec); }
public static List<InputRow> inputRowsWithDimensions(final List<String> dimensions) { final MapInputRowParser parser = new MapInputRowParser( new TimeAndDimsParseSpec( new TimestampSpec(TIMESTAMP_COLUMN, "iso", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(dimensions), dimensions.isEmpty() ? ImmutableList.of("t", "c1", "c2") : null, null ) ) ); return ImmutableList.of( parser.parseBatch(ImmutableMap.of("t", "2000-01-01", "c1", "9", "c2", ImmutableList.of("a"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2000-01-02", "c1", "10.1", "c2", ImmutableList.of())).get(0), parser.parseBatch(ImmutableMap.of("t", "2000-01-03", "c1", "2", "c2", ImmutableList.of(""))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-01", "c1", "1", "c2", ImmutableList.of("a", "c"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-02", "c1", "4", "c2", ImmutableList.of("abc"))).get(0), parser.parseBatch(ImmutableMap.of("t", "2001-01-03", "c1", "5")).get(0) ); }
parser = new MapInputRowParser( new TimeAndDimsParseSpec( new TimestampSpec("timestamp", "auto", null),
@Test public void testMapInputRowParserSerde() throws Exception final MapInputRowParser parser = new MapInputRowParser( new JSONParseSpec( new TimestampSpec("timeposix", "posix", null),
@Test public void testMapInputRowParserNumbersSerde() throws Exception final MapInputRowParser parser = new MapInputRowParser( new JSONParseSpec( new TimestampSpec("timemillis", "millis", null),
@Before public void setUp() { req = EasyMock.createMock(HttpServletRequest.class); eventReceiverFirehoseFactory = new EventReceiverFirehoseFactory( SERVICE_NAME, CAPACITY, MAX_IDLE_TIME, null, new DefaultObjectMapper(), new DefaultObjectMapper(), register, AuthTestUtils.TEST_AUTHORIZER_MAPPER ); firehose = (EventReceiverFirehoseFactory.EventReceiverFirehose) eventReceiverFirehoseFactory.connect( new MapInputRowParser( new JSONParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("d1")), null, null), null, null ) ), null ); }
@Before public void setUp() throws Exception { req = EasyMock.createMock(HttpServletRequest.class); eventReceiverFirehoseFactory = new EventReceiverFirehoseFactory( SERVICE_NAME, CAPACITY, MAX_IDLE_TIME, null, new DefaultObjectMapper(), new DefaultObjectMapper(), register, AuthTestUtils.TEST_AUTHORIZER_MAPPER ); firehose = (EventReceiverFirehoseFactory.EventReceiverFirehose) eventReceiverFirehoseFactory.connect( new MapInputRowParser( new JSONParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("d1")), null, null), null, null ) ), null ); }
"test_ds", TestHelper.makeJsonMapper().convertValue( new MapInputRowParser( new TimeAndDimsParseSpec( new TimestampSpec("t", "auto", null),
for (InputRowParser parser : Arrays.<InputRowParser>asList( ROW_PARSER, new MapInputRowParser( new JSONParseSpec( new TimestampSpec(TIME_COLUMN, "auto", null),
@Test(expected = ISE.class) public void testDuplicateRegistering() { EventReceiverFirehoseFactory eventReceiverFirehoseFactory2 = new EventReceiverFirehoseFactory( SERVICE_NAME, CAPACITY, MAX_IDLE_TIME, null, new DefaultObjectMapper(), new DefaultObjectMapper(), register, AuthTestUtils.TEST_AUTHORIZER_MAPPER ); EventReceiverFirehoseFactory.EventReceiverFirehose firehose2 = (EventReceiverFirehoseFactory.EventReceiverFirehose) eventReceiverFirehoseFactory2 .connect( new MapInputRowParser( new JSONParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("d1")), null, null), null, null ) ), null ); }
private static class TestFirehoseFactory implements FirehoseFactory<InputRowParser> { public TestFirehoseFactory() { } @Override @SuppressWarnings("unchecked") public Firehose connect(InputRowParser parser, File temporaryDirectory) throws ParseException { return new TestFirehose(parser); } }
new MapInputRowParser( new JSONParseSpec( new TimestampSpec("ts", "auto", null),
new MapInputRowParser( new JSONParseSpec( new TimestampSpec("ts", "auto", null),
@Override public InputRowParser withParseSpec(ParseSpec parseSpec) { return new MapInputRowParser(parseSpec); } }
@JsonCreator public StringInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, @JsonProperty("encoding") String encoding ) { this.parseSpec = Preconditions.checkNotNull(parseSpec, "parseSpec"); this.mapParser = new MapInputRowParser(parseSpec); if (encoding != null) { this.charset = Charset.forName(encoding); } else { this.charset = DEFAULT_CHARSET; } }
public static List<InputRow> parseGenericRecord( GenericRecord record, ParseSpec parseSpec, ObjectFlattener<GenericRecord> avroFlattener ) { return new MapInputRowParser(parseSpec).parseBatch(avroFlattener.flatten(record)); } }