@Nullable @Override public InputRow parse(@Nullable final String input) { return transformer.transform(super.parse(input)); }
public HadoopyStringInputRowParser(@JsonProperty("parseSpec") ParseSpec parseSpec) { this.parser = new StringInputRowParser(parseSpec, null); }
@Override public StringInputRowParser withParseSpec(ParseSpec parseSpec) { return new StringInputRowParser(parseSpec, getEncoding()); }
@Override public List<InputRow> parseBatch(ByteBuffer input) { return Utils.nullableListOf(parseMap(buildStringKeyMap(input))); }
@Override public List<InputRow> parseBatch(Object input) { if (input instanceof Text) { return ImmutableList.of(parser.parse(((Text) input).toString())); } else if (input instanceof BytesWritable) { BytesWritable valueBytes = (BytesWritable) input; return parser.parseBatch(ByteBuffer.wrap(valueBytes.getBytes(), 0, valueBytes.getLength())); } else { throw new IAE("can't convert type [%s] to InputRow", input.getClass().getName()); } }
@Test public void testDisableJavaScript2() { final JavaScriptParseSpec parseSpec = new JavaScriptParseSpec( new TimestampSpec("timestamp", "auto", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas( ImmutableList.of( "dim1", "dim2" ) ), null, null ), "func", new JavaScriptConfig(false) ); final StringInputRowParser parser = new StringInputRowParser(parseSpec, "UTF-8"); expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); parser.parse(""); } }
new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null), final InputRow row1bb = parser.parseBatch( ByteBuffer.wrap("{\"time\":\"2000-01-01\",\"dimA\":\"foo\"}".getBytes(StandardCharsets.UTF_8)) ).get(0); Assert.assertEquals("foofoo", row1bb.getRaw("expr")); final InputRow row1string = parser.parse("{\"time\":\"2000-01-01\",\"dimA\":\"foo\"}"); Assert.assertEquals(DateTimes.of("2000-01-01"), row1string.getTimestamp()); Assert.assertEquals("foo", row1string.getRaw("dimA")); Assert.assertEquals("foofoo", row1string.getRaw("expr")); final InputRow row2 = parser.parseBatch( ByteBuffer.wrap("{\"time\":\"2000-01-01\",\"dimA\":\"x\"}".getBytes(StandardCharsets.UTF_8)) ).get(0);
); final StringInputRowParser parser = new StringInputRowParser( new DelimitedParseSpec( new TimestampSpec("timestamp", "yyyyMMddHH", null), .setIndexSchema( new IncrementalIndexSchema.Builder() .withDimensionsSpec(parser.getParseSpec().getDimensionsSpec()) .withMetrics(AGGREGATORS.toArray(new AggregatorFactory[0])) .build() ) { for (String line : rows) { index.add(parser.parse(line));
@Test public void testDisableJavaScript() { final JavaScriptParseSpec parseSpec = new JavaScriptParseSpec( new TimestampSpec("timestamp", "auto", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas( ImmutableList.of( "dim1", "dim2" ) ), null, null ), "func", new JavaScriptConfig(false) ); final StringInputRowParser parser = new StringInputRowParser(parseSpec, "UTF-8"); expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); parser.startFileFromBeginning(); }
parser = jsonMapper.readValue(jsonFile, StringInputRowParser.class); parser.initializeParser(); if (resource != null) { final CharSource source; source = Resources.asByteSource(new File(resource).toURI().toURL()).asCharSource( Charset.forName( parser.getEncoding() source = Resources.asByteSource(loader.getResource(resource)).asCharSource( Charset.forName( parser.getEncoding()
@Nullable private Map<String, Object> parseString(@Nullable String inputString) { initializeParser(); return parser.parseToMap(inputString); }
public <T> InputRowParser<T> decorate(final InputRowParser<T> parser) { // Always decorates, even if the transformSpec is a no-op. This is so fromInputRowParser can insist that the // parser is a transforming parser, and possibly help detect coding errors where someone forgot to call "decorate". if (parser instanceof StringInputRowParser) { // Hack to support the fact that some callers use special methods in StringInputRowParser, such as // parse(String) and startFileFromBeginning. return (InputRowParser<T>) new TransformingStringInputRowParser( parser.getParseSpec(), ((StringInputRowParser) parser).getEncoding(), this ); } else { return new TransformingInputRowParser<>(parser, this); } }
private LineIterator getNextLineIterator() { if (lineIterator != null) { lineIterator.close(); } final LineIterator iterator = lineIterators.next(); parser.startFileFromBeginning(); return iterator; }
@Override public List<InputRow> parseBatch(final ByteBuffer input) { return super.parseBatch(input).stream().map(transformer::transform).collect(Collectors.toList()); }
final StringInputRowParser parser = new StringInputRowParser( new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), ); final InputRow parsed = parser2.parse( "{\"blah\":[4,5,6], \"newmet\":5, \"foo\":{\"bar1\":\"aaa\", \"bar2\":\"bbb\"}, \"baz\":[1,2,3], \"timestamp\":\"2999\", \"foo.bar1\":\"Hello world!\", \"hey\":[{\"barx\":\"asdf\"}], \"met\":{\"a\":456}}" );
@Override public List<InputRow> parseBatch(Object input) { if (input instanceof Text) { return ImmutableList.of(parser.parse(((Text) input).toString())); } else if (input instanceof BytesWritable) { BytesWritable valueBytes = (BytesWritable) input; return parser.parseBatch(ByteBuffer.wrap(valueBytes.getBytes(), 0, valueBytes.getLength())); } else { throw new IAE("can't convert type [%s] to InputRow", input.getClass().getName()); } }
parser = jsonMapper.readValue(jsonFile, StringInputRowParser.class); parser.initializeParser(); if (resource != null) { final CharSource source; source = Resources.asByteSource(new File(resource).toURI().toURL()).asCharSource( Charset.forName( parser.getEncoding() source = Resources.asByteSource(loader.getResource(resource)).asCharSource( Charset.forName( parser.getEncoding()
public void startFileFromBeginning() { initializeParser(); parser.startFileFromBeginning(); }
@Override public List<InputRow> parseBatch(ByteBuffer input) { return Utils.nullableListOf(parseMap(buildStringKeyMap(input))); }
public <T> InputRowParser<T> decorate(final InputRowParser<T> parser) { // Always decorates, even if the transformSpec is a no-op. This is so fromInputRowParser can insist that the // parser is a transforming parser, and possibly help detect coding errors where someone forgot to call "decorate". if (parser instanceof StringInputRowParser) { // Hack to support the fact that some callers use special methods in StringInputRowParser, such as // parse(String) and startFileFromBeginning. return (InputRowParser<T>) new TransformingStringInputRowParser( parser.getParseSpec(), ((StringInputRowParser) parser).getEncoding(), this ); } else { return new TransformingInputRowParser<>(parser, this); } }