public HadoopyStringInputRowParser(@JsonProperty("parseSpec") ParseSpec parseSpec) { this.parser = new StringInputRowParser(parseSpec, null); }
@Override public StringInputRowParser withParseSpec(ParseSpec parseSpec) { return new StringInputRowParser(parseSpec, getEncoding()); }
public FileIteratingFirehoseTest(List<String> texts, int numSkipHeaderRows) { parser = new StringInputRowParser( new CSVParseSpec( new TimestampSpec("ts", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("x")), null, null), ",", ImmutableList.of("ts", "x"), false, numSkipHeaderRows ), null ); this.inputs = texts; this.expectedResults = inputs.stream() .map(input -> input.split("\n")) .flatMap(lines -> { final List<String> filteredLines = Arrays .stream(lines) .filter(line -> line.length() > 0) .map(line -> line.split(",")[1]) .collect(Collectors.toList()); final int numRealSkippedRows = Math.min(filteredLines.size(), numSkipHeaderRows); IntStream.range(0, numRealSkippedRows).forEach(i -> filteredLines.set(i, null)); return filteredLines.stream(); }) .collect(Collectors.toList()); }
private InputRow testCharsetParseHelper(Charset charset) throws Exception { final StringInputRowParser parser = new StringInputRowParser( new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo", "bar")), null, null), null, null ), charset.name() ); final ByteBufferInputRowParser parser2 = jsonMapper.readValue( jsonMapper.writeValueAsBytes(parser), ByteBufferInputRowParser.class ); final InputRow parsed = parser2.parseBatch( ByteBuffer.wrap( "{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"3000\"}".getBytes(charset) ) ).get(0); return parsed; }
@Test public void testConnect() throws IOException try (final Firehose firehose = factory.connect(new StringInputRowParser( new CSVParseSpec( new TimestampSpec(
@Test public void testStringInputRowParserSerde() throws Exception { final StringInputRowParser parser = new StringInputRowParser( new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo", "bar")), null, null), null, null ), null ); final ByteBufferInputRowParser parser2 = jsonMapper.readValue( jsonMapper.writeValueAsBytes(parser), ByteBufferInputRowParser.class ); final InputRow parsed = parser2.parseBatch( ByteBuffer.wrap(StringUtils.toUtf8("{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"2000\"}")) ).get(0); Assert.assertEquals(ImmutableList.of("foo", "bar"), parsed.getDimensions()); Assert.assertEquals(ImmutableList.of("x"), parsed.getDimension("foo")); Assert.assertEquals(ImmutableList.of("y"), parsed.getDimension("bar")); Assert.assertEquals(DateTimes.of("2000").getMillis(), parsed.getTimestampFromEpoch()); }
DATA_SOURCE, objectMapper.convertValue( new StringInputRowParser(parseSpec, null), Map.class ),
"test", jsonMapper.convertValue( new StringInputRowParser( parseSpec != null ? parseSpec : DEFAULT_PARSE_SPEC, null
@Test public void testDisableJavaScript() { final JavaScriptParseSpec parseSpec = new JavaScriptParseSpec( new TimestampSpec("timestamp", "auto", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas( ImmutableList.of( "dim1", "dim2" ) ), null, null ), "func", new JavaScriptConfig(false) ); final StringInputRowParser parser = new StringInputRowParser(parseSpec, "UTF-8"); expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); parser.startFileFromBeginning(); }
@Test public void testDisableJavaScript2() { final JavaScriptParseSpec parseSpec = new JavaScriptParseSpec( new TimestampSpec("timestamp", "auto", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas( ImmutableList.of( "dim1", "dim2" ) ), null, null ), "func", new JavaScriptConfig(false) ); final StringInputRowParser parser = new StringInputRowParser(parseSpec, "UTF-8"); expectedException.expect(CoreMatchers.instanceOf(IllegalStateException.class)); expectedException.expectMessage("JavaScript is disabled"); parser.parse(""); } }
public static IncrementalIndex loadIncrementalIndex( final IncrementalIndex retVal, final CharSource source ) throws IOException { final StringInputRowParser parser = new StringInputRowParser( new DelimitedParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(DIMENSION_SCHEMAS, null, null), "\t", "\u0001", Arrays.asList(COLUMNS), false, 0 ), "utf8" ); return loadIncrementalIndex(() -> retVal, source, parser); }
new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null),
new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null),
new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null),
new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null),
); final StringInputRowParser parser = new StringInputRowParser( new DelimitedParseSpec( new TimestampSpec("timestamp", "yyyyMMddHH", null),
@Test public void testDefaultExclusions() { Map<String, Object> parser = jsonMapper.convertValue( new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA")), null, null), null, null ), null ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); DataSchema schema = new DataSchema( "test", parser, new AggregatorFactory[]{ new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2"), }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper ); Assert.assertEquals( ImmutableSet.of("time", "col1", "col2", "metric1", "metric2"), schema.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions() ); }
new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null),
.buildOnheap(); StringInputRowParser parser = new StringInputRowParser( new CSVParseSpec( new TimestampSpec("timestamp", "iso", null),
.buildOnheap(); StringInputRowParser parser = new StringInputRowParser( new CSVParseSpec( new TimestampSpec("timestamp", "iso", null),