@Override public ParseSpec withDimensionsSpec(DimensionsSpec spec) { return new JSONParseSpec(getTimestampSpec(), spec, getFlattenSpec(), getFeatureSpec()); }
@Override public ParseSpec withTimestampSpec(TimestampSpec spec) { return new JSONParseSpec(spec, getDimensionsSpec(), getFlattenSpec(), getFeatureSpec()); }
public Parser getFieldDiscoveryParser() { List<JSONPathFieldSpec> fields = new ArrayList<>(); JSONPathSpec flattenSpec = new JSONPathSpec(true, fields); JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), flattenSpec, null ); return spec.makeParser(); }
public Parser getFlatParser() { JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null), null, null ); return spec.makeParser(); }
JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null),
JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("ts", "iso", null), new DimensionsSpec(null, null, null),
private InputRow testCharsetParseHelper(Charset charset) throws Exception { final StringInputRowParser parser = new StringInputRowParser( new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo", "bar")), null, null), null, null ), charset.name() ); final ByteBufferInputRowParser parser2 = jsonMapper.readValue( jsonMapper.writeValueAsBytes(parser), ByteBufferInputRowParser.class ); final InputRow parsed = parser2.parseBatch( ByteBuffer.wrap( "{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"3000\"}".getBytes(charset) ) ).get(0); return parsed; }
@Test public void testStringInputRowParserSerde() throws Exception { final StringInputRowParser parser = new StringInputRowParser( new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo", "bar")), null, null), null, null ), null ); final ByteBufferInputRowParser parser2 = jsonMapper.readValue( jsonMapper.writeValueAsBytes(parser), ByteBufferInputRowParser.class ); final InputRow parsed = parser2.parseBatch( ByteBuffer.wrap(StringUtils.toUtf8("{\"foo\":\"x\",\"bar\":\"y\",\"qux\":\"z\",\"timestamp\":\"2000\"}")) ).get(0); Assert.assertEquals(ImmutableList.of("foo", "bar"), parsed.getDimensions()); Assert.assertEquals(ImmutableList.of("x"), parsed.getDimension("foo")); Assert.assertEquals(ImmutableList.of("y"), parsed.getDimension("bar")); Assert.assertEquals(DateTimes.of("2000").getMillis(), parsed.getTimestampFromEpoch()); }
new JSONParseSpec( new TimestampSpec("timeposix", "posix", null), new DimensionsSpec(
@Test public void testSerde() throws IOException { HashMap<String, Boolean> feature = new HashMap<String, Boolean>(); feature.put("ALLOW_UNQUOTED_CONTROL_CHARS", true); JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")), null, null), null, feature ); final JSONParseSpec serde = (JSONParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("timestamp", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals(Arrays.asList("bar", "foo"), serde.getDimensionsSpec().getDimensionNames()); Assert.assertEquals(feature, serde.getFeatureSpec()); } }
@Test public void testParseRowWithConditional() { final JSONParseSpec parseSpec = new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("foo")), null, null), new JSONPathSpec( true, ImmutableList.of( new JSONPathFieldSpec(JSONPathFieldType.PATH, "foo", "$.[?(@.maybe_object)].maybe_object.foo.test"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "baz", "$.maybe_object_2.foo.test"), new JSONPathFieldSpec(JSONPathFieldType.PATH, "bar", "$.[?(@.something_else)].something_else.foo") ) ), null ); final Map<String, Object> expected = new HashMap<>(); expected.put("foo", new ArrayList()); expected.put("baz", null); expected.put("bar", Collections.singletonList("test")); final Parser<String, Object> parser = parseSpec.makeParser(); final Map<String, Object> parsedRow = parser.parseToMap("{\"something_else\": {\"foo\": \"test\"}}"); Assert.assertNotNull(parsedRow); Assert.assertEquals(expected, parsedRow); }
new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of(
@Test public void testParseRow() final JSONParseSpec parseSpec = new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")), null, null),
new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(
@Before public void setUp() throws Exception { req = EasyMock.createMock(HttpServletRequest.class); eventReceiverFirehoseFactory = new EventReceiverFirehoseFactory( SERVICE_NAME, CAPACITY, MAX_IDLE_TIME, null, new DefaultObjectMapper(), new DefaultObjectMapper(), register, AuthTestUtils.TEST_AUTHORIZER_MAPPER ); firehose = (EventReceiverFirehoseFactory.EventReceiverFirehose) eventReceiverFirehoseFactory.connect( new MapInputRowParser( new JSONParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("d1")), null, null), null, null ) ), null ); }
@Before public void setUp() { req = EasyMock.createMock(HttpServletRequest.class); eventReceiverFirehoseFactory = new EventReceiverFirehoseFactory( SERVICE_NAME, CAPACITY, MAX_IDLE_TIME, null, new DefaultObjectMapper(), new DefaultObjectMapper(), register, AuthTestUtils.TEST_AUTHORIZER_MAPPER ); firehose = (EventReceiverFirehoseFactory.EventReceiverFirehose) eventReceiverFirehoseFactory.connect( new MapInputRowParser( new JSONParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("d1")), null, null), null, null ) ), null ); }
new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(
new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(
@Test public void testDefaultExclusions() { Map<String, Object> parser = jsonMapper.convertValue( new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA")), null, null), null, null ), null ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); DataSchema schema = new DataSchema( "test", parser, new AggregatorFactory[]{ new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2"), }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper ); Assert.assertEquals( ImmutableSet.of("time", "col1", "col2", "metric1", "metric2"), schema.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions() ); }
@Test(expected = ISE.class) public void testDuplicateRegistering() { EventReceiverFirehoseFactory eventReceiverFirehoseFactory2 = new EventReceiverFirehoseFactory( SERVICE_NAME, CAPACITY, MAX_IDLE_TIME, null, new DefaultObjectMapper(), new DefaultObjectMapper(), register, AuthTestUtils.TEST_AUTHORIZER_MAPPER ); EventReceiverFirehoseFactory.EventReceiverFirehose firehose2 = (EventReceiverFirehoseFactory.EventReceiverFirehose) eventReceiverFirehoseFactory2 .connect( new MapInputRowParser( new JSONParseSpec( new TimestampSpec( "timestamp", "auto", null ), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("d1")), null, null), null, null ) ), null ); }