@Override public List<InputRow> parseBatch(Map<String, Object> theMap) { final List<String> dimensions; if (!this.dimensions.isEmpty()) { dimensions = this.dimensions; } else { dimensions = Lists.newArrayList( Sets.difference(theMap.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) ); } final DateTime timestamp; try { timestamp = parseSpec.getTimestampSpec().extractTimestamp(theMap); if (timestamp == null) { final String input = theMap.toString(); throw new NullPointerException( StringUtils.format( "Null timestamp in input: %s", input.length() < 100 ? input : input.substring(0, 100) + "..." ) ); } } catch (Exception e) { throw new ParseException(e, "Unparseable timestamp found! Event: %s", theMap); } return ImmutableList.of(new MapBasedInputRow(timestamp, dimensions, theMap)); }
/** * imitate avro extension {@link org.apache.druid.data.input.avro.AvroParsers#parseGenericRecord} */ @Override public List<InputRow> parseBatch(GenericRecord record) { Map<String, Object> row = recordFlattener.flatten(record); final List<String> dimensions; if (!this.dimensions.isEmpty()) { dimensions = this.dimensions; } else { dimensions = Lists.newArrayList( Sets.difference(row.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) ); } // check for parquet Date // https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#date LogicalType logicalType = determineTimestampSpecLogicalType(record.getSchema(), timestampSpec.getTimestampColumn()); DateTime dateTime; if (logicalType instanceof LogicalTypes.Date) { int daysSinceEpoch = (Integer) record.get(timestampSpec.getTimestampColumn()); dateTime = DateTimes.utc(TimeUnit.DAYS.toMillis(daysSinceEpoch)); } else { // Fall back to a binary format that will be parsed using joda-time dateTime = timestampSpec.extractTimestamp(row); } return ImmutableList.of(new MapBasedInputRow(dateTime, dimensions, row)); }
} else { dimensions = Lists.newArrayList( Sets.difference(record.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) );
} else { dimensions = Lists.newArrayList( Sets.difference(record.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) );
} else { dimensions = Lists.newArrayList( Sets.difference(map.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) );
firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") );
byteBufferInputRowParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") );
.getParseSpec() .getDimensionsSpec() .getDimensionExclusions()
dims = getUniqueDimensions( timeLineSegments, inputRowParser.getParseSpec().getDimensionsSpec().getDimensionExclusions() );
schema.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions() );
@Test public void testDefaultExclusions() { Map<String, Object> parser = jsonMapper.convertValue( new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA")), null, null), null, null ), null ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); DataSchema schema = new DataSchema( "test", parser, new AggregatorFactory[]{ new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2"), }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper ); Assert.assertEquals( ImmutableSet.of("time", "col1", "col2", "metric1", "metric2"), schema.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions() ); }
@Override public List<InputRow> parseBatch(Map<String, Object> theMap) { final List<String> dimensions = parseSpec.getDimensionsSpec().hasCustomDimensions() ? parseSpec.getDimensionsSpec().getDimensionNames() : Lists.newArrayList( Sets.difference( theMap.keySet(), parseSpec.getDimensionsSpec() .getDimensionExclusions() ) ); final DateTime timestamp; try { timestamp = parseSpec.getTimestampSpec().extractTimestamp(theMap); if (timestamp == null) { final String input = theMap.toString(); throw new NullPointerException( StringUtils.format( "Null timestamp in input: %s", input.length() < 100 ? input : input.substring(0, 100) + "..." ) ); } } catch (Exception e) { throw new ParseException(e, "Unparseable timestamp found! Event: %s", theMap); } return ImmutableList.of(new MapBasedInputRow(timestamp.getMillis(), dimensions, theMap)); }
firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") );
.getParseSpec() .getDimensionsSpec() .getDimensionExclusions()
dims = getUniqueDimensions( timeLineSegments, inputRowParser.getParseSpec().getDimensionsSpec().getDimensionExclusions() );