@Test public void testDefaultTimestampSpec() { expectedException.expect(NullPointerException.class); expectedException.expectMessage("parseSpec requires timestampSpec"); @SuppressWarnings("unused") // expected exception final ParseSpec spec = new DelimitedParseSpec( null, new DimensionsSpec( DimensionsSpec.getDefaultSchemas(Collections.singletonList("a")), Lists.newArrayList("B", "B"), new ArrayList<>() ), ",", null, Arrays.asList("a", "B"), false, 0 ); }
@Override public List<InputRow> parseBatch(Map<String, Object> theMap) { final List<String> dimensions; if (!this.dimensions.isEmpty()) { dimensions = this.dimensions; } else { dimensions = Lists.newArrayList( Sets.difference(theMap.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) ); } final DateTime timestamp; try { timestamp = parseSpec.getTimestampSpec().extractTimestamp(theMap); if (timestamp == null) { final String input = theMap.toString(); throw new NullPointerException( StringUtils.format( "Null timestamp in input: %s", input.length() < 100 ? input : input.substring(0, 100) + "..." ) ); } } catch (Exception e) { throw new ParseException(e, "Unparseable timestamp found! Event: %s", theMap); } return ImmutableList.of(new MapBasedInputRow(timestamp, dimensions, theMap)); }
@JsonCreator public MapInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec ) { this.parseSpec = parseSpec; this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); }
"%s-%s", baseDataSource, DigestUtils.sha1Hex(dimensionsSpec.toString()).substring(0, 8) for (DimensionSchema schema : dimensionsSpec.getDimensions()) { dimensions.add(schema.getName());
if (!(dimensionsSpec.hasCustomDimensions() && dimensionsSpec.getDimensionNames().contains(timestampColumn))) { dimensionExclusions.add(timestampColumn); metSet.add(aggregator.getName()); final Set<String> dimSet = Sets.newHashSet(dimensionsSpec.getDimensionNames()); final Set<String> overlap = Sets.intersection(metSet, dimSet); if (!overlap.isEmpty()) { .withDimensionsSpec( dimensionsSpec .withDimensionExclusions( Sets.difference(dimensionExclusions, dimSet)
if (dimensions != null) { dims = dimensions; } else if (inputRowParser.getParseSpec().getDimensionsSpec().hasCustomDimensions()) { dims = inputRowParser.getParseSpec().getDimensionsSpec().getDimensionNames(); } else { dims = getUniqueDimensions( timeLineSegments, inputRowParser.getParseSpec().getDimensionsSpec().getDimensionExclusions() );
@Test public void testSerde() throws IOException { HashMap<String, Boolean> feature = new HashMap<String, Boolean>(); feature.put("ALLOW_UNQUOTED_CONTROL_CHARS", true); JSONParseSpec spec = new JSONParseSpec( new TimestampSpec("timestamp", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("bar", "foo")), null, null), null, feature ); final JSONParseSpec serde = (JSONParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("timestamp", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals(Arrays.asList("bar", "foo"), serde.getDimensionsSpec().getDimensionNames()); Assert.assertEquals(feature, serde.getFeatureSpec()); } }
public DimensionsSpec withDimensionExclusions(Set<String> dimExs) { return new DimensionsSpec( dimensions, ImmutableList.copyOf(Sets.union(dimensionExclusions, dimExs)), null ); }
@Test public void testDefaultExclusions() { Map<String, Object> parser = jsonMapper.convertValue( new StringInputRowParser( new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(ImmutableList.of("dimB", "dimA")), null, null), null, null ), null ), JacksonUtils.TYPE_REFERENCE_MAP_STRING_OBJECT ); DataSchema schema = new DataSchema( "test", parser, new AggregatorFactory[]{ new DoubleSumAggregatorFactory("metric1", "col1"), new DoubleSumAggregatorFactory("metric2", "col2"), }, new ArbitraryGranularitySpec(Granularities.DAY, ImmutableList.of(Intervals.of("2014/2015"))), null, jsonMapper ); Assert.assertEquals( ImmutableSet.of("time", "col1", "col2", "metric1", "metric2"), schema.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions() ); }
firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") ); firehoseParser.getParseSpec() .getDimensionsSpec() .withDimensionExclusions( newDimExclus
Assert.assertTrue(parser.getParseSpec() instanceof TimeAndDimsParseSpec); Assert.assertEquals( new HashSet<>(expectedDimensionsSpec.getDimensions()), new HashSet<>(parser.getParseSpec().getDimensionsSpec().getDimensions()) ); new HashSet<>(expectedDimensionsSpec.getDimensionNames()), new HashSet<>(ingestSegmentFirehoseFactory.getDimensions()) );
public static Map<String, IndexSerdeTypeHelper> getTypeHelperMap(DimensionsSpec dimensionsSpec) { Map<String, IndexSerdeTypeHelper> typeHelperMap = new HashMap<>(); for (DimensionSchema dimensionSchema : dimensionsSpec.getDimensions()) { IndexSerdeTypeHelper typeHelper; switch (dimensionSchema.getValueType()) { case STRING: typeHelper = STRING_HELPER; break; case LONG: typeHelper = LONG_HELPER; break; case FLOAT: typeHelper = FLOAT_HELPER; break; case DOUBLE: typeHelper = DOUBLE_HELPER; break; default: throw new IAE("Invalid type: [%s]", dimensionSchema.getValueType()); } typeHelperMap.put(dimensionSchema.getName(), typeHelper); } return typeHelperMap; }
for (DimensionSchema dimSchema : dimensionsSpec.getDimensions()) { ValueType type = TYPE_MAP.get(dimSchema.getValueType()); String dimName = dimSchema.getName(); List<SpatialDimensionSchema> spatialDimensions = dimensionsSpec.getSpatialDimensions(); if (!spatialDimensions.isEmpty()) { this.rowTransformers.add(new SpatialDimensionRowTransformer(spatialDimensions));
public static List<DimensionSchema> getDefaultSchemas( final List<String> dimNames, final DimensionSchema.MultiValueHandling multiValueHandling ) { return dimNames.stream() .map(input -> new StringDimensionSchema(input, multiValueHandling, true)) .collect(Collectors.toList()); }
FireHydrant lastHydrant = hydrants.get(numHydrants - 1); newCount = lastHydrant.getCount() + 1; if (!indexSchema.getDimensionsSpec().hasCustomDimensions()) { Map<String, ColumnCapabilitiesImpl> oldCapabilities; if (lastHydrant.hasSwapped()) {
if (updatedIngestionSpec.getDimensions() == null) { List<String> dims; if (config.getParser().getParseSpec().getDimensionsSpec().hasCustomDimensions()) { dims = config.getParser().getParseSpec().getDimensionsSpec().getDimensionNames(); } else { Set<String> dimSet = Sets.newHashSet( .getParseSpec() .getDimensionsSpec() .getDimensionExclusions()
@Test public void testSerde() throws IOException { RegexParseSpec spec = new RegexParseSpec( new TimestampSpec("abc", "iso", null), new DimensionsSpec(DimensionsSpec.getDefaultSchemas(Collections.singletonList("abc")), null, null), "\u0001", Collections.singletonList("abc"), "abc" ); final RegexParseSpec serde = (RegexParseSpec) jsonMapper.readValue( jsonMapper.writeValueAsString(spec), ParseSpec.class ); Assert.assertEquals("abc", serde.getTimestampSpec().getTimestampColumn()); Assert.assertEquals("iso", serde.getTimestampSpec().getTimestampFormat()); Assert.assertEquals("abc", serde.getPattern()); Assert.assertEquals("\u0001", serde.getListDelimiter()); Assert.assertEquals(Collections.singletonList("abc"), serde.getDimensionsSpec().getDimensionNames()); } }
@Deprecated public DimensionsSpec withSpatialDimensions(List<SpatialDimensionSchema> spatials) { return new DimensionsSpec(dimensions, ImmutableList.copyOf(dimensionExclusions), spatials); }
new JSONParseSpec( new TimestampSpec("time", "auto", null), new DimensionsSpec( DimensionsSpec.getDefaultSchemas(ImmutableList.of("time", "dimA", "dimB", "col2")), ImmutableList.of("dimC"), null schema.getParser().getParseSpec().getDimensionsSpec().getDimensionExclusions() );
if (!(dimensionsSpec.hasCustomDimensions() && dimensionsSpec.getDimensionNames().contains(timestampColumn))) { dimensionExclusions.add(timestampColumn); metSet.add(aggregator.getName()); final Set<String> dimSet = Sets.newHashSet(dimensionsSpec.getDimensionNames()); final Set<String> overlap = Sets.intersection(metSet, dimSet); if (!overlap.isEmpty()) { .withDimensionsSpec( dimensionsSpec .withDimensionExclusions( Sets.difference(dimensionExclusions, dimSet)