@JsonCreator public MapInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec ) { this.parseSpec = parseSpec; this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); }
@VisibleForTesting static String typeStringFromParseSpec(ParseSpec parseSpec) { StringBuilder builder = new StringBuilder("struct<"); builder.append(parseSpec.getTimestampSpec().getTimestampColumn()).append(":string"); // the typeString seems positionally dependent, so repeated timestamp column causes incorrect mapping if (parseSpec.getDimensionsSpec().getDimensionNames().size() > 0) { builder.append(","); builder.append(String.join( ":string,", parseSpec.getDimensionsSpec() .getDimensionNames() .stream() .filter(s -> !s.equals(parseSpec.getTimestampSpec().getTimestampColumn())) .collect(Collectors.toList()))); builder.append(":string"); } builder.append(">"); return builder.toString(); }
parser = parseSpec.makeParser(); initDescriptor(); } else { dimensions = Lists.newArrayList( Sets.difference(record.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) ); parseSpec.getTimestampSpec().extractTimestamp(record), dimensions, record
final DimensionsSpec dimensionsSpec = inputRowParser.getParseSpec().getDimensionsSpec(); final TimestampSpec timestampSpec = inputRowParser.getParseSpec().getTimestampSpec(); .withDimensionsSpec( dimensionsSpec .withDimensionExclusions(
firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") ); .withDimensionsSpec( firehoseParser.getParseSpec() .getDimensionsSpec() .withDimensionExclusions( newDimExclus
public Builder withTimestampSpec(InputRowParser parser) { if (parser != null && parser.getParseSpec() != null && parser.getParseSpec().getTimestampSpec() != null) { this.timestampSpec = parser.getParseSpec().getTimestampSpec(); } else { this.timestampSpec = new TimestampSpec(null, null, null); } return this; }
@Override public int hashCode() { return parseSpec.hashCode(); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } NoopInputRowParser that = (NoopInputRowParser) o; return parseSpec.equals(that.parseSpec); }
public void initializeParser() { if (parser == null) { // parser should be created when it is really used to avoid unnecessary initialization of the underlying // parseSpec. parser = parseSpec.makeParser(); } }
final DimensionsSpec dimensionsSpec = inputRowParser.getParseSpec().getDimensionsSpec(); final TimestampSpec timestampSpec = inputRowParser.getParseSpec().getTimestampSpec(); .withDimensionsSpec( dimensionsSpec .withDimensionExclusions(
byteBufferInputRowParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") ); .withDimensionsSpec( byteBufferInputRowParser.getParseSpec() .getDimensionsSpec() .withDimensionExclusions( newDimExclus
public void verify() { Preconditions.checkNotNull(schema.getDataSchema().getDataSource(), "dataSource"); Preconditions.checkNotNull(schema.getDataSchema().getParser().getParseSpec(), "parseSpec"); Preconditions.checkNotNull(schema.getDataSchema().getParser().getParseSpec().getTimestampSpec(), "timestampSpec"); Preconditions.checkNotNull(schema.getDataSchema().getGranularitySpec(), "granularitySpec"); Preconditions.checkNotNull(pathSpec, "inputSpec"); Preconditions.checkNotNull(schema.getTuningConfig().getWorkingPath(), "workingPath"); Preconditions.checkNotNull(schema.getIOConfig().getSegmentOutputPath(), "segmentOutputPath"); Preconditions.checkNotNull(schema.getTuningConfig().getVersion(), "version"); }
@Override public int hashCode() { return Objects.hash(super.hashCode(), flattenSpec); } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } NestedDataParseSpec that = (NestedDataParseSpec) o; return Objects.equals(flattenSpec, that.flattenSpec); }
public void initializeParser() { if (parser == null) { // parser should be created when it is really used to avoid unnecessary initialization of the underlying // parseSpec. parser = parseSpec.makeParser(); } }
@JsonCreator public ProtobufInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, @JsonProperty("descriptor") String descriptorFilePath, @JsonProperty("protoMessageType") String protoMessageType ) { this.parseSpec = parseSpec; this.descriptorFilePath = descriptorFilePath; this.protoMessageType = protoMessageType; this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); }
@JsonCreator public ParquetAvroHadoopInputRowParser( @JsonProperty("parseSpec") ParseSpec parseSpec, @JsonProperty("binaryAsString") Boolean binaryAsString ) { this.parseSpec = parseSpec; this.timestampSpec = parseSpec.getTimestampSpec(); this.dimensions = parseSpec.getDimensionsSpec().getDimensionNames(); this.binaryAsString = binaryAsString == null ? false : binaryAsString; final JSONPathSpec flattenSpec; if (parseSpec != null && (parseSpec instanceof AvroParseSpec)) { flattenSpec = ((AvroParseSpec) parseSpec).getFlattenSpec(); } else { flattenSpec = JSONPathSpec.DEFAULT; } this.recordFlattener = ObjectFlatteners.create( flattenSpec, new AvroFlattenerMaker(false, this.binaryAsString) ); }
parser = parseSpec.makeParser(); } else { dimensions = Lists.newArrayList( Sets.difference(record.keySet(), parseSpec.getDimensionsSpec().getDimensionExclusions()) ); parseSpec.getTimestampSpec().extractTimestamp(record), dimensions, record
firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") ); .withDimensionsSpec( firehoseParser.getParseSpec() .getDimensionsSpec() .withDimensionExclusions( newDimExclus
private static IncrementalIndex makeIncrementalIndex( Bucket theBucket, AggregatorFactory[] aggs, HadoopDruidIndexerConfig config, Iterable<String> oldDimOrder, Map<String, ColumnCapabilitiesImpl> oldCapabilities ) { final HadoopTuningConfig tuningConfig = config.getSchema().getTuningConfig(); final IncrementalIndexSchema indexSchema = new IncrementalIndexSchema.Builder() .withMinTimestamp(theBucket.time.getMillis()) .withTimestampSpec(config.getSchema().getDataSchema().getParser().getParseSpec().getTimestampSpec()) .withDimensionsSpec(config.getSchema().getDataSchema().getParser()) .withQueryGranularity(config.getSchema().getDataSchema().getGranularitySpec().getQueryGranularity()) .withMetrics(aggs) .withRollup(config.getSchema().getDataSchema().getGranularitySpec().isRollup()) .build(); IncrementalIndex newIndex = new IncrementalIndex.Builder() .setIndexSchema(indexSchema) .setReportParseExceptions(!tuningConfig.isIgnoreInvalidRows()) // only used by OffHeapIncrementalIndex .setMaxRowCount(tuningConfig.getRowFlushBoundary()) .setMaxBytesInMemory(TuningConfigs.getMaxBytesInMemoryOrDefault(tuningConfig.getMaxBytesInMemory())) .buildOnheap(); if (oldDimOrder != null && !indexSchema.getDimensionsSpec().hasCustomDimensions()) { newIndex.loadDimensionIterable(oldDimOrder, oldCapabilities); } return newIndex; }