@Override public List<InputRow> parseBatch(final T row) { return parser.parseBatch(row).stream().map(transformer::transform).collect(Collectors.toList()); }
@Override public ParseSpec getParseSpec() { return parser.getParseSpec(); }
if (inputRowParser.getParseSpec() != null) { final DimensionsSpec dimensionsSpec = inputRowParser.getParseSpec().getDimensionsSpec(); final TimestampSpec timestampSpec = inputRowParser.getParseSpec().getTimestampSpec(); cachedParser = inputRowParser.withParseSpec( inputRowParser.getParseSpec() .withDimensionsSpec( dimensionsSpec
@Override @SuppressWarnings("unchecked") public InputRowParser<T> withParseSpec(final ParseSpec parseSpec) { return new TransformingInputRowParser<>(parser.withParseSpec(parseSpec), transformSpec); }
/** * Parse an input into list of {@link InputRow}. List can contains null for rows that should be thrown away, * or throws {@code ParseException} if the input is unparseable. This method should never return null otherwise * lots of things will break. */ @NotNull default List<InputRow> parseBatch(T input) { return Utils.nullableListOf(parse(input)); }
@Override @SuppressWarnings("unchecked") public InputRowParser<T> withParseSpec(final ParseSpec parseSpec) { return new TransformingInputRowParser<>(parser.withParseSpec(parseSpec), transformSpec); }
/** * Parse an input into list of {@link InputRow}. List can contains null for rows that should be thrown away, * or throws {@code ParseException} if the input is unparseable. This method should never return null otherwise * lots of things will break. */ @NotNull default List<InputRow> parseBatch(T input) { return Utils.nullableListOf(parse(input)); }
@Nullable @Override public InputRow nextRow() { Map<String, Object> mapToParse = lineIterator.next(); return (InputRow) Iterators.getOnlyElement(parser.parseBatch(mapToParse).iterator()); }
public Builder withTimestampSpec(InputRowParser parser) { if (parser != null && parser.getParseSpec() != null && parser.getParseSpec().getTimestampSpec() != null) { this.timestampSpec = parser.getParseSpec().getTimestampSpec(); } else { this.timestampSpec = new TimestampSpec(null, null, null); } return this; }
firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") ); final InputRowParser<ByteBuffer> theParser = firehoseParser.withParseSpec( firehoseParser.getParseSpec() .withDimensionsSpec( firehoseParser.getParseSpec() .getDimensionsSpec() .withDimensionExclusions(
public static InputRow createRow(final ImmutableMap<String, ?> map) { return PARSER.parseBatch((Map<String, Object>) map).get(0); }
public Builder withDimensionsSpec(InputRowParser parser) { if (parser != null && parser.getParseSpec() != null && parser.getParseSpec().getDimensionsSpec() != null) { this.dimensionsSpec = parser.getParseSpec().getDimensionsSpec(); } else { this.dimensionsSpec = new DimensionsSpec(null, null, null); } return this; }
byteBufferInputRowParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") ); final InputRowParser<ByteBuffer> theParser = byteBufferInputRowParser.withParseSpec( byteBufferInputRowParser.getParseSpec() .withDimensionsSpec( byteBufferInputRowParser.getParseSpec() .getDimensionsSpec() .withDimensionExclusions(
public static InputRow createRow(final ImmutableMap<String, ?> map, InputRowParser<Map<String, Object>> parser) { return parser.parseBatch((Map<String, Object>) map).get(0); }
public void verify() { Preconditions.checkNotNull(schema.getDataSchema().getDataSource(), "dataSource"); Preconditions.checkNotNull(schema.getDataSchema().getParser().getParseSpec(), "parseSpec"); Preconditions.checkNotNull(schema.getDataSchema().getParser().getParseSpec().getTimestampSpec(), "timestampSpec"); Preconditions.checkNotNull(schema.getDataSchema().getGranularitySpec(), "granularitySpec"); Preconditions.checkNotNull(pathSpec, "inputSpec"); Preconditions.checkNotNull(schema.getTuningConfig().getWorkingPath(), "workingPath"); Preconditions.checkNotNull(schema.getIOConfig().getSegmentOutputPath(), "segmentOutputPath"); Preconditions.checkNotNull(schema.getTuningConfig().getVersion(), "version"); }
firehoseParser.getParseSpec().getDimensionsSpec().getDimensionExclusions(), Sets.newHashSet("feed") ); final InputRowParser<ByteBuffer> theParser = firehoseParser.withParseSpec( firehoseParser.getParseSpec() .withDimensionsSpec( firehoseParser.getParseSpec() .getDimensionsSpec() .withDimensionExclusions(
@Nullable @Override public InputRow nextRow() { if (nextIterator.hasNext()) { return nextIterator.next(); } for (Map.Entry<MessageQueue, ConcurrentSkipListSet<MessageExt>> entry : messageQueueTreeSetMap.entrySet()) { if (!entry.getValue().isEmpty()) { MessageExt message = entry.getValue().pollFirst(); nextIterator = theParser.parseBatch(ByteBuffer.wrap(message.getBody())).iterator(); windows .computeIfAbsent(entry.getKey(), k -> new ConcurrentSkipListSet<>()) .add(message.getQueueOffset()); return nextIterator.next(); } } // should never happen. throw new RuntimeException("Unexpected Fatal Error! There should have been one row available."); }
public <T> InputRowParser<T> decorate(final InputRowParser<T> parser) { // Always decorates, even if the transformSpec is a no-op. This is so fromInputRowParser can insist that the // parser is a transforming parser, and possibly help detect coding errors where someone forgot to call "decorate". if (parser instanceof StringInputRowParser) { // Hack to support the fact that some callers use special methods in StringInputRowParser, such as // parse(String) and startFileFromBeginning. return (InputRowParser<T>) new TransformingStringInputRowParser( parser.getParseSpec(), ((StringInputRowParser) parser).getEncoding(), this ); } else { return new TransformingInputRowParser<>(parser, this); } }
if (inputRowParser.getParseSpec() != null) { final DimensionsSpec dimensionsSpec = inputRowParser.getParseSpec().getDimensionsSpec(); final TimestampSpec timestampSpec = inputRowParser.getParseSpec().getTimestampSpec(); cachedParser = inputRowParser.withParseSpec( inputRowParser.getParseSpec() .withDimensionsSpec( dimensionsSpec
@Nullable @Override public InputRow nextRow() { try { if (!nextIterator.hasNext()) { final byte[] message = iter.next().message(); if (message == null) { return null; } nextIterator = theParser.parseBatch(ByteBuffer.wrap(message)).iterator(); } return nextIterator.next(); } catch (InvalidMessageException e) { /* IF the CRC is caused within the wire transfer, this is not the best way to handel CRC. Probably it is better to shutdown the fireHose without commit and start it again. */ log.error(e, "Message failed its checksum and it is corrupt, will skip it"); return null; } }