@Override public ValueWithRecordId<ValueT> decode(InputStream inStream, Context context) throws IOException { return new ValueWithRecordId<>( valueCoder.decode(inStream), idCoder.decode(inStream, context)); }
@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<T>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); }
@ProcessElement public void processElement(ProcessContext c) { c.output(c.element().getValue()); } }
@Override public void flatMap( WindowedValue<ValueWithRecordId<T>> value, Collector<WindowedValue<T>> collector) throws Exception { collector.collect(value.withValue(value.getValue().getValue())); } }
/** Emit the current element from the given Reader. The reader is guaranteed to have data. */ private void emitElement( SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> ctx, UnboundedSource.UnboundedReader<OutputT> reader) { // make sure that reader state update and element emission are atomic // with respect to snapshots OutputT item = reader.getCurrent(); byte[] recordId = reader.getCurrentRecordId(); Instant timestamp = reader.getCurrentTimestamp(); WindowedValue<ValueWithRecordId<OutputT>> windowedValue = WindowedValue.of( new ValueWithRecordId<>(item, recordId), timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING); ctx.collectWithTimestamp(windowedValue, timestamp.getMillis()); }
@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<T>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); }
@Override public void flatMap( WindowedValue<ValueWithRecordId<T>> value, Collector<WindowedValue<T>> collector) throws Exception { collector.collect(value.withValue(value.getValue().getValue())); } }
/** Emit the current element from the given Reader. The reader is guaranteed to have data. */ private void emitElement( SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> ctx, UnboundedSource.UnboundedReader<OutputT> reader) { // make sure that reader state update and element emission are atomic // with respect to snapshots OutputT item = reader.getCurrent(); byte[] recordId = reader.getCurrentRecordId(); Instant timestamp = reader.getCurrentTimestamp(); WindowedValue<ValueWithRecordId<OutputT>> windowedValue = WindowedValue.of( new ValueWithRecordId<>(item, recordId), timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING); ctx.collectWithTimestamp(windowedValue, timestamp.getMillis()); }
@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<T>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); }
@ProcessElement public void processElement(ProcessContext c) { c.output(c.element().getValue().getValue()); } }));
/** * Emit the current element from the given Reader. The reader is guaranteed to have data. */ private void emitElement( SourceContext<WindowedValue<ValueWithRecordId<OutputT>>> ctx, UnboundedSource.UnboundedReader<OutputT> reader) { // make sure that reader state update and element emission are atomic // with respect to snapshots synchronized (ctx.getCheckpointLock()) { OutputT item = reader.getCurrent(); byte[] recordId = reader.getCurrentRecordId(); Instant timestamp = reader.getCurrentTimestamp(); WindowedValue<ValueWithRecordId<OutputT>> windowedValue = WindowedValue.of(new ValueWithRecordId<>(item, recordId), timestamp, GlobalWindow.INSTANCE, PaneInfo.NO_FIRING); ctx.collectWithTimestamp(windowedValue, timestamp.getMillis()); } }
@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<String>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); } }, TypeInformation.of(ByteBuffer.class));
@Override public void flatMap(WindowedValue<ValueWithRecordId<T>> value, Collector<WindowedValue<T>> collector) throws Exception { collector.collect(value.withValue(value.getValue().getValue())); }
WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key1, key1.getBytes())))); WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key2, key2.getBytes())))); WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key1, key1.getBytes())))); WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key2, key2.getBytes())))); WindowedValue.valueInGlobalWindow(new ValueWithRecordId<>(key3, key3.getBytes()))));
private KeyedOneInputStreamOperatorTestHarness< ByteBuffer, WindowedValue<ValueWithRecordId<String>>, WindowedValue<String>> getDebupingHarness() throws Exception { DedupingOperator<String> operator = new DedupingOperator<>(); return new KeyedOneInputStreamOperatorTestHarness<>( operator, value -> ByteBuffer.wrap(value.getValue().getId()), TypeInformation.of(ByteBuffer.class)); } }
@Override public void collect(StreamRecord<WindowedValue<ValueWithRecordId<T>>> record) { emittedElements.add(record.getValue().getValue().getValue()); count++; if (count >= elementsToConsumeLimit) { throw new SuccessException(); } }
new StreamRecord<>( WindowedValue.valueInGlobalWindow( new ValueWithRecordId<>(key1, key1.getBytes(StandardCharsets.UTF_8))))); new ValueWithRecordId<>(key2, key2.getBytes(StandardCharsets.UTF_8))))); new ValueWithRecordId<>(key1, key1.getBytes(StandardCharsets.UTF_8))))); new StreamRecord<>( WindowedValue.valueInGlobalWindow( new ValueWithRecordId<>(key2, key2.getBytes(StandardCharsets.UTF_8))))); new ValueWithRecordId<>(key3, key3.getBytes(StandardCharsets.UTF_8)))));
private KeyedOneInputStreamOperatorTestHarness< ByteBuffer, WindowedValue<ValueWithRecordId<String>>, WindowedValue<String>> getDebupingHarness() throws Exception { DedupingOperator<String> operator = new DedupingOperator<>(); return new KeyedOneInputStreamOperatorTestHarness<>( operator, value -> ByteBuffer.wrap(value.getValue().getId()), TypeInformation.of(ByteBuffer.class)); } }
@Override public void collect(StreamRecord<WindowedValue<ValueWithRecordId<T>>> record) { emittedElements.add(record.getValue().getValue().getValue()); count++; if (count >= elementsToConsumeLimit) { throw new SuccessException(); } }
new StreamRecord<>( WindowedValue.valueInGlobalWindow( new ValueWithRecordId<>(key1, key1.getBytes(StandardCharsets.UTF_8))))); new ValueWithRecordId<>(key2, key2.getBytes(StandardCharsets.UTF_8))))); new ValueWithRecordId<>(key1, key1.getBytes(StandardCharsets.UTF_8))))); new StreamRecord<>( WindowedValue.valueInGlobalWindow( new ValueWithRecordId<>(key2, key2.getBytes(StandardCharsets.UTF_8))))); new ValueWithRecordId<>(key3, key3.getBytes(StandardCharsets.UTF_8)))));