@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo( Coder<T> coder, WindowingStrategy<?, ?> windowingStrategy) { WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( coder, windowingStrategy.getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo( Coder<T> coder, WindowingStrategy<?, ?> windowingStrategy) { WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder(coder, windowingStrategy.getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo( Coder<T> coder, WindowingStrategy<?, ?> windowingStrategy) { WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder(coder, windowingStrategy.getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
@Override public void addOutput(String name, PCollection<?> value) { translator.producers.put(value, translator.currentTransform); // Wrap the PCollection element Coder inside a WindowedValueCoder. Coder<?> coder = WindowedValue.getFullCoder( value.getCoder(), value.getWindowingStrategy().getWindowFn().windowCoder()); addOutput(name, value, coder); }
public <T> Coder<WindowedValue<T>> getWindowedInputCoder(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); return WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); }
public <T> Coder<WindowedValue<T>> getWindowedInputCoder(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); return WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); }
public <T> Coder<WindowedValue<T>> getCoder(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); return WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); }
public ProcessFn( DoFn<InputT, OutputT> fn, Coder<InputT> elementCoder, Coder<RestrictionT> restrictionCoder, WindowingStrategy<InputT, ?> inputWindowingStrategy) { this.fn = fn; this.elementCoder = elementCoder; this.restrictionCoder = restrictionCoder; this.inputWindowingStrategy = inputWindowingStrategy; this.elementTag = StateTags.value( "element", WindowedValue.getFullCoder( elementCoder, inputWindowingStrategy.getWindowFn().windowCoder())); this.restrictionTag = StateTags.value("restriction", restrictionCoder); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
private void translateStreamingImpulse( String id, RunnerApi.Pipeline pipeline, StreamingTranslationContext context) { RunnerApi.PTransform pTransform = pipeline.getComponents().getTransformsOrThrow(id); TypeInformation<WindowedValue<byte[]>> typeInfo = new CoderTypeInformation<>( WindowedValue.getFullCoder(ByteArrayCoder.of(), GlobalWindow.Coder.INSTANCE)); ObjectMapper objectMapper = new ObjectMapper(); final int intervalMillis; final int messageCount; try { JsonNode config = objectMapper.readTree(pTransform.getSpec().getPayload().toByteArray()); intervalMillis = config.path("interval_ms").asInt(100); messageCount = config.path("message_count").asInt(0); } catch (IOException e) { throw new RuntimeException("Failed to parse configuration for streaming impulse", e); } SingleOutputStreamOperator<WindowedValue<byte[]>> source = context .getExecutionEnvironment() .addSource(new StreamingImpulseSource(intervalMillis, messageCount)) .returns(typeInfo); context.addDataStream(Iterables.getOnlyElement(pTransform.getOutputsMap().values()), source); }
@Test public void testFullWindowedValueCoderIsSerializableWithWellKnownCoderType() { CoderProperties.coderSerializable( WindowedValue.getFullCoder(GlobalWindow.Coder.INSTANCE, GlobalWindow.Coder.INSTANCE)); }
@SuppressWarnings("unchecked") public <T> TypeInformation<WindowedValue<T>> getTypeInfo(PCollection<T> collection) { Coder<T> valueCoder = collection.getCoder(); WindowedValue.FullWindowedValueCoder<T> windowedValueCoder = WindowedValue.getFullCoder( valueCoder, collection.getWindowingStrategy().getWindowFn().windowCoder()); return new CoderTypeInformation<>(windowedValueCoder); }
public void putDataset(PValue pvalue, Dataset dataset, boolean forceCache) { try { dataset.setName(pvalue.getName()); } catch (IllegalStateException e) { // name not set, ignore } if ((forceCache || shouldCache(pvalue)) && pvalue instanceof PCollection) { // we cache only PCollection Coder<?> coder = ((PCollection<?>) pvalue).getCoder(); Coder<? extends BoundedWindow> wCoder = ((PCollection<?>) pvalue).getWindowingStrategy().getWindowFn().windowCoder(); dataset.cache(storageLevel(), WindowedValue.getFullCoder(coder, wCoder)); } datasets.put(pvalue, dataset); leaves.add(dataset); }
private void translateImpulse( String id, RunnerApi.Pipeline pipeline, StreamingTranslationContext context) { RunnerApi.PTransform pTransform = pipeline.getComponents().getTransformsOrThrow(id); TypeInformation<WindowedValue<byte[]>> typeInfo = new CoderTypeInformation<>( WindowedValue.getFullCoder(ByteArrayCoder.of(), GlobalWindow.Coder.INSTANCE)); boolean keepSourceAlive = !context.getPipelineOptions().isShutdownSourcesOnFinalWatermark(); SingleOutputStreamOperator<WindowedValue<byte[]>> source = context .getExecutionEnvironment() .addSource(new ImpulseSourceFunction(keepSourceAlive)) .returns(typeInfo); context.addDataStream(Iterables.getOnlyElement(pTransform.getOutputsMap().values()), source); }
private static void translateImpulse( PTransformNode transform, RunnerApi.Pipeline pipeline, BatchTranslationContext context) { TypeInformation<WindowedValue<byte[]>> typeInformation = new CoderTypeInformation<>( WindowedValue.getFullCoder(ByteArrayCoder.of(), GlobalWindow.Coder.INSTANCE)); DataSource<WindowedValue<byte[]>> dataSource = new DataSource<>( context.getExecutionEnvironment(), new ImpulseInputFormat(), typeInformation, transform.getTransform().getUniqueName()); context.addDataSet( Iterables.getOnlyElement(transform.getTransform().getOutputsMap().values()), dataSource); }
private void translateImpulse( String id, RunnerApi.Pipeline pipeline, StreamingTranslationContext context) { RunnerApi.PTransform pTransform = pipeline.getComponents().getTransformsOrThrow(id); TypeInformation<WindowedValue<byte[]>> typeInfo = new CoderTypeInformation<>( WindowedValue.getFullCoder(ByteArrayCoder.of(), GlobalWindow.Coder.INSTANCE)); boolean keepSourceAlive = !context.getPipelineOptions().isShutdownSourcesOnFinalWatermark(); SingleOutputStreamOperator<WindowedValue<byte[]>> source = context .getExecutionEnvironment() .addSource(new ImpulseSourceFunction(keepSourceAlive), "Impulse") .returns(typeInfo); context.addDataStream(Iterables.getOnlyElement(pTransform.getOutputsMap().values()), source); }
private static void translateImpulse( PTransformNode transform, RunnerApi.Pipeline pipeline, BatchTranslationContext context) { TypeInformation<WindowedValue<byte[]>> typeInformation = new CoderTypeInformation<>( WindowedValue.getFullCoder(ByteArrayCoder.of(), GlobalWindow.Coder.INSTANCE)); DataSource<WindowedValue<byte[]>> dataSource = new DataSource<>( context.getExecutionEnvironment(), new ImpulseInputFormat(), typeInformation, transform.getTransform().getUniqueName()) .name("Impulse"); context.addDataSet( Iterables.getOnlyElement(transform.getTransform().getOutputsMap().values()), dataSource); }
@Test public void testComplexCoderSerializable() { CoderProperties.coderSerializable( WindowedValue.getFullCoder( KvCoder.of(ShardedKeyCoder.of(StringUtf8Coder.of()), TableRowInfoCoder.of()), IntervalWindow.getCoder())); }
/** * @param edge IR edge to add. * @param elementCoder element coder. * @param windowCoder window coder. */ void addEdge(final IREdge edge, final Coder elementCoder, final Coder windowCoder) { edge.setProperty(KeyExtractorProperty.of(new BeamKeyExtractor())); if (elementCoder instanceof KvCoder) { Coder keyCoder = ((KvCoder) elementCoder).getKeyCoder(); edge.setProperty(KeyEncoderProperty.of(new BeamEncoderFactory(keyCoder))); edge.setProperty(KeyDecoderProperty.of(new BeamDecoderFactory(keyCoder))); } final WindowedValue.FullWindowedValueCoder coder = WindowedValue.getFullCoder(elementCoder, windowCoder); edge.setProperty(EncoderProperty.of(new BeamEncoderFactory<>(coder))); edge.setProperty(DecoderProperty.of(new BeamDecoderFactory<>(coder))); builder.connectVertices(edge); }