/** * Constructs an {@link OutputDescriptor} instance. Hdfs output has no key. Value type is determined by * writer class (see {@link HdfsSystemDescriptor#withWriterClassName}). * * @param streamId id of the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ HdfsOutputDescriptor(String streamId, SystemDescriptor systemDescriptor) { super(streamId, new NoOpSerde(), systemDescriptor); } }
/** * Constructs an {@link InputDescriptor} instance. Hdfs input has no key. Value type is determined by * reader type (see {@link HdfsSystemDescriptor#withReaderType}). * * @param streamId id of the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ HdfsInputDescriptor(String streamId, SystemDescriptor systemDescriptor) { super(streamId, new NoOpSerde(), systemDescriptor, null); } }
/** * Constructs an {@link InputDescriptor} instance. * * @param streamId id of the stream * @param valueSerde serde the values in the messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ <T> KinesisInputDescriptor(String streamId, Serde<T> valueSerde, SystemDescriptor systemDescriptor) { super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor, null); }
KV<Serde, Serde> getOrCreateStreamSerdes(String streamId, Serde serde) { Serde keySerde, valueSerde; KV<Serde, Serde> currentSerdePair = streamSerdes.get(streamId); if (serde instanceof KVSerde) { keySerde = ((KVSerde) serde).getKeySerde(); valueSerde = ((KVSerde) serde).getValueSerde(); } else { keySerde = new NoOpSerde(); valueSerde = serde; } if (currentSerdePair == null) { if (keySerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the key serde for stream " + streamId + ". Keys will not be (de)serialized"); } if (valueSerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the value serde for stream " + streamId + ". Values will not be (de)serialized"); } streamSerdes.put(streamId, KV.of(keySerde, valueSerde)); } else if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for streamId: %s is already defined. Cannot change it to " + "different serdes.", streamId)); } return streamSerdes.get(streamId); }
KV<Serde, Serde> getOrCreateStreamSerdes(String streamId, Serde serde) { Serde keySerde, valueSerde; KV<Serde, Serde> currentSerdePair = streamSerdes.get(streamId); if (serde instanceof KVSerde) { keySerde = ((KVSerde) serde).getKeySerde(); valueSerde = ((KVSerde) serde).getValueSerde(); } else { keySerde = new NoOpSerde(); valueSerde = serde; } if (currentSerdePair == null) { if (keySerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the key serde for stream " + streamId + ". Keys will not be (de)serialized"); } if (valueSerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the value serde for stream " + streamId + ". Values will not be (de)serialized"); } streamSerdes.put(streamId, KV.of(keySerde, valueSerde)); } else if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for streamId: %s is already defined. Cannot change it to " + "different serdes.", streamId)); } return streamSerdes.get(streamId); }
KV<Serde, Serde> getOrCreateStreamSerdes(String streamId, Serde serde) { Serde keySerde, valueSerde; KV<Serde, Serde> currentSerdePair = streamSerdes.get(streamId); if (serde instanceof KVSerde) { keySerde = ((KVSerde) serde).getKeySerde(); valueSerde = ((KVSerde) serde).getValueSerde(); } else { keySerde = new NoOpSerde(); valueSerde = serde; } if (currentSerdePair == null) { if (keySerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the key serde for stream " + streamId + ". Keys will not be (de)serialized"); } if (valueSerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the value serde for stream " + streamId + ". Values will not be (de)serialized"); } streamSerdes.put(streamId, KV.of(keySerde, valueSerde)); } else if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for streamId: %s is already defined. Cannot change it to " + "different serdes.", streamId)); } return streamSerdes.get(streamId); }
KV<Serde, Serde> getOrCreateStreamSerdes(String streamId, Serde serde) { Serde keySerde, valueSerde; KV<Serde, Serde> currentSerdePair = streamSerdes.get(streamId); if (serde instanceof KVSerde) { keySerde = ((KVSerde) serde).getKeySerde(); valueSerde = ((KVSerde) serde).getValueSerde(); } else { keySerde = new NoOpSerde(); valueSerde = serde; } if (currentSerdePair == null) { if (keySerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the key serde for stream " + streamId + ". Keys will not be (de)serialized"); } if (valueSerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the value serde for stream " + streamId + ". Values will not be (de)serialized"); } streamSerdes.put(streamId, KV.of(keySerde, valueSerde)); } else if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for streamId: %s is already defined. Cannot change it to " + "different serdes.", streamId)); } return streamSerdes.get(streamId); }
KV<Serde, Serde> getOrCreateStreamSerdes(String streamId, Serde serde) { Serde keySerde, valueSerde; KV<Serde, Serde> currentSerdePair = streamSerdes.get(streamId); if (serde instanceof KVSerde) { keySerde = ((KVSerde) serde).getKeySerde(); valueSerde = ((KVSerde) serde).getValueSerde(); } else { keySerde = new NoOpSerde(); valueSerde = serde; } if (currentSerdePair == null) { if (keySerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the key serde for stream " + streamId + ". Keys will not be (de)serialized"); } if (valueSerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the value serde for stream " + streamId + ". Values will not be (de)serialized"); } streamSerdes.put(streamId, KV.of(keySerde, valueSerde)); } else if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for streamId: %s is already defined. Cannot change it to " + "different serdes.", streamId)); } return streamSerdes.get(streamId); }
/** * Constructs an {@link InputDescriptor} instance. * * @param streamId id of the stream * @param namespace namespace for the Event Hubs entity to consume from, not null * @param entityPath entity path for the Event Hubs entity to consume from, not null * @param valueSerde serde the values in the messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ EventHubsInputDescriptor(String streamId, String namespace, String entityPath, Serde valueSerde, SystemDescriptor systemDescriptor) { super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor, null); this.namespace = StringUtils.stripToNull(namespace); this.entityPath = StringUtils.stripToNull(entityPath); if (this.namespace == null || this.entityPath == null) { throw new ConfigException(String.format("Missing namespace and entity path Event Hubs input descriptor in " // + "system: {%s}, stream: {%s}", getSystemName(), streamId)); } }
void translate(final TableScan tableScan, final TranslatorContext context, Map<String, DelegatingSystemDescriptor> systemDescriptors, Map<String, MessageStream<KV<Object, Object>>> inputMsgStreams) { StreamApplicationDescriptor streamAppDesc = context.getStreamAppDescriptor(); List<String> tableNameParts = tableScan.getTable().getQualifiedName(); String sourceName = SqlIOConfig.getSourceFromSourceParts(tableNameParts); Validate.isTrue(relMsgConverters.containsKey(sourceName), String.format("Unknown source %s", sourceName)); SqlIOConfig sqlIOConfig = systemStreamConfig.get(sourceName); final String systemName = sqlIOConfig.getSystemName(); final String streamName = sqlIOConfig.getStreamName(); final String source = sqlIOConfig.getSource(); KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericInputDescriptor<KV<Object, Object>> isd = sd.getInputDescriptor(streamName, noOpKVSerde); MessageStream<KV<Object, Object>> inputStream = inputMsgStreams.computeIfAbsent(source, v -> streamAppDesc.getInputStream(isd)); MessageStream<SamzaSqlRelMessage> samzaSqlRelMessageStream = inputStream.map(new ScanMapFunction(sourceName, queryId)); context.registerMessageStream(tableScan.getId(), samzaSqlRelMessageStream); } }
/** * Constructs an {@link OutputDescriptor} instance. * * @param streamId id of the stream * @param namespace namespace for the Event Hubs entity to produce to, not null * @param entityPath entity path for the Event Hubs entity to produce to, not null * @param valueSerde serde the values in the messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ EventHubsOutputDescriptor(String streamId, String namespace, String entityPath, Serde valueSerde, SystemDescriptor systemDescriptor) { super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor); this.namespace = StringUtils.stripToNull(namespace); this.entityPath = StringUtils.stripToNull(entityPath); if (this.namespace == null || this.entityPath == null) { throw new ConfigException(String.format("Missing namespace and entity path Event Hubs output descriptor in " // + "system: {%s}, stream: {%s}", getSystemName(), streamId)); } }
private void sendToOutputStream(String queryLogicalId, String logicalOpId, String sinkStream, StreamApplicationDescriptor appDesc, TranslatorContext translatorContext, RelNode node, int queryId) { SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(sinkStream); MessageStream<SamzaSqlRelMessage> stream = translatorContext.getMessageStream(node.getId()); MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(queryLogicalId, logicalOpId, sinkStream, queryId)); Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor(); if (!tableDescriptor.isPresent()) { KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); String systemName = sinkConfig.getSystemName(); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamId(), noOpKVSerde); OutputStream stm = outputMsgStreams.computeIfAbsent(sinkConfig.getSource(), v -> appDesc.getOutputStream(osd)); outputStream.sendTo(stm); } else { Table outputTable = appDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource(); throw new SamzaException(msg); } outputStream.sendTo(outputTable); } } }
private void sendToOutputStream(StreamApplicationDescriptor appDesc, TranslatorContext context, RelNode node, int queryId) { SqlIOConfig sinkConfig = sqlConfig.getOutputSystemStreamConfigsBySource().get(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG); MessageStream<SamzaSqlRelMessage> stream = context.getMessageStream(node.getId()); MessageStream<KV<Object, Object>> outputStream = stream.map(new OutputMapFunction(SamzaSqlApplicationConfig.SAMZA_SYSTEM_LOG, queryId)); Optional<TableDescriptor> tableDescriptor = sinkConfig.getTableDescriptor(); if (!tableDescriptor.isPresent()) { KVSerde<Object, Object> noOpKVSerde = KVSerde.of(new NoOpSerde<>(), new NoOpSerde<>()); String systemName = sinkConfig.getSystemName(); DelegatingSystemDescriptor sd = systemDescriptors.computeIfAbsent(systemName, DelegatingSystemDescriptor::new); GenericOutputDescriptor<KV<Object, Object>> osd = sd.getOutputDescriptor(sinkConfig.getStreamName(), noOpKVSerde); if (OutputMapFunction.logOutputStream == null) { OutputMapFunction.logOutputStream = appDesc.getOutputStream(osd); } outputStream.sendTo(OutputMapFunction.logOutputStream); } else { Table outputTable = appDesc.getTable(tableDescriptor.get()); if (outputTable == null) { String msg = "Failed to obtain table descriptor of " + sinkConfig.getSource(); throw new SamzaException(msg); } outputStream.sendTo(outputTable); } } }
intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); GenericOutputDescriptor<KV<Object, Object>> intermediateOutput1 = system1Descriptor.getOutputDescriptor( intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); GenericOutputDescriptor<KV<Object, Object>> intermediateBroacastOutput1 = system1Descriptor.getOutputDescriptor( intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); inputDescriptors.put(intermediateStream1, intermediateInput1); outputDescriptors.put(intermediateStream1, intermediateOutput1);
InputOperatorSpec testInput = new InputOperatorSpec(inputStreamId1, new NoOpSerde(), new NoOpSerde(), null, true, inputStreamId1); StreamOperatorSpec filterOp = OperatorSpecs.createFilterOperatorSpec(m -> true, "test-filter-2"); OutputStreamImpl outputStream1 = new OutputStreamImpl(outputStreamId, null, null, true); filterOp.registerNextOperatorSpec(outputSpec); String streamId2 = "test-input-2"; InputOperatorSpec testInput2 = new InputOperatorSpec(streamId2, new NoOpSerde(), new NoOpSerde(), null, true, "test-input-4"); StreamOperatorSpec testMap = OperatorSpecs.createMapOperatorSpec(m -> m, "test-map-5"); SinkOperatorSpec testSink = OperatorSpecs.createSinkOperatorSpec((m, mc, tc) -> { }, "test-sink-6");
@Test public void testJoinOperatorSpec() { InputOperatorSpec leftOpSpec = new InputOperatorSpec( "test-input-1", new NoOpSerde<>(), new NoOpSerde<>(), null, false, "op0"); InputOperatorSpec rightOpSpec = new InputOperatorSpec( "test-input-2", new NoOpSerde<>(), new NoOpSerde<>(), null, false, "op1"); Serde<Object> objSerde = new Serde<Object>() { @Override public Object fromBytes(byte[] bytes) { return null; } @Override public byte[] toBytes(Object object) { return new byte[0]; } }; JoinFunction<String, Object, Object, TestOutputMessageEnvelope> joinFn = new TestJoinFunction(); JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope> joinOperatorSpec = new JoinOperatorSpec<>(leftOpSpec, rightOpSpec, joinFn, new StringSerde("UTF-8"), objSerde, objSerde, 50000, "op2"); JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope> joinOpCopy = (JoinOperatorSpec<String, Object, Object, TestOutputMessageEnvelope>) OperatorSpecTestUtils.copyOpSpec(joinOperatorSpec); assertNotEquals("Expected deserialized copy of operator spec should not be the same as the original operator spec", joinOperatorSpec, joinOpCopy); assertTrue(joinOperatorSpec.isClone(joinOpCopy)); assertTrue(joinOpCopy.getLeftInputOpSpec().isClone(leftOpSpec)); assertTrue(joinOpCopy.getRightInputOpSpec().isClone(rightOpSpec)); }
InputDescriptor inputDescriptor = systemDescriptor.getInputDescriptor(streamId, new NoOpSerde<>()); MessageStream<SamzaSqlRelMessage> samzaSqlRelMessageStream = inputMsgStreams.computeIfAbsent(source, v -> streamAppDesc.getInputStream(inputDescriptor))