@Override public boolean createStream(StreamSpec streamSpec) { createdStreamName = streamSpec.getPhysicalName(); return true; }
@Override public boolean createStream(StreamSpec streamSpec) { createdStreamName = streamSpec.getPhysicalName(); return true; }
@Override public boolean clearStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); KafkaSystemAdminUtilsScala.clearStream(streamSpec, getZkConnection()); Map<String, List<PartitionInfo>> topicsMetadata = getTopicMetadata(ImmutableSet.of(streamSpec.getPhysicalName())); return topicsMetadata.get(streamSpec.getPhysicalName()).isEmpty(); }
StreamSpec getStreamSpec() { StreamSpec spec = (partitions == PARTITIONS_UNKNOWN) ? streamSpec : streamSpec.copyWithPartitionCount(partitions); if (isIntermediate) { String physicalName = StreamManager.createUniqueNameForBatch(spec.getPhysicalName(), config); if (!physicalName.equals(spec.getPhysicalName())) { spec = spec.copyWithPhysicalName(physicalName); } } return spec; }
StreamSpec getStreamSpec() { StreamSpec spec = (partitions == PARTITIONS_UNKNOWN) ? streamSpec : streamSpec.copyWithPartitionCount(partitions); if (isIntermediate) { String physicalName = StreamManager.createUniqueNameForBatch(spec.getPhysicalName(), config); if (!physicalName.equals(spec.getPhysicalName())) { spec = spec.copyWithPhysicalName(physicalName); } } return spec; }
StreamSpec getStreamSpec() { StreamSpec spec = (partitions == PARTITIONS_UNKNOWN) ? streamSpec : streamSpec.copyWithPartitionCount(partitions); if (isIntermediate) { String physicalName = StreamManager.createUniqueNameForBatch(spec.getPhysicalName(), config); if (!physicalName.equals(spec.getPhysicalName())) { spec = spec.copyWithPhysicalName(physicalName); } } return spec; }
StreamSpec getStreamSpec() { StreamSpec spec = (partitions == PARTITIONS_UNKNOWN) ? streamSpec : streamSpec.copyWithPartitionCount(partitions); if (isIntermediate) { String physicalName = StreamManager.createUniqueNameForBatch(spec.getPhysicalName(), config); if (!physicalName.equals(spec.getPhysicalName())) { spec = spec.copyWithPhysicalName(physicalName); } } return spec; }
StreamSpec getStreamSpec() { StreamSpec spec = (partitions == PARTITIONS_UNKNOWN) ? streamSpec : streamSpec.copyWithPartitionCount(partitions); if (isIntermediate) { String physicalName = StreamManager.createUniqueNameForBatch(spec.getPhysicalName(), config); if (!physicalName.equals(spec.getPhysicalName())) { spec = spec.copyWithPhysicalName(physicalName); } } return spec; }
@Override public boolean clearStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); String topicName = streamSpec.getPhysicalName(); try { DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(ImmutableSet.of(topicName)); deleteTopicsResult.all().get(KAFKA_ADMIN_OPS_TIMEOUT_MS, TimeUnit.MILLISECONDS); } catch (Exception e) { LOG.error("Failed to delete topic {} with exception {}.", topicName, e); return false; } return true; }
@Override public boolean createStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); return KafkaSystemAdminUtilsScala.createStream(toKafkaSpec(streamSpec), getZkConnection()); }
@Override public boolean createStream(StreamSpec streamSpec) { LOG.info("Creating Kafka topic: {} on system: {}", streamSpec.getPhysicalName(), streamSpec.getSystemName()); return KafkaSystemAdminUtilsScala.createStream(toKafkaSpec(streamSpec), getZkConnection()); }
private String streamSpecToJson(StreamSpec streamSpec) { return String.format(STREAM_SPEC_JSON_FORMAT, streamSpec.getId(), streamSpec.getId(), streamSpec.getSystemName(), streamSpec.getPhysicalName()); } }
@Test public void testGetStreamSpec_Batch() { Map<String, String> config = new HashMap<>(); config.put(ApplicationConfig.APP_MODE, ApplicationConfig.ApplicationMode.BATCH.name()); config.put(ApplicationConfig.APP_RUN_ID, "123"); StreamEdge edge = new StreamEdge(spec, true, false, new MapConfig(config)); assertEquals(edge.getStreamSpec().getPhysicalName(), spec.getPhysicalName() + "-123"); }
@Test public void testGetStreamWithoutPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.SYSTEM(), TEST_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getPhysicalName()); }
@Test public void testGetStreamWithPhysicalNameInConfig() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, StreamConfig.SYSTEM(), TEST_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); }
@Test public void testGetStreamPhysicalNameArgSpecialCharacters() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME_SPECIAL_CHARS, StreamConfig.SYSTEM(), TEST_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_PHYSICAL_NAME_SPECIAL_CHARS, spec.getPhysicalName()); }
@Test public void testGetStreamPhysicalNameArgNull() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), null, StreamConfig.SYSTEM(), TEST_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertNull(spec.getPhysicalName()); }
@Test public void testGetStreamSystemNameArgValid() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, // This should be ignored because of the explicit arg StreamConfig.SYSTEM(), TEST_SYSTEM); // This too StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME, spec.getPhysicalName()); assertEquals(TEST_SYSTEM, spec.getSystemName()); }
@Test public void testGetStreamPhysicalNameArgSimple() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME2, // This should be ignored because of the explicit arg StreamConfig.SYSTEM(), TEST_SYSTEM); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(STREAM_ID, spec.getId()); assertEquals(TEST_PHYSICAL_NAME2, spec.getPhysicalName()); assertEquals(TEST_SYSTEM, spec.getSystemName()); }
@Test public void testBasicConstructor() { StreamSpec streamSpec = new StreamSpec("dummyId", "dummyPhysicalName", "dummySystemName", 1); assertEquals("dummyId", streamSpec.getId()); assertEquals("dummyPhysicalName", streamSpec.getPhysicalName()); assertEquals("dummySystemName", streamSpec.getSystemName()); assertEquals(1, streamSpec.getPartitionCount()); // SystemStream should use the physical name, not the streamId. SystemStream systemStream = new SystemStream("dummySystemName", "dummyPhysicalName"); assertEquals(systemStream, streamSpec.toSystemStream()); }