/** * Builds a map from SystemStreamPartition to oldest offset for changelogs. */ private Map<SystemStream, String> getChangeLogOldestOffsetsForPartition(Partition partition, Map<SystemStream, SystemStreamMetadata> inputStreamMetadata) { Map<SystemStream, String> retVal = new HashMap<>(); // NOTE: do not use Collectors.Map because of https://bugs.openjdk.java.net/browse/JDK-8148463 inputStreamMetadata.entrySet() .stream() .filter(x -> x.getValue().getSystemStreamPartitionMetadata().get(partition) != null) .forEach(e -> retVal.put(e.getKey(), e.getValue().getSystemStreamPartitionMetadata().get(partition).getOldestOffset())); return retVal; }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
SystemStreamMetadata.SystemStreamPartitionMetadata partition0Metadata = systemStreamPartitionMetadata.get(new Partition(0)); assertEquals("oldest offset for partition 0", partition0Metadata.getOldestOffset(), KAFKA_BEGINNING_OFFSET_FOR_PARTITION0.toString()); assertEquals("upcoming offset for partition 0", partition0Metadata.getUpcomingOffset(), SystemStreamMetadata.SystemStreamPartitionMetadata partition1Metadata = systemStreamPartitionMetadata.get(new Partition(1)); assertEquals("oldest offset for partition 1", partition1Metadata.getOldestOffset(), KAFKA_BEGINNING_OFFSET_FOR_PARTITION1.toString()); assertEquals("upcoming offset for partition 1", partition1Metadata.getUpcomingOffset(),
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
@Test public void testGetSystemStreamMetaDataForTopicWithNoMessage() { // The topic with no messages will have beginningOffset = 0 and endOffset = 0 when(mockKafkaConsumer.beginningOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); when(mockKafkaConsumer.endOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC)); assertEquals("metadata should return for 1 topic", metadataMap.size(), 1); // verify the metadata streamName assertEquals("the stream name should be " + VALID_TOPIC, metadataMap.get(VALID_TOPIC).getStreamName(), VALID_TOPIC); // verify the offset for each partition Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> systemStreamPartitionMetadata = metadataMap.get(VALID_TOPIC).getSystemStreamPartitionMetadata(); assertEquals("there are 2 partitions", systemStreamPartitionMetadata.size(), 2); SystemStreamMetadata.SystemStreamPartitionMetadata partition0Metadata = systemStreamPartitionMetadata.get(new Partition(0)); assertEquals("oldest offset for partition 0", partition0Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 0", partition0Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition0Metadata.getNewestOffset(), null); SystemStreamMetadata.SystemStreamPartitionMetadata partition1Metadata = systemStreamPartitionMetadata.get(new Partition(1)); assertEquals("oldest offset for partition 1", partition1Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 1", partition1Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition1Metadata.getNewestOffset(), null); }
@Ignore("Integration Test") @Test public void testGetStreamMetadata() { EventHubSystemFactory eventHubSystemFactory = new EventHubSystemFactory(); SystemAdmin eventHubSystemAdmin = eventHubSystemFactory.getAdmin(SYSTEM_NAME, MockEventHubConfigFactory.getEventHubConfig(EventHubSystemProducer.PartitioningMethod.EVENT_HUB_HASHING)); Set<String> streams = new HashSet<>(); streams.add(STREAM_NAME1); streams.add(STREAM_NAME2); Map<String, SystemStreamMetadata> metadataMap = eventHubSystemAdmin.getSystemStreamMetadata(streams); for (String stream : streams) { Assert.assertTrue(metadataMap.containsKey(stream)); Assert.assertEquals(stream, metadataMap.get(stream).getStreamName()); Assert.assertNotNull(metadataMap.get(stream).getSystemStreamPartitionMetadata()); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadataMap = metadataMap.get(stream).getSystemStreamPartitionMetadata(); Assert.assertTrue(partitionMetadataMap.size() >= MIN_EVENTHUB_ENTITY_PARTITION); Assert.assertTrue(partitionMetadataMap.size() <= MAX_EVENTHUB_ENTITY_PARTITION); partitionMetadataMap.forEach((partition, metadata) -> { Assert.assertEquals(EventHubSystemConsumer.START_OF_STREAM, metadata.getOldestOffset()); Assert.assertNotSame(EventHubSystemConsumer.END_OF_STREAM, metadata.getNewestOffset()); String expectedUpcomingOffset = String.valueOf(Long.parseLong(metadata.getNewestOffset()) + 1); Assert.assertEquals(expectedUpcomingOffset, metadata.getUpcomingOffset()); }); } }
/** * @param offsetType * The type of offset to get. Either oldest, newest, or upcoming. * @return The corresponding offset for the offset type requested. */ public String getOffset(OffsetType offsetType) { if (offsetType.equals(OffsetType.OLDEST)) { return getOldestOffset(); } else if (offsetType.equals(OffsetType.NEWEST)) { return getNewestOffset(); } else if (offsetType.equals(OffsetType.UPCOMING)) { return getUpcomingOffset(); } else { throw new SamzaException("Invalid offset type defined " + offsetType + "."); } }
/** * @param offsetType * The type of offset to get. Either oldest, newest, or upcoming. * @return The corresponding offset for the offset type requested. */ public String getOffset(OffsetType offsetType) { if (offsetType.equals(OffsetType.OLDEST)) { return getOldestOffset(); } else if (offsetType.equals(OffsetType.NEWEST)) { return getNewestOffset(); } else if (offsetType.equals(OffsetType.UPCOMING)) { return getUpcomingOffset(); } else { throw new SamzaException("Invalid offset type defined " + offsetType + "."); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }