@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> metadata = new HashMap<String, SystemStreamMetadata>(); for (String streamName : streamNames) { metadata.put(streamName, new SystemStreamMetadata(streamName, FAKE_PARTITION_METADATA)); } return metadata; }
private Stream<SystemStreamPartition> mapSSMToSSP(Map.Entry<SystemStream, SystemStreamMetadata> ssMs) { return ssMs.getValue() .getSystemStreamPartitionMetadata() .keySet() .stream() .map(partition -> new SystemStreamPartition(ssMs.getKey(), partition)); }
@Ignore("Integration Test") @Test public void testGetStreamMetadata() { EventHubSystemFactory eventHubSystemFactory = new EventHubSystemFactory(); SystemAdmin eventHubSystemAdmin = eventHubSystemFactory.getAdmin(SYSTEM_NAME, MockEventHubConfigFactory.getEventHubConfig(EventHubSystemProducer.PartitioningMethod.EVENT_HUB_HASHING)); Set<String> streams = new HashSet<>(); streams.add(STREAM_NAME1); streams.add(STREAM_NAME2); Map<String, SystemStreamMetadata> metadataMap = eventHubSystemAdmin.getSystemStreamMetadata(streams); for (String stream : streams) { Assert.assertTrue(metadataMap.containsKey(stream)); Assert.assertEquals(stream, metadataMap.get(stream).getStreamName()); Assert.assertNotNull(metadataMap.get(stream).getSystemStreamPartitionMetadata()); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadataMap = metadataMap.get(stream).getSystemStreamPartitionMetadata(); Assert.assertTrue(partitionMetadataMap.size() >= MIN_EVENTHUB_ENTITY_PARTITION); Assert.assertTrue(partitionMetadataMap.size() <= MAX_EVENTHUB_ENTITY_PARTITION); partitionMetadataMap.forEach((partition, metadata) -> { Assert.assertEquals(EventHubSystemConsumer.START_OF_STREAM, metadata.getOldestOffset()); Assert.assertNotSame(EventHubSystemConsumer.END_OF_STREAM, metadata.getNewestOffset()); String expectedUpcomingOffset = String.valueOf(Long.parseLong(metadata.getNewestOffset()) + 1); Assert.assertEquals(expectedUpcomingOffset, metadata.getUpcomingOffset()); }); } }
System.out.println("STARTING1"); assertEquals("the stream name should be " + VALID_TOPIC, metadataMap.get(VALID_TOPIC).getStreamName(), VALID_TOPIC); System.out.println("STARTING2"); metadataMap.get(VALID_TOPIC).getSystemStreamPartitionMetadata(); assertEquals("there are 2 partitions", systemStreamPartitionMetadata.size(), 2); System.out.println("STARTING3");
/** * Builds a map from SystemStreamPartition to oldest offset for changelogs. */ private Map<SystemStream, String> getChangeLogOldestOffsetsForPartition(Partition partition, Map<SystemStream, SystemStreamMetadata> inputStreamMetadata) { Map<SystemStream, String> retVal = new HashMap<>(); // NOTE: do not use Collectors.Map because of https://bugs.openjdk.java.net/browse/JDK-8148463 inputStreamMetadata.entrySet() .stream() .filter(x -> x.getValue().getSystemStreamPartitionMetadata().get(partition) != null) .forEach(e -> retVal.put(e.getKey(), e.getValue().getSystemStreamPartitionMetadata().get(partition).getOldestOffset())); return retVal; }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> metadata = new HashMap<String, SystemStreamMetadata>(); for (String streamName : streamNames) { metadata.put(streamName, new SystemStreamMetadata(streamName, FAKE_PARTITION_METADATA)); } return metadata; }
@Test public void testGetSystemStreamMetaDataForTopicWithNoMessage() { // The topic with no messages will have beginningOffset = 0 and endOffset = 0 when(mockKafkaConsumer.beginningOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); when(mockKafkaConsumer.endOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC)); assertEquals("metadata should return for 1 topic", metadataMap.size(), 1); // verify the metadata streamName assertEquals("the stream name should be " + VALID_TOPIC, metadataMap.get(VALID_TOPIC).getStreamName(), VALID_TOPIC); // verify the offset for each partition Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> systemStreamPartitionMetadata = metadataMap.get(VALID_TOPIC).getSystemStreamPartitionMetadata(); assertEquals("there are 2 partitions", systemStreamPartitionMetadata.size(), 2); SystemStreamMetadata.SystemStreamPartitionMetadata partition0Metadata = systemStreamPartitionMetadata.get(new Partition(0)); assertEquals("oldest offset for partition 0", partition0Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 0", partition0Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition0Metadata.getNewestOffset(), null); SystemStreamMetadata.SystemStreamPartitionMetadata partition1Metadata = systemStreamPartitionMetadata.get(new Partition(1)); assertEquals("oldest offset for partition 1", partition1Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 1", partition1Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition1Metadata.getNewestOffset(), null); }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return streamNames.stream() .collect(Collectors.toMap(Function.identity(), streamName -> new SystemStreamMetadata(streamName, Collections.singletonMap(new Partition(0), new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null))))); }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }