assertEquals("oldest offset for partition 0", partition0Metadata.getOldestOffset(), KAFKA_BEGINNING_OFFSET_FOR_PARTITION0.toString()); assertEquals("upcoming offset for partition 0", partition0Metadata.getUpcomingOffset(), KAFKA_END_OFFSET_FOR_PARTITION0.toString()); assertEquals("newest offset for partition 0", partition0Metadata.getNewestOffset(), assertEquals("oldest offset for partition 1", partition1Metadata.getOldestOffset(), KAFKA_BEGINNING_OFFSET_FOR_PARTITION1.toString()); assertEquals("upcoming offset for partition 1", partition1Metadata.getUpcomingOffset(), KAFKA_END_OFFSET_FOR_PARTITION1.toString()); assertEquals("newest offset for partition 1", partition1Metadata.getNewestOffset(),
@Test public void testGetSystemStreamMetaDataForTopicWithNoMessage() { // The topic with no messages will have beginningOffset = 0 and endOffset = 0 when(mockKafkaConsumer.beginningOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); when(mockKafkaConsumer.endOffsets(ImmutableList.of(testTopicPartition0, testTopicPartition1))).thenReturn( ImmutableMap.of(testTopicPartition0, 0L, testTopicPartition1, 0L)); Map<String, SystemStreamMetadata> metadataMap = kafkaSystemAdmin.getSystemStreamMetadata(ImmutableSet.of(VALID_TOPIC)); assertEquals("metadata should return for 1 topic", metadataMap.size(), 1); // verify the metadata streamName assertEquals("the stream name should be " + VALID_TOPIC, metadataMap.get(VALID_TOPIC).getStreamName(), VALID_TOPIC); // verify the offset for each partition Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> systemStreamPartitionMetadata = metadataMap.get(VALID_TOPIC).getSystemStreamPartitionMetadata(); assertEquals("there are 2 partitions", systemStreamPartitionMetadata.size(), 2); SystemStreamMetadata.SystemStreamPartitionMetadata partition0Metadata = systemStreamPartitionMetadata.get(new Partition(0)); assertEquals("oldest offset for partition 0", partition0Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 0", partition0Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition0Metadata.getNewestOffset(), null); SystemStreamMetadata.SystemStreamPartitionMetadata partition1Metadata = systemStreamPartitionMetadata.get(new Partition(1)); assertEquals("oldest offset for partition 1", partition1Metadata.getOldestOffset(), "0"); assertEquals("upcoming offset for partition 1", partition1Metadata.getUpcomingOffset(), "0"); assertEquals("newest offset is not set due to abnormal upcoming offset", partition1Metadata.getNewestOffset(), null); }
@Ignore("Integration Test") @Test public void testGetStreamMetadata() { EventHubSystemFactory eventHubSystemFactory = new EventHubSystemFactory(); SystemAdmin eventHubSystemAdmin = eventHubSystemFactory.getAdmin(SYSTEM_NAME, MockEventHubConfigFactory.getEventHubConfig(EventHubSystemProducer.PartitioningMethod.EVENT_HUB_HASHING)); Set<String> streams = new HashSet<>(); streams.add(STREAM_NAME1); streams.add(STREAM_NAME2); Map<String, SystemStreamMetadata> metadataMap = eventHubSystemAdmin.getSystemStreamMetadata(streams); for (String stream : streams) { Assert.assertTrue(metadataMap.containsKey(stream)); Assert.assertEquals(stream, metadataMap.get(stream).getStreamName()); Assert.assertNotNull(metadataMap.get(stream).getSystemStreamPartitionMetadata()); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadataMap = metadataMap.get(stream).getSystemStreamPartitionMetadata(); Assert.assertTrue(partitionMetadataMap.size() >= MIN_EVENTHUB_ENTITY_PARTITION); Assert.assertTrue(partitionMetadataMap.size() <= MAX_EVENTHUB_ENTITY_PARTITION); partitionMetadataMap.forEach((partition, metadata) -> { Assert.assertEquals(EventHubSystemConsumer.START_OF_STREAM, metadata.getOldestOffset()); Assert.assertNotSame(EventHubSystemConsumer.END_OF_STREAM, metadata.getNewestOffset()); String expectedUpcomingOffset = String.valueOf(Long.parseLong(metadata.getNewestOffset()) + 1); Assert.assertEquals(expectedUpcomingOffset, metadata.getUpcomingOffset()); }); } }
/** * @param offsetType * The type of offset to get. Either oldest, newest, or upcoming. * @return The corresponding offset for the offset type requested. */ public String getOffset(OffsetType offsetType) { if (offsetType.equals(OffsetType.OLDEST)) { return getOldestOffset(); } else if (offsetType.equals(OffsetType.NEWEST)) { return getNewestOffset(); } else if (offsetType.equals(OffsetType.UPCOMING)) { return getUpcomingOffset(); } else { throw new SamzaException("Invalid offset type defined " + offsetType + "."); } }
/** * @param offsetType * The type of offset to get. Either oldest, newest, or upcoming. * @return The corresponding offset for the offset type requested. */ public String getOffset(OffsetType offsetType) { if (offsetType.equals(OffsetType.OLDEST)) { return getOldestOffset(); } else if (offsetType.equals(OffsetType.NEWEST)) { return getNewestOffset(); } else if (offsetType.equals(OffsetType.UPCOMING)) { return getUpcomingOffset(); } else { throw new SamzaException("Invalid offset type defined " + offsetType + "."); } }