String newestOffset = MultiFileHdfsReader.generateOffset(lengthList.size() - 1, String.valueOf(lengthList.get(lengthList.size() - 1))); SystemStreamPartitionMetadata metadata = new SystemStreamPartitionMetadata(oldestOffset, newestOffset, null); partitionMetadataMap.put(partition, metadata); partitionDescriptorMap.get(streamName).put(partition, pathList);
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
new SystemStreamMetadata.SystemStreamPartitionMetadata("0", "50", "51"); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); partitionMetadata.put(new Partition(0), sspMetadata);
new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null) { String msg = "getSystemStreamPartitionCounts does not populate SystemStreaMetadata info. Only number of partitions";
new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null) { String msg = "getSystemStreamPartitionCounts does not populate SystemStreaMetadata info. Only number of partitions";
new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null) { String msg = "getSystemStreamPartitionCounts does not populate SystemStreaMetadata info. Only number of partitions";
SystemStreamPartition ssp = new SystemStreamPartition(SYSTEM_NAME, streamName, partition); systemStreamPartitionSet.add(ssp); String offset = metadata.getOldestOffset(); systemConsumer.register(ssp, offset); });
String newestOffset = ehPartitionInfo.getLastEnqueuedOffset(); String upcomingOffset = EventHubSystemConsumer.END_OF_STREAM; SystemStreamPartitionMetadata sspMetadata = new SystemStreamPartitionMetadata(startingOffset, newestOffset, upcomingOffset); sspMetadataMap.put(new Partition(Integer.parseInt(partition)), sspMetadata);
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
Collectors.toMap(Function.identity(), ssp -> partitionMetadata.get(ssp.getPartition()).getOldestOffset()));
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
SystemStreamMetadata.SystemStreamPartitionMetadata partition0Metadata = systemStreamPartitionMetadata.get(new Partition(0)); assertEquals("oldest offset for partition 0", partition0Metadata.getOldestOffset(), KAFKA_BEGINNING_OFFSET_FOR_PARTITION0.toString()); assertEquals("upcoming offset for partition 0", partition0Metadata.getUpcomingOffset(), KAFKA_END_OFFSET_FOR_PARTITION0.toString()); assertEquals("newest offset for partition 0", partition0Metadata.getNewestOffset(), Long.toString(KAFKA_END_OFFSET_FOR_PARTITION0 - 1)); System.out.println("STARTING4"); SystemStreamMetadata.SystemStreamPartitionMetadata partition1Metadata = systemStreamPartitionMetadata.get(new Partition(1)); assertEquals("oldest offset for partition 1", partition1Metadata.getOldestOffset(), KAFKA_BEGINNING_OFFSET_FOR_PARTITION1.toString()); assertEquals("upcoming offset for partition 1", partition1Metadata.getUpcomingOffset(), KAFKA_END_OFFSET_FOR_PARTITION1.toString()); assertEquals("newest offset for partition 1", partition1Metadata.getNewestOffset(), Long.toString(KAFKA_END_OFFSET_FOR_PARTITION1 - 1));
/** * Retrieves the oldest offset in the coordinator stream, and registers the * coordinator stream with the SystemConsumer using the earliest offset. */ public void register() { if (isStarted) { log.info("Coordinator stream partition {} has already been registered. Skipping.", coordinatorSystemStreamPartition); return; } log.debug("Attempting to register: {}", coordinatorSystemStreamPartition); Set<String> streamNames = new HashSet<String>(); String streamName = coordinatorSystemStreamPartition.getStream(); streamNames.add(streamName); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(streamNames); log.info(String.format("Got metadata %s", systemStreamMetadataMap.toString())); if (systemStreamMetadataMap == null) { throw new SamzaException("Received a null systemStreamMetadataMap from the systemAdmin. This is illegal."); } SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); if (systemStreamMetadata == null) { throw new SamzaException("Expected " + streamName + " to be in system stream metadata."); } SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); if (systemStreamPartitionMetadata == null) { throw new SamzaException("Expected metadata for " + coordinatorSystemStreamPartition + " to exist."); } String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); log.debug("Registering {} with offset {}", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> metadataMap = new HashMap<>(); Map<String, Set<Partition>> partitionMap = MSG_QUEUES.entrySet() .stream() .filter(entry -> streamNames.contains(entry.getKey().getSystemStream().getStream())) .map(e -> e.getKey()).<Map<String, Set<Partition>>>collect(HashMap::new, (m, ssp) -> { if (m.get(ssp.getStream()) == null) { m.put(ssp.getStream(), new HashSet<>()); } m.get(ssp.getStream()).add(ssp.getPartition()); }, (m1, m2) -> { m2.forEach((k, v) -> { if (m1.get(k) == null) { m1.put(k, v); } else { m1.get(k).addAll(v); } }); }); partitionMap.forEach((k, v) -> { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetaMap = v.stream().<Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata>>collect(HashMap::new, (m, p) -> { m.put(p, new SystemStreamMetadata.SystemStreamPartitionMetadata("", "", "")); }, (m1, m2) -> m1.putAll(m2)); metadataMap.put(k, new SystemStreamMetadata(k, partitionMetaMap)); }); return metadataMap; }
SystemAdmin systemAdmin = mock(MySystemAdmin.class); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("1", "2", "3"); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition1Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("11", "12", "13"); SystemStreamMetadata.SystemStreamPartitionMetadata otherStreamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("21", "22", "23"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of(