/** * Builds a map from SystemStreamPartition to oldest offset for changelogs. */ private Map<SystemStream, String> getChangeLogOldestOffsetsForPartition(Partition partition, Map<SystemStream, SystemStreamMetadata> inputStreamMetadata) { Map<SystemStream, String> retVal = new HashMap<>(); // NOTE: do not use Collectors.Map because of https://bugs.openjdk.java.net/browse/JDK-8148463 inputStreamMetadata.entrySet() .stream() .filter(x -> x.getValue().getSystemStreamPartitionMetadata().get(partition) != null) .forEach(e -> retVal.put(e.getKey(), e.getValue().getSystemStreamPartitionMetadata().get(partition).getOldestOffset())); return retVal; }
private Stream<SystemStreamPartition> mapSSMToSSP(Map.Entry<SystemStream, SystemStreamMetadata> ssMs) { return ssMs.getValue() .getSystemStreamPartitionMetadata() .keySet() .stream() .map(partition -> new SystemStreamPartition(ssMs.getKey(), partition)); }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
private int getPartitionCount(SystemStream systemStream) { return PARTITION_COUNT_CACHE.computeIfAbsent(systemStream, ss -> { SystemStreamMetadata metadata = metadataCache.getSystemStreamMetadata(ss, true); if (metadata == null) { throw new SamzaException("Unable to find metadata for stream " + systemStream); } return metadata.getSystemStreamPartitionMetadata().size(); }); } }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
Map<String, Integer> getStreamPartitionCounts(String systemName, Set<String> streamNames) { Map<String, Integer> streamToPartitionCount = new HashMap<>(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); if (systemAdmin == null) { throw new SamzaException(String.format("System %s does not exist.", systemName)); } // retrieve the metadata for the streams in this system Map<String, SystemStreamMetadata> streamToMetadata = systemAdmin.getSystemStreamMetadata(streamNames); // set the partitions of a stream to its StreamEdge streamToMetadata.forEach((stream, data) -> streamToPartitionCount.put(stream, data.getSystemStreamPartitionMetadata().size())); return streamToPartitionCount; }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
private void registerConsumer() { LOG.debug("Attempting to register system stream partition: {}", coordinatorSystemStreamPartition); String streamName = coordinatorSystemStreamPartition.getStream(); Map<String, SystemStreamMetadata> systemStreamMetadataMap = systemAdmin.getSystemStreamMetadata(Sets.newHashSet(streamName)); SystemStreamMetadata systemStreamMetadata = systemStreamMetadataMap.get(streamName); Preconditions.checkNotNull(systemStreamMetadata, String.format("System stream metadata does not exist for stream: %s.", streamName)); SystemStreamPartitionMetadata systemStreamPartitionMetadata = systemStreamMetadata.getSystemStreamPartitionMetadata().get(coordinatorSystemStreamPartition.getPartition()); Preconditions.checkNotNull(systemStreamPartitionMetadata, String.format("System stream partition metadata does not exist for: %s.", coordinatorSystemStreamPartition)); String startingOffset = systemStreamPartitionMetadata.getOldestOffset(); LOG.info("Registering system stream partition: {} with offset: {}.", coordinatorSystemStreamPartition, startingOffset); systemConsumer.register(coordinatorSystemStreamPartition, startingOffset); } }
@Override public void validateStream(StreamSpec streamSpec) throws StreamValidationException { LOG.info("About to validate stream = " + streamSpec); String streamName = streamSpec.getPhysicalName(); SystemStreamMetadata systemStreamMetadata = getSystemStreamMetadata(Collections.singleton(streamName)).get(streamName); if (systemStreamMetadata == null) { throw new StreamValidationException( "Failed to obtain metadata for stream " + streamName + ". Validation failed."); } int actualPartitionCounter = systemStreamMetadata.getSystemStreamPartitionMetadata().size(); int expectedPartitionCounter = streamSpec.getPartitionCount(); LOG.info("actualCount=" + actualPartitionCounter + "; expectedCount=" + expectedPartitionCounter); if (actualPartitionCounter != expectedPartitionCounter) { throw new StreamValidationException( String.format("Mismatch of partitions for stream %s. Expected %d, got %d. Validation failed.", streamName, expectedPartitionCounter, actualPartitionCounter)); } }
@Before public void setup() throws Exception { when(mockStreamMetadataCache.getStreamMetadata(argThat(new ArgumentMatcher<scala.collection.immutable.Set<SystemStream>>() { @Override public boolean matches(Object argument) { scala.collection.immutable.Set<SystemStream> set = (scala.collection.immutable.Set<SystemStream>) argument; return set.equals(inputStreamSet); } }), anyBoolean())).thenReturn(mockStreamMetadataMap); when(mockStreamMetadata.getSystemStreamPartitionMetadata()).thenReturn(mockSspMetadataMap); PowerMockito.whenNew(TaskAssignmentManager.class).withAnyArguments().thenReturn(mockTaskManager); when(mockTaskManager.readTaskAssignment()).thenReturn(Collections.EMPTY_MAP); }
@Test public void testShouldGetASinglePartition() { SinglePartitionWithoutOffsetsSystemAdmin admin = new SinglePartitionWithoutOffsetsSystemAdmin(); Set<String> streamNames = new HashSet<String>(); streamNames.add("a"); streamNames.add("b"); Map<String, SystemStreamMetadata> metadata = admin.getSystemStreamMetadata(streamNames); assertEquals(2, metadata.size()); SystemStreamMetadata metadata1 = metadata.get("a"); SystemStreamMetadata metadata2 = metadata.get("b"); assertEquals(1, metadata1.getSystemStreamPartitionMetadata().size()); assertEquals(1, metadata2.getSystemStreamPartitionMetadata().size()); assertNull(metadata.get(new SystemStreamPartition("test-system", "c", new Partition(0)))); } }