@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> metadata = new HashMap<String, SystemStreamMetadata>(); for (String streamName : streamNames) { metadata.put(streamName, new SystemStreamMetadata(streamName, FAKE_PARTITION_METADATA)); } return metadata; }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> metadata = new HashMap<String, SystemStreamMetadata>(); for (String streamName : streamNames) { metadata.put(streamName, new SystemStreamMetadata(streamName, FAKE_PARTITION_METADATA)); } return metadata; }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
private SystemStreamMetadata constructSystemStreamMetadata( String streamName, Map<SystemStreamPartition, List<IncomingMessageEnvelope>> sspToMessagesForSystem) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = sspToMessagesForSystem .entrySet() .stream() .collect(Collectors.toMap(entry -> entry.getKey().getPartition(), entry -> { String oldestOffset = "0"; String newestOffset = String.valueOf(entry.getValue().size()); String upcomingOffset = String.valueOf(entry.getValue().size() + 1); return new SystemStreamMetadata.SystemStreamPartitionMetadata(oldestOffset, newestOffset, upcomingOffset); })); return new SystemStreamMetadata(streamName, partitionMetadata); }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return streamNames.stream() .collect(Collectors.toMap(Function.identity(), streamName -> new SystemStreamMetadata(streamName, Collections.singletonMap(new Partition(0), new SystemStreamMetadata.SystemStreamPartitionMetadata(null, null, null))))); }
@Override public Map<String, SystemStreamMetadata> apply(ExponentialSleepStrategy.RetryLoop loop) { Map<String, SystemStreamMetadata> allMetadata = new HashMap<>(); streamNames.forEach(streamName -> { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); List<PartitionInfo> partitionInfos = metadataConsumer.partitionsFor(streamName); LOG.debug("Stream {} has partitions {}", streamName, partitionInfos); partitionInfos.forEach(partitionInfo -> { partitionMetadata.put(new Partition(partitionInfo.partition()), dummySspm); }); allMetadata.put(streamName, new SystemStreamMetadata(streamName, partitionMetadata)); }); loop.done(); return allMetadata; } };
@Override public Map<String, SystemStreamMetadata> apply(ExponentialSleepStrategy.RetryLoop loop) { Map<String, SystemStreamMetadata> allMetadata = new HashMap<>(); streamNames.forEach(streamName -> { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); List<PartitionInfo> partitionInfos = metadataConsumer.partitionsFor(streamName); LOG.debug("Stream {} has partitions {}", streamName, partitionInfos); partitionInfos.forEach(partitionInfo -> { partitionMetadata.put(new Partition(partitionInfo.partition()), dummySspm); }); allMetadata.put(streamName, new SystemStreamMetadata(streamName, partitionMetadata)); }); loop.done(); return allMetadata; } };
@Override public Map<String, SystemStreamMetadata> apply(ExponentialSleepStrategy.RetryLoop loop) { Map<String, SystemStreamMetadata> allMetadata = new HashMap<>(); streamNames.forEach(streamName -> { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetadata = new HashMap<>(); List<PartitionInfo> partitionInfos = metadataConsumer.partitionsFor(streamName); LOG.debug("Stream {} has partitions {}", streamName, partitionInfos); partitionInfos.forEach(partitionInfo -> { partitionMetadata.put(new Partition(partitionInfo.partition()), dummySspm); }); allMetadata.put(streamName, new SystemStreamMetadata(streamName, partitionMetadata)); }); loop.done(); return allMetadata; } };
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> map = new HashMap<>(); for (String stream : streamNames) { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> m = new HashMap<>(); for (int i = 0; i < streamToPartitions.get(stream); i++) { m.put(new Partition(i), new SystemStreamMetadata.SystemStreamPartitionMetadata("", "", "")); } map.put(stream, new SystemStreamMetadata(stream, m)); } return map; }
/** * * Fetch metadata from hdfs system for a set of streams. This has the potential side effect * to persist partition description to the staging directory on hdfs if staging directory * is not empty. See getStagingDirectory on {@link HdfsConfig} * * @param streamNames * The streams to to fetch metadata for. * @return A map from stream name to SystemStreamMetadata for each stream * requested in the parameter set. */ @Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> systemStreamMetadataMap = new HashMap<>(); streamNames.forEach(streamName -> { systemStreamMetadataMap.put(streamName, new SystemStreamMetadata(streamName, directoryPartitioner .getPartitionMetadataMap(streamName, obtainPartitionDescriptorMap(stagingDirectory, streamName)))); if (!partitionDescriptorExists(streamName)) { persistPartitionDescriptor(streamName, directoryPartitioner.getPartitionDescriptor(streamName)); } }); return systemStreamMetadataMap; }
/** * Create a new {@link SystemAdmin} that returns the provided oldest and newest offsets for its topics */ private SystemAdmin newAdmin(String oldestOffset, String newestOffset) { SystemStreamMetadata checkpointTopicMetadata = new SystemStreamMetadata(CHECKPOINT_TOPIC, ImmutableMap.of(new Partition(0), new SystemStreamPartitionMetadata(oldestOffset, newestOffset, Integer.toString(Integer.parseInt(newestOffset) + 1)))); SystemAdmin mockAdmin = mock(SystemAdmin.class); when(mockAdmin.getSystemStreamMetadata(Collections.singleton(CHECKPOINT_TOPIC))).thenReturn( ImmutableMap.of(CHECKPOINT_TOPIC, checkpointTopicMetadata)); return mockAdmin; }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { return streamNames.stream() .collect(Collectors.toMap(Function.identity(), streamName -> new SystemStreamMetadata(streamName, Collections.singletonMap(new Partition(0), new SystemStreamMetadata.SystemStreamPartitionMetadata(null, END_OF_STREAM_OFFSET, null))))); }
@Override public Map<String, SystemStreamMetadata> getSystemStreamMetadata(Set<String> streamNames) { Map<String, SystemStreamMetadata> metadataMap = new HashMap<>(); Map<String, Set<Partition>> partitionMap = MSG_QUEUES.entrySet() .stream() .filter(entry -> streamNames.contains(entry.getKey().getSystemStream().getStream())) .map(e -> e.getKey()).<Map<String, Set<Partition>>>collect(HashMap::new, (m, ssp) -> { if (m.get(ssp.getStream()) == null) { m.put(ssp.getStream(), new HashSet<>()); } m.get(ssp.getStream()).add(ssp.getPartition()); }, (m1, m2) -> { m2.forEach((k, v) -> { if (m1.get(k) == null) { m1.put(k, v); } else { m1.get(k).addAll(v); } }); }); partitionMap.forEach((k, v) -> { Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> partitionMetaMap = v.stream().<Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata>>collect(HashMap::new, (m, p) -> { m.put(p, new SystemStreamMetadata.SystemStreamPartitionMetadata("", "", "")); }, (m1, m2) -> m1.putAll(m2)); metadataMap.put(k, new SystemStreamMetadata(k, partitionMetaMap)); }); return metadataMap; }
private SystemStreamMetadata createSystemStreamMetadata(String stream) { LOG.info("create stream metadata for stream {} based on aws stream", stream); Map<Partition, SystemStreamMetadata.SystemStreamPartitionMetadata> metadata = new HashMap<>(); AmazonKinesisClient client = null; try { ClientConfiguration clientConfig = kConfig.getAWSClientConfig(system); AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard() .withCredentials(kConfig.credentialsProviderForStream(system, stream)) .withClientConfiguration(clientConfig); builder.setRegion(kConfig.getRegion(system, stream).getName()); client = (AmazonKinesisClient) builder.build(); StreamDescription desc = client.describeStream(stream).getStreamDescription(); IntStream.range(0, desc.getShards().size()) .forEach(i -> metadata.put(new Partition(i), SYSTEM_STREAM_PARTITION_METADATA)); } catch (Exception e) { String errMsg = "couldn't load metadata for stream " + stream; LOG.error(errMsg, e); throw new SamzaException(errMsg, e); } finally { if (client != null) { client.shutdown(); } } return new SystemStreamMetadata(stream, metadata); }
/** * Given some SSPs, but missing metadata for one of the SSPs, getSSPMetadata should delegate to * getSystemStreamMetadata and only fill in results for the SSPs that have metadata. */ @Test public void testGetSSPMetadataMissingPartition() { SystemStreamPartition streamPartition0 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(0)); SystemStreamPartition streamPartition1 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(1)); SystemAdmin systemAdmin = mock(MySystemAdmin.class); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("1", "2", "3"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of(new Partition(0), streamPartition0Metadata)))); Set<SystemStreamPartition> ssps = ImmutableSet.of(streamPartition0, streamPartition1); when(systemAdmin.getSSPMetadata(ssps)).thenCallRealMethod(); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> expected = ImmutableMap.of(streamPartition0, streamPartition0Metadata); assertEquals(expected, systemAdmin.getSSPMetadata(ssps)); verify(systemAdmin).getSystemStreamMetadata(ImmutableSet.of(STREAM)); }
/** * Given some SSPs, but missing metadata for one of the streams, getSSPMetadata should delegate to * getSystemStreamMetadata and only fill in results for the SSPs corresponding to streams with metadata. */ @Test public void testGetSSPMetadataMissingStream() { SystemStreamPartition streamPartition0 = new SystemStreamPartition(SYSTEM, STREAM, new Partition(0)); SystemStreamPartition otherStreamPartition0 = new SystemStreamPartition(SYSTEM, OTHER_STREAM, new Partition(0)); SystemAdmin systemAdmin = mock(MySystemAdmin.class); SystemStreamMetadata.SystemStreamPartitionMetadata streamPartition0Metadata = new SystemStreamMetadata.SystemStreamPartitionMetadata("1", "2", "3"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of(new Partition(0), streamPartition0Metadata)))); Set<SystemStreamPartition> ssps = ImmutableSet.of(streamPartition0, otherStreamPartition0); when(systemAdmin.getSSPMetadata(ssps)).thenCallRealMethod(); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> expected = ImmutableMap.of(streamPartition0, streamPartition0Metadata); assertEquals(expected, systemAdmin.getSSPMetadata(ssps)); verify(systemAdmin).getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM)); }
new SystemStreamMetadata.SystemStreamPartitionMetadata("21", "22", "23"); when(systemAdmin.getSystemStreamMetadata(ImmutableSet.of(STREAM, OTHER_STREAM))).thenReturn(ImmutableMap.of( STREAM, new SystemStreamMetadata(STREAM, ImmutableMap.of( new Partition(0), streamPartition0Metadata, new Partition(1), streamPartition1Metadata)), OTHER_STREAM, new SystemStreamMetadata(OTHER_STREAM, ImmutableMap.of( new Partition(0), otherStreamPartition0Metadata)))); Set<SystemStreamPartition> ssps = ImmutableSet.of(streamPartition0, streamPartition1, otherStreamPartition0);