@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
private String serializeSystemStreamPartition(SystemStreamPartition ssp) { // Create the Json string for SystemStreamPartition Map<String, String> sspMap = new HashMap<>(); sspMap.put(SYSTEM_PROP_NAME, ssp.getSystem()); sspMap.put(STREAM_PROP_NAME, ssp.getStream()); sspMap.put(PARTITION_PROP_NAME, String.valueOf(ssp.getPartition().getPartitionId())); return new String(jsonSerde.toBytes(sspMap)); }
@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
@Override public void serialize(SystemStreamPartition systemStreamPartition, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException, JsonProcessingException { Map<String, Object> systemStreamPartitionMap = new HashMap<String, Object>(); systemStreamPartitionMap.put("system", systemStreamPartition.getSystem()); systemStreamPartitionMap.put("stream", systemStreamPartition.getStream()); systemStreamPartitionMap.put("partition", systemStreamPartition.getPartition()); jsonGenerator.writeObject(systemStreamPartitionMap); } }
/** * Fetch system stream metadata for the given streams. * * @param systemName system name * @param streamNames set of input streams * * @return a {@link Map} of stream to {@link SystemStreamMetadata} */ Map<String, SystemStreamMetadata> getSystemStreamMetadata(String systemName, Set<String> streamNames) { Map<String, Map<SystemStreamPartition, List<IncomingMessageEnvelope>>> result = bufferedMessages.entrySet() .stream() .filter(entry -> systemName.equals(entry.getKey().getSystem()) && streamNames.contains(entry.getKey().getStream())) .collect(Collectors.groupingBy(entry -> entry.getKey().getStream(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); return result.entrySet() .stream() .collect(Collectors.toMap( Map.Entry::getKey, entry -> constructSystemStreamMetadata(entry.getKey(), entry.getValue()))); }
/** * Gets the starting offsets for the {@link SystemStreamPartition}s belonging to all the side input stores. * If the local file offset is available and is greater than the oldest available offset from source, uses it, * else falls back to oldest offset in the source. * * @param fileOffsets offsets from the local offset file * @param oldestOffsets oldest offsets from the source * @return a {@link Map} of {@link SystemStreamPartition} to offset */ @VisibleForTesting Map<SystemStreamPartition, String> getStartingOffsets( Map<SystemStreamPartition, String> fileOffsets, Map<SystemStreamPartition, String> oldestOffsets) { Map<SystemStreamPartition, String> startingOffsets = new HashMap<>(); sspsToStores.keySet().forEach(ssp -> { String fileOffset = fileOffsets.get(ssp); String oldestOffset = oldestOffsets.get(ssp); startingOffsets.put(ssp, StorageManagerUtil.getStartingOffset( ssp, systemAdmins.getSystemAdmin(ssp.getSystem()), fileOffset, oldestOffset)); }); return startingOffsets; }
/** * Gets the starting offsets for the {@link SystemStreamPartition}s belonging to all the side input stores. * If the local file offset is available and is greater than the oldest available offset from source, uses it, * else falls back to oldest offset in the source. * * @param fileOffsets offsets from the local offset file * @param oldestOffsets oldest offsets from the source * @return a {@link Map} of {@link SystemStreamPartition} to offset */ @VisibleForTesting Map<SystemStreamPartition, String> getStartingOffsets( Map<SystemStreamPartition, String> fileOffsets, Map<SystemStreamPartition, String> oldestOffsets) { Map<SystemStreamPartition, String> startingOffsets = new HashMap<>(); sspsToStores.keySet().forEach(ssp -> { String fileOffset = fileOffsets.get(ssp); String oldestOffset = oldestOffsets.get(ssp); startingOffsets.put(ssp, StorageManagerUtil.getStartingOffset( ssp, systemAdmins.getSystemAdmin(ssp.getSystem()), fileOffset, oldestOffset)); }); return startingOffsets; }
/** * Constructs a Samza stream partition object based upon an existing Samza stream partition. * @param other Reference to an already existing Samza stream partition. */ public SystemStreamPartition(SystemStreamPartition other) { this(other.getSystem(), other.getStream(), other.getPartition()); }
/** * Constructs a Samza stream partition object based upon an existing Samza stream partition. * @param other Reference to an already existing Samza stream partition. */ public SystemStreamPartition(SystemStreamPartition other) { this(other.getSystem(), other.getStream(), other.getPartition()); }
private void maybeRefreshMetadata(SystemStreamPartition requestedSSP) { synchronized (this.metadataRefreshLock) { Instant refreshRequestedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); if (shouldRefresh(requestedSSP, refreshRequestedAt)) { String system = requestedSSP.getSystem(); Set<SystemStreamPartition> sspsToFetchFor = new HashSet<>(); sspsToFetchFor.add(requestedSSP); for (SystemStreamPartition sspToPrefetch : this.sspsToPrefetch) { if (system.equals(sspToPrefetch.getSystem()) && shouldRefresh(sspToPrefetch, refreshRequestedAt)) { sspsToFetchFor.add(sspToPrefetch); } } SystemAdmin systemAdmin = this.systemAdmins.getSystemAdmin(system); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> fetchedMetadata = systemAdmin.getSSPMetadata(sspsToFetchFor); Instant updatedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); // we want to add an entry even if there was no metadata, so iterate over sspsToFetchFor sspsToFetchFor.forEach(ssp -> this.cache.put(ssp, new CacheEntry(fetchedMetadata.get(ssp), updatedAt))); } } }
private void maybeRefreshMetadata(SystemStreamPartition requestedSSP) { synchronized (this.metadataRefreshLock) { Instant refreshRequestedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); if (shouldRefresh(requestedSSP, refreshRequestedAt)) { String system = requestedSSP.getSystem(); Set<SystemStreamPartition> sspsToFetchFor = new HashSet<>(); sspsToFetchFor.add(requestedSSP); for (SystemStreamPartition sspToPrefetch : this.sspsToPrefetch) { if (system.equals(sspToPrefetch.getSystem()) && shouldRefresh(sspToPrefetch, refreshRequestedAt)) { sspsToFetchFor.add(sspToPrefetch); } } SystemAdmin systemAdmin = this.systemAdmins.getSystemAdmin(system); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> fetchedMetadata = systemAdmin.getSSPMetadata(sspsToFetchFor); Instant updatedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); // we want to add an entry even if there was no metadata, so iterate over sspsToFetchFor sspsToFetchFor.forEach(ssp -> this.cache.put(ssp, new CacheEntry(fetchedMetadata.get(ssp), updatedAt))); } } }
private void maybeRefreshMetadata(SystemStreamPartition requestedSSP) { synchronized (this.metadataRefreshLock) { Instant refreshRequestedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); if (shouldRefresh(requestedSSP, refreshRequestedAt)) { String system = requestedSSP.getSystem(); Set<SystemStreamPartition> sspsToFetchFor = new HashSet<>(); sspsToFetchFor.add(requestedSSP); for (SystemStreamPartition sspToPrefetch : this.sspsToPrefetch) { if (system.equals(sspToPrefetch.getSystem()) && shouldRefresh(sspToPrefetch, refreshRequestedAt)) { sspsToFetchFor.add(sspToPrefetch); } } SystemAdmin systemAdmin = this.systemAdmins.getSystemAdmin(system); Map<SystemStreamPartition, SystemStreamMetadata.SystemStreamPartitionMetadata> fetchedMetadata = systemAdmin.getSSPMetadata(sspsToFetchFor); Instant updatedAt = Instant.ofEpochMilli(this.clock.currentTimeMillis()); // we want to add an entry even if there was no metadata, so iterate over sspsToFetchFor sspsToFetchFor.forEach(ssp -> this.cache.put(ssp, new CacheEntry(fetchedMetadata.get(ssp), updatedAt))); } } }
public Partition(SystemStreamPartition systemStreamPartition) { this(systemStreamPartition.getSystem(), systemStreamPartition.getStream(), systemStreamPartition.getPartition().getPartitionId()); }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }
@Override public void serialize(SystemStreamPartition ssp, JsonGenerator jgen, SerializerProvider provider) throws IOException { String sspString = ssp.getSystem() + "." + ssp.getStream() + "." + String.valueOf(ssp.getPartition().getPartitionId()); jgen.writeFieldName(sspString); } }