@Override public void addStream(StreamId streamId) throws Exception { streams.put(streamId.getNamespace(), streamId.getEntityName()); }
@Override public void removeStream(StreamId streamId) throws Exception { streams.remove(streamId.getNamespace(), streamId.getEntityName()); }
private static String makeStreamKey(StreamId stream) { return Joiner.on('.').join("stream", stream.getNamespace(), stream.getEntityName()); } }
private ExecutorService createScanExecutor(StreamId streamId) { ThreadFactory threadFactory = Threads.newDaemonThreadFactory(String.format("stream-%s-%s-consumer-scanner-", streamId.getNamespace(), streamId.getEntityName())); ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 20, 60, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), threadFactory); executor.allowCoreThreadTimeOut(true); return executor; } }
protected StreamSizeAggregator(StreamId streamId, long baseCount, int streamThresholdMB, Cancellable cancellable) { this.streamWriterSizes = Maps.newHashMap(); this.streamBaseCount = new AtomicLong(baseCount); this.streamInitSize = baseCount; this.streamThresholdMB = new AtomicInteger(streamThresholdMB); this.cancellable = cancellable; this.streamId = streamId; this.streamFeed = new NotificationFeedId(streamId.getNamespace(), Constants.Notification.Stream.STREAM_FEED_CATEGORY, String.format("%sSize", streamId.getEntityName())); }
protected StreamSizeAggregator(StreamId streamId, long baseCount, int streamThresholdMB, Cancellable cancellable) { this.streamId = streamId; this.streamInitSize = baseCount; this.streamBaseCount = new AtomicLong(baseCount); this.cancellable = cancellable; this.streamFeed = new NotificationFeedId( streamId.getNamespace(), Constants.Notification.Stream.STREAM_FEED_CATEGORY, String.format("%sSize", streamId.getEntityName())); this.streamThresholdMB = new AtomicInteger(streamThresholdMB); }
@Override public void dropAll(StreamId streamId, String namespace, Iterable<Long> groupIds) throws IOException { // A bit hacky to assume namespace is formed by appId.flowId. See AbstractDataFabricFacade // String namespace = String.format("%s.%s", // programId.getApplicationId(), // programId.getId()); String invalidNamespaceError = String.format("Namespace string %s must be of the form <app>.<flow>", namespace); Iterator<String> namespaceParts = Splitter.on('.').split(namespace).iterator(); Preconditions.checkArgument(namespaceParts.hasNext(), invalidNamespaceError); String appId = namespaceParts.next(); Preconditions.checkArgument(namespaceParts.hasNext(), invalidNamespaceError); String flowId = namespaceParts.next(); queueService.truncateAllWithPrefix(QueueName.prefixForFlow(new FlowId(streamId.getNamespace(), appId, flowId))); } }
private StreamViewId deserializeViewId(JsonObject id) { StreamId streamId = deserializeStreamId(id.getAsJsonObject("stream")); String view = id.get("id").getAsString(); return new StreamViewId(streamId.getNamespace(), streamId.getStream(), view); } }
private TableId getTableId(StreamId streamId, String namespace) { return TableId.from(streamId.getNamespace(), String.format("%s.%s.%s", tablePrefix, streamId.getEntityName(), namespace)); }
@Override public boolean streamExists(StreamId streamId) throws Exception { return streams.containsEntry(streamId.getNamespace(), streamId.getEntityName()); }
private StreamViewId deserializeViewId(JsonObject id) { StreamId streamId = deserializeStreamId(id); String view = id.get("view").getAsString(); return new StreamViewId(streamId.getNamespace(), streamId.getStream(), view); }
public static QueueName fromStream(StreamId streamId) { return fromStream(streamId.getNamespace(), streamId.getEntityName()); }
protected QueryHandle doEnableExploreStream(StreamId stream, String tableName, FormatSpecification format) throws ExploreException { HttpResponse response = doPost(String.format( "namespaces/%s/data/explore/streams/%s/tables/%s/enable", stream.getNamespace(), stream.getEntityName(), tableName), format == null ? null : GSON.toJson(format), null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot enable explore on stream %s with table %s. Reason: %s", stream.getEntityName(), tableName, response)); }
protected QueryHandle doDisableExploreStream(StreamId stream, String tableName) throws ExploreException { HttpResponse response = doPost(String.format("namespaces/%s/data/explore/streams/%s/tables/%s/disable", stream.getNamespace(), stream.getEntityName(), tableName), null, null); if (response.getResponseCode() == HttpURLConnection.HTTP_OK) { return QueryHandle.fromId(parseResponseAsMap(response, "handle")); } throw new ExploreException(String.format("Cannot disable explore on stream %s with table %s. Reason: %s", stream.getEntityName(), tableName, response)); }
private void addStream(MDSKey.Builder keyBuilder, StreamId stream) { keyBuilder.add(STREAM_MARKER) .add(stream.getNamespace()) .add(stream.getEntityName()); }
private MDSKey getKey(StreamId id) { return new MDSKey.Builder() .add(TYPE_STREAM_VIEW, id.getNamespace(), id.getEntityName()) .build(); }
private MDSKey getKey(StreamId streamId) { return new MDSKey.Builder().add(TYPE_STREAM, streamId.getNamespace(), streamId.getEntityName()).build(); }
/** * Create the public {@link NotificationFeedId}s that concerns the stream with configuration {@code config}. * * @param config config of the stream to create feeds for */ private void createStreamFeeds(StreamConfig config) { try { NotificationFeedInfo streamFeed = new NotificationFeedInfo( config.getStreamId().getNamespace(), Constants.Notification.Stream.STREAM_FEED_CATEGORY, String.format("%sSize", config.getStreamId().getEntityName()), String.format("Size updates feed for Stream %s every %dMB", config.getStreamId(), config.getNotificationThresholdMB())); notificationFeedManager.createFeed(streamFeed); } catch (NotificationFeedException e) { LOG.error("Cannot create feed for Stream {}", config.getStreamId(), e); } }
@Override public MDSKey getKey(StreamId streamId) { return new MDSKey.Builder() .add(streamId.getNamespace()) .add(streamId.getEntityName()) .build(); }
Constants.Explore.STREAM_NAMESPACE, streamId.getNamespace(), Constants.Explore.FORMAT_SPEC, GSON.toJson(formatSpec));