@Override @Nullable public RollbackDetail publish(StoreRequest request) throws TopicNotFoundException, IOException { return getMessagingService().publish(request); }
@Override @Nullable public RollbackDetail publish(StoreRequest request) throws TopicNotFoundException, IOException { return getMessagingService().publish(request); }
@Override public boolean commitTx() throws Exception { // One potential improvement in future we can make is to publish concurrently for different topics using a executor. for (Map.Entry<TopicId, StoreRequestBuilder> entry : txPublishRequests.entrySet()) { // If the payload of publish request is empty, no need to publish if (!entry.getValue().hasPayload()) { continue; } rollbackDetails.put(entry.getKey(), messagingService.publish(entry.getValue().build())); } return true; }
@Override public boolean commitTx() throws Exception { // One potential improvement in future we can make is to publish concurrently for different topics using a executor. for (Map.Entry<TopicId, StoreRequestBuilder> entry : txPublishRequests.entrySet()) { // If the payload of publish request is empty, no need to publish if (!entry.getValue().hasPayload()) { continue; } rollbackDetails.put(entry.getKey(), messagingService.publish(entry.getValue().build())); } return true; }
private void doRegisterAll(Iterable<? extends EntityId> users, EntityId entityId) throws Exception { // Only record usage from program StoreRequest request = StoreRequestBuilder.of(topic).addPayloads( StreamSupport.stream(users.spliterator(), false) .filter(ProgramId.class::isInstance) .map(ProgramId.class::cast) .map(id -> new MetadataMessage(MetadataMessage.Type.USAGE, id, GSON.toJsonTree(new DatasetUsage(entityId)))) .map(GSON::toJson) .map(s -> s.getBytes(StandardCharsets.UTF_8)) .iterator() ).build(); Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } }
@Override public void publish(EntityId publisher, MetadataOperation operation) { MetadataMessage message = new MetadataMessage(Type.METADATA_OPERATION, publisher, GSON.toJsonTree(operation)); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); LOG.trace("Publishing message {} to topic {}", message, topic); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish metadata operation: " + operation, e); } }
@Override public void publish(ProgramRunId programRunId, MetadataOperation operation) { MetadataMessage message = new MetadataMessage(Type.METADATA_OPERATION, programRunId, GSON.toJsonTree(operation)); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); LOG.trace("Publishing message {} to topic {}", message, topic); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish metadata operation: " + operation, e); } }
@Override public void publish(EntityId entityId, DatasetInstanceOperation datasetInstanceOperation) { MetadataMessage message = new MetadataMessage(Type.DATASET_OPERATION, entityId, GSON.toJsonTree(datasetInstanceOperation)); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); LOG.trace("Publishing message {} to topic {}", message, topic); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish metadata operation: " + datasetInstanceOperation, e); } } }
@Override public void setWorkflowToken(ProgramRunId workflowRunId, WorkflowToken token) { MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_TOKEN, workflowRunId, GSON.toJsonTree(token)); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { // Don't log the workflow token, as it can be large and may contain sensitive data throw new RuntimeException("Failed to publish workflow token for workflow run " + workflowRunId, e); } }
private void publish(MetadataMessage message) { StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { LOG.trace("Failed to publish metadata message: {}", message); ProgramRunId programRunId = (ProgramRunId) message.getEntityId(); throw new RuntimeException(String.format("Failed to publish metadata message of type '%s' for program " + "run '%s'.", message.getType(), programRunId), e); } } }
private void publish(MetadataMessage message) { StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { LOG.trace("Failed to publish metadata message: {}", message); ProgramRunId programRunId = (ProgramRunId) message.getEntityId(); throw new RuntimeException(String.format("Failed to publish metadata message of type '%s' for program " + "run '%s'.", message.getType(), programRunId), e); } } }
@Override public void setWorkflowToken(ProgramRunId workflowRunId, WorkflowToken token) { MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_TOKEN, workflowRunId, GSON.toJsonTree(token)); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { // Don't log the workflow token, as it can be large and may contain sensitive data throw new RuntimeException("Failed to publish workflow token for workflow run " + workflowRunId, e); } }
@Override public void publish(TopicId topicId, Iterator<byte[]> payloads) throws IOException, TopicNotFoundException { messagingService.publish(StoreRequestBuilder.of(topicId).addPayloads(payloads).build()); } }
@Override public void publish(TopicId topicId, Iterator<byte[]> payloads) throws IOException, TopicNotFoundException { messagingService.publish(StoreRequestBuilder.of(topicId).addPayloads(payloads).build()); } }
@Override public void addWorkflowNodeState(ProgramRunId workflowRunId, WorkflowNodeStateDetail state) { MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_STATE, workflowRunId, GSON.toJsonTree(state)); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish workflow node state for workflow run " + workflowRunId + "of node " + state.getNodeId() + " with state " + state.getNodeStatus(), e); } } }
@Override public void addWorkflowNodeState(ProgramRunId workflowRunId, WorkflowNodeStateDetail state) { MetadataMessage message = new MetadataMessage(MetadataMessage.Type.WORKFLOW_STATE, workflowRunId, GSON.toJsonTree(state)); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish workflow node state for workflow run " + workflowRunId + "of node " + state.getNodeId() + " with state " + state.getNodeStatus(), e); } } }
@Override public void register(ProgramId programId, DatasetId datasetId) { MetadataMessage message = new MetadataMessage(MetadataMessage.Type.USAGE, programId, GSON.toJsonTree(new DatasetUsage(datasetId))); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish usage for " + datasetId + " for program " + programId, e); } }
@Override public void register(ProgramId programId, DatasetId datasetId) { MetadataMessage message = new MetadataMessage(MetadataMessage.Type.USAGE, programId, GSON.toJsonTree(new DatasetUsage(datasetId))); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish usage for " + datasetId + " for program " + programId, e); } }
@Override public void register(ProgramId programId, StreamId streamId) { MetadataMessage message = new MetadataMessage(MetadataMessage.Type.USAGE, programId, GSON.toJsonTree(new DatasetUsage(streamId))); StoreRequest request = StoreRequestBuilder.of(topic).addPayload(GSON.toJson(message)).build(); try { Retries.callWithRetries(() -> messagingService.publish(request), retryStrategy, Retries.ALWAYS_TRUE); } catch (Exception e) { throw new RuntimeException("Failed to publish usage for " + streamId + " for program " + programId, e); } }
private void publishNotification(TopicId topicId, NamespaceId namespaceId, String dataset) throws Exception { DatasetId datasetId = namespaceId.dataset(dataset); PartitionKey partitionKey = PartitionKey.builder().addIntField("part1", 1).build(); Notification notification = Notification.forPartitions(datasetId, ImmutableList.of(partitionKey)); messagingService.publish(StoreRequestBuilder.of(topicId).addPayload(GSON.toJson(notification)).build()); }