@Override protected Map<String, String> getAttributes(final FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final Map<String,String> attributes = new HashMap<>(3); attributes.put("tcp.sender", sender); attributes.put("tcp.port", String.valueOf(port)); return attributes; }
@Override protected String getTransitUri(FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender; final String transitUri = new StringBuilder().append("beats").append("://").append(senderHost).append(":") .append(port).toString(); return transitUri; }
@Override protected void init(final ProcessorInitializationContext context) { final List<PropertyDescriptor> descriptors = new ArrayList<>(); descriptors.add(NETWORK_INTF_NAME); descriptors.add(PORT); descriptors.add(RECV_BUFFER_SIZE); descriptors.add(MAX_MESSAGE_QUEUE_SIZE); descriptors.add(MAX_SOCKET_BUFFER_SIZE); descriptors.add(CHARSET); descriptors.addAll(getAdditionalProperties()); this.descriptors = Collections.unmodifiableList(descriptors); final Set<Relationship> relationships = new HashSet<>(); relationships.add(REL_SUCCESS); relationships.addAll(getAdditionalRelationships()); this.relationships = Collections.unmodifiableSet(relationships); }
final E event = getMessage(true, true, session); if (event == null) { break; final String batchKey = getBatchKey(event); FlowFileEventBatch batch = batches.get(batchKey); batch = new FlowFileEventBatch(session.create(), new ArrayList<E>()); batches.put(batchKey, batch); batch.getEvents().add(event); final byte[] rawMessage = event.getData(); FlowFile appendedFlowFile = session.append(batch.getFlowFile(), new OutputStreamCallback() { @Override public void process(final OutputStream out) throws IOException { batch.setFlowFile(appendedFlowFile); getLogger().error("Failed to write contents of the message to FlowFile due to {}; will re-queue message and try again", new Object[] {e.getMessage()}, e); errorEvents.offer(event);
@Override public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { final int maxBatchSize = context.getProperty(MAX_BATCH_SIZE).asInteger(); final Map<String,FlowFileEventBatch> batches = getBatches(session, maxBatchSize, messageDemarcatorBytes); FlowFile flowFile = entry.getValue().getFlowFile(); final List<E> events = entry.getValue().getEvents(); getLogger().debug("No data written to FlowFile from batch {}; removing FlowFile", new Object[] {entry.getKey()}); continue; final Map<String,String> attributes = getAttributes(entry.getValue()); flowFile = session.putAllAttributes(flowFile, attributes); getLogger().debug("Transferring {} to success", new Object[] {flowFile}); session.transfer(flowFile, REL_SUCCESS); session.adjustCounter("FlowFiles Transferred to Success", 1L, false); final String transitUri = getTransitUri(entry.getValue()); session.getProvenanceReporter().receive(flowFile, transitUri); postProcess(context, session, allEvents);
@Override @OnScheduled public void onScheduled(ProcessContext context) throws IOException { super.onScheduled(context); // wanted to ensure charset was already populated here relpEncoder = new RELPEncoder(charset); }
@Override @OnScheduled public void onScheduled(ProcessContext context) throws IOException { super.onScheduled(context); final String msgDemarcator = context.getProperty(MESSAGE_DELIMITER).getValue().replace("\\n", "\n").replace("\\r", "\r").replace("\\t", "\t"); messageDemarcatorBytes = msgDemarcator.getBytes(charset); }
@Override protected void init(final ProcessorInitializationContext context) { final List<PropertyDescriptor> descriptors = new ArrayList<>(); descriptors.add(NETWORK_INTF_NAME); descriptors.add(PORT); descriptors.add(RECV_BUFFER_SIZE); descriptors.add(MAX_MESSAGE_QUEUE_SIZE); descriptors.add(MAX_SOCKET_BUFFER_SIZE); descriptors.add(CHARSET); descriptors.add(MAX_BATCH_SIZE); descriptors.add(MESSAGE_DELIMITER); descriptors.addAll(getAdditionalProperties()); this.descriptors = Collections.unmodifiableList(descriptors); final Set<Relationship> relationships = new HashSet<>(); relationships.add(REL_SUCCESS); relationships.addAll(getAdditionalRelationships()); this.relationships = Collections.unmodifiableSet(relationships); }
event = events.poll(getLongPollTimeout(), TimeUnit.MILLISECONDS); } else { event = events.poll();
@Override protected String getTransitUri(FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender; final String transitUri = new StringBuilder().append("tcp").append("://").append(senderHost).append(":") .append(port).toString(); return transitUri; }
@Override protected String getTransitUri(FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender; final String transitUri = new StringBuilder().append("relp").append("://").append(senderHost).append(":") .append(port).toString(); return transitUri; }
@Override @OnScheduled public void onScheduled(ProcessContext context) throws IOException { super.onScheduled(context); // wanted to ensure charset was already populated here lumberjackEncoder = new LumberjackEncoder(); }
@Override @OnScheduled public void onScheduled(ProcessContext context) throws IOException { super.onScheduled(context); this.pollTimeout = context.getProperty(POLL_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS); }
@Override protected Map<String, String> getAttributes(final FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final Map<String,String> attributes = new HashMap<>(3); attributes.put(UDP_SENDER_ATTR, sender); attributes.put(UDP_PORT_ATTR, String.valueOf(port)); return attributes; }
@Override protected String getTransitUri(FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender; final String transitUri = new StringBuilder().append("lumberjack").append("://").append(senderHost).append(":") .append(port).toString(); return transitUri; }
@Override @OnScheduled public void onScheduled(ProcessContext context) throws IOException { super.onScheduled(context); // wanted to ensure charset was already populated here beatsEncoder = new BeatsEncoder(); }
@Override protected String getTransitUri(FlowFileEventBatch batch) { final String sender = batch.getEvents().get(0).getSender(); final String senderHost = sender.startsWith("/") && sender.length() > 1 ? sender.substring(1) : sender; final String transitUri = new StringBuilder().append("udp").append("://").append(senderHost).append(":") .append(port).toString(); return transitUri; }
@Override protected Map<String, String> getAttributes(FlowFileEventBatch batch) { final List<BeatsEvent> events = batch.getEvents(); // the sender and command will be the same for all events based on the batch key final String sender = events.get(0).getSender(); final int numAttributes = events.size() == 1 ? 5 : 4; final Map<String, String> attributes = new HashMap<>(numAttributes); attributes.put(beatsAttributes.SENDER.key(), sender); attributes.put(beatsAttributes.PORT.key(), String.valueOf(port)); attributes.put(CoreAttributes.MIME_TYPE.key(), "application/json"); // if there was only one event then we can pass on the transaction // NOTE: we could pass on all the transaction ids joined together if (events.size() == 1) { attributes.put(beatsAttributes.SEQNUMBER.key(), String.valueOf(events.get(0).getSeqNumber())); } return attributes; }
@Override protected Map<String, String> getAttributes(FlowFileEventBatch batch) { final List<RELPEvent> events = batch.getEvents(); // the sender and command will be the same for all events based on the batch key final String sender = events.get(0).getSender(); final String command = events.get(0).getCommand(); final int numAttributes = events.size() == 1 ? 5 : 4; final Map<String,String> attributes = new HashMap<>(numAttributes); attributes.put(RELPAttributes.COMMAND.key(), command); attributes.put(RELPAttributes.SENDER.key(), sender); attributes.put(RELPAttributes.PORT.key(), String.valueOf(port)); attributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain"); // if there was only one event then we can pass on the transaction // NOTE: we could pass on all the transaction ids joined together if (events.size() == 1) { attributes.put(RELPAttributes.TXNR.key(), String.valueOf(events.get(0).getTxnr())); } return attributes; }
@Override protected Map<String, String> getAttributes(FlowFileEventBatch batch) { final List<LumberjackEvent> events = batch.getEvents(); // the sender and command will be the same for all events based on the batch key final String sender = events.get(0).getSender(); final int numAttributes = events.size() == 1 ? 5 : 4; final Map<String, String> attributes = new HashMap<>(numAttributes); attributes.put(LumberjackAttributes.SENDER.key(), sender); attributes.put(LumberjackAttributes.PORT.key(), String.valueOf(port)); attributes.put(CoreAttributes.MIME_TYPE.key(), "text/plain"); // if there was only one event then we can pass on the transaction // NOTE: we could pass on all the transaction ids joined together if (events.size() == 1) { attributes.put(LumberjackAttributes.SEQNUMBER.key(), String.valueOf(events.get(0).getSeqNumber())); // Convert the serialized fields from JSON String serialFields = String.valueOf(events.get(0).getFields()); Gson jsonObject = new Gson(); Map<String, String> fields = jsonObject.fromJson(serialFields, Map.class); for (Map.Entry<String, String> entry : fields.entrySet()) { attributes.put(LumberjackAttributes.FIELDS.key().concat(".").concat(entry.getKey()), entry.getValue()); } } return attributes; }