How to use

Journal

in

org.graylog2.shared.journal

  @Nullable
  @Override
  public Journal.Entry apply(RawMessageEvent input) {
    try {
      if (log.isTraceEnabled()) {
        log.trace("Journalling message {}", input.getMessageId());
      }
      final byte[] messageIdBytes = input.getMessageIdBytes();
      final byte[] encodedRawMessage = input.getEncodedRawMessage();
      // stats
      final int size = encodedRawMessage.length;
      bytesWritten += size;
      byteCounter.inc(size);
      // clear for gc and to avoid promotion to tenured space
      input.setMessageIdBytes(null);
      input.setEncodedRawMessage(null);
      // convert to journal entry
      return journal.createEntry(messageIdBytes, encodedRawMessage);
    } catch (Exception e) {
      log.error("Unable to convert RawMessageEvent to Journal.Entry - skipping event", e);
      return null;
    }
  }
}

@Override
public void write(List<Message> messages) throws Exception {
  long maxOffset = Long.MIN_VALUE;
  for (final Message message : messages) {
    maxOffset = Math.max(message.getJournalOffset(), maxOffset);
  }
  journal.markJournalOffsetCommitted(maxOffset);
  messagesDiscarded.mark(messages.size());
}

final List<Journal.JournalReadEntry> encodedRawMessages = journal.read(remainingCapacity);
if (encodedRawMessages.isEmpty()) {
  log.debug("No messages to read from Journal, waiting until the writer adds more messages.");
      journal.markJournalOffsetCommitted(encodedRawMessage.getOffset());
      continue;

private void writeToJournal(Converter converter, List<Journal.Entry> entries) {
  final long lastOffset = journal.write(entries);
  log.debug("Processed batch, wrote {} bytes, last journal offset: {}, signalling reader.",
      converter.getBytesWritten(),
      lastOffset);
  journalFilled.release();
}

private void writeToJournal(Converter converter, List<Journal.Entry> entries) {
  final long lastOffset = journal.write(entries);
  log.debug("Processed batch, wrote {} bytes, last journal offset: {}, signalling reader.",
      converter.getBytesWritten(),
      lastOffset);
  journalFilled.release();
}

@Override
public void write(Message message) throws Exception {
  journal.markJournalOffsetCommitted(message.getJournalOffset());
  messagesDiscarded.mark();
}

final List<Journal.JournalReadEntry> encodedRawMessages = journal.read(remainingCapacity);
if (encodedRawMessages.isEmpty()) {
  log.debug("No messages to read from Journal, waiting until the writer adds more messages.");
      journal.markJournalOffsetCommitted(encodedRawMessage.getOffset());
      continue;

private void writeToJournal(Converter converter, List<Journal.Entry> entries) {
  final long lastOffset = journal.write(entries);
  log.debug("Processed batch, wrote {} bytes, last journal offset: {}, signalling reader.",
      converter.getBytesWritten(),
      lastOffset);
  journalFilled.release();
}

  @Nullable
  @Override
  public Journal.Entry apply(RawMessageEvent input) {
    try {
      if (log.isTraceEnabled()) {
        log.trace("Journalling message {}", input.getMessageId());
      }
      final byte[] messageIdBytes = input.getMessageIdBytes();
      final byte[] encodedRawMessage = input.getEncodedRawMessage();
      // stats
      final int size = encodedRawMessage.length;
      bytesWritten += size;
      byteCounter.inc(size);
      // clear for gc and to avoid promotion to tenured space
      input.setMessageIdBytes(null);
      input.setEncodedRawMessage(null);
      // convert to journal entry
      return journal.createEntry(messageIdBytes, encodedRawMessage);
    } catch (Exception e) {
      log.error("Unable to convert RawMessageEvent to Journal.Entry - skipping event", e);
      return null;
    }
  }
}

@Override
public void write(Message message) throws Exception {
  journal.markJournalOffsetCommitted(message.getJournalOffset());
  messagesWritten.mark();
}

final List<Journal.JournalReadEntry> encodedRawMessages = journal.read(remainingCapacity);
if (encodedRawMessages.isEmpty()) {
  log.debug("No messages to read from Journal, waiting until the writer adds more messages.");
      journal.markJournalOffsetCommitted(encodedRawMessage.getOffset());
      continue;

  @Nullable
  @Override
  public Journal.Entry apply(RawMessageEvent input) {
    try {
      if (log.isTraceEnabled()) {
        log.trace("Journalling message {}", input.getMessageId());
      }
      final byte[] messageIdBytes = input.getMessageIdBytes();
      final byte[] encodedRawMessage = input.getEncodedRawMessage();
      // stats
      final int size = encodedRawMessage.length;
      bytesWritten += size;
      byteCounter.inc(size);
      // clear for gc and to avoid promotion to tenured space
      input.setMessageIdBytes(null);
      input.setEncodedRawMessage(null);
      // convert to journal entry
      return journal.createEntry(messageIdBytes, encodedRawMessage);
    } catch (Exception e) {
      log.error("Unable to convert RawMessageEvent to Journal.Entry - skipping event", e);
      return null;
    }
  }
}

@Override
public void write(List<Message> messages) throws Exception {
  long maxOffset = Long.MIN_VALUE;
  for (final Message message : messages) {
    maxOffset = Math.max(message.getJournalOffset(), maxOffset);
  }
  journal.markJournalOffsetCommitted(maxOffset);
  messagesWritten.mark(messages.size());
}

private void potentiallyDropFilteredMessage(Message message) {
  if (message.getFilterOut()) {
    log.debug("[{}] marked message to be discarded. Dropping message.", message.getId());
    filteredOutMessages.mark();
    journal.markJournalOffsetCommitted(message.getJournalOffset());
  }
}

@Override
public Messages process(Messages messages) {
  for (final MessageFilter filter : filterRegistry) {
    for (Message msg : messages) {
      final String timerName = name(filter.getClass(), "executionTime");
      final Timer timer = metricRegistry.timer(timerName);
      final Timer.Context timerContext = timer.time();
      try {
        LOG.debug("Applying filter [{}] on message <{}>.", filter.getName(), msg.getId());
        if (filter.filter(msg)) {
          LOG.debug("Filter [{}] marked message <{}> to be discarded. Dropping message.",
              filter.getName(),
              msg.getId());
          msg.setFilterOut(true);
          filteredOutMessages.mark();
          journal.markJournalOffsetCommitted(msg.getJournalOffset());
        }
      } catch (Exception e) {
        LOG.error("Could not apply filter [" + filter.getName() + "] on message <" + msg.getId() + ">: ",
            e);
      } finally {
        final long elapsedNanos = timerContext.stop();
        msg.recordTiming(serverStatus, timerName, elapsedNanos);
      }
    }
  }
  return messages;
}

@Override
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
  final Timer.Context context = decodeTime.time();
  try {
    processMessage(event);
  } catch (Exception e) {
    final RawMessage rawMessage = event.getRaw();
    LOG.error("Error processing message " + rawMessage, ExceptionUtils.getRootCause(e));
    // Mark message as processed to avoid keeping it in the journal.
    journal.markJournalOffsetCommitted(rawMessage.getJournalOffset());
    // always clear the event fields, even if they are null, to avoid later stages to process old messages.
    // basically this will make sure old messages are cleared out early.
    event.clearMessages();
  } finally {
    if (event.getMessage() != null) {
      event.getMessage().recordTiming(serverStatus, "decode", context.stop());
    } else if (event.getMessages() != null) {
      for (final Message message : event.getMessages()) {
        message.recordTiming(serverStatus, "decode", context.stop());
      }
    }
    // aid garbage collection to collect the raw message early (to avoid promoting it to later generations).
    event.clearRaw();
  }
}

@Override
public void write(Message message) throws Exception {
  journal.markJournalOffsetCommitted(message.getJournalOffset());
  messagesDiscarded.mark();
}

@Override
public void write(List<Message> messages) throws Exception {
  long maxOffset = Long.MIN_VALUE;
  for (final Message message : messages) {
    maxOffset = Math.max(message.getJournalOffset(), maxOffset);
  }
  journal.markJournalOffsetCommitted(maxOffset);
  messagesWritten.mark(messages.size());
}

@Override
public void write(List<Message> messages) throws Exception {
  long maxOffset = Long.MIN_VALUE;
  for (final Message message : messages) {
    maxOffset = Math.max(message.getJournalOffset(), maxOffset);
  }
  journal.markJournalOffsetCommitted(maxOffset);
  messagesDiscarded.mark(messages.size());
}

@Override
public void write(Message message) throws Exception {
  journal.markJournalOffsetCommitted(message.getJournalOffset());
  messagesWritten.mark();
}