@Override public long getTimestamp(MessageAndMetadata<byte[], byte[]> kafkaMessage) { return kafkaMessage.timestamp(); }
@Override public void process(final OutputStream out) throws IOException { if (!firstMessage) { out.write(demarcatorBytes); } out.write(mam.message()); } });
public EmitState emit(SpoutOutputCollector collector) { if (emittingMessages.isEmpty()) { fillMessages(); } int count = 0; while (true) { MessageAndOffset toEmitMsg = emittingMessages.pollFirst(); if (toEmitMsg == null) { return EmitState.EMIT_END; } count ++; Iterable<List<Object>> tups = generateTuples(toEmitMsg.message()); if (tups != null) { for (List<Object> tuple : tups) { LOG.debug("emit message {}", new String(Utils.toByteArray(toEmitMsg.message().payload()))); collector.emit(tuple, new KafkaMessageId(partition, toEmitMsg.offset())); } if(count>=config.batchSendCount) { break; } } else { ack(toEmitMsg.offset()); } } if (emittingMessages.isEmpty()) { return EmitState.EMIT_END; } else { return EmitState.EMIT_MORE; } }
private Iterable<MessageAndOffset> buildOffsetFilteringIterable(final ByteBufferMessageSet messageAndOffsets, final long startOffset, final long endOffset) { return Iterables.filter(messageAndOffsets, input -> { // Filter messages that are either null or have an offset ∉ [startOffset; endOffset[ if (input == null || input.offset() < startOffset || (endOffset <= input.offset() && endOffset != -1)) { return false; } // Check the message's checksum // TODO We might want to have better handling of this situation, maybe try to fetch the message again? if (!input.message().isValid()) { LOGGER.warn("Discarded message with invalid checksum in partition {} of topic {}", _partition, _topic); return false; } return true; }); }
private void fillMessages() { ByteBufferMessageSet msgs; try { long start = System.currentTimeMillis(); msgs = consumer.fetchMessages(partition, emittingOffset + 1); if (msgs == null) { LOG.error("fetch null message from offset {}", emittingOffset); return; } int count = 0; for (MessageAndOffset msg : msgs) { count += 1; emittingMessages.add(msg); emittingOffset = msg.offset(); pendingOffsets.add(emittingOffset); LOG.debug("fillmessage fetched a message:{}, offset:{}", msg.message().toString(), msg.offset()); } long end = System.currentTimeMillis(); LOG.info("fetch message from partition:"+partition+", offset:" + emittingOffset+", size:"+msgs.sizeInBytes()+", count:"+count +", time:"+(end-start)); } catch (Exception e) { e.printStackTrace(); LOG.error(e.getMessage(),e); } }
public int getMessageLengthAtIndex(int index) { return messageList.get(index).message().payloadSize(); }
@SuppressWarnings("unchecked") public Iterable<List<Object>> generateTuples(Message msg) { Iterable<List<Object>> tups = null; ByteBuffer payload = msg.payload(); if (payload == null) { return null; } tups = Arrays.asList(Utils.tuple(Utils.toByteArray(payload))); return tups; }
@Override public boolean advanceNextPosition() { while (true) { if (cursorOffset >= split.getEnd()) { return endOfData(); // Split end is exclusive. } // Create a fetch request openFetchRequest(); while (messageAndOffsetIterator.hasNext()) { MessageAndOffset currentMessageAndOffset = messageAndOffsetIterator.next(); long messageOffset = currentMessageAndOffset.offset(); if (messageOffset >= split.getEnd()) { return endOfData(); // Past our split end. Bail. } if (messageOffset >= cursorOffset) { return nextRow(currentMessageAndOffset); } } messageAndOffsetIterator = null; } }
public long getNextStreamMessageOffsetAtIndex(int index) { return messageList.get(index).nextOffset(); } }
public void pushToStream(String message) { int streamNo = (int) this.nextStream.incrementAndGet() % this.queues.size(); AtomicLong offset = this.offsets.get(streamNo); BlockingQueue<FetchedDataChunk> queue = this.queues.get(streamNo); AtomicLong thisOffset = new AtomicLong(offset.incrementAndGet()); List<Message> seq = Lists.newArrayList(); seq.add(new Message(message.getBytes(Charsets.UTF_8))); ByteBufferMessageSet messageSet = new ByteBufferMessageSet(NoCompressionCodec$.MODULE$, offset, JavaConversions.asScalaBuffer(seq)); FetchedDataChunk chunk = new FetchedDataChunk(messageSet, new PartitionTopicInfo("topic", streamNo, queue, thisOffset, thisOffset, new AtomicInteger(1), "clientId"), thisOffset.get()); queue.add(chunk); }
@Override protected void processMessage(MessageAndMetadata<byte[], byte[]> message) { this.messages.add(new String(message.message(), Charsets.UTF_8)); }
@Nullable @Override public InputRow nextRow() { try { if (!nextIterator.hasNext()) { final byte[] message = iter.next().message(); if (message == null) { return null; } nextIterator = theParser.parseBatch(ByteBuffer.wrap(message)).iterator(); } return nextIterator.next(); } catch (InvalidMessageException e) { /* IF the CRC is caused within the wire transfer, this is not the best way to handel CRC. Probably it is better to shutdown the fireHose without commit and start it again. */ log.error(e, "Message failed its checksum and it is corrupt, will skip it"); return null; } }
private ByteArrayBasedKafkaRecord getMockMessageAndOffset(ByteBuffer payload) { MessageAndOffset mockMessageAndOffset = mock(MessageAndOffset.class); Message mockMessage = mock(Message.class); when(mockMessage.payload()).thenReturn(payload); when(mockMessageAndOffset.message()).thenReturn(mockMessage); return new Kafka08ConsumerRecord(mockMessageAndOffset); }
private void testRecordsWritten(int totalSuccessful, String topic) throws UnsupportedEncodingException { final ConsumerIterator<byte[], byte[]> iterator = kafkaTestHelper.getIteratorForTopic(topic); for (int i = 0; i < totalSuccessful; ++i) { String message = new String(iterator.next().message(), "UTF-8"); log.debug(String.format("%d of %d: Message consumed: %s", (i+1), totalSuccessful, message)); } }