private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException( String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException(String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
fetchResponse = consumer.fetch(fetchRequest);
FetchResponse fetchResponse = _simpleConsumer.fetch( new FetchRequestBuilder().minBytes(100000).maxWait(timeoutMillis) .addFetch(_topic, _partition, startOffset, 500000).build());
fetchResponse = simpleConsumer.fetch(req); } catch (Exception e) { if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException
FetchResponse fetchResponse = consumer.fetch(req); if (fetchResponse.hasError()) { short errorCode = fetchResponse.errorCode(split.getTopicName(), split.getPartitionId());
MAX_MESSAGE_SIZE_BYTES) .build(); FetchResponse response = consumer.fetch(request); if (response.hasError()) { consumer.close();
response = consumer.fetch(request);
private boolean refreshTopicMetadataAndRetryFetch(FetchRequest fetchRequest, long tempTime) { try { refreshTopicMetadata(); FetchResponse fetchResponse = simpleConsumer.fetch(fetchRequest); if (fetchResponse.hasError()) { log.warn("Error encountered during fetch request retry from Kafka"); log.warn("Error Code generated : " + fetchResponse.errorCode(kafkaRequest.getTopic(), kafkaRequest.getPartition())); return false; } return processFetchResponse(fetchResponse, tempTime); } catch (Exception e) { log.info("Exception generated during fetch for topic " + kafkaRequest.getTopic() + ". This topic will be skipped."); return false; } }
@Test public void testFetchFailure() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException, IOException { SimpleConsumer consumer = EasyMock.createNiceMock(SimpleConsumer.class); EasyMock.expect(consumer.fetch((FetchRequest) EasyMock.anyObject())).andReturn(null); EasyMock.expectLastCall().times(2); EasyMock.expect(consumer.send((TopicMetadataRequest) EasyMock.anyObject())).andThrow(new RuntimeException()); EasyMock.replay(consumer); Field field = kafkaReader.getClass().getDeclaredField("simpleConsumer"); field.setAccessible(true); field.set(kafkaReader, consumer); kafkaReader.fetch(); EasyMock.verify(consumer); } }
fetchResponse = simpleConsumer.fetch(fetchRequest); if (fetchResponse.hasError()) { String message =
private SimpleConsumer mockSimpleConsumer(TopicMetadataResponse metadataResponse, OffsetResponse offsetResponse, FetchResponse fetchResponse) { SimpleConsumer simpleConsumer = EasyMock.createMock(SimpleConsumer.class); mocks.add(simpleConsumer); EasyMock.expect(simpleConsumer.send((TopicMetadataRequest)EasyMock.anyObject())).andReturn(metadataResponse).times(1); EasyMock.expect(simpleConsumer.getOffsetsBefore((OffsetRequest)EasyMock.anyObject())).andReturn(offsetResponse).anyTimes(); simpleConsumer.close(); EasyMock.expectLastCall().andVoid().anyTimes(); EasyMock.expect(simpleConsumer.clientId()).andReturn(KAFKA_CLIENT_ID).times(1); EasyMock.expect(simpleConsumer.fetch((FetchRequest)EasyMock.anyObject())).andReturn(fetchResponse).times(1); EasyMock.expect(simpleConsumer.host()).andReturn("dummyHost").anyTimes(); EasyMock.expect(simpleConsumer.port()).andReturn(8888).anyTimes(); return simpleConsumer; }
DEFAULT_SO_TIMEOUT, DEFAULT_BUFFER_SIZE, "foo"); FetchRequest req = new FetchRequestBuilder().addFetch(sinkTopic, 0, 0, DEFAULT_BUFFER_SIZE).build(); FetchResponse resp = consumer.fetch(req); ByteBufferMessageSet ms = resp.messageSet(sinkTopic, 0); assertTrue(ms.validBytes() > 0);
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException( String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException(String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException(String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
private synchronized FetchResponse getFetchResponseForFetchRequest(FetchRequest fetchRequest, KafkaPartition partition) { SimpleConsumer consumer = getSimpleConsumer(partition.getLeader().getHostAndPort()); FetchResponse fetchResponse = consumer.fetch(fetchRequest); if (fetchResponse.hasError()) { throw new RuntimeException( String.format("error code %d", fetchResponse.errorCode(partition.getTopicName(), partition.getId()))); } return fetchResponse; }
public static void main(String[] args) throws Exception { final String topic = "topic1"; String clientID = "DemoLowLevelConsumer1"; SimpleConsumer simpleConsumer = new SimpleConsumer("kafka0", 9092, 100000, 64 * 1000000, clientID); FetchRequest req = new FetchRequestBuilder().clientId(clientID) .addFetch(topic, 0, 0L, 50).addFetch(topic, 1, 0L, 5000).addFetch(topic, 2, 0L, 1000000).build(); FetchResponse fetchResponse = simpleConsumer.fetch(req); ByteBufferMessageSet messageSet = (ByteBufferMessageSet) fetchResponse.messageSet(topic, 0); for (MessageAndOffset messageAndOffset : messageSet) { ByteBuffer payload = messageAndOffset.message().payload(); long offset = messageAndOffset.offset(); byte[] bytes = new byte[payload.limit()]; payload.get(bytes); System.out.println("Offset:" + offset + ", Payload:" + new String(bytes, "UTF-8")); } }
/** * Makes a call to kafka to fetch messages. */ private FetchResponse fetchMessages(SimpleConsumer consumer, long offset) { FetchRequest request = new FetchRequestBuilder() .clientId(consumer.clientId()) .addFetch(topicPart.getTopic(), topicPart.getPartition(), offset, FETCH_SIZE) .maxWait(MAX_WAIT) .build(); return consumer.fetch(request); }
/** * Makes a call to kafka to fetch messages. */ private FetchResponse fetchMessages(SimpleConsumer consumer, long offset) { FetchRequest request = new FetchRequestBuilder() .clientId(consumer.clientId()) .addFetch(topicPart.getTopic(), topicPart.getPartition(), offset, FETCH_SIZE) .maxWait(MAX_WAIT) .build(); return consumer.fetch(request); }