private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) { LOG.info(String.format("Fetching topic metadata from broker %s", broker)); SimpleConsumer consumer = null; try { consumer = getSimpleConsumer(broker); for (int i = 0; i < this.fetchTopicRetries; i++) { try { return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata(); } catch (Exception e) { LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e); try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { LOG.warn("Caught InterruptedException: " + e2); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) { log.info(String.format("Fetching topic metadata from broker %s", broker)); SimpleConsumer consumer = null; try { consumer = getSimpleConsumer(broker); for (int i = 0; i < this.fetchTopicRetries; i++) { try { return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata(); } catch (Exception e) { log.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e); try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { log.warn("Caught InterruptedException: " + e2); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
for (TopicMetadata item : consumer.send(new TopicMetadataRequest(Collections.<String>emptyList())).topicsMetadata()) { if (item.errorCode() != ErrorMapping.NoError()) {
_simpleConsumer.send(new TopicMetadataRequest(Collections.singletonList(_topic))); try { _leader = null;
@Override public int getNumPartitions(String topic) { SimpleConsumer consumer = null; try { consumer = createConsumer( mConfig.getKafkaSeedBrokerHost(), mConfig.getKafkaSeedBrokerPort(), "partitionLookup"); List<String> topics = new ArrayList<String>(); topics.add(topic); TopicMetadataRequest request = new TopicMetadataRequest(topics); TopicMetadataResponse response = consumer.send(request); if (response.topicsMetadata().size() != 1) { throw new RuntimeException("Expected one metadata for topic " + topic + " found " + response.topicsMetadata().size()); } TopicMetadata topicMetadata = response.topicsMetadata().get(0); return topicMetadata.partitionsMetadata().size(); } finally { if (consumer != null) { consumer.close(); } } }
for (TopicMetadata item : consumer.send(new TopicMetadataRequest(topics)).topicsMetadata()) { if (item.errorCode() != ErrorMapping.NoError()) {
kafka.javaapi.TopicMetadataResponse resp = null; try { resp = consumer.send(req); } catch (Exception e) { errors += 1;
topicMetadataResponse = _simpleConsumer.send(new TopicMetadataRequest(Collections.singletonList(_topic))); } catch (Exception e) { _currentState.handleConsumerException(e);
private HostAndPort findLeader(TopicPartition topicPartition) { SimpleConsumer consumer = null; try { LOG.debug("looking up leader for topic {} partition {}", topicPartition.getTopic(), topicPartition.getPartition()); consumer = createConsumer( mConfig.getKafkaSeedBrokerHost(), mConfig.getKafkaSeedBrokerPort(), "leaderLookup"); List<String> topics = new ArrayList<String>(); topics.add(topicPartition.getTopic()); TopicMetadataRequest request = new TopicMetadataRequest(topics); TopicMetadataResponse response = consumer.send(request); List<TopicMetadata> metaData = response.topicsMetadata(); for (TopicMetadata item : metaData) { for (PartitionMetadata part : item.partitionsMetadata()) { if (part.partitionId() == topicPartition.getPartition()) { return HostAndPort.fromParts(part.leader().host(), part.leader().port()); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
log.info("Finding new leader from Kafka brokers, try broker [%s]", broker.toString()); consumer = new SimpleConsumer(broker.getHostText(), broker.getPort(), SO_TIMEOUT_MILLIS, BUFFER_SIZE, leaderLookupClientId); TopicMetadataResponse resp = consumer.send(new TopicMetadataRequest(Collections.singletonList(topic)));
TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest);
for (int iter = 0; iter < NUM_TRIES_TOPIC_METADATA; iter++) { try { topicMetadataList = consumer.send(new TopicMetadataRequest(metaRequestTopics)).topicsMetadata(); fetchMetaDataSucceeded = true; break;
private Map<String, Long> getTopicOffsets(List<String> topics) { ArrayList<HostAndPort> nodes = new ArrayList<>(config.getNodes()); Collections.shuffle(nodes); SimpleConsumer simpleConsumer = consumerManager.getConsumer(nodes.get(0)); TopicMetadataRequest topicMetadataRequest = new TopicMetadataRequest(topics); TopicMetadataResponse topicMetadataResponse = simpleConsumer.send(topicMetadataRequest); ImmutableMap.Builder<String, Long> builder = ImmutableMap.builder(); for (TopicMetadata metadata : topicMetadataResponse.topicsMetadata()) { for (PartitionMetadata part : metadata.partitionsMetadata()) { LOGGER.debug(format("Adding Partition %s/%s", metadata.topic(), part.partitionId())); Broker leader = part.leader(); if (leader == null) { // Leader election going on... LOGGER.warn(format("No leader for partition %s/%s found!", metadata.topic(), part.partitionId())); } else { HostAndPort leaderHost = HostAndPort.fromParts(leader.host(), leader.port()); SimpleConsumer leaderConsumer = consumerManager.getConsumer(leaderHost); long offset = findAllOffsets(leaderConsumer, metadata.topic(), part.partitionId())[0]; builder.put(metadata.topic(), offset); } } } return builder.build(); } }
@Test public void testFetchFailure() throws SecurityException, NoSuchFieldException, IllegalArgumentException, IllegalAccessException, IOException { SimpleConsumer consumer = EasyMock.createNiceMock(SimpleConsumer.class); EasyMock.expect(consumer.fetch((FetchRequest) EasyMock.anyObject())).andReturn(null); EasyMock.expectLastCall().times(2); EasyMock.expect(consumer.send((TopicMetadataRequest) EasyMock.anyObject())).andThrow(new RuntimeException()); EasyMock.replay(consumer); Field field = kafkaReader.getClass().getDeclaredField("simpleConsumer"); field.setAccessible(true); field.set(kafkaReader, consumer); kafkaReader.fetch(); EasyMock.verify(consumer); } }
private void refreshTopicMetadata() { TopicMetadataRequest request = new TopicMetadataRequest(Collections.singletonList(kafkaRequest.getTopic())); TopicMetadataResponse response; try { response = simpleConsumer.send(request); } catch (Exception e) { log.error("Exception caught when refreshing metadata for topic " + request.topics().get(0) + ": " + e.getMessage()); return; } TopicMetadata metadata = response.topicsMetadata().get(0); for (PartitionMetadata partitionMetadata : metadata.partitionsMetadata()) { if (partitionMetadata.partitionId() == kafkaRequest.getPartition()) { simpleConsumer = new SimpleConsumer(partitionMetadata.leader().host(), partitionMetadata.leader().port(), CamusJob.getKafkaTimeoutValue(context), CamusJob.getKafkaBufferSize(context), CamusJob.getKafkaClientName(context)); break; } } }
private void updateLeaderMap() { for (String broker : brokerList) { try { SimpleConsumer consumer = getSimpleConsumer(broker); TopicMetadataRequest req = new TopicMetadataRequest(auditTopics); kafka.javaapi.TopicMetadataResponse resp = consumer.send(req); List<TopicMetadata> metaData = resp.topicsMetadata(); for (TopicMetadata tmd : metaData) { for (PartitionMetadata pmd : tmd.partitionsMetadata()) { TopicAndPartition topicAndPartition = new TopicAndPartition(tmd.topic(), pmd.partitionId()); partitionLeader.put(topicAndPartition, getHostPort(pmd.leader())); } } } catch (Exception e) { logger.warn("Got exception to get metadata from broker=" + broker, e); } } }
mocks.add(simpleConsumer); EasyMock.expect(simpleConsumer.clientId()).andReturn(DUMMY_VALUE).times(2); EasyMock.expect(simpleConsumer.send((TopicMetadataRequest) EasyMock.anyObject())).andThrow( new RuntimeException("No TopicMD")); EasyMock.expect(simpleConsumer.send((TopicMetadataRequest) EasyMock.anyObject())).andReturn(topicMetadataResponse); simpleConsumer.close(); EasyMock.expectLastCall().andVoid().anyTimes();
@Test(expected = RuntimeException.class) public void testWithThreeRetries() { List<Object> mocks = new ArrayList<Object>(); Configuration configuration = EasyMock.createMock(Configuration.class); mocks.add(configuration); EasyMock.expect(configuration.get(EasyMock.anyString())).andReturn(DUMMY_VALUE).anyTimes(); JobContext jobContext = EasyMock.createMock(JobContext.class); mocks.add(jobContext); EasyMock.expect(jobContext.getConfiguration()).andReturn(configuration).anyTimes(); SimpleConsumer simpleConsumer = EasyMock.createMock(SimpleConsumer.class); mocks.add(simpleConsumer); EasyMock.expect(simpleConsumer.clientId()).andReturn(DUMMY_VALUE) .times(EtlInputFormat.NUM_TRIES_TOPIC_METADATA + 1); Exception ex = new RuntimeException("No TopicMeta"); EasyMock.expect(simpleConsumer.send((TopicMetadataRequest) EasyMock.anyObject())).andThrow(ex) .times(EtlInputFormat.NUM_TRIES_TOPIC_METADATA); simpleConsumer.close(); EasyMock.expectLastCall().andVoid().anyTimes(); EasyMock.replay(mocks.toArray()); EtlInputFormat inputFormat = new EtlInputFormatForUnitTest(); EtlInputFormatForUnitTest.consumerType = EtlInputFormatForUnitTest.ConsumerType.MOCK; EtlInputFormatForUnitTest.consumer = simpleConsumer; List<TopicMetadata> actualTopicMetadatas = inputFormat.getKafkaMetadata(jobContext, new ArrayList<String>()); EasyMock.verify(mocks.toArray()); }
private SimpleConsumer mockConsumerThrowsExceptionForOffsetRangeCall(TopicMetadataResponse metadataResponse) { SimpleConsumer simpleConsumer = EasyMock.createMock(SimpleConsumer.class); mocks.add(simpleConsumer); EasyMock.expect(simpleConsumer.send((TopicMetadataRequest)EasyMock.anyObject())) .andReturn(metadataResponse).times(1); EasyMock.expect(simpleConsumer.getOffsetsBefore((OffsetRequest) EasyMock.anyObject())) .andThrow(new RuntimeException()).times(3); EasyMock.expect(simpleConsumer.clientId()).andReturn(KAFKA_CLIENT_ID).times(1); simpleConsumer.close(); EasyMock.expectLastCall().andVoid().times(2); EasyMock.expect(simpleConsumer.host()).andReturn("dummyHost").times(4); EasyMock.expect(simpleConsumer.port()).andReturn(8888).times(4); return simpleConsumer; }
private SimpleConsumer mockSimpleConsumer(TopicMetadataResponse metadataResponse, OffsetResponse offsetResponse, FetchResponse fetchResponse) { SimpleConsumer simpleConsumer = EasyMock.createMock(SimpleConsumer.class); mocks.add(simpleConsumer); EasyMock.expect(simpleConsumer.send((TopicMetadataRequest)EasyMock.anyObject())).andReturn(metadataResponse).times(1); EasyMock.expect(simpleConsumer.getOffsetsBefore((OffsetRequest)EasyMock.anyObject())).andReturn(offsetResponse).anyTimes(); simpleConsumer.close(); EasyMock.expectLastCall().andVoid().anyTimes(); EasyMock.expect(simpleConsumer.clientId()).andReturn(KAFKA_CLIENT_ID).times(1); EasyMock.expect(simpleConsumer.fetch((FetchRequest)EasyMock.anyObject())).andReturn(fetchResponse).times(1); EasyMock.expect(simpleConsumer.host()).andReturn("dummyHost").anyTimes(); EasyMock.expect(simpleConsumer.port()).andReturn(8888).anyTimes(); return simpleConsumer; }