Tabnine Logo
com.netflix.suro.sink.kafka
Code IndexAdd Tabnine to your IDE (free)

How to use com.netflix.suro.sink.kafka

Best Java code snippets using com.netflix.suro.sink.kafka (Showing top 16 results out of 315)

origin: com.netflix.suro/suro-kafka-producer

@Override
public void init(List<KafkaMetric> metrics) {
  for (KafkaMetric metric : metrics) {
    addMetric(metric);
  }
}
origin: com.netflix.suro/suro-kafka-producer

@Override
public String toString() {
  return String.format("routingKey: %s, payload byte size: %d",
      getRoutingKey(),
      getPayload().length);
}
origin: com.netflix.suro/suro-kafka-producer

@Override
public void open() {
  setName(KafkaSink.class.getSimpleName() + "-" + clientId);
  start();
}
origin: com.netflix.suro/suro-kafka

@Override
protected void write(List<Message> msgList) {
  final List<KeyedMessage<Long, byte[]>> kafkaMsgList = new ArrayList<KeyedMessage<Long, byte[]>>();
  for (Message m : msgList) {
    SuroKeyedMessage keyedMessage = (SuroKeyedMessage) m;
    kafkaMsgList.add(new KeyedMessage<Long, byte[]>(
        keyedMessage.getRoutingKey(),
        keyedMessage.getKey(),
        keyedMessage.getPayload()));
  }
  senders.submit(new Runnable() {
    @Override
    public void run() {
      producer.send(kafkaMsgList);
    }
  });
}
origin: com.netflix.suro/suro-kafka-producer

for( Message m : msgList ){
  SuroKeyedMessage sKeyedMsg = (SuroKeyedMessage) m;
  msgCopies.add( new SuroKeyedMessage( sKeyedMsg.getKey(), 
                     new Message( m.getRoutingKey(), m.getPayload() )));
origin: com.netflix.suro/suro-kafka

@Override
public void writeTo(MessageContainer message) {
  long key = msgId++;
  if (!keyTopicMap.isEmpty()) {
    try {
      Map<String, Object> msgMap = message.getEntity(new TypeReference<Map<String, Object>>() {});
      Object keyField = msgMap.get(keyTopicMap.get(message.getRoutingKey()));
      if (keyField != null) {
        key = keyField.hashCode();
      }
    } catch (Exception e) {
      log.error("Exception on getting key field: " + e.getMessage());
    }
  }
  enqueue(new SuroKeyedMessage(key, message.getMessage()));
}
origin: com.netflix.suro/suro-kafka-producer

@Override
public void writeTo(MessageContainer message) {
  long key = msgId++;
  if (!keyTopicMap.isEmpty()) {
    try {
      Map<String, Object> msgMap = message.getEntity(new TypeReference<Map<String, Object>>() {});
      Object keyField = msgMap.get(keyTopicMap.get(message.getRoutingKey()));
      if (keyField != null) {
        key = keyField.hashCode();
      }
    } catch (Exception e) {
      QueuedSink.log.error("Exception on getting key field: " + e.getMessage());
    }
  }
  QueuedSink.log.trace( "KafkaSink writeTo()" );
  receivedCount.incrementAndGet();
  enqueue(new SuroKeyedMessage(key, message.getMessage()));
}
origin: com.netflix.suro/suro-kafka-producer

@Override
public void writeTo(final MessageContainer message) {
  queuedRecords.incrementAndGet();
  DynamicCounter.increment(
    MonitorConfig
      .builder("queuedRecord")
      .withTag(TagKey.ROUTING_KEY, message.getRoutingKey())
      .build());
  runRecordCounterListener();
  if (metadataFetchedTopicSet.contains(message.getRoutingKey())) {
    sendMessage(message);
  } else {
    if(!metadataWaitingQueue.offer(message)) {
      dropMessage(message.getRoutingKey(), "metadataWaitingQueueFull");
    }
  }
}
origin: com.netflix.suro/suro-kafka

@Override
public void open() {
  setName(KafkaSink.class.getSimpleName() + "-" + clientId);
  start();
}
origin: com.netflix.suro/suro-kafka-producer

private void sendMessage(final MessageContainer message) {
  try {
    List<PartitionInfo> partitionInfos = producer.partitionsFor(message.getRoutingKey());
    int partition = retentionPartitioner.getKey(message.getRoutingKey(), partitionInfos);
    dropMessage(message.getRoutingKey(), e.getClass().getName());
origin: com.netflix.suro/suro-kafka-producer

  private void dropMessage(final String routingKey, final String reason) {
    DynamicCounter.increment(
      MonitorConfig
        .builder("droppedRecord")
        .withTag(TagKey.ROUTING_KEY, routingKey)
        .withTag(TagKey.DROPPED_REASON, reason)
        .build());
    droppedRecords.incrementAndGet();
    runRecordCounterListener();
  }
}
origin: com.netflix.suro/suro-kafka-producer

setServoReporter();
origin: com.netflix.suro/suro-kafka-producer

initialize(
    "kafka_" + clientId,
    queue4Sink == null ? new MemoryQueue4Sink(10000) : queue4Sink,
origin: com.netflix.suro/suro-kafka

initialize("kafka_" + clientId, queue4Sink == null ? new MemoryQueue4Sink(10000) : queue4Sink, batchSize, batchTimeout);
origin: com.netflix.suro/suro-kafka

@Override
public String toString() {
  return String.format("routingKey: %s, payload byte size: %d",
      getRoutingKey(),
      getPayload().length);
}
origin: com.netflix.suro/suro-kafka-producer

@Override
public void metricChange(KafkaMetric metric) {
  addMetric(metric);
}
com.netflix.suro.sink.kafka

Most used classes

  • KafkaSink
    Kafka 0.8 Sink
  • SuroKeyedMessage
  • KafkaRetentionPartitioner
  • KafkaSinkV2
    Kafka 0.8.2 Sink, using new Java-native producer, rather than Scala produer. Requests are re-queued
  • ServoReporter
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now