@Override public Double getValue() { return (double) queue.pctFull(); }
@Override protected Result check() throws Exception { Double ret = (double) queue.pctFull(); if (ret > 0.9) { return HealthCheck.Result.unhealthy(name + QUEUE_IS_FULL); } else { return healthy; } }
public boolean checkIfUnderFlowCtrl(DisruptorQueue queue) { return queue.pctFull() < lowMark && queue.cacheSize() == 0; }
private boolean isOutboundTaskAvailable(int taskId) { boolean ret = false; DisruptorQueue targetQueue = workerData.getInnerTaskTransfer().get(taskId); if (targetQueue != null) { float queueLoadRatio = targetQueue.pctFull(); if (queueLoadRatio < loadMark) { ret = true; } } else { WorkerSlot slot = taskNodePort.get(taskId); if (slot != null) { IConnection connection = nodePortToSocket.get(slot); if (connection != null) { ret = connection.available(taskId); } } } if (!ret) { LOG.debug("taskId:{} is unavailable", taskId); } return ret; }
@Override protected int getActiveTask(RandomRange randomrange, List<Integer> outTasks) { int index = randomrange.nextInt(); int size = outTasks.size(); int i; for (i = 0; i < size; i++) { Integer taskId = outTasks.get(index); boolean taskStatus = workerData.isOutboundTaskActive(taskId); DisruptorQueue exeQueue = (workerData.getInnerTaskTransfer().get(taskId)); float queueLoadRatio = exeQueue != null ? exeQueue.pctFull() : 0; if (taskStatus && queueLoadRatio < loadMark) break; else index = randomrange.nextInt(); } return (i < size ? index : -1); }
public void flowCtrl(Channel channel, DisruptorQueue queue, int taskId, byte[] message) { boolean initFlowCtrl = false; final Set<String> remoteAddrs = flowCtrlClients.get(taskId); synchronized (remoteAddrs) { if (remoteAddrs.size() > 0) { queue.publishCache(message); if(remoteAddrs.add(channel.getRemoteAddress().toString())) JStormUtils.sendFlowControlRequest(channel, taskId, 1); } else { queue.publish(message); if (queue.pctFull() > highMark) { remoteAddrs.add(channel.getRemoteAddress().toString()); JStormUtils.sendFlowControlRequest(channel, taskId, 1); initFlowCtrl = true; } } } if (initFlowCtrl) queue.publishCallback(new BackpressureCallback(this, taskId)); }
public void deserializeTuple(KryoTupleDeserializer deserializer, byte[] serMsg, DisruptorQueue queue) { Tuple tuple = deserializer.deserialize(serMsg); if (tuple != null) { if (JStormDebugger.isDebugRecv(tuple.getMessageId())) { LOG.info(idStr + " receive " + tuple.toString()); } //queue.publish(tuple); if (isBackpressureEnable) { if (backpressureStatus) { while (queue.pctFull() > lowMark) { JStormUtils.sleepMs(1); } queue.publish(tuple); backpressureStatus = false; } else { queue.publish(tuple); if (queue.pctFull() > highMark) { backpressureStatus = true; } } } else { queue.publish(tuple); } } } }
public boolean deserializer(KryoTupleDeserializer deserializer, boolean forceConsume) { //LOG.debug("start Deserializer of task, {}", taskId); boolean isIdling = true; DisruptorQueue exeQueue = innerTaskTransfer.get(taskId); if (!taskStatus.isShutdown()) { if ((deserializeQueue.population() > 0 && exeQueue.pctFull() < 1.0) || forceConsume) { try { List<Object> objects = deserializeQueue.retreiveAvailableBatch(); for (Object object : objects) { deserialize(deserializer, (byte[]) object, exeQueue); } isIdling = false; } catch (InterruptedException e) { LOG.error("InterruptedException " + e.getCause()); return true; } catch (TimeoutException e) { return true; } catch (Throwable e) { if (Utils.exceptionCauseIsInstanceOf(KryoException.class, e)) { throw new RuntimeException(e); } else if (!taskStatus.isShutdown()) { LOG.error("Unknown exception ", e); } } } } else { task.unregisterDeserializeQueue(); } return isIdling; }
@Override public Double getValue() { return (double) queue.pctFull(); }
@Override protected Result check() throws Exception { Double ret = (double) queue.pctFull(); if (ret > 0.9) { return HealthCheck.Result.unhealthy(name + QUEUE_IS_FULL); } else { return healthy; } }
private boolean isOutboundTaskAvailable(int taskId) { boolean ret = false; DisruptorQueue targetQueue = workerData.getInnerTaskTransfer().get(taskId); if (targetQueue != null) { float queueLoadRatio = targetQueue.pctFull(); if (queueLoadRatio < loadMark) { ret = true; } } else { WorkerSlot slot = taskNodeport.get(taskId); if (slot != null) { IConnection connection = nodeportSocket.get(slot); if (connection != null) { ret = connection.available(); } } } if (ret == false) { LOG.debug("taskId:{} is unavailable", taskId); } return ret; }
public <T> Object execute(T... args) { if (monitorQueue.pctFull() < lowMark && monitorQueue.cacheSize() == 0) { HashSet<String> remoteAddrs = remoteClientsUnderFlowCtrl.get(taskId); //LOG.debug("Release flow ctrl for task-{} for {}", taskId, remoteAddrs); synchronized (remoteAddrs) { for (String remoteAddr : remoteAddrs) { Channel channel = stormChannelGroup.getChannel(remoteAddr); if (channel == null) { continue; } // send back backpressure flow control request to source client ByteBuffer buffer = ByteBuffer.allocate(Integer.SIZE + 1); buffer.put((byte) 0); // 1-> start flow control; 0-> stop flow control buffer.putInt(taskId); TaskMessage flowCtrlMsg = new TaskMessage(TaskMessage.BACK_PRESSURE_REQUEST, 0, buffer.array()); channel.write(flowCtrlMsg); } remoteAddrs.clear(); } return true; } else { return false; } }
@Override protected int getActiveTask(RandomRange randomrange, List<Integer> outTasks) { int index = randomrange.nextInt(); int size = outTasks.size(); int i = 0; for (i = 0; i < size; i++) { Integer taskId = outTasks.get(index); boolean taskStatus = workerData.isOutboundTaskActive(taskId); DisruptorQueue exeQueue = (workerData.getInnerTaskTransfer().get(taskId)); float queueLoadRatio = exeQueue != null ? exeQueue.pctFull() : 0; if (taskStatus && queueLoadRatio < loadMark) break; else index = randomrange.nextInt(); } return (i < size ? index : -1); }
private void flowCtrl(Channel channel, String addr, DisruptorQueue queue, int taskId, byte[] message) { boolean isFlowCtrl = false; boolean isInitFlowCtrl = false; if (queue.pctFull() > lowMark || queue.cacheSize() > 0) { HashSet<String> remoteAddrs = remoteClientsUnderFlowCtrl.get(taskId); synchronized (remoteAddrs) { if (remoteAddrs.isEmpty()) { if (queue.pctFull() >= highMark) { remoteAddrs.add(addr); isInitFlowCtrl = true;
protected void deserializeTuple(KryoTupleDeserializer deserializer, byte[] serMsg, DisruptorQueue queue) { Tuple tuple = deserializer.deserialize(serMsg); if (tuple != null) { if (JStormDebugger.isDebugRecv(tuple.getMessageId())) { LOG.info(idStr + " receive " + tuple.toString()); } //queue.publish(tuple); if (isBackpressureEnable) { if (backpressureStatus) { while (queue.pctFull() > lowMark) { JStormUtils.sleepMs(1); } queue.publish(tuple); backpressureStatus = false; } else { queue.publish(tuple); if (queue.pctFull() > highMark) { backpressureStatus = true; } } } else { queue.publish(tuple); } } } }
public boolean deserializer(KryoTupleDeserializer deserializer, boolean forceConsume) { //LOG.debug("start Deserializer of task, {}", taskId); boolean isIdling = true; DisruptorQueue exeQueue = innerTaskTransfer.get(taskId); if (!taskStatus.isShutdown()) { if ((deserializeQueue.population() > 0 && exeQueue.pctFull() < 1.0) || forceConsume) { try { List<Object> objects = deserializeQueue.retreiveAvailableBatch(); for (Object object : objects) { deserialize(deserializer, (byte[]) object, exeQueue); } isIdling = false; } catch (InterruptedException e) { LOG.error("InterruptedException " + e.getCause()); return isIdling; } catch (TimeoutException e) { return isIdling; } catch (Throwable e) { if (Utils.exceptionCauseIsInstanceOf(KryoException.class, e)) { throw new RuntimeException(e); } else if (!taskStatus.isShutdown()) { LOG.error("Unknow exception ", e); } } } } else { task.unregisterDeserializeQueue(); } return isIdling; }