public void logInterruptedException(final InterruptedException interruptedException) { LOG.warn("Worker interrupt signalled, not performing callback for task {} (message id: {})", taskMessage.getTaskId(), messageId, interruptedException); }
@Override public byte[] getData() { return taskMessage.getTaskData(); }
private static TaskMessage buildTaskMessage(Document document, long workflowId, String projectId, String storageDirectory) throws CodecException { TaskData taskData = new TaskData(); taskData.setDocument(document); taskData.setOutputPartialReference(storageDirectory); taskData.setWorkflowId(String.valueOf(workflowId)); taskData.setExecutePolicyOnClassifiedDocuments(true); taskData.setProjectId(projectId); byte[] serializedTaskData = codec.serialise(taskData); TaskMessage taskMessage = new TaskMessage(); taskMessage.setTaskData(serializedTaskData); taskMessage.setContext(new HashMap<>()); taskMessage.setTaskId(String.valueOf(messageCount.addAndGet(1)) + "-" + document.getReference()); taskMessage.setTaskClassifier("PolicyWorker"); taskMessage.setTaskApiVersion(1); taskMessage.setTaskStatus(TaskStatus.NEW_TASK); taskMessage.setTo(destinationQueue); return taskMessage; } }
/** * Deserialize message to the worker-under-test result using configured {@link Codec} implementation. * * @param message the message * @param resultClass the result class * @return the t result * @throws CodecException the codec exception */ protected TResult deserializeMessage(TaskMessage message, Class<TResult> resultClass) throws CodecException { if (message.getTaskStatus() != TaskStatus.RESULT_SUCCESS && message.getTaskStatus() != TaskStatus.RESULT_FAILURE) { throw new AssertionError("Task status was failure."); } TResult workerResult = codec.deserialise(message.getTaskData(), resultClass); return workerResult; }
protected void checkTaskMessageReturnedTaskStatus(TaskMessage resultWrapper, String expectedTaskId, TaskStatus expectedStatus) { //check that this has the task ID we specified when adding to Input Queue Assert.assertEquals(expectedTaskId, resultWrapper.getTaskId()); TaskStatus status = resultWrapper.getTaskStatus(); //check that task status is x Assert.assertEquals(expectedStatus, status); }
/** * Checks whether tracking of this task message should end when publishing to the specified queue. If tracking is to end then this * method removes and returns the tracking info from the task message. * * @param queueMsgId the reference to the message this task arrived on * @param queueToSend the queue to which the message is to be published * @param tm task message whose tracking info is to be checked * @return if tracking of the message terminates on publishing to the specified queue then the removed tracking info is returned; * otherwise null is returned and the tracking info is not removed from the message */ private TrackingInfo checkForTrackingTermination(final String queueMsgId, final String queueToSend, TaskMessage tm) { Objects.requireNonNull(queueMsgId); Objects.requireNonNull(tm); // queueToSend can be null for a dead end worker final TrackingInfo tracking = tm.getTracking(); if (tracking != null) { final String trackTo = tracking.getTrackTo(); if ((trackTo == null && queueToSend == null) || (trackTo != null && trackTo.equalsIgnoreCase(queueToSend))) { LOG.debug("Task {} (message id: {}): removing tracking info from this message as tracking ends on publishing to the queue {}.", tm.getTaskId(), queueMsgId, queueToSend); tm.setTracking(null); } } return tracking; } }
private void debugCurrentQueueItems() { logger.warn("Message queue currently contains: " + consumerImpl.getDelivery().size()); // do we have any items on the queue at all, if so, output what we have? for (Map.Entry<String, Delivery> deliveredItem : consumerImpl.getDelivery().entrySet()) { Delivery tmpDel = deliveredItem.getValue(); TaskMessage tm = getTaskMessageFromDelivery(tmpDel); logger.warn( String.format("Queue contains other item: {%s} TrackingInfo jobTaskId: {%s}", tm.getTaskId(), tm.getTracking() == null ? "null" : tm.getTracking().getJobTaskId())); } }
Objects.requireNonNull(responseMessage); LOG.debug("Task {} complete (message id: {})", responseMessage.getTaskId(), queueMsgId); LOG.debug("Setting destination {} in task {} (message id: {})", queue, responseMessage.getTaskId(), queueMsgId); responseMessage.setTo(queue); checkForTrackingTermination(queueMsgId, queue, responseMessage); try { workerQueue.publish(queueMsgId, output, queue, Collections.emptyMap(), responseMessage.getPriority() == null ? 0 : responseMessage.getPriority()); stats.getOutputSizes().update(output.length); if (TaskStatus.isSuccessfulResponse(responseMessage.getTaskStatus())) { stats.incrementTasksSucceeded(); } else { LOG.error("Cannot publish data for task {}, rejecting", responseMessage.getTaskId(), e); abandon(queueMsgId, e);
private TaskMessage createResponseMessage(final boolean includeTaskContext, final WorkerResponse response) { final Map<String, byte[]> responseContext = createFullResponseContext(includeTaskContext, response.getContext()); final String responseMessageType = response.getMessageType(); // Check if a tracking change is required. If empty string then no further changes required. final TrackingInfo trackingInfo; if ("".equals(response.getTrackTo())) { // No tracking changes required. trackingInfo = taskMessage.getTracking(); } else { // Set trackTo field to the specified value. trackingInfo = getTrackingInfoWithChanges(response.getTrackTo()); } final TaskMessage responseMessage = new TaskMessage( taskMessage.getTaskId(), responseMessageType, response.getApiVersion(), response.getData(), response.getTaskStatus(), responseContext, response.getQueueReference(), trackingInfo, new TaskSourceInfo(getWorkerName(responseMessageType), getWorkerVersion())); responseMessage.setPriority(priorityManager.getResponsePriority(taskMessage)); return responseMessage; }
/** * Updates the specified {@link TaskMessage} with the specified subtask identifier. */ private static void updateTaskId(final TaskMessage responseMessage, final int subtaskId, final boolean isFinalResponse) { // Put together the suffix to be added final String subtaskSuffix; { final StringBuilder builder = new StringBuilder(); builder.append('.'); builder.append(subtaskId); if (isFinalResponse) { builder.append('*'); } subtaskSuffix = builder.toString(); } // Update the task id responseMessage.setTaskId(responseMessage.getTaskId() + subtaskSuffix); // Update the tracking info final TrackingInfo taskMessageTracking = responseMessage.getTracking(); if (taskMessageTracking != null) { final String trackingTaskId = taskMessageTracking.getJobTaskId(); if (trackingTaskId != null) { final TrackingInfo trackingInfo = new TrackingInfo(taskMessageTracking); trackingInfo.setJobTaskId(trackingTaskId + subtaskSuffix); responseMessage.setTracking(trackingInfo); } } }
@Override public void setResponse(final InvalidTaskException invalidTaskException) { if (invalidTaskException == null) { throw new IllegalArgumentException(); } incrementResponseCount(true); LOG.error("Task data is invalid for {}, returning status {}", taskMessage.getTaskId(), TaskStatus.INVALID_TASK, invalidTaskException); final String taskClassifier = MoreObjects.firstNonNull(taskMessage.getTaskClassifier(), ""); final String invalidTaskExceptionMessage = invalidTaskException.getMessage(); final byte[] taskData = invalidTaskExceptionMessage == null ? new byte[]{} : invalidTaskExceptionMessage.getBytes(StandardCharsets.UTF_8); final Map<String, byte[]> context = MoreObjects.firstNonNull( taskMessage.getContext(), Collections.<String, byte[]>emptyMap()); final TaskMessage invalidResponse = new TaskMessage( MoreObjects.firstNonNull(taskMessage.getTaskId(), ""), taskClassifier, taskMessage.getTaskApiVersion(), taskData, TaskStatus.INVALID_TASK, context, workerFactory.getInvalidTaskQueue(), taskMessage.getTracking(), new TaskSourceInfo(getWorkerName(taskClassifier), getWorkerVersion())); completeResponse(invalidResponse); }
trackingReport.jobTaskId = tm.getTracking().getJobTaskId(); final TaskStatus taskStatus = tm.getTaskStatus(); final String trackToPipe = tm.getTracking().getTrackTo(); final String toPipe = tm.getTo(); failure.failureTime = new Date(); failure.failureSource = getWorkerName(tm); final byte[] taskData = tm.getTaskData(); if (taskData != null) { failure.failureMessage = new String(taskData, StandardCharsets.UTF_8);
@Override public void send(final String queueMsgId, final TaskMessage responseMessage) { Objects.requireNonNull(queueMsgId); Objects.requireNonNull(responseMessage); LOG.debug("Sending task {} complete (message id: {})", responseMessage.getTaskId(), queueMsgId); final String queue = responseMessage.getTo(); checkForTrackingTermination(queueMsgId, queue, responseMessage); final byte[] output; try { output = codec.serialise(responseMessage); } catch (final CodecException ex) { throw new RuntimeException(ex); } final int priority = responseMessage.getPriority() == null ? 0 : responseMessage.getPriority(); try { workerQueue.publish("-1", output, queue, Collections.emptyMap(), priority); } catch (final QueueException ex) { throw new RuntimeException(ex); } }
@Override public void forward(String queueMsgId, String queue, TaskMessage forwardedMessage, Map<String, Object> headers) { Objects.requireNonNull(queueMsgId); Objects.requireNonNull(forwardedMessage); // queue can be null for a dead end worker LOG.debug("Task {} (message id: {}) being forwarded to queue {}", forwardedMessage.getTaskId(), queueMsgId, queue); checkForTrackingTermination(queueMsgId, queue, forwardedMessage); try { // If the queue is null, acknowledge the task rather than forwarding it if (queue == null) { workerQueue.acknowledgeTask(queueMsgId); } else { // Else forward the task byte[] output = codec.serialise(forwardedMessage); workerQueue.publish(queueMsgId, output, queue, headers, forwardedMessage.getPriority() == null ? 0 : forwardedMessage.getPriority()); stats.incrementTasksForwarded(); //TODO - I'm guessing this stat should not be updated for forwarded messages: // stats.getOutputSizes().update(output.length); } } catch (CodecException | QueueException e) { LOG.error("Cannot publish data for forwarded task {}, rejecting", forwardedMessage.getTaskId(), e); abandon(queueMsgId, e); } }
private void registerNewTaskImpl(final String queueMsgId, final byte[] taskMessage, Map<String, Object> headers) throws InvalidTaskException, TaskRejectedException { try { TaskMessage tm = codec.deserialise(taskMessage, TaskMessage.class, DecodeMethod.LENIENT); LOG.debug("Received task {} (message id: {})", tm.getTaskId(), queueMsgId); boolean poison = isTaskPoisoned(headers); validateTaskMessage(tm); boolean taskIsActive = checkStatus(tm); if (taskIsActive) { if (tm.getTo() != null && tm.getTo().equalsIgnoreCase(workerQueue.getInputQueue())) { LOG.debug("Task {} (message id: {}) on input queue {} {}", tm.getTaskId(), queueMsgId, workerQueue.getInputQueue(), (tm.getTo() != null) ? "is intended for this worker" : "has no explicit destination, therefore assuming it is intended for this worker"); executor.executeTask(tm, queueMsgId, poison, headers, codec); } else { LOG.debug("Task {} (message id: {}) is not intended for this worker: input queue {} does not match message destination queue {}", tm.getTaskId(), queueMsgId, workerQueue.getInputQueue(), tm.getTo()); executor.forwardTask(tm, queueMsgId, headers); } } else { LOG.debug("Task {} is no longer active. The task message (message id: {}) will not be executed", tm.getTaskId(), queueMsgId); executor.discardTask(tm, queueMsgId); } } catch (CodecException e) { throw new InvalidTaskException("Queue data did not deserialise to a TaskMessage", e); } catch (InvalidJobTaskIdException ijte) { throw new InvalidTaskException("TaskMessage contains an invalid job task identifier", ijte); } }
/** * Create task message. * * @param workerTask the worker task * @param taskId the task id * @return the task message * @throws CodecException the codec exception */ public TaskMessage create(final Object workerTask, TrackingInfo tracking, final String taskId) throws CodecException { Map<java.lang.String, byte[]> context = Collections.singletonMap(CONTEXT_KEY, CONTEXT_DATA); TaskMessage msg = new TaskMessage(taskId, workerName, apiVersion, codec.serialise(workerTask), TaskStatus.NEW_TASK, context, workerInputQueueName, tracking); return msg; } }
@Override public TaskStatus getStatus() { return taskMessage.getTaskStatus(); }
@Override public TrackingInfo getTrackingInfo() { return taskMessage.getTracking(); }
/** * Checks whether a task is still active. If a status check cannot be performed then the task is assumed to be active. Checking * status may result in a change to the tracking info on the supplied task message. * * @param tm task message to be checked to verify whether the task is still active * @return true if the task is still active, false otherwise */ private boolean checkStatus(TaskMessage tm) throws InvalidJobTaskIdException { Objects.requireNonNull(tm); TrackingInfo tracking = tm.getTracking(); if (tracking != null) { Date statusCheckTime = tracking.getStatusCheckTime(); if (statusCheckTime == null || statusCheckTime.getTime() <= System.currentTimeMillis()) { return performJobStatusCheck(tm); } LOG.debug("Task {} active status is not being checked - it is not yet time for the status check to be performed: status check due at {}", tm.getTaskId(), statusCheckTime); } else { LOG.debug("Task {} active status is not being checked - the task message does not have tracking info", tm.getTaskId()); } //By default a task is considered to be active. return true; }
@Override public void processDelivery(Delivery delivery) { System.out.print("New delivery"); try { TaskMessage taskMessage = codec.deserialise(delivery.getMessageData(), TaskMessage.class, DecodeMethod.LENIENT); System.out.println(taskMessage.getTaskId() + ", status: " + taskMessage.getTaskStatus()); synchronized (syncLock) { resultHandler.handleResult(taskMessage); } } catch (CodecException e) { e.printStackTrace(); throw new AssertionError("Failed: " + e.getMessage()); } catch (Exception e) { e.printStackTrace(); throw new AssertionError("Failed: " + e.getMessage()); } }