@SuppressWarnings("unchecked") private Workflow initializeWorkflow() throws Exception { Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader()); if (!Workflow.class.isAssignableFrom(clz)) { throw new IllegalStateException(String.format("%s is not Workflow.", clz)); } Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz; final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create(); // set metrics Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics())); if (!(workflow instanceof ProgramLifecycle)) { return workflow; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null)); workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false); workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); return workflow; }
@SuppressWarnings("unchecked") private Workflow initializeWorkflow() throws Exception { Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader()); if (!Workflow.class.isAssignableFrom(clz)) { throw new IllegalStateException(String.format("%s is not Workflow.", clz)); } Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz; final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create(); // set metrics Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics())); if (!(workflow instanceof ProgramLifecycle)) { return workflow; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null)); workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false); workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); return workflow; }
@SuppressWarnings("unchecked") private void destroyWorkflow() { if (!(workflow instanceof ProgramLifecycle)) { return; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "destroy"); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.destroyProgram((ProgramLifecycle) workflow, txControl, false); try { workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); } catch (Throwable t) { LOG.error("Failed to store the final workflow token of Workflow {}", workflowRunId, t); } if (ProgramStatus.COMPLETED != workflowContext.getState().getStatus()) { return; } try { Set<Operation> fieldLineageOperations = workflowContext.getFieldLineageOperations(); if (!fieldLineageOperations.isEmpty()) { FieldLineageInfo info = new FieldLineageInfo(fieldLineageOperations); fieldLineageWriter.write(workflowRunId, info); } } catch (Throwable t) { LOG.debug("Failed to emit the field lineage operations for Workflow {}", workflowRunId, t); } }
@SuppressWarnings("unchecked") private void destroyWorkflow() { if (!(workflow instanceof ProgramLifecycle)) { return; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "destroy"); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.destroyProgram((ProgramLifecycle) workflow, txControl, false); try { workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); } catch (Throwable t) { LOG.error("Failed to store the final workflow token of Workflow {}", workflowRunId, t); } if (ProgramStatus.COMPLETED != workflowContext.getState().getStatus()) { return; } try { Set<Operation> fieldLineageOperations = workflowContext.getFieldLineageOperations(); if (!fieldLineageOperations.isEmpty()) { FieldLineageInfo info = new FieldLineageInfo(fieldLineageOperations); fieldLineageWriter.write(workflowRunId, info); } } catch (Throwable t) { LOG.debug("Failed to emit the field lineage operations for Workflow {}", workflowRunId, t); } }
private void executeNode(ApplicationSpecification appSpec, WorkflowNode node, InstantiatorFactory instantiator, ClassLoader classLoader, WorkflowToken token) throws Exception { WorkflowNodeType nodeType = node.getType(); ((BasicWorkflowToken) token).setCurrentNode(node.getNodeId()); switch (nodeType) { case ACTION: WorkflowActionNode actionNode = (WorkflowActionNode) node; if (SchedulableProgramType.CUSTOM_ACTION == actionNode.getProgram().getProgramType()) { executeCustomAction(actionNode, instantiator, classLoader, token); } else { executeAction(actionNode, token); } break; case FORK: executeFork(appSpec, (WorkflowForkNode) node, instantiator, classLoader, token); break; case CONDITION: executeCondition(appSpec, (WorkflowConditionNode) node, instantiator, classLoader, token); break; default: break; } }
private void executeNode(ApplicationSpecification appSpec, WorkflowNode node, InstantiatorFactory instantiator, ClassLoader classLoader, WorkflowToken token) throws Exception { WorkflowNodeType nodeType = node.getType(); ((BasicWorkflowToken) token).setCurrentNode(node.getNodeId()); switch (nodeType) { case ACTION: WorkflowActionNode actionNode = (WorkflowActionNode) node; if (SchedulableProgramType.CUSTOM_ACTION == actionNode.getProgram().getProgramType()) { executeCustomAction(actionNode, instantiator, classLoader, token); } else { executeAction(actionNode, token); } break; case FORK: executeFork(appSpec, (WorkflowForkNode) node, instantiator, classLoader, token); break; case CONDITION: executeCondition(appSpec, (WorkflowConditionNode) node, instantiator, classLoader, token); break; default: break; } }
@Test public void testNonUpdatableWorkflowToken() { BasicWorkflowToken token = new BasicWorkflowToken(0); token.setCurrentNode("node"); try { token.put("a", "b"); Assert.fail("Workflow token update should fail because the token is non-updatable."); } catch (IllegalStateException e) { assertSizeExceededErrorMessage(e); } }
@Test public void testUpdateWithLargeRecord() { BasicWorkflowToken token = new BasicWorkflowToken(1); token.setCurrentNode("node"); try { token.put("k", generateDataInKb(1024)); Assert.fail("Workflow token update should fail because token size should have exceeded limit."); } catch (IllegalStateException e) { assertSizeExceededErrorMessage(e); } }
token.setCurrentNode("node1"); token.put("key", "value");
@Test public void testSerDeserScheduleInfo() { BasicWorkflowToken token = new BasicWorkflowToken(1); token.setCurrentNode("node"); token.put("tokenKey", "tokenVal"); List<TriggerInfo> triggerInfos = ImmutableList.of( new DefaultProgramStatusTriggerInfo("ns", Specifications.from(new WorkflowAppWithFork()), ProgramType.WORKFLOW, WorkflowAppWithFork.WorkflowWithFork.class.getSimpleName(), RunIds.generate(), ProgramStatus.COMPLETED, token, Collections.emptyMap()), new DefaultPartitionTriggerInfo("ns", "ds", 10, 11), new DefaultTimeTriggerInfo("1 * * * *", 0L) ); TriggeringScheduleInfo scheduleInfo = new DefaultTriggeringScheduleInfo("schedule", "description", triggerInfos, ImmutableMap.of("key", "value")); String scheduleInfoJson = GSON.toJson(scheduleInfo); TriggeringScheduleInfo deserializedScheduleInfo = GSON.fromJson(scheduleInfoJson, TriggeringScheduleInfo.class); Assert.assertEquals(scheduleInfoJson, GSON.toJson(deserializedScheduleInfo)); DefaultProgramStatusTriggerInfo expectedProgramStatusTriggerInfo = (DefaultProgramStatusTriggerInfo) triggerInfos.get(0); DefaultProgramStatusTriggerInfo deserializedProgramStatusTriggerInfo = (DefaultProgramStatusTriggerInfo) deserializedScheduleInfo.getTriggerInfos().get(0); Assert.assertEquals(expectedProgramStatusTriggerInfo.getApplicationSpecification().getName(), deserializedProgramStatusTriggerInfo.getApplicationSpecification().getName()); Assert.assertEquals(expectedProgramStatusTriggerInfo.getWorkflowToken().getAll(), deserializedProgramStatusTriggerInfo.getWorkflowToken().getAll()); } }
@Test public void testMergeLargeWorkflowToken() { BasicWorkflowToken token1 = new BasicWorkflowToken(1); token1.setCurrentNode("node1"); // total size of token1 after this operation is 512KB token1.put(generateDataInKb(1), generateDataInKb(511)); // add an additional 2 bytes, just so we have size > max size (and not equal to max size) after merge token1.put("k", "v"); BasicWorkflowToken token2 = new BasicWorkflowToken(1); token2.setCurrentNode("node1"); // total size of token2 after this operation is 512KB token2.put(generateDataInKb(1), generateDataInKb(511)); // should succeed, because token1 already contains the NodeValue being merged token1.mergeToken(token2); // set a different node in token2 and add the same data token2.setCurrentNode("node2"); // token2 is at capacity after the following operation token2.put(generateDataInKb(1), generateDataInKb(511)); // merging should now fail, because token1 now does not contain the NodeValue being merged try { token1.mergeToken(token2); Assert.fail("Workflow token merging should fail because token size should have exceeded limit."); } catch (IllegalStateException e) { assertSizeExceededErrorMessage(e); } }
@Test public void testRepeatedPutAtSameNode() { BasicWorkflowToken token = new BasicWorkflowToken(1); token.setCurrentNode("node1"); // after this put, size would be 512KB token.put(generateDataInKb(1), generateDataInKb(511)); // after another successful put at a different node, size would be 1024KB. Workflow token would be at capacity. token.setCurrentNode("node2"); token.put(generateDataInKb(1), generateDataInKb(511)); // should succeed because the entry at key k1 should be replaced token.put(generateDataInKb(1), generateDataInKb(511)); // now should fail, because even though we're updating node2's value, we're adding an extra KB try { token.put(generateDataInKb(1), generateDataInKb(512)); Assert.fail("Workflow token update at existing key should fail because token size should have exceeded limit."); } catch (IllegalStateException e) { assertSizeExceededErrorMessage(e); } }