@Override public String getId() { return getFlowId(); }
private boolean isFlowRunningHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getProjectId() == projectId && ref.getSecond().getFlowId().equals(flowId)) { return true; } } return false; }
private boolean isFlowRunningHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getProjectId() == projectId && ref.getSecond().getFlowId().equals(flowId)) { return true; } } return false; }
private List<Integer> getRunningFlowsHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { final List<Integer> executionIds = new ArrayList<>(); for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getFlowId().equals(flowId) && ref.getSecond().getProjectId() == projectId) { executionIds.add(ref.getFirst().getExecId()); } } return executionIds; }
private List<Integer> getRunningFlowsHelper(final int projectId, final String flowId, final Collection<Pair<ExecutionReference, ExecutableFlow>> collection) { final List<Integer> executionIds = new ArrayList<>(); for (final Pair<ExecutionReference, ExecutableFlow> ref : collection) { if (ref.getSecond().getFlowId().equals(flowId) && ref.getSecond().getProjectId() == projectId) { executionIds.add(ref.getFirst().getExecId()); } } return executionIds; }
final HashMap<String, Object> flowInfo = new HashMap<>(); flowInfo.put("execId", flow.getExecutionId()); flowInfo.put("flowId", flow.getFlowId()); flowInfo.put("projectId", flow.getProjectId()); flowInfo.put("status", flow.getStatus().toString());
@Test public void testFetchActiveFlowByProject() throws Exception { initializeUnfinishedFlows(); final List<Integer> executions = this.controller .getRunningFlows(this.flow2.getProjectId(), this.flow2.getFlowId()); assertThat(executions.contains(this.flow2.getExecutionId())).isTrue(); assertThat(executions.contains(this.flow3.getExecutionId())).isTrue(); assertThat(this.controller.isFlowRunning(this.flow2.getProjectId(), this.flow2.getFlowId())) .isTrue(); assertThat(this.controller.isFlowRunning(this.flow3.getProjectId(), this.flow3.getFlowId())) .isTrue(); }
@Override public void resumeFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { final Pair<ExecutionReference, ExecutableFlow> pair = this.runningExecutions.get().get(exFlow.getExecutionId()); if (pair == null) { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } this.apiGateway .callWithReferenceByUser(pair.getFirst(), ConnectorParams.RESUME_ACTION, userId); } }
@Override public void pauseFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { final Pair<ExecutionReference, ExecutableFlow> pair = this.runningExecutions.get().get(exFlow.getExecutionId()); if (pair == null) { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } this.apiGateway .callWithReferenceByUser(pair.getFirst(), ConnectorParams.PAUSE_ACTION, userId); } }
@Override public void alertOnError(final ExecutableFlow flow, final String... extraReasons) { final EmailMessage message = this.messageCreator.createMessage(); final MailCreator mailCreator = getMailCreator(flow); List<ExecutableFlow> last72hoursExecutions = new ArrayList<>(); if (flow.getStartTime() > 0) { final long startTime = flow.getStartTime() - Duration.ofHours(72).toMillis(); try { last72hoursExecutions = this.executorLoader.fetchFlowHistory(flow.getProjectId(), flow .getFlowId(), startTime); } catch (final ExecutorManagerException e) { logger.error("unable to fetch past executions", e); } } final boolean mailCreated = mailCreator.createErrorEmail(flow, last72hoursExecutions, message, this.azkabanName, this.scheme, this.clientHostname, this.clientPortNumber, extraReasons); sendEmail(message, mailCreated, "error email message for execution " + flow.getExecutionId()); }
public synchronized void uploadExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { final String INSERT_EXECUTABLE_FLOW = "INSERT INTO execution_flows " + "(project_id, flow_id, version, status, submit_time, submit_user, update_time) " + "values (?,?,?,?,?,?,?)"; final long submitTime = System.currentTimeMillis(); flow.setStatus(Status.PREPARING); flow.setSubmitTime(submitTime); /** * Why we need a transaction to get last insert ID? * Because "SELECT LAST_INSERT_ID()" needs to have the same connection * as inserting the new entry. * See https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_last-insert-id */ final SQLTransaction<Long> insertAndGetLastID = transOperator -> { transOperator.update(INSERT_EXECUTABLE_FLOW, flow.getProjectId(), flow.getFlowId(), flow.getVersion(), Status.PREPARING.getNumVal(), submitTime, flow.getSubmitUser(), submitTime); transOperator.getConnection().commit(); return transOperator.getLastInsertId(); }; try { final long id = this.dbOperator.transaction(insertAndGetLastID); logger.info("Flow given " + flow.getFlowId() + " given id " + id); flow.setExecutionId((int) id); updateExecutableFlow(flow); } catch (final SQLException e) { throw new ExecutorManagerException("Error creating execution.", e); } }
@Override public void pauseFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { final Pair<ExecutionReference, ExecutableFlow> pair = this.executorLoader.fetchActiveFlowByExecId(exFlow.getExecutionId()); if (pair == null) { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } this.apiGateway .callWithReferenceByUser(pair.getFirst(), ConnectorParams.PAUSE_ACTION, userId); } }
/** * If a flow is already dispatched to an executor, cancel by calling Executor. Else if it's still * queued in DB, remove it from DB queue and finalize. {@inheritDoc} */ @Override public void cancelFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { final Map<Integer, Pair<ExecutionReference, ExecutableFlow>> unfinishedFlows = this.executorLoader .fetchUnfinishedFlows(); if (unfinishedFlows.containsKey(exFlow.getExecutionId())) { final Pair<ExecutionReference, ExecutableFlow> pair = unfinishedFlows .get(exFlow.getExecutionId()); if (pair.getFirst().getExecutor().isPresent()) { // Flow is already dispatched to an executor, so call that executor to cancel the flow. this.apiGateway .callWithReferenceByUser(pair.getFirst(), ConnectorParams.CANCEL_ACTION, userId); } else { // Flow is still queued, need to finalize it and update the status in DB. ExecutionControllerUtils.finalizeFlow(this.executorLoader, this.alerterHolder, exFlow, "Cancelled before dispatching to executor", null); } } else { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } } }
@Override public void resumeFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { final Pair<ExecutionReference, ExecutableFlow> pair = this.executorLoader.fetchActiveFlowByExecId(exFlow.getExecutionId()); if (pair == null) { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } this.apiGateway .callWithReferenceByUser(pair.getFirst(), ConnectorParams.RESUME_ACTION, userId); } }
private void assertTwoFlowSame(final ExecutableFlow flow1, final ExecutableFlow flow2, final boolean compareFlowData) { assertThat(flow1.getExecutionId()).isEqualTo(flow2.getExecutionId()); assertThat(flow1.getStatus()).isEqualTo(flow2.getStatus()); assertThat(flow1.getEndTime()).isEqualTo(flow2.getEndTime()); assertThat(flow1.getStartTime()).isEqualTo(flow2.getStartTime()); assertThat(flow1.getSubmitTime()).isEqualTo(flow2.getSubmitTime()); assertThat(flow1.getFlowId()).isEqualTo(flow2.getFlowId()); assertThat(flow1.getProjectId()).isEqualTo(flow2.getProjectId()); assertThat(flow1.getVersion()).isEqualTo(flow2.getVersion()); assertThat(flow1.getSubmitUser()).isEqualTo(flow2.getSubmitUser()); if (compareFlowData) { assertThat(flow1.getExecutionOptions().getFailureAction()) .isEqualTo(flow2.getExecutionOptions().getFailureAction()); assertThat(new HashSet<>(flow1.getEndNodes())).isEqualTo(new HashSet<>(flow2.getEndNodes())); } }
@Ignore @Test public void testFetchActiveFlowByProject() throws Exception { testSetUpForRunningFlows(); final List<Integer> executions = this.manager.getRunningFlows(this.flow1.getProjectId(), this.flow1.getFlowId()); Assert.assertTrue(executions.contains(this.flow1.getExecutionId())); Assert .assertTrue(this.manager.isFlowRunning(this.flow1.getProjectId(), this.flow1.getFlowId())); }
/** * if flows was dispatched to an executor, cancel by calling Executor else if flow is still in * queue, remove from queue and finalize {@inheritDoc} * * @see azkaban.executor.ExecutorManagerAdapter#cancelFlow(azkaban.executor.ExecutableFlow, * java.lang.String) */ @Override public void cancelFlow(final ExecutableFlow exFlow, final String userId) throws ExecutorManagerException { synchronized (exFlow) { if (this.runningExecutions.get().containsKey(exFlow.getExecutionId())) { final Pair<ExecutionReference, ExecutableFlow> pair = this.runningExecutions.get().get(exFlow.getExecutionId()); this.apiGateway.callWithReferenceByUser(pair.getFirst(), ConnectorParams.CANCEL_ACTION, userId); } else if (this.queuedFlows.hasExecution(exFlow.getExecutionId())) { this.queuedFlows.dequeue(exFlow.getExecutionId()); this.executionFinalizer .finalizeFlow(exFlow, "Cancelled before dispatching to executor", null); } else { throw new ExecutorManagerException("Execution " + exFlow.getExecutionId() + " of flow " + exFlow.getFlowId() + " isn't running."); } } }
assertThat(flow.getProjectId()).isEqualTo(info.getProjectId()); assertThat(flow.getVersion()).isEqualTo(info.getVersion()); assertThat(flow.getFlowId()).isEqualTo(info.getFlowId());
(flow1.getProjectId(), flow1.getFlowId(), dateUtil.dateStrToLong("2018-09-01 09:00:00")); final List<ExecutableFlow> expected = new ArrayList<>(); expected.add(flow1);
@Test public void testCreateExecutableFlow() throws Exception { final Flow flow = FlowUtils.getFlow(this.project, "jobe"); final ExecutableFlow exFlow = FlowUtils.createExecutableFlow(this.project, flow); Assert.assertEquals(exFlow.getProjectId(), this.project.getId()); Assert.assertEquals(exFlow.getFlowId(), flow.getId()); Assert.assertEquals(exFlow.getProxyUsers(), this.project.getProxyUsers()); }