private static Pair<ExecutionReference, ExecutableFlow> getExecutableFlowMetadataHelper( final ResultSet rs) throws SQLException { final Flow flow = new Flow(rs.getString("flow_id")); final Project project = new Project(rs.getInt("project_id"), null); project.setVersion(rs.getInt("version")); final ExecutableFlow exFlow = new ExecutableFlow(project, flow); exFlow.setExecutionId(rs.getInt("exec_id")); exFlow.setStatus(Status.fromInteger(rs.getInt("status"))); exFlow.setSubmitTime(rs.getLong("submit_time")); exFlow.setStartTime(rs.getLong("start_time")); exFlow.setEndTime(rs.getLong("end_time")); exFlow.setSubmitUser(rs.getString("submit_user")); return getPairWithExecutorInfo(rs, exFlow); }
private ExecutableFlow createExecutableFlow(final String flowName, final int priority, final long updateTime, final int executionId) throws IOException { final ExecutableFlow execFlow = TestUtils.createTestExecutableFlow("exectest1", flowName); execFlow.setUpdateTime(updateTime); execFlow.setExecutionId(executionId); if (priority > 0) { execFlow.getExecutionOptions().getFlowParameters() .put(ExecutionOptions.FLOW_PRIORITY, String.valueOf(priority)); } return execFlow; }
private Pair<ExecutionReference, ExecutableFlow> createExecutablePair( final String flowName, final int execId) throws IOException { final ExecutableFlow execFlow = TestUtils.createTestExecutableFlow("exectest1", flowName); execFlow.setExecutionId(execId); final ExecutionReference ref = new ExecutionReference(execId); return new Pair<>(ref, execFlow); }
@Test public void testSelectAndUpdateExecution() throws Exception { final ExecutableFlow flow = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow.setExecutionId(1); this.executionFlowDao.uploadExecutableFlow(flow); final Executor executor = this.executorDao.addExecutor("localhost", 12345); assertThat(this.executionFlowDao.selectAndUpdateExecution(executor.getId())).isEqualTo(flow .getExecutionId()); assertThat(this.executorDao.fetchExecutorByExecutionId(flow.getExecutionId())).isEqualTo (executor); }
@Before public void setUp() throws Exception { this.props = new Props(); this.props.put(ConfigurationKeys.AZKABAN_EXECUTOR_MAX_FAILURE_COUNT, 2); this.props.put(ConfigurationKeys.AZKABAN_ADMIN_ALERT_EMAIL, AZ_ADMIN_ALERT_EMAIL); this.loader = mock(ExecutorLoader.class); this.mailAlerter = mock(Alerter.class); this.alerterHolder = mock(AlerterHolder.class); this.apiGateway = mock(ExecutorApiGateway.class); this.executorHealthChecker = new ExecutorHealthChecker(this.props, this.loader, this .apiGateway, this.alerterHolder); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow1.setExecutionId(EXECUTION_ID_11); this.flow1.setStatus(Status.RUNNING); this.executor1 = new Executor(1, "localhost", 12345, true); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
public synchronized void uploadExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { final String INSERT_EXECUTABLE_FLOW = "INSERT INTO execution_flows " + "(project_id, flow_id, version, status, submit_time, submit_user, update_time) " + "values (?,?,?,?,?,?,?)"; final long submitTime = System.currentTimeMillis(); flow.setStatus(Status.PREPARING); flow.setSubmitTime(submitTime); /** * Why we need a transaction to get last insert ID? * Because "SELECT LAST_INSERT_ID()" needs to have the same connection * as inserting the new entry. * See https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_last-insert-id */ final SQLTransaction<Long> insertAndGetLastID = transOperator -> { transOperator.update(INSERT_EXECUTABLE_FLOW, flow.getProjectId(), flow.getFlowId(), flow.getVersion(), Status.PREPARING.getNumVal(), submitTime, flow.getSubmitUser(), submitTime); transOperator.getConnection().commit(); return transOperator.getLastInsertId(); }; try { final long id = this.dbOperator.transaction(insertAndGetLastID); logger.info("Flow given " + flow.getFlowId() + " given id " + id); flow.setExecutionId((int) id); updateExecutableFlow(flow); } catch (final SQLException e) { throw new ExecutorManagerException("Error creating execution.", e); } }
@Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); this.execution = new ExecutableFlow(); this.execution.setExecutionId(EXECUTION_ID_77); this.activeExecutor = new Executor(1, "activeExecutor-1", 9999, true); this.runningExecutions = new RunningExecutions(); this.runningExecutions.get().put(EXECUTION_ID_77, new Pair<>( new ExecutionReference(EXECUTION_ID_77, this.activeExecutor), this.execution)); this.updater = new RunningExecutionsUpdater(this.updaterStage, this.alerterHolder, this.commonMetrics, this.apiGateway, this.runningExecutions, this.executionFinalizer, this.executorLoader); when(this.alerterHolder.get("email")).thenReturn(this.mailAlerter); }
@Test(expected = ExecutorManagerException.class) public void testTooManySubmitFlows() throws Exception { testSetUpForRunningFlows(); final ExecutableFlow flow1 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow1.setExecutionId(101); final ExecutableFlow flow2 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow2.setExecutionId(102); final ExecutableFlow flow3 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow3.setExecutionId(103); final ExecutableFlow flow4 = TestUtils .createTestExecutableFlowFromYaml("basicyamlshelltest", "bashSleep"); flow4.setExecutionId(104); this.manager.submitExecutableFlow(flow1, this.user.getUserId()); verify(this.loader).uploadExecutableFlow(flow1); this.manager.submitExecutableFlow(flow2, this.user.getUserId()); verify(this.loader).uploadExecutableFlow(flow2); this.manager.submitExecutableFlow(flow3, this.user.getUserId()); this.manager.submitExecutableFlow(flow4, this.user.getUserId()); }
this.flow3 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); this.flow4 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); this.flow1.setExecutionId(1); this.flow2.setExecutionId(2); this.flow3.setExecutionId(3); this.flow4.setExecutionId(4); this.ref1 = new ExecutionReference(this.flow1.getExecutionId(), null); this.ref2 = new ExecutionReference(this.flow2.getExecutionId(), executor2);
private void testSetUpForRunningFlows() throws Exception { this.loader = mock(ExecutorLoader.class); this.apiGateway = mock(ExecutorApiGateway.class); this.user = TestUtils.getTestUser(); this.props.put(Constants.ConfigurationKeys.USE_MULTIPLE_EXECUTORS, "true"); //To test runningFlows, AZKABAN_QUEUEPROCESSING_ENABLED should be set to true //so that flows will be dispatched to executors. this.props.put(Constants.ConfigurationKeys.QUEUEPROCESSING_ENABLED, "true"); // allow two concurrent runs give one Flow this.props.put(Constants.ConfigurationKeys.MAX_CONCURRENT_RUNS_ONEFLOW, 2); final List<Executor> executors = new ArrayList<>(); final Executor executor1 = new Executor(1, "localhost", 12345, true); final Executor executor2 = new Executor(2, "localhost", 12346, true); executors.add(executor1); executors.add(executor2); when(this.loader.fetchActiveExecutors()).thenReturn(executors); this.manager = createExecutorManager(); this.flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); this.flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); this.flow1.setExecutionId(1); this.flow2.setExecutionId(2); final ExecutionReference ref1 = new ExecutionReference(this.flow1.getExecutionId(), executor1); final ExecutionReference ref2 = new ExecutionReference(this.flow2.getExecutionId(), executor2); this.activeFlows.put(this.flow1.getExecutionId(), new Pair<>(ref1, this.flow1)); this.activeFlows.put(this.flow2.getExecutionId(), new Pair<>(ref2, this.flow2)); when(this.loader.fetchActiveFlows()).thenReturn(this.activeFlows); }
@Test public void testQueuedFlows() throws Exception { final ExecutorManager manager = createMultiExecutorManagerInstance(); final ExecutableFlow flow1 = TestUtils.createTestExecutableFlow("exectest1", "exec1"); flow1.setExecutionId(1); final ExecutableFlow flow2 = TestUtils.createTestExecutableFlow("exectest1", "exec2"); flow2.setExecutionId(2); final User testUser = TestUtils.getTestUser(); manager.submitExecutableFlow(flow1, testUser.getUserId()); manager.submitExecutableFlow(flow2, testUser.getUserId()); final List<Integer> testFlows = Arrays.asList(flow1.getExecutionId(), flow2.getExecutionId()); final List<Pair<ExecutionReference, ExecutableFlow>> queuedFlowsDB = this.loader.fetchQueuedFlows(); Assert.assertEquals(queuedFlowsDB.size(), testFlows.size()); // Verify things are correctly setup in db for (final Pair<ExecutionReference, ExecutableFlow> pair : queuedFlowsDB) { Assert.assertTrue(testFlows.contains(pair.getSecond().getExecutionId())); } // Verify running flows using old definition of "running" flows i.e. a // non-dispatched flow is also considered running final List<Integer> managerActiveFlows = manager.getRunningFlows() .stream().map(ExecutableFlow::getExecutionId).collect(Collectors.toList()); Assert.assertTrue(managerActiveFlows.containsAll(testFlows) && testFlows.containsAll(managerActiveFlows)); // Verify getQueuedFlowIds method Assert.assertEquals("[1, 2]", manager.getQueuedFlowIds()); }
flow.setExecutionId(10);
@Test public void createErrorEmail() throws Exception { setJobStatus(Status.FAILED); this.executableFlow.setEndTime(END_TIME_MILLIS); this.executableFlow.setStatus(Status.FAILED); final List<ExecutableFlow> executableFlows = new ArrayList<>(); final ExecutableFlow executableFlow1 = new ExecutableFlow(this.project, this.flow); executableFlow1.setExecutionId(1); executableFlow1.setStartTime(START_TIME_MILLIS); executableFlow1.setEndTime(END_TIME_MILLIS); executableFlow1.setStatus(Status.FAILED); executableFlows.add(executableFlow1); final ExecutableFlow executableFlow2 = new ExecutableFlow(this.project, this.flow); executableFlow2.setExecutionId(2); executableFlow2.setStartTime(START_TIME_MILLIS); executableFlow2.setEndTime(END_TIME_MILLIS); executableFlow2.setStatus(Status.SUCCEEDED); executableFlows.add(executableFlow2); assertTrue(this.mailCreator.createErrorEmail( this.executableFlow, executableFlows, this.message, this.azkabanName, this.scheme, this .clientHostname, this.clientPortNumber)); assertEquals("Flow 'mail-creator-test' has failed on unit-tests", this.message.getSubject()); assertThat(TestUtils.readResource("errorEmail.html", this)) .isEqualToIgnoringWhitespace(this.message.getBody()); }
exFlow.setExecutionId(101);
final Flow flow = this.project.getFlow("jobe"); final ExecutableFlow exFlow = new ExecutableFlow(this.project, flow); exFlow.setExecutionId(101);
private ExecutableFlow getExecutableFlowMetadata( ExecutableFlow fullExFlow) { final Flow flow = new Flow(fullExFlow.getId()); final Project project = new Project(fullExFlow.getProjectId(), null); project.setVersion(fullExFlow.getVersion()); flow.setVersion(fullExFlow.getVersion()); final ExecutableFlow metadata = new ExecutableFlow(project, flow); metadata.setExecutionId(fullExFlow.getExecutionId()); metadata.setStatus(fullExFlow.getStatus()); metadata.setSubmitTime(fullExFlow.getSubmitTime()); metadata.setStartTime(fullExFlow.getStartTime()); metadata.setEndTime(fullExFlow.getEndTime()); metadata.setSubmitUser(fullExFlow.getSubmitUser()); return metadata; }
public synchronized void uploadExecutableFlow(final ExecutableFlow flow) throws ExecutorManagerException { final String INSERT_EXECUTABLE_FLOW = "INSERT INTO execution_flows " + "(project_id, flow_id, version, status, submit_time, submit_user, update_time) " + "values (?,?,?,?,?,?,?)"; final long submitTime = System.currentTimeMillis(); flow.setStatus(Status.PREPARING); /** * Why we need a transaction to get last insert ID? * Because "SELECT LAST_INSERT_ID()" needs to have the same connection * as inserting the new entry. * See https://dev.mysql.com/doc/refman/5.7/en/information-functions.html#function_last-insert-id */ final SQLTransaction<Long> insertAndGetLastID = transOperator -> { transOperator.update(INSERT_EXECUTABLE_FLOW, flow.getProjectId(), flow.getFlowId(), flow.getVersion(), Status.PREPARING.getNumVal(), submitTime, flow.getSubmitUser(), submitTime); transOperator.getConnection().commit(); return transOperator.getLastInsertId(); }; try { final long id = this.dbOperator.transaction(insertAndGetLastID); logger.info("Flow given " + flow.getFlowId() + " given id " + id); flow.setExecutionId((int) id); updateExecutableFlow(flow); } catch (final SQLException e) { throw new ExecutorManagerException("Error creating execution.", e); } }
flow.setExecutionId((int)id);