private static void printJobStatusMessages(List<JobStatusMessage> jobs) { SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss"); Comparator<JobStatusMessage> startTimeComparator = (o1, o2) -> (int) (o1.getStartTime() - o2.getStartTime()); Comparator<Map.Entry<JobStatus, List<JobStatusMessage>>> statusComparator = (o1, o2) -> String.CASE_INSENSITIVE_ORDER.compare(o1.getKey().toString(), o2.getKey().toString()); Map<JobStatus, List<JobStatusMessage>> jobsByState = jobs.stream().collect(Collectors.groupingBy(JobStatusMessage::getJobState)); jobsByState.entrySet().stream() .sorted(statusComparator) .map(Map.Entry::getValue).flatMap(List::stream).sorted(startTimeComparator) .forEachOrdered(job -> System.out.println(dateFormat.format(new Date(job.getStartTime())) + " : " + job.getJobId() + " : " + job.getJobName() + " (" + job.getJobState() + ")")); }
@Override public CompletableFuture<Collection<JobStatusMessage>> listJobs() { return sendRequest(JobsOverviewHeaders.getInstance()) .thenApply( (multipleJobsDetails) -> multipleJobsDetails .getJobs() .stream() .map(detail -> new JobStatusMessage( detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime())) .collect(Collectors.toList())); }
if (job.getJobState() == JobStatus.RUNNING) { jobId = job.getJobId(); LOG.info("Job running. ID: " + jobId); break;
final List<JobStatusMessage> terminatedJobs = new ArrayList<>(); jobDetails.forEach(details -> { if (details.getJobState() == JobStatus.CREATED) { scheduledJobs.add(details); } else if (!details.getJobState().isGloballyTerminalState()) { runningJobs.add(details); } else {
public JobID findJobId(ActorGateway jobManagerGateway, String jobName) throws Exception { Future<Object> response = jobManagerGateway.ask( JobManagerMessages.getRequestRunningJobsStatus(), askTimeout); Object result; try { result = Await.result(response, askTimeout); } catch (Exception e) { throw new Exception("Could not retrieve running jobs from the JobManager.", e); } if (result instanceof RunningJobsStatus) { List<JobStatusMessage> jobs = ((RunningJobsStatus) result).getStatusMessages(); for (JobStatusMessage rj : jobs) { if (rj.getJobState().equals(JobStatus.RUNNING) || rj.getJobState().equals(JobStatus.RESTARTING)) { if (rj.getJobName().equals(jobName)) return rj.getJobId(); } } } throw new Exception("Could not find job"); }
if (status.getJobName().equals(id)) { return status.getJobId();
@Override public int compare(JobStatusMessage o1, JobStatusMessage o2) { return (int)(o1.getStartTime()-o2.getStartTime()); } };
private JobID waitForJobToBeReady() throws InterruptedException, ExecutionException { while (true) { JobStatusMessage jobStatus = Iterables.getFirst(flinkCluster.listJobs().get(), null); if (jobStatus != null && jobStatus.getJobState() == JobStatus.RUNNING) { return jobStatus.getJobId(); } Thread.sleep(100); } }
private static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); }
private static void printJobStatusMessages(List<JobStatusMessage> jobs) { SimpleDateFormat dateFormat = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss"); Comparator<JobStatusMessage> startTimeComparator = (o1, o2) -> (int) (o1.getStartTime() - o2.getStartTime()); Comparator<Map.Entry<JobStatus, List<JobStatusMessage>>> statusComparator = (o1, o2) -> String.CASE_INSENSITIVE_ORDER.compare(o1.getKey().toString(), o2.getKey().toString()); Map<JobStatus, List<JobStatusMessage>> jobsByState = jobs.stream().collect(Collectors.groupingBy(JobStatusMessage::getJobState)); jobsByState.entrySet().stream() .sorted(statusComparator) .map(Map.Entry::getValue).flatMap(List::stream).sorted(startTimeComparator) .forEachOrdered(job -> System.out.println(dateFormat.format(new Date(job.getStartTime())) + " : " + job.getJobId() + " : " + job.getJobName() + " (" + job.getJobState() + ")")); }
@After public void afterTest() throws Exception { for (JobStatusMessage jobStatusMessage : flinkCluster.listJobs().get()) { if (jobStatusMessage.getJobState() == JobStatus.RUNNING) { flinkCluster.cancelJob(jobStatusMessage.getJobId()).get(); } } }
private static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); } }
/** * Lists the currently running and finished jobs on the cluster. * * @return future collection of running and finished jobs * @throws Exception if no connection to the cluster could be established */ public CompletableFuture<Collection<JobStatusMessage>> listJobs() throws Exception { final ActorGateway jobManager = getJobManagerGateway(); Future<Object> response = jobManager.ask(new RequestJobDetails(true, false), timeout); CompletableFuture<Object> responseFuture = FutureUtils.toJava(response); return responseFuture.thenApply((responseMessage) -> { if (responseMessage instanceof MultipleJobsDetails) { MultipleJobsDetails details = (MultipleJobsDetails) responseMessage; final Collection<JobDetails> jobDetails = details.getJobs(); Collection<JobStatusMessage> flattenedDetails = new ArrayList<>(jobDetails.size()); jobDetails.forEach(detail -> flattenedDetails.add(new JobStatusMessage(detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime()))); return flattenedDetails; } else { throw new CompletionException( new IllegalStateException("Unknown JobManager response of type " + responseMessage.getClass())); } }); }
Comparator<JobStatusMessage> startTimeComparator = (o1, o2) -> (int) (o1.getStartTime() - o2.getStartTime()); if (details.getJobState() == JobStatus.CREATED) { scheduledJobs.add(details); } else if (!details.getJobState().isGloballyTerminalState()) { runningJobs.add(details); System.out.println(dateFormat.format(new Date(runningJob.getStartTime())) + " : " + runningJob.getJobId() + " : " + runningJob.getJobName() + " (" + runningJob.getJobState() + ")"); System.out.println(dateFormat.format(new Date(scheduledJob.getStartTime())) + " : " + scheduledJob.getJobId() + " : " + scheduledJob.getJobName());
private static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); } }
private static ClusterClient<String> createClusterClient() throws Exception { final ClusterClient<String> clusterClient = mock(ClusterClient.class); when(clusterClient.listJobs()).thenReturn(CompletableFuture.completedFuture(Arrays.asList( new JobStatusMessage(new JobID(), "job1", JobStatus.RUNNING, 1L), new JobStatusMessage(new JobID(), "job2", JobStatus.CREATED, 1L), new JobStatusMessage(new JobID(), "job3", JobStatus.SUSPENDING, 3L), new JobStatusMessage(new JobID(), "job4", JobStatus.SUSPENDING, 2L), new JobStatusMessage(new JobID(), "job5", JobStatus.FINISHED, 3L) ))); return clusterClient; } }
if (running && (rj.getJobState().equals(JobStatus.RUNNING) || rj.getJobState().equals(JobStatus.RESTARTING))) { runningJobs.add(rj); if (scheduled && rj.getJobState().equals(JobStatus.CREATED)) { scheduledJobs.add(rj); System.out.println(df.format(new Date(rj.getStartTime())) + " : " + rj.getJobId() + " : " + rj.getJobName() + " (" + rj.getJobState() + ")"); System.out.println(df.format(new Date(rj.getStartTime())) + " : " + rj.getJobId() + " : " + rj.getJobName());
public static List<JobID> getRunningJobs(ClusterClient<?> client) throws Exception { Collection<JobStatusMessage> statusMessages = client.listJobs().get(); return statusMessages.stream() .filter(status -> !status.getJobState().isGloballyTerminalState()) .map(JobStatusMessage::getJobId) .collect(Collectors.toList()); }
@Override public CompletableFuture<Collection<JobStatusMessage>> listJobs() { return sendRequest(JobsOverviewHeaders.getInstance()) .thenApply( (multipleJobsDetails) -> multipleJobsDetails .getJobs() .stream() .map(detail -> new JobStatusMessage( detail.getJobId(), detail.getJobName(), detail.getStatus(), detail.getStartTime())) .collect(Collectors.toList())); }
@Test public void testListJobs() throws Exception { try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(new TestListJobsHandler())) { RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort()); try { CompletableFuture<Collection<JobStatusMessage>> jobDetailsFuture = restClusterClient.listJobs(); Collection<JobStatusMessage> jobDetails = jobDetailsFuture.get(); Iterator<JobStatusMessage> jobDetailsIterator = jobDetails.iterator(); JobStatusMessage job1 = jobDetailsIterator.next(); JobStatusMessage job2 = jobDetailsIterator.next(); Assert.assertNotEquals("The job status should not be equal.", job1.getJobState(), job2.getJobState()); } finally { restClusterClient.shutdown(); } } }