/** * Returns a {@link JobVertexID} to vertex name mapping for the given graph. */ private Map<JobVertexID, String> rememberIds(JobGraph jobGraph) { final Map<JobVertexID, String> ids = new HashMap<>(); for (JobVertex vertex : jobGraph.getVertices()) { ids.put(vertex.getID(), vertex.getName()); } return ids; }
/** * Verifies that each {@link JobVertexID} of the {@link JobGraph} is contained in the given map * and mapped to the same vertex name. */ private void verifyIdsEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) { // Verify same number of vertices assertEquals(jobGraph.getNumberOfVertices(), ids.size()); // Verify that all IDs->name mappings are identical for (JobVertex vertex : jobGraph.getVertices()) { String expectedName = ids.get(vertex.getID()); assertNotNull(expectedName); assertEquals(expectedName, vertex.getName()); } }
String containerTaskName = container.getName(); if (containerTaskName.startsWith("CHAIN ")) { container.setName(containerTaskName + " -> " + chainedTask.getTaskName());
if (jobVertex.getName().contains("test_source")) { assertTrue(jobVertex.getMinResources().equals(resource1)); } else if (jobVertex.getName().contains("Iteration_Source")) { assertTrue(jobVertex.getPreferredResources().equals(resource2)); } else if (jobVertex.getName().contains("test_flatMap")) { assertTrue(jobVertex.getMinResources().equals(resource3.merge(resource4))); } else if (jobVertex.getName().contains("Iteration_Tail")) { assertTrue(jobVertex.getPreferredResources().equals(ResourceSpec.DEFAULT)); } else if (jobVertex.getName().contains("test_sink")) { assertTrue(jobVertex.getMinResources().equals(resource5));
streamConfig.getStreamOperator(Thread.currentThread().getContextClassLoader()); try (MockEnvironment environment = createMockEnvironment(chainedVertex.getName())) { StreamTask<Integer, StreamMap<Integer, Integer>> mockTask = createMockTask(streamConfig, environment); OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = createOperatorChain(streamConfig, environment, mockTask);
streamConfig.getStreamOperator(Thread.currentThread().getContextClassLoader()); try (MockEnvironment environment = createMockEnvironment(chainedVertex.getName())) { StreamTask<Integer, StreamMap<Integer, Integer>> mockTask = createMockTask(streamConfig, environment); OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = createOperatorChain(streamConfig, environment, mockTask);
@Test public void testTempInIterationTest() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class); DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 1, 0); DataSet<Tuple2<Long, Long>> update = iteration.getWorkset() .join(iteration.getSolutionSet()).where(0).equalTo(0) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); iteration.closeWith(update, update) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jg = jgg.compileJobGraph(oPlan); boolean solutionSetUpdateChecked = false; for(JobVertex v : jg.getVertices()) { if(v.getName().equals("SolutionSet Delta")) { // check if input of solution set delta is temped TaskConfig tc = new TaskConfig(v.getConfiguration()); assertTrue(tc.isInputAsynchronouslyMaterialized(0)); solutionSetUpdateChecked = true; } } assertTrue(solutionSetUpdateChecked); }
@Override public String toString() { return jobVertex.getName(); } }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
private String generateDebugString() { return "ExecutionJobVertex" + "(" + jobVertex.getName() + " | " + jobVertex.getID() + ")" + "{" + "parallelism=" + parallelism + ", maxParallelism=" + getMaxParallelism() + ", maxParallelismConfigured=" + maxParallelismConfigured + '}'; }
@Override public void acceptWithException(JobVertex jobVertex, Integer newParallelism) throws FlinkException { if (jobVertex.getMaxParallelism() < newParallelism) { throw new FlinkException("Cannot rescale vertex " + jobVertex.getName() + " because its maximum parallelism " + jobVertex.getMaxParallelism() + " is smaller than the new parallelism " + newParallelism + '.'); } else { jobVertex.setParallelism(newParallelism); } } },
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); }
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); }
public ArchivedExecutionJobVertex(ExecutionJobVertex jobVertex) { this.taskVertices = new ArchivedExecutionVertex[jobVertex.getTaskVertices().length]; for (int x = 0; x < taskVertices.length; x++) { taskVertices[x] = jobVertex.getTaskVertices()[x].archive(); } archivedUserAccumulators = jobVertex.getAggregatedUserAccumulatorsStringified(); this.id = jobVertex.getJobVertexId(); this.name = jobVertex.getJobVertex().getName(); this.parallelism = jobVertex.getParallelism(); this.maxParallelism = jobVertex.getMaxParallelism(); }