@Override public ClusterSpecification getClusterSpecification(CommandLine commandLine) { return new ClusterSpecification.ClusterSpecificationBuilder().createClusterSpecification(); } }
@Override public StandaloneClusterId getClusterId() { return StandaloneClusterId.getInstance(); }
public ClusterSpecification createClusterSpecification() { return new ClusterSpecification( masterMemoryMB, taskManagerMemoryMB, numberTaskManagers, slotsPerTaskManager); } }
public static ClusterSpecification fromConfiguration(Configuration configuration) { int slots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1); int jobManagerMemoryMb = ConfigurationUtils.getJobManagerHeapMemory(configuration).getMebiBytes(); int taskManagerMemoryMb = ConfigurationUtils.getTaskManagerHeapMemory(configuration).getMebiBytes(); return new ClusterSpecificationBuilder() .setMasterMemoryMB(jobManagerMemoryMb) .setTaskManagerMemoryMB(taskManagerMemoryMb) .setNumberTaskManagers(1) .setSlotsPerTaskManager(slots) .createClusterSpecification(); }
/** * Tests the specifying heap memory with config default value for job manager and task manager. */ @Test public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception { final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(1024)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024)); }
@Test public void testCorrectSettingOfMaxSlots() throws Exception { String[] params = new String[] {"-yn", "2", "-ys", "3"}; FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true); AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine); final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine); // each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased. assertEquals(3, clusterSpecification.getSlotsPerTaskManager()); assertEquals(2, clusterSpecification.getNumberTaskManagers()); }
@Override public StandaloneClusterDescriptor createClusterDescriptor( CommandLine commandLine) throws FlinkException { final Configuration effectiveConfiguration = applyCommandLineOptionsToConfiguration(commandLine); return new StandaloneClusterDescriptor(effectiveConfiguration); }
@Override public RestClusterClient<StandaloneClusterId> retrieve(StandaloneClusterId standaloneClusterId) throws ClusterRetrieveException { try { return new RestClusterClient<>(config, standaloneClusterId); } catch (Exception e) { throw new ClusterRetrieveException("Couldn't retrieve standalone cluster", e); } }
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
@Override public ClusterClient<ApplicationId> deploySessionCluster(ClusterSpecification clusterSpecification) throws ClusterDeploymentException { try { return deployInternal( clusterSpecification, "Flink session cluster", getYarnSessionClusterEntrypoint(), null, false); } catch (Exception e) { throw new ClusterDeploymentException("Couldn't deploy Yarn session cluster", e); } }
private RestClusterClient<ApplicationId> deploySessionCluster(YarnClusterDescriptor yarnClusterDescriptor) throws ClusterDeploymentException { final int containerMemory = 256; final ClusterClient<ApplicationId> yarnClusterClient = yarnClusterDescriptor.deploySessionCluster( new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(containerMemory) .setTaskManagerMemoryMB(containerMemory) .setSlotsPerTaskManager(1) .createClusterSpecification()); assertThat(yarnClusterClient, is(instanceOf(RestClusterClient.class))); return (RestClusterClient<ApplicationId>) yarnClusterClient; }
@Override public ClusterSpecification getClusterSpecification(CommandLine commandLine) { return new ClusterSpecification.ClusterSpecificationBuilder().createClusterSpecification(); } }
@Override @Nullable public StandaloneClusterId getClusterId(CommandLine commandLine) { return StandaloneClusterId.getInstance(); }
/** * Tests the specifying heap memory with arbitrary unit for job manager and task manager. */ @Test public void testHeapMemoryPropertyWithArbitraryUnit() throws Exception { final String[] args = new String[] { "-yn", "2", "-yjm", "1g", "-ytm", "2g" }; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(1024)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048)); }
@Override public ClusterClient<ApplicationId> deployJobCluster( ClusterSpecification clusterSpecification, JobGraph jobGraph, boolean detached) throws ClusterDeploymentException { // this is required because the slots are allocated lazily jobGraph.setAllowQueuedScheduling(true); try { return deployInternal( clusterSpecification, "Flink per-job cluster", getYarnJobClusterEntrypoint(), jobGraph, detached); } catch (Exception e) { throw new ClusterDeploymentException("Could not deploy Yarn job cluster.", e); } }
@Test public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentException { final Configuration flinkConfiguration = new Configuration(); flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0); YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor( flinkConfiguration, yarnConfiguration, temporaryFolder.getRoot().getAbsolutePath(), yarnClient, true); clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath())); ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(1) .setTaskManagerMemoryMB(1) .setNumberTaskManagers(1) .setSlotsPerTaskManager(Integer.MAX_VALUE) .createClusterSpecification(); try { clusterDescriptor.deploySessionCluster(clusterSpecification); fail("The deploy call should have failed."); } catch (ClusterDeploymentException e) { // we expect the cause to be an IllegalConfigurationException if (!(e.getCause() instanceof IllegalConfigurationException)) { throw e; } } finally { clusterDescriptor.close(); } }
private RestClusterClient<StandaloneClusterId> createRestClusterClient(final int port) throws Exception { final Configuration clientConfig = new Configuration(restConfig); clientConfig.setInteger(RestOptions.PORT, port); return new RestClusterClient<>( clientConfig, createRestClient(), StandaloneClusterId.getInstance(), (attempt) -> 0, null); }
/** * Tests the specifying heap memory without unit for job manager and task manager. */ @Test public void testHeapMemoryPropertyWithoutUnit() throws Exception { final String[] args = new String[] { "-yn", "2", "-yjm", "1024", "-ytm", "2048" }; final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli( new Configuration(), tmp.getRoot().getAbsolutePath(), "y", "yarn"); final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false); final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine); assertThat(clusterSpecification.getMasterMemoryMB(), is(1024)); assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048)); }
ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder() .setMasterMemoryMB(1) .setTaskManagerMemoryMB(1) .setNumberTaskManagers(1) .setSlotsPerTaskManager(1) .createClusterSpecification(); } catch (ClusterDeploymentException e) { if (!(e.getCause() instanceof IllegalConfigurationException)) { throw e;
@Before public void setUp() throws Exception { final Configuration clientConfig = new Configuration(); clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0); clientConfig.setLong(RestOptions.RETRY_DELAY, 0); clientConfig.addAll(CLUSTER.getClientConfiguration()); client = new RestClusterClient<>( clientConfig, StandaloneClusterId.getInstance() ); }