/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
/** * Computes the parameters to be used to start a TaskManager Java process. * * @param config The Flink configuration. * @param containerMemoryMB The size of the complete container, in megabytes. * @return The parameters to start the TaskManager processes with. */ public static ContaineredTaskManagerParameters create( Configuration config, long containerMemoryMB, int numSlots) { // (1) try to compute how much memory used by container final long cutoffMB = calculateCutoffMB(config, containerMemoryMB); // (2) split the remaining Java memory between heap and off-heap final long heapSizeMB = TaskManagerServices.calculateHeapSizeMB(containerMemoryMB - cutoffMB, config); // use the cut-off memory for off-heap (that was its intention) final long offHeapSizeMB = containerMemoryMB - heapSizeMB; return create(config, containerMemoryMB, (int)heapSizeMB, (int)offHeapSizeMB, numSlots, 0); }
int numSlots) { final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);
int numSlots) { final long cutoffMB = calculateCutoffMB(config, containerMemoryMB);
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }
/** * Method to validate cluster specification before deploy it, it will throw * an {@link FlinkException} if the {@link ClusterSpecification} is invalid. * * @param clusterSpecification cluster specification to check against the configuration of the * AbstractYarnClusterDescriptor * @throws FlinkException if the cluster cannot be started with the provided {@link ClusterSpecification} */ private void validateClusterSpecification(ClusterSpecification clusterSpecification) throws FlinkException { try { final long taskManagerMemorySize = clusterSpecification.getTaskManagerMemoryMB(); // We do the validation by calling the calculation methods here // Internally these methods will check whether the cluster can be started with the provided // ClusterSpecification and the configured memory requirements final long cutoff = ContaineredTaskManagerParameters.calculateCutoffMB(flinkConfiguration, taskManagerMemorySize); TaskManagerServices.calculateHeapSizeMB(taskManagerMemorySize - cutoff, flinkConfiguration); } catch (IllegalArgumentException iae) { throw new FlinkException("Cannot fulfill the minimum memory requirements with the provided " + "cluster specification. Please increase the memory of the cluster.", iae); } }