@Override public StepSummary getClusterActiveStep(String clusterId, AwsParamsDto awsParamsDto) { ListStepsRequest listStepsRequest = new ListStepsRequest().withClusterId(clusterId).withStepStates(StepState.RUNNING); List<StepSummary> stepSummaryList = emrOperations.listStepsRequest(getEmrClient(awsParamsDto), listStepsRequest).getSteps(); return !stepSummaryList.isEmpty() ? stepSummaryList.get(0) : null; }
@Override public String addEmrStep(String clusterId, StepConfig emrStepConfig, AwsParamsDto awsParamsDto) throws Exception { List<StepConfig> steps = new ArrayList<>(); steps.add(emrStepConfig); // Add the job flow request AddJobFlowStepsRequest jobFlowStepRequest = new AddJobFlowStepsRequest(clusterId, steps); List<String> emrStepIds = emrOperations.addJobFlowStepsRequest(getEmrClient(awsParamsDto), jobFlowStepRequest); return emrStepIds.get(0); }
@Override public Cluster getEmrClusterById(String clusterId, AwsParamsDto awsParams) { Cluster cluster = null; if (StringUtils.isNotBlank(clusterId)) { DescribeClusterResult describeClusterResult = emrOperations.describeClusterRequest(getEmrClient(awsParams), new DescribeClusterRequest().withClusterId(clusterId)); if (describeClusterResult != null && describeClusterResult.getCluster() != null) { cluster = describeClusterResult.getCluster(); } } return cluster; }
@Test public void addEmrMasterSecurityGroupsCallsEc2AddSecurityGroup() throws Exception { String clusterName = "clusterName"; List<String> securityGroups = Arrays.asList("securityGroup"); AwsParamsDto awsParams = getAwsParamsDto(); String ec2InstanceId = "ec2InstanceId"; ListClustersResult listClustersResult = new ListClustersResult(); listClustersResult.setClusters(new ArrayList<>()); ClusterSummary clusterSummary = new ClusterSummary(); clusterSummary.setId("clusterId"); clusterSummary.setName(clusterName); listClustersResult.getClusters().add(clusterSummary); when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult); ListInstancesResult listInstancesResult = new ListInstancesResult(); listInstancesResult.setInstances(new ArrayList<>()); Instance instance = new Instance(); instance.setEc2InstanceId(ec2InstanceId); listInstancesResult.getInstances().add(instance); when(mockEmrOperations.listClusterInstancesRequest(any(), any())).thenReturn(listInstancesResult); emrDao.addEmrMasterSecurityGroups(clusterName, securityGroups, awsParams); verify(mockEc2Dao).addSecurityGroupsToEc2Instance(eq(ec2InstanceId), eq(securityGroups), any()); verifyNoMoreInteractions(mockEc2Dao); }
@Test public void terminateEmrCluster() throws Exception { String clusterName = "clusterName"; boolean overrideTerminationProtection = false; String clusterId = "clusterId"; ListClustersResult listClustersResult = new ListClustersResult(); listClustersResult.setClusters(new ArrayList<>()); ClusterSummary clusterSummary = new ClusterSummary(); clusterSummary.setId(clusterId); clusterSummary.setName(clusterName); listClustersResult.getClusters().add(clusterSummary); when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult); emrDao.terminateEmrCluster(clusterId, overrideTerminationProtection, getAwsParamsDto()); // Assert that terminateEmrCluster was called with these parameters ONCE verify(mockEmrOperations).terminateEmrCluster(any(), eq(clusterId), eq(overrideTerminationProtection)); }
@Override public String createEmrCluster(String clusterName, EmrClusterDefinition emrClusterDefinition, AwsParamsDto awsParams) { RunJobFlowRequest runJobFlowRequest = getRunJobFlowRequest(clusterName, emrClusterDefinition); LOGGER.info("runJobFlowRequest={}", jsonHelper.objectToJson(runJobFlowRequest)); String clusterId = emrOperations.runEmrJobFlow(getEmrClient(awsParams), runJobFlowRequest); LOGGER.info("EMR cluster started. emrClusterId=\"{}\"", clusterId); return clusterId; }
clusterSummary.setName(clusterName); listClustersResult.setClusters(Arrays.asList(clusterSummary)); when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult); when(mockEmrOperations.addJobFlowStepsRequest(any(), any())).thenAnswer(new Answer<List<String>>()
@Test public void getActiveEmrClusterByNameAssertReturnNullWhenClusterNameIsBlank() throws Exception { String clusterName = ""; when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(new ListClustersResult()); assertNull(emrDao.getActiveEmrClusterByName(clusterName, getAwsParamsDto())); }
@Override public Instance getEmrMasterInstance(String clusterId, AwsParamsDto awsParams) throws Exception { // Get the master EC2 instance ListInstancesRequest listInstancesRequest = new ListInstancesRequest().withClusterId(clusterId).withInstanceGroupTypes(InstanceGroupType.MASTER); List<Instance> instances = emrOperations.listClusterInstancesRequest(getEmrClient(awsParams), listInstancesRequest).getInstances(); // Throw error in case there are no master instances found yet if (instances.size() == 0) { throw new IllegalArgumentException("No master instances found for the cluster \"" + clusterId + "\"."); } // EMR has only one master node. return instances.get(0); }
@Override public Step getClusterStep(String clusterId, String stepId, AwsParamsDto awsParamsDto) { DescribeStepRequest describeStepRequest = new DescribeStepRequest().withClusterId(clusterId).withStepId(stepId); return emrOperations.describeStepRequest(getEmrClient(awsParamsDto), describeStepRequest).getStep(); }
@Override public ListInstanceFleetsResult getListInstanceFleetsResult(String clusterId, AwsParamsDto awsParams) { return emrOperations.listInstanceFleets(getEmrClient(awsParams), new ListInstanceFleetsRequest().withClusterId(clusterId)); }
@Override public void terminateEmrCluster(String clusterId, boolean overrideTerminationProtection, AwsParamsDto awsParams) { emrOperations.terminateEmrCluster(getEmrClient(awsParams), clusterId, overrideTerminationProtection); }
@Test public void addEmrMasterSecurityGroupsThrowWhenNoInstancesFound() throws Exception { String clusterName = "clusterName"; List<String> securityGroups = Arrays.asList("securityGroup"); AwsParamsDto awsParams = getAwsParamsDto(); ListClustersResult listClustersResult = new ListClustersResult(); listClustersResult.setClusters(new ArrayList<>()); ClusterSummary clusterSummary = new ClusterSummary(); clusterSummary.setId("clusterId"); clusterSummary.setName(clusterName); listClustersResult.getClusters().add(clusterSummary); when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult); when(mockEmrOperations.listClusterInstancesRequest(any(), any())).thenReturn(new ListInstancesResult()); try { emrDao.addEmrMasterSecurityGroups(clusterName, securityGroups, awsParams); fail(); } catch (Exception e) { assertEquals(IllegalArgumentException.class, e.getClass()); assertEquals("No master instances found for the cluster \"" + clusterName + "\".", e.getMessage()); } }
@Override public String createEmrCluster(String clusterName, EmrClusterDefinition emrClusterDefinition, AwsParamsDto awsParams) { RunJobFlowRequest runJobFlowRequest = getRunJobFlowRequest(clusterName, emrClusterDefinition); LOGGER.info("runJobFlowRequest={}", jsonHelper.objectToJson(runJobFlowRequest)); String clusterId = emrOperations.runEmrJobFlow(getEmrClient(awsParams), runJobFlowRequest); LOGGER.info("EMR cluster started. emrClusterId=\"{}\"", clusterId); return clusterId; }
emrOperations.listEmrClusters(getEmrClient(awsParams), listClustersRequest.withMarker(markerForListClusters));
@Override public Instance getEmrMasterInstance(String clusterId, AwsParamsDto awsParams) throws Exception { // Get the master EC2 instance ListInstancesRequest listInstancesRequest = new ListInstancesRequest().withClusterId(clusterId).withInstanceGroupTypes(InstanceGroupType.MASTER); List<Instance> instances = emrOperations.listClusterInstancesRequest(getEmrClient(awsParams), listInstancesRequest).getInstances(); // Throw error in case there are no master instances found yet if (instances.size() == 0) { throw new IllegalArgumentException("No master instances found for the cluster \"" + clusterId + "\"."); } // EMR has only one master node. return instances.get(0); }
@Override public Step getClusterStep(String clusterId, String stepId, AwsParamsDto awsParamsDto) { DescribeStepRequest describeStepRequest = new DescribeStepRequest().withClusterId(clusterId).withStepId(stepId); return emrOperations.describeStepRequest(getEmrClient(awsParamsDto), describeStepRequest).getStep(); }
@Override public ListInstanceFleetsResult getListInstanceFleetsResult(String clusterId, AwsParamsDto awsParams) { return emrOperations.listInstanceFleets(getEmrClient(awsParams), new ListInstanceFleetsRequest().withClusterId(clusterId)); }
@Override public void terminateEmrCluster(String clusterId, boolean overrideTerminationProtection, AwsParamsDto awsParams) { emrOperations.terminateEmrCluster(getEmrClient(awsParams), clusterId, overrideTerminationProtection); }
@Override public StepSummary getClusterActiveStep(String clusterId, AwsParamsDto awsParamsDto) { ListStepsRequest listStepsRequest = new ListStepsRequest().withClusterId(clusterId).withStepStates(StepState.RUNNING); List<StepSummary> stepSummaryList = emrOperations.listStepsRequest(getEmrClient(awsParamsDto), listStepsRequest).getSteps(); return !stepSummaryList.isEmpty() ? stepSummaryList.get(0) : null; }