/** * Update a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void updateFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("updateFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowId flowId = new FlowId().setFlowGroup(flowConfig.getId().getFlowGroup()) .setFlowName(flowConfig.getId().getFlowName()); UpdateRequest<FlowConfig> updateRequest = _flowconfigsRequestBuilders.update().id(new ComplexResourceKey<>(flowId, new EmptyRecord())) .input(flowConfig).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(updateRequest); response.getResponse(); }
@Finder("latestFlowStatus") public List<FlowStatus> getLatestFlowStatus(@Context PagingContext context, @QueryParam("flowId") FlowId flowId) { LOG.info("getLatestFlowStatus called with flowGroup " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); gobblin.service.monitoring.FlowStatus latestFlowStatus = _flowStatusGenerator.getLatestFlowStatus(flowId.getFlowName(), flowId.getFlowGroup()); if (latestFlowStatus != null) { return Collections.singletonList(convertFlowStatus(latestFlowStatus)); } // will return 404 status code return null; }
@SuppressWarnings("DLS_DEAD_LOCAL_STORE") private static void testGobblinService(GobblinServiceManager gobblinServiceManager) { FlowConfigClient client = new FlowConfigClient(String.format("http://localhost:%s/", gobblinServiceManager.restliServer.getPort())); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); final String TEST_GROUP_NAME = "testGroup1"; final String TEST_FLOW_NAME = "testFlow1"; final String TEST_SCHEDULE = "0 1/0 * ? * *"; final String TEST_TEMPLATE_URI = "FS:///templates/test.template"; FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { client.createFlowConfig(flowConfig); } catch (RemoteInvocationException e) { throw new RuntimeException(e); } } }
/** * Get the latest flow status * @param flowId identifier of flow status to get * @return a {@link FlowStatus} with the flow status * @throws RemoteInvocationException */ public FlowStatus getLatestFlowStatus(FlowId flowId) throws RemoteInvocationException { LOG.debug("getFlowConfig with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); FindRequest<FlowStatus> findRequest = _flowstatusesRequestBuilders.findByLatestFlowStatus().flowIdParam(flowId).build(); Response<CollectionResponse<FlowStatus>> response = _restClient.get().sendRequest(findRequest).getResponse(); List<FlowStatus> flowStatusList = response.getEntity().getElements(); if (flowStatusList.isEmpty()) { return null; } else { Preconditions.checkArgument(flowStatusList.size() == 1); return flowStatusList.get(0); } }
String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); flowPropsAsStringMap.putAll(Maps.fromProperties(flowProps)); return flowConfig.setId(new FlowId().setFlowGroup(flowGroup).setFlowName(flowName)) .setProperties(flowPropsAsStringMap); } catch (URISyntaxException e) {
/** * Create a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void createFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("createFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); CreateIdRequest<ComplexResourceKey<FlowId, EmptyRecord>, FlowConfig> request = _flowconfigsRequestBuilders.create().input(flowConfig).build(); ResponseFuture<IdResponse<ComplexResourceKey<FlowId, EmptyRecord>>> flowConfigResponseFuture = _restClient.get().sendRequest(request); flowConfigResponseFuture.getResponse(); }
FlowId flowId = new FlowId().setFlowName(monitoringFlowStatus.getFlowName()) .setFlowGroup(monitoringFlowStatus.getFlowGroup()); long flowStartTime = Long.MAX_VALUE; long flowEndTime = -1L; .setId(new FlowStatusId().setFlowGroup(flowId.getFlowGroup()).setFlowName(flowId.getFlowName()) .setFlowExecutionId(monitoringFlowStatus.getFlowExecutionId())) .setExecutionStatistics(new FlowStatistics().setExecutionStartTime(flowStartTime)
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsRequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new EmptyRecord())).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); URI flowUri = null; if (!flowGroup.equals(flowConfig.getId().getFlowGroup()) || !flowName.equals(flowConfig.getId().getFlowName())) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "flowName and flowGroup cannot be changed in update", null);
/** * Delete a configured flow. Running flows are not affected. The schedule will be removed for scheduled flows. * @param key composite key containing flow group and flow name that identifies the flow to remove from the * {@link FlowCatalog} * @return {@link UpdateResponse} */ @Override public UpdateResponse delete(ComplexResourceKey<FlowId, EmptyRecord> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); URI flowUri = null; LOG.info("Delete called with flowGroup " + flowGroup + " flowName " + flowName); try { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowGroup + "/" + flowName, null, null); FlowSpec flowSpec = (FlowSpec) getFlowCatalog().getSpec(flowUri); getFlowCatalog().remove(flowUri); return new UpdateResponse(HttpStatus.S_200_OK); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowUri, e); } catch (SpecNotFoundException e) { logAndThrowRestLiServiceException(HttpStatus.S_404_NOT_FOUND, "Flow does not exist: flowGroup " + flowGroup + " flowName " + flowName, null); } return null; }
/** * Get a flow configuration * @param flowId identifier of flow configuration to get * @return a {@link FlowConfig} with the flow configuration * @throws RemoteInvocationException */ public FlowConfig getFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("getFlowConfig with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); GetRequest<FlowConfig> getRequest = _flowconfigsRequestBuilders.get() .id(new ComplexResourceKey<>(flowId, new EmptyRecord())).build(); Response<FlowConfig> response = _restClient.get().sendRequest(getRequest).getResponse(); return response.getEntity(); }
/** * Create a flow configuration that the service will forward to execution instances for execution * @param flowConfig flow configuration * @return {@link CreateResponse} */ @Override public CreateResponse create(FlowConfig flowConfig) { LOG.info("Create called with flowName " + flowConfig.getId().getFlowName()); LOG.info("ReadyToUse is: " + readyToUse); LOG.info("FlowCatalog is: " + getFlowCatalog()); if (!readyToUse && getFlowCatalog() == null) { throw new RuntimeException("Not ready for use."); } try { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowConfig.getId().getFlowGroup() + "/" + flowConfig.getId().getFlowName(), null, null); if (getFlowCatalog().getSpec(flowUri) != null) { logAndThrowRestLiServiceException(HttpStatus.S_409_CONFLICT, "Flow with the same name already exists: " + flowUri, null); } } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getId().getFlowName(), e); } catch (SpecNotFoundException e) { // okay if flow does not exist } getFlowCatalog().put(createFlowSpecForConfig(flowConfig)); return new CreateResponse(new ComplexResourceKey<>(flowConfig.getId(), new EmptyRecord()), HttpStatus.S_201_CREATED); }
/** * Build a {@link FlowSpec} from a {@link FlowConfig} * @param flowConfig flow configuration * @return {@link FlowSpec} created with attributes from flowConfig */ private FlowSpec createFlowSpecForConfig(FlowConfig flowConfig) { ConfigBuilder configBuilder = ConfigBuilder.create() .addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowConfig.getId().getFlowGroup()) .addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowConfig.getId().getFlowName()); if (flowConfig.hasSchedule()) { Schedule schedule = flowConfig.getSchedule(); configBuilder.addPrimitive(ConfigurationKeys.JOB_SCHEDULE_KEY, schedule.getCronSchedule()); configBuilder.addPrimitive(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, schedule.isRunImmediately()); } Config config = configBuilder.build(); Config configWithFallback = config.withFallback(ConfigFactory.parseMap(flowConfig.getProperties())); URI templateURI = null; try { templateURI = new URI(flowConfig.getTemplateUris()); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getTemplateUris(), e); } return FlowSpec.builder().withConfig(configWithFallback).withTemplate(templateURI).build(); }