@SuppressWarnings("DLS_DEAD_LOCAL_STORE") private static void testGobblinService(GobblinServiceManager gobblinServiceManager) { FlowConfigClient client = new FlowConfigClient(String.format("http://localhost:%s/", gobblinServiceManager.restliServer.getPort())); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); final String TEST_GROUP_NAME = "testGroup1"; final String TEST_FLOW_NAME = "testFlow1"; final String TEST_SCHEDULE = "0 1/0 * ? * *"; final String TEST_TEMPLATE_URI = "FS:///templates/test.template"; FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { client.createFlowConfig(flowConfig); } catch (RemoteInvocationException e) { throw new RuntimeException(e); } } }
/** * Build a {@link FlowSpec} from a {@link FlowConfig} * @param flowConfig flow configuration * @return {@link FlowSpec} created with attributes from flowConfig */ private FlowSpec createFlowSpecForConfig(FlowConfig flowConfig) { ConfigBuilder configBuilder = ConfigBuilder.create() .addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowConfig.getId().getFlowGroup()) .addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowConfig.getId().getFlowName()); if (flowConfig.hasSchedule()) { Schedule schedule = flowConfig.getSchedule(); configBuilder.addPrimitive(ConfigurationKeys.JOB_SCHEDULE_KEY, schedule.getCronSchedule()); configBuilder.addPrimitive(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, schedule.isRunImmediately()); } Config config = configBuilder.build(); Config configWithFallback = config.withFallback(ConfigFactory.parseMap(flowConfig.getProperties())); URI templateURI = null; try { templateURI = new URI(flowConfig.getTemplateUris()); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getTemplateUris(), e); } return FlowSpec.builder().withConfig(configWithFallback).withTemplate(templateURI).build(); }
/** * Create a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void createFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("createFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); CreateIdRequest<ComplexResourceKey<FlowId, EmptyRecord>, FlowConfig> request = _flowconfigsRequestBuilders.create().input(flowConfig).build(); ResponseFuture<IdResponse<ComplexResourceKey<FlowId, EmptyRecord>>> flowConfigResponseFuture = _restClient.get().sendRequest(request); flowConfigResponseFuture.getResponse(); }
/** * Create a flow configuration that the service will forward to execution instances for execution * @param flowConfig flow configuration * @return {@link CreateResponse} */ @Override public CreateResponse create(FlowConfig flowConfig) { LOG.info("Create called with flowName " + flowConfig.getId().getFlowName()); LOG.info("ReadyToUse is: " + readyToUse); LOG.info("FlowCatalog is: " + getFlowCatalog()); if (!readyToUse && getFlowCatalog() == null) { throw new RuntimeException("Not ready for use."); } try { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowConfig.getId().getFlowGroup() + "/" + flowConfig.getId().getFlowName(), null, null); if (getFlowCatalog().getSpec(flowUri) != null) { logAndThrowRestLiServiceException(HttpStatus.S_409_CONFLICT, "Flow with the same name already exists: " + flowUri, null); } } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getId().getFlowName(), e); } catch (SpecNotFoundException e) { // okay if flow does not exist } getFlowCatalog().put(createFlowSpecForConfig(flowConfig)); return new CreateResponse(new ComplexResourceKey<>(flowConfig.getId(), new EmptyRecord()), HttpStatus.S_201_CREATED); }
"/" + flowGroup + "/" + flowName, null, null); FlowSpec spec = (FlowSpec) getFlowCatalog().getSpec(flowUri); FlowConfig flowConfig = new FlowConfig(); Properties flowProps = spec.getConfigAsProperties(); Schedule schedule = null; flowConfig.setTemplateUris(flowProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH)); } else if (spec.getTemplateURIs().isPresent()) { flowConfig.setTemplateUris(StringUtils.join(spec.getTemplateURIs().get(), ",")); } else { flowConfig.setTemplateUris("NA"); flowConfig.setSchedule(schedule); flowPropsAsStringMap.putAll(Maps.fromProperties(flowProps)); return flowConfig.setId(new FlowId().setFlowGroup(flowGroup).setFlowName(flowName)) .setProperties(flowPropsAsStringMap); } catch (URISyntaxException e) { logAndThrowRestLiServiceException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowName, e);
/** * Update a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void updateFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("updateFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowId flowId = new FlowId().setFlowGroup(flowConfig.getId().getFlowGroup()) .setFlowName(flowConfig.getId().getFlowName()); UpdateRequest<FlowConfig> updateRequest = _flowconfigsRequestBuilders.update().id(new ComplexResourceKey<>(flowId, new EmptyRecord())) .input(flowConfig).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(updateRequest); response.getResponse(); }