private void testFlowSpec(FlowConfig flowConfig) { try { FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); } catch (FlowConfigLoggedException e) { Assert.fail("Should not get to here"); } }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)}. * * Please refer to {@link FlowConfigResourceLocalHandler#createFlowConfig(FlowConfig)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)} in non-balance mode. */ private void handleAdd(String msg) throws IOException { FlowConfig config = FlowConfigUtils.deserializeFlowConfig(msg); if (this.flowCatalogLocalCommit) { // in balance mode, flow spec is already added in flow catalog on standby node. FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(config); log.info("Only handle add {} scheduling because flow catalog is committed locally on standby.", flowSpec); jobScheduler.onAddSpec(flowSpec); } else { resourceHandler.createFlowConfig(config); } }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)}. * * Please refer to {@link FlowConfigResourceLocalHandler#updateFlowConfig(FlowId, FlowConfig)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} in non-balance mode. */ private void handleUpdate(String msg) throws IOException { FlowConfig config = FlowConfigUtils.deserializeFlowConfig(msg); if (flowCatalogLocalCommit) { // in balance mode, flow spec is already updated in flow catalog on standby node. FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(config); log.info("Only handle update {} scheduling because flow catalog is committed locally on standby.", flowSpec); jobScheduler.onUpdateSpec(flowSpec); } else { resourceHandler.updateFlowConfig(config.getId(), config); } }
/** * Update flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public UpdateResponse updateFlowConfig(FlowId flowId, FlowConfig flowConfig, boolean triggerListener) { log.info("[GAAS-REST] Update called with flowGroup {} flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); if (!flowId.getFlowGroup().equals(flowConfig.getId().getFlowGroup()) || !flowId.getFlowName().equals(flowConfig.getId().getFlowName())) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "flowName and flowGroup cannot be changed in update", null); } this.flowCatalog.put(createFlowSpecForConfig(flowConfig), triggerListener); return new UpdateResponse(HttpStatus.S_200_OK); }
.setProperties(new StringMap(flowProperties)); FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_NAME_KEY), TEST_FLOW_NAME); setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig);
public void testFlowConfigWithoutTemplateUri() { FlowConfig flowConfig = new FlowConfig().setId(new FlowId() .setFlowName("SN_CRMSYNC") .setFlowGroup("DYNAMICS-USER-123456789")); flowConfig.setSchedule(new Schedule() .setCronSchedule("0 58 2/12 ? * * *")); Properties properties = new Properties(); properties.put("gobblin.flow.sourceIdentifier", "dynamicsCrm"); properties.put("gobblin.flow.destinationIdentifier", "espresso"); flowConfig.setProperties(new StringMap(Maps.fromProperties(properties))); try { FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); Assert.fail("Should not get to here"); } catch (RequiredFieldNotPresentException e) { Assert.assertTrue(true, "templateUri cannot be empty"); } testSerDer(flowConfig); } }
/** * Add flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public CreateResponse createFlowConfig(FlowConfig flowConfig, boolean triggerListener) throws FlowConfigLoggedException { log.info("[GAAS-REST] Create called with flowGroup " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowSpec flowSpec = createFlowSpecForConfig(flowConfig); // Existence of a flow spec in the flow catalog implies that the flow is currently running. // If the new flow spec has a schedule we should allow submission of the new flow to accept the new schedule. // However, if the new flow spec does not have a schedule, we should allow submission only if it is not running. if (!flowConfig.hasSchedule() && this.flowCatalog.exists(flowSpec.getUri())) { return new CreateResponse(new ComplexResourceKey<>(flowConfig.getId(), new EmptyRecord()), HttpStatus.S_409_CONFLICT); } else { this.flowCatalog.put(flowSpec, triggerListener); return new CreateResponse(new ComplexResourceKey<>(flowConfig.getId(), new EmptyRecord()), HttpStatus.S_201_CREATED); } }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)}. * * Please refer to {@link FlowConfigResourceLocalHandler#createFlowConfig(FlowConfig)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onAddSpec(Spec)} in non-balance mode. */ private void handleAdd(String msg) throws IOException { FlowConfig config = FlowConfigUtils.deserializeFlowConfig(msg); if (this.flowCatalogLocalCommit) { // in balance mode, flow spec is already added in flow catalog on standby node. FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(config); log.info("Only handle add {} scheduling because flow catalog is committed locally on standby.", flowSpec); jobScheduler.onAddSpec(flowSpec); } else { resourceHandler.createFlowConfig(config); } }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)}. * * Please refer to {@link FlowConfigResourceLocalHandler#updateFlowConfig(FlowId, FlowConfig)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} in non-balance mode. */ private void handleUpdate(String msg) throws IOException { FlowConfig config = FlowConfigUtils.deserializeFlowConfig(msg); if (flowCatalogLocalCommit) { // in balance mode, flow spec is already updated in flow catalog on standby node. FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(config); log.info("Only handle update {} scheduling because flow catalog is committed locally on standby.", flowSpec); jobScheduler.onUpdateSpec(flowSpec); } else { resourceHandler.updateFlowConfig(config.getId(), config); } }