How to use org.apache.gobblin.service

/**
 * Create a flow configuration that the service will forward to execution instances for execution
 * @param flowConfig flow configuration
 * @return {@link CreateResponse}
 */
@Override
public CreateResponse create(FlowConfig flowConfig) {
 return this.getFlowConfigResourceHandler().createFlowConfig(flowConfig);
}

/**
 * Update flowConfig locally and trigger all listeners
 */
public UpdateResponse updateFlowConfig(FlowId flowId, FlowConfig flowConfig) throws FlowConfigLoggedException {
 return updateFlowConfig(flowId, flowConfig, true);
}

@Override
public FlowConfig getFlowConfig(FlowId flowId)
  throws FlowConfigLoggedException {
 return this.localHandler.getFlowConfig(flowId);
}

@Test (dependsOnMethods = "testCreateAgain")
public void testGet() throws Exception {
 FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME);
 FlowConfig flowConfig = _client.getFlowConfig(flowId);
 Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
 Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME);
 Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE );
 Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
 Assert.assertFalse(flowConfig.getSchedule().isRunImmediately());
 // Add this asssert back when getFlowSpec() is changed to return the raw flow spec
 //Assert.assertEquals(flowConfig.getProperties().size(), 1);
 Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1");
}

@Test
public void testCreate() throws Exception {
 Map<String, String> flowProperties = Maps.newHashMap();
 flowProperties.put("param1", "value1");
 FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME))
   .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE).
     setRunImmediately(true))
   .setProperties(new StringMap(flowProperties));
 _client.createFlowConfig(flowConfig);
}

/**
 * Retrieve the flow configuration with the given key
 * @param key flow config id key containing group name and flow name
 * @return {@link FlowConfig} with flow configuration
 */
@Override
public FlowConfig get(ComplexResourceKey<FlowId, FlowStatusId> key) {
 String flowGroup = key.getKey().getFlowGroup();
 String flowName = key.getKey().getFlowName();
 FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName);
 return this.getFlowConfigResourceHandler().getFlowConfig(flowId);
}

/**
 * Update the flow configuration with the specified key. Running flows are not affected.
 * An error is raised if the flow configuration does not exist.
 * @param key composite key containing group name and flow name that identifies the flow to update
 * @param flowConfig new flow configuration
 * @return {@link UpdateResponse}
 */
@Override
public UpdateResponse update(ComplexResourceKey<FlowId, FlowStatusId> key, FlowConfig flowConfig) {
 String flowGroup = key.getKey().getFlowGroup();
 String flowName = key.getKey().getFlowName();
 FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName);
 return this.getFlowConfigResourceHandler().updateFlowConfig(flowId, flowConfig);
}

/**
 * Delete a configured flow. Running flows are not affected. The schedule will be removed for scheduled flows.
 * @param key composite key containing flow group and flow name that identifies the flow to remove from the flow catalog
 * @return {@link UpdateResponse}
 */
@Override
public UpdateResponse delete(ComplexResourceKey<FlowId, FlowStatusId> key) {
 String flowGroup = key.getKey().getFlowGroup();
 String flowName = key.getKey().getFlowName();
 FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName);
 return this.getFlowConfigResourceHandler().deleteFlowConfig(flowId, getHeaders());
}

/**
 * Delete a configured flow. Running flows are not affected. The schedule will be removed for scheduled flows.
 * @param key composite key containing flow group and flow name that identifies the flow to remove from the flow catalog
 * @return {@link UpdateResponse}
 */
@Override
public UpdateResponse delete(ComplexResourceKey<FlowId, EmptyRecord> key) {
 String flowGroup = key.getKey().getFlowGroup();
 String flowName = key.getKey().getFlowName();
 FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName);
 return this.flowConfigsResourceHandler.deleteFlowConfig(flowId, getHeaders());
}

/**
 * Update flowConfig locally and trigger all listeners iff @param triggerListener is set to true
 */
public UpdateResponse updateFlowConfig(FlowId flowId, FlowConfig flowConfig, boolean triggerListener) {
 log.info("[GAAS-REST] Update called with flowGroup {} flowName {}", flowId.getFlowGroup(), flowId.getFlowName());
 if (!flowId.getFlowGroup().equals(flowConfig.getId().getFlowGroup()) || !flowId.getFlowName().equals(flowConfig.getId().getFlowName())) {
  throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST,
    "flowName and flowGroup cannot be changed in update", null);
 }
 this.flowCatalog.put(createFlowSpecForConfig(flowConfig), triggerListener);
 return new UpdateResponse(HttpStatus.S_200_OK);
}

/**
 * Delete flowConfig locally and trigger all listeners iff @param triggerListener is set to true
 */
public UpdateResponse deleteFlowConfig(FlowId flowId, Properties header, boolean triggerListener) throws FlowConfigLoggedException {
 log.info("[GAAS-REST] Delete called with flowGroup {} flowName {}", flowId.getFlowGroup(), flowId.getFlowName());
 URI flowUri = null;
 try {
  flowUri = createFlowSpecUri(flowId);
  this.flowCatalog.remove(flowUri, header, triggerListener);
  return new UpdateResponse(HttpStatus.S_200_OK);
 } catch (URISyntaxException e) {
  throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowUri, e);
 }
}

/**
 * Create a flow configuration that the service will forward to execution instances for execution
 * @param flowConfig flow configuration
 * @return {@link CreateResponse}
 */
@Override
public CreateResponse create(FlowConfig flowConfig) {
 List<ServiceRequester> requestorList = this.requesterService.findRequesters(this);
 try {
  String serialized = this.requesterService.serialize(requestorList);
  flowConfig.getProperties().put(RequesterService.REQUESTER_LIST, serialized);
  LOG.info("Rest requester list is " + serialized);
 } catch (IOException e) {
  throw new FlowConfigLoggedException(HttpStatus.S_401_UNAUTHORIZED,
    "cannot get who is the requester", e);
 }
 return this.flowConfigsResourceHandler.createFlowConfig(flowConfig);
}

private void initializeWatermarks() {
 initializeLowWatermarks();
 initializeHighWatermarks();
}

public SimpleKafkaSpecExecutor(Config config, Optional<Logger> log) {
 super(config, log);
 specProducer = new SimpleKafkaSpecProducer(config, log);
}

/**
 * Add flowConfig locally and trigger all listeners
 */
public CreateResponse createFlowConfig(FlowConfig flowConfig) throws FlowConfigLoggedException {
 return this.createFlowConfig(flowConfig, true);
}

/**
 * Delete flowConfig locally and trigger all listeners
 */
public UpdateResponse deleteFlowConfig(FlowId flowId, Properties header)  throws FlowConfigLoggedException {
 return deleteFlowConfig(flowId, header, true);
}

@Override
public void close() throws IOException {
 shutDown();
}

@Test (dependsOnMethods = "testCreateAgain")
public void testGet() throws Exception {
 FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME);
 FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(flowId);
 Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
 Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME);
 Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE);
 Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
 Assert.assertFalse(flowConfig.getSchedule().isRunImmediately());
 // Add this assert back when getFlowSpec() is changed to return the raw flow spec
 //Assert.assertEquals(flowConfig.getProperties().size(), 1);
 Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1");
}

/**
 * Retrieve the flow configuration with the given key
 * @param key flow config id key containing group name and flow name
 * @return {@link FlowConfig} with flow configuration
 */
@Override
public FlowConfig get(ComplexResourceKey<FlowId, EmptyRecord> key) {
 String flowGroup = key.getKey().getFlowGroup();
 String flowName = key.getKey().getFlowName();
 FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName);
 return this.flowConfigsResourceHandler.getFlowConfig(flowId);
}

/**
 * Update the flow configuration with the specified key. Running flows are not affected.
 * An error is raised if the flow configuration does not exist.
 * @param key composite key containing group name and flow name that identifies the flow to update
 * @param flowConfig new flow configuration
 * @return {@link UpdateResponse}
 */
@Override
public UpdateResponse update(ComplexResourceKey<FlowId, EmptyRecord> key, FlowConfig flowConfig) {
 String flowGroup = key.getKey().getFlowGroup();
 String flowName = key.getKey().getFlowName();
 FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName);
 return this.flowConfigsResourceHandler.updateFlowConfig(flowId, flowConfig);
}