public static URI createFlowSpecUri (FlowId flowId) throws URISyntaxException { URI flowCatalogURI = new URI("gobblin-flow", null, "/", null, null); URI flowUri = new URI(flowCatalogURI.getScheme(), flowCatalogURI.getAuthority(), "/" + flowId.getFlowGroup() + "/" + flowId.getFlowName(), null, null); return flowUri; }
public static String serializeFlowId(FlowId id) throws IOException { Properties properties = new Properties(); properties.setProperty(FLOWCONFIG_ID_NAME, id.getFlowName()); properties.setProperty(FLOWCONFIG_ID_GROUP, id.getFlowGroup()); return PropertiesUtils.serialize(properties); }
/** * Update flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public UpdateResponse updateFlowConfig(FlowId flowId, FlowConfig flowConfig, boolean triggerListener) { log.info("[GAAS-REST] Update called with flowGroup {} flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); if (!flowId.getFlowGroup().equals(flowConfig.getId().getFlowGroup()) || !flowId.getFlowName().equals(flowConfig.getId().getFlowName())) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "flowName and flowGroup cannot be changed in update", null); } this.flowCatalog.put(createFlowSpecForConfig(flowConfig), triggerListener); return new UpdateResponse(HttpStatus.S_200_OK); }
@Finder("latestFlowStatus") public List<FlowStatus> getLatestFlowStatus(@Context PagingContext context, @QueryParam("flowId") FlowId flowId) { LOG.info("getLatestFlowStatus called with flowGroup " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); org.apache.gobblin.service.monitoring.FlowStatus latestFlowStatus = _flowStatusGenerator.getLatestFlowStatus(flowId.getFlowName(), flowId.getFlowGroup()); if (latestFlowStatus != null) { return Collections.singletonList(convertFlowStatus(latestFlowStatus)); } // will return 404 status code return null; }
/** * Delete flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public UpdateResponse deleteFlowConfig(FlowId flowId, Properties header, boolean triggerListener) throws FlowConfigLoggedException { log.info("[GAAS-REST] Delete called with flowGroup {} flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); URI flowUri = null; try { flowUri = createFlowSpecUri(flowId); this.flowCatalog.remove(flowUri, header, triggerListener); return new UpdateResponse(HttpStatus.S_200_OK); } catch (URISyntaxException e) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowUri, e); } }
public static String serializeFlowConfig(FlowConfig flowConfig) throws IOException { Properties properties = ConfigUtils.configToProperties(ConfigFactory.parseMap(flowConfig.getProperties())); properties.setProperty(FLOWCONFIG_ID_NAME, flowConfig.getId().getFlowName()); properties.setProperty(FLOWCONFIG_ID_GROUP, flowConfig.getId().getFlowGroup()); if (flowConfig.hasSchedule()) { properties.setProperty(FLOWCONFIG_SCHEDULE_CRON, flowConfig.getSchedule().getCronSchedule()); properties.setProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY, Boolean.toString(flowConfig.getSchedule().isRunImmediately())); } if (flowConfig.hasTemplateUris()) { properties.setProperty(FLOWCONFIG_TEMPLATEURIS, flowConfig.getTemplateUris()); } return PropertiesUtils.serialize(properties); }
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfigWithStateStore(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig and state store with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsV2RequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).setHeader(DELETE_STATE_STORE_KEY, Boolean.TRUE.toString()).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfigWithStateStore(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig and state store with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsRequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new EmptyRecord())).setHeader(DELETE_STATE_STORE_KEY, Boolean.TRUE.toString()).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
/** * Create a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void createFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("createFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); CreateIdRequest<ComplexResourceKey<FlowId, EmptyRecord>, FlowConfig> request = _flowconfigsRequestBuilders.create().input(flowConfig).build(); ResponseFuture<IdResponse<ComplexResourceKey<FlowId, EmptyRecord>>> flowConfigResponseFuture = _restClient.get().sendRequest(request); flowConfigResponseFuture.getResponse(); }
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig with groupName {}, flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsV2RequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsRequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new EmptyRecord())).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
/** * Retrieve the flow configuration with the given key * @param key flow config id key containing group name and flow name * @return {@link FlowConfig} with flow configuration */ @Override public FlowConfig get(ComplexResourceKey<FlowId, EmptyRecord> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName); return this.flowConfigsResourceHandler.getFlowConfig(flowId); }
/** * Update the flow configuration with the specified key. Running flows are not affected. * An error is raised if the flow configuration does not exist. * @param key composite key containing group name and flow name that identifies the flow to update * @param flowConfig new flow configuration * @return {@link UpdateResponse} */ @Override public UpdateResponse update(ComplexResourceKey<FlowId, EmptyRecord> key, FlowConfig flowConfig) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName); return this.flowConfigsResourceHandler.updateFlowConfig(flowId, flowConfig); }
/** * Retrieve the flow configuration with the given key * @param key flow config id key containing group name and flow name * @return {@link FlowConfig} with flow configuration */ @Override public FlowConfig get(ComplexResourceKey<FlowId, FlowStatusId> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName); return this.getFlowConfigResourceHandler().getFlowConfig(flowId); }
/** * Delete a configured flow. Running flows are not affected. The schedule will be removed for scheduled flows. * @param key composite key containing flow group and flow name that identifies the flow to remove from the flow catalog * @return {@link UpdateResponse} */ @Override public UpdateResponse delete(ComplexResourceKey<FlowId, EmptyRecord> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName); return this.flowConfigsResourceHandler.deleteFlowConfig(flowId, getHeaders()); }
/** * Delete a configured flow. Running flows are not affected. The schedule will be removed for scheduled flows. * @param key composite key containing flow group and flow name that identifies the flow to remove from the flow catalog * @return {@link UpdateResponse} */ @Override public UpdateResponse delete(ComplexResourceKey<FlowId, FlowStatusId> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName); return this.getFlowConfigResourceHandler().deleteFlowConfig(flowId, getHeaders()); }
/** * Update the flow configuration with the specified key. Running flows are not affected. * An error is raised if the flow configuration does not exist. * @param key composite key containing group name and flow name that identifies the flow to update * @param flowConfig new flow configuration * @return {@link UpdateResponse} */ @Override public UpdateResponse update(ComplexResourceKey<FlowId, FlowStatusId> key, FlowConfig flowConfig) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName); return this.getFlowConfigResourceHandler().updateFlowConfig(flowId, flowConfig); }
@Test (dependsOnMethods = "testCreateAgain") public void testGet() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); FlowConfig flowConfig = _client.getFlowConfig(flowId); Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE ); Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI); Assert.assertFalse(flowConfig.getSchedule().isRunImmediately()); // Add this asssert back when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 1); Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1"); }
@Test (dependsOnMethods = "testCreateAgain") public void testGet() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(flowId); Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE); Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI); Assert.assertFalse(flowConfig.getSchedule().isRunImmediately()); // Add this assert back when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 1); Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1"); }
@Test (dependsOnMethods = "testUpdate") public void testDelete() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); // make sure flow config exists FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(flowId); Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME); this.flowConfigClient.deleteFlowConfig(flowId); try { this.flowConfigClient.getFlowConfig(flowId); } catch (RestLiResponseException e) { Assert.assertEquals(e.getStatus(), HttpStatus.NOT_FOUND_404); return; } Assert.fail("Get should have gotten a 404 error"); }