private void addTopologyInputs(Map<String, SpoutSpec> spouts, Map stormConf, String topologyOwner, AtlasEntity topology, AtlasEntityExtInfo entityExtInfo) { List<AtlasEntity> inputs = new ArrayList<>(); for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) { Serializable instance = Utils.javaDeserialize(entry.getValue().get_spout_object().get_serialized_java(), Serializable.class); String dsType = instance.getClass().getSimpleName(); AtlasEntity dsEntity = addDataSet(dsType, topologyOwner, instance, stormConf, entityExtInfo); if (dsEntity != null) { inputs.add(dsEntity); } } topology.setAttribute("inputs", AtlasTypeUtil.getAtlasObjectIds(inputs)); }
private void addTopologyOutputs(StormTopology stormTopology, String topologyOwner, Map stormConf, AtlasEntity topology, AtlasEntityExtInfo entityExtInfo) { List<AtlasEntity> outputs = new ArrayList<>(); Map<String, Bolt> bolts = stormTopology.get_bolts(); Set<String> boltNames = StormTopologyUtil.getTerminalUserBoltNames(stormTopology); for (String boltName : boltNames) { Serializable instance = Utils.javaDeserialize(bolts.get(boltName).get_bolt_object().get_serialized_java(), Serializable.class); String dsType = instance.getClass().getSimpleName(); AtlasEntity dsEntity = addDataSet(dsType, topologyOwner, instance, stormConf, entityExtInfo); if (dsEntity != null) { outputs.add(dsEntity); } } topology.setAttribute("outputs", AtlasTypeUtil.getAtlasObjectIds(outputs)); }
/** * This is the client-side hook that storm fires when a topology is added. * * @param topologyInfo topology info * @param stormConf configuration * @param stormTopology a storm topology */ @Override public void notify(TopologyInfo topologyInfo, Map stormConf, StormTopology stormTopology) { LOG.info("Collecting metadata for a new storm topology: {}", topologyInfo.get_name()); try { String user = getUser(topologyInfo.get_owner(), null); AtlasEntity topology = createTopologyInstance(topologyInfo, stormConf); AtlasEntitiesWithExtInfo entity = new AtlasEntitiesWithExtInfo(topology); addTopologyDataSets(stormTopology, topologyInfo.get_owner(), stormConf, topology, entity); // create the graph for the topology List<AtlasEntity> graphNodes = createTopologyGraph(stormTopology, stormTopology.get_spouts(), stormTopology.get_bolts()); if (CollectionUtils.isNotEmpty(graphNodes)) { // add the connection from topology to the graph topology.setAttribute("nodes", AtlasTypeUtil.getAtlasObjectIds(graphNodes)); for (AtlasEntity graphNode : graphNodes) { entity.addReferredEntity(graphNode); } } List<HookNotification> hookNotifications = Collections.singletonList(new EntityCreateRequestV2(user, entity)); notifyEntities(hookNotifications, null); } catch (Exception e) { throw new RuntimeException("Atlas hook is unable to process the topology.", e); } }
private void createOrUpdateTableInstance(HBaseOperationContext hbaseOperationContext) { AtlasEntity nameSpace = buildNameSpace(hbaseOperationContext); AtlasEntity table = buildTable(hbaseOperationContext, nameSpace); List<AtlasEntity> columnFamilies = buildColumnFamilies(hbaseOperationContext, nameSpace, table); table.setAttribute(ATTR_COLUMNFAMILIES, AtlasTypeUtil.getAtlasObjectIds(columnFamilies)); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(table); entities.addReferredEntity(nameSpace); if (CollectionUtils.isNotEmpty(columnFamilies)) { for (AtlasEntity columnFamily : columnFamilies) { entities.addReferredEntity(columnFamily); } } switch (hbaseOperationContext.getOperation()) { case CREATE_TABLE: LOG.info("Create Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityCreateRequestV2(hbaseOperationContext.getUser(), entities)); break; case ALTER_TABLE: LOG.info("Modify Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityUpdateRequestV2(hbaseOperationContext.getUser(), entities)); break; } }
private void createOrUpdateTableInstance(HBaseOperationContext hbaseOperationContext) { AtlasEntity nameSpace = buildNameSpace(hbaseOperationContext); AtlasEntity table = buildTable(hbaseOperationContext, nameSpace); List<AtlasEntity> columnFamilies = buildColumnFamilies(hbaseOperationContext, nameSpace, table); table.setAttribute(ATTR_COLUMNFAMILIES, AtlasTypeUtil.getAtlasObjectIds(columnFamilies)); AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo(table); entities.addReferredEntity(nameSpace); if (CollectionUtils.isNotEmpty(columnFamilies)) { for (AtlasEntity columnFamily : columnFamilies) { entities.addReferredEntity(columnFamily); } } switch (hbaseOperationContext.getOperation()) { case CREATE_TABLE: LOG.info("Create Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityCreateRequestV2(hbaseOperationContext.getUser(), entities)); break; case ALTER_TABLE: LOG.info("Modify Table {}", table.getAttribute(REFERENCEABLE_ATTRIBUTE_NAME)); hbaseOperationContext.addMessage(new EntityUpdateRequestV2(hbaseOperationContext.getUser(), entities)); break; } }