public void setExEntity(AtlasEntity exEntity) { this.exEntity = exEntity; this.atlasGuid = exEntity.getGuid(); }
private Referenceable toReferenceable(AtlasObjectId id) { return StringUtils.isEmpty(id.getGuid()) ? new Referenceable(id.getTypeName(), id.getUniqueAttributes()) : new Referenceable(id.getGuid(), id.getTypeName(), id.getUniqueAttributes()); }
try { final AtlasEntity.AtlasEntityWithExtInfo entityExt = searchEntityDef(id); final AtlasEntity entity = entityExt.getEntity(); if (AtlasEntity.Status.DELETED.equals(entity.getStatus())) { return null; final Map<String, Object> uniqueAttrs = Collections.singletonMap(ATTR_QUALIFIED_NAME, entity.getAttribute(ATTR_QUALIFIED_NAME)); return new Tuple<>(new AtlasObjectId(id.getGuid(), id.getTypeName(), uniqueAttrs), entity); } catch (AtlasServiceException e) { logger.warn("Failed to search entity by id {}, due to {}", id, e);
private AtlasEntity createOrUpdateRootGroupPortEntity(boolean isInput, String qualifiedName, String portName) { final Map<AtlasObjectId, AtlasEntity> ports = isInput ? rootInputPortEntities : rootOutputPortEntities; final Optional<AtlasObjectId> existingPortId = findIdByQualifiedName(ports.keySet(), qualifiedName); final String typeName = isInput ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT; if (existingPortId.isPresent()) { final AtlasEntity entity = ports.get(existingPortId.get()); final String portGuid = entity.getGuid(); stillExistingEntityGuids.add(portGuid); final Object currentName = entity.getAttribute(ATTR_NAME); if (isUpdated(currentName, portName)) { // Update port name and set updated flag. entity.setAttribute(ATTR_NAME, portName); updatedEntityGuids.add(portGuid); updateAudit.add(String.format("Name of %s %s changed from %s to %s", entity.getTypeName(), portGuid, currentName, portName)); } return entity; } else { final AtlasEntity entity = new AtlasEntity(typeName); entity.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); entity.setAttribute(ATTR_NAME, portName); entity.setAttribute(ATTR_QUALIFIED_NAME, qualifiedName); final AtlasObjectId portId = new AtlasObjectId(typeName, ATTR_QUALIFIED_NAME, qualifiedName); ports.put(portId, entity); return entity; } }
final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Created FlowPath entities mutation response={}", mutationResponse); final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments(); createdEntities.forEach(entity -> { final String guid = entity.getGuid(); entity.setGuid(guidAssignments.get(guid)); final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME))); final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId); path.setExEntity(entity); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities); final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities); logger.debug("Updated FlowPath entities mutation response={}", mutationResponse); updatedEntities.forEach(entity -> { final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME))); final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId); path.setExEntity(entity); return changedEntities.entrySet().stream() .filter(entry -> !DELETED.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()) .map(path -> new AtlasObjectId(path.getGuid(), TYPE_NIFI_FLOW_PATH, Collections.singletonMap(ATTR_QUALIFIED_NAME, path.getAttribute(ATTR_QUALIFIED_NAME)))) .collect(Collectors.toSet());
private AtlasEntity registerNiFiFlowEntity(final NiFiFlow nifiFlow) throws AtlasServiceException { final List<AtlasEntity> entities = new ArrayList<>(); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(entities); if (!nifiFlow.isMetadataUpdated()) { // Nothing has been changed, return existing entity. return nifiFlow.getExEntity(); } // Create parent flow entity using existing NiFiFlow entity if available, so that common properties are taken over. final AtlasEntity flowEntity = nifiFlow.getExEntity() != null ? new AtlasEntity(nifiFlow.getExEntity()) : new AtlasEntity(); flowEntity.setTypeName(TYPE_NIFI_FLOW); flowEntity.setVersion(1L); flowEntity.setAttribute(ATTR_NAME, nifiFlow.getFlowName()); flowEntity.setAttribute(ATTR_QUALIFIED_NAME, nifiFlow.toQualifiedName(nifiFlow.getRootProcessGroupId())); flowEntity.setAttribute(ATTR_URL, nifiFlow.getUrl()); flowEntity.setAttribute(ATTR_DESCRIPTION, nifiFlow.getDescription()); // If flowEntity is not persisted yet, then store nifi_flow entity to make nifiFlowId available for other entities. if (flowEntity.getGuid().startsWith("-")) { entities.add(flowEntity); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Registered a new nifi_flow entity, mutation response={}", mutationResponse); final String assignedNiFiFlowGuid = mutationResponse.getGuidAssignments().get(flowEntity.getGuid()); flowEntity.setGuid(assignedNiFiFlowGuid); nifiFlow.setAtlasGuid(assignedNiFiFlowGuid); } return flowEntity; }
public Tuple<AtlasObjectId, AtlasEntity> getOrCreateQueue(String destinationComponentId) { final String qualifiedName = toQualifiedName(destinationComponentId); final Optional<AtlasObjectId> existingQueueId = findIdByQualifiedName(queues.keySet(), qualifiedName); if (existingQueueId.isPresent()) { final AtlasEntity entity = queues.get(existingQueueId.get()); stillExistingEntityGuids.add(entity.getGuid()); return new Tuple<>(existingQueueId.get(), entity); } else { final AtlasObjectId queueId = new AtlasObjectId(TYPE_NIFI_QUEUE, ATTR_QUALIFIED_NAME, qualifiedName); final AtlasEntity queue = new AtlasEntity(TYPE_NIFI_QUEUE); queue.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); queue.setAttribute(ATTR_QUALIFIED_NAME, qualifiedName); queue.setAttribute(ATTR_NAME, "queue"); queue.setAttribute(ATTR_DESCRIPTION, "Input queue for " + destinationComponentId); queues.put(queueId, queue); return new Tuple<>(queueId, queue); } }
final AtlasObjectId flowId = new AtlasObjectId(TYPE_NIFI_FLOW, ATTR_QUALIFIED_NAME, qualifiedName); final AtlasEntity.AtlasEntityWithExtInfo nifiFlowExt = searchEntityDef(flowId); if (nifiFlowExt == null || nifiFlowExt.getEntity() == null) { return null; final AtlasEntity nifiFlowEntity = nifiFlowExt.getEntity(); final Map<String, Object> attributes = nifiFlowEntity.getAttributes(); final NiFiFlow nifiFlow = new NiFiFlow(rootProcessGroupId); nifiFlow.setExEntity(nifiFlowEntity); nifiFlow.setDescription(toStr(attributes.get(ATTR_DESCRIPTION))); nifiFlow.getQueues().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_QUEUES)))); nifiFlow.getRootInputPortEntities().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_INPUT_PORTS)))); nifiFlow.getRootOutputPortEntities().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_OUTPUT_PORTS)))); final String pathQualifiedName = toStr(flowPathEntity.getAttribute(ATTR_QUALIFIED_NAME)); final NiFiFlowPath flowPath = new NiFiFlowPath(getComponentIdFromQualifiedName(pathQualifiedName)); if (flowPathEntity.hasAttribute(ATTR_URL)) { final Matcher urlMatcher = FLOW_PATH_URL_PATTERN.matcher(toStr(flowPathEntity.getAttribute(ATTR_URL))); if (urlMatcher.matches()) { flowPath.setGroupId(urlMatcher.group(1)); flowPath.setName(toStr(flowPathEntity.getAttribute(ATTR_NAME))); flowPath.getInputs().addAll(toQualifiedNameIds(toAtlasObjectIds(flowPathEntity.getAttribute(ATTR_INPUTS))).keySet()); flowPath.getOutputs().addAll(toQualifiedNameIds(toAtlasObjectIds(flowPathEntity.getAttribute(ATTR_OUTPUTS))).keySet()); flowPath.startTrackingChanges(nifiFlow);
try { metrics.dataSetSearched++; final AtlasEntity.AtlasEntityWithExtInfo refExt = atlasClient.searchEntityDef(new AtlasObjectId(guid, typeName)); final String qualifiedName = (String) refExt.getEntity().getAttribute(ATTR_QUALIFIED_NAME); typedQualifiedNameToRef.put(toTypedQualifiedName(typeName, qualifiedName), new Referenceable(guid, typeName, Collections.EMPTY_MAP)); return qualifiedName;
private Tuple<EntityChangeType, AtlasEntity> toAtlasEntity(EntityChangeType changeType, final NiFiFlowPath path) { final AtlasEntity entity = EntityChangeType.CREATED.equals(changeType) ? new AtlasEntity() : new AtlasEntity(path.getExEntity()); entity.setTypeName(TYPE_NIFI_FLOW_PATH); entity.setVersion(1L); entity.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); final StringBuilder name = new StringBuilder(); final StringBuilder description = new StringBuilder(); path.getProcessComponentIds().forEach(pid -> { final String componentName = getProcessComponentName(pid); if (name.length() > 0) { name.append(", "); description.append(", "); } name.append(componentName); description.append(String.format("%s::%s", componentName, pid)); }); path.setName(name.toString()); entity.setAttribute(ATTR_NAME, name.toString()); entity.setAttribute(ATTR_DESCRIPTION, description.toString()); // Use first processor's id as qualifiedName. entity.setAttribute(ATTR_QUALIFIED_NAME, toQualifiedName(path.getId())); entity.setAttribute(ATTR_URL, path.createDeepLinkURL(getUrl())); final boolean inputsChanged = setChangedIOIds(path, entity, true); final boolean outputsChanged = setChangedIOIds(path, entity, false); // Even iff there's no flow path metadata changed, if any IO is changed then the pass should be updated. EntityChangeType finalChangeType = EntityChangeType.AS_IS.equals(changeType) ? (path.isMetadataUpdated() || inputsChanged || outputsChanged ? EntityChangeType.UPDATED : EntityChangeType.AS_IS) : changeType; return new Tuple<>(finalChangeType, entity); }
flowEntity.setAttribute(ATTR_FLOW_PATHS, remainingPathIds); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_QUEUES, updatedDataSetEntities.get(TYPE_NIFI_QUEUE)); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_INPUT_PORTS, updatedDataSetEntities.get(TYPE_NIFI_INPUT_PORT)); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_OUTPUT_PORTS, updatedDataSetEntities.get(TYPE_NIFI_OUTPUT_PORT)); shouldUpdateNiFiFlow = true; final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(entities); entities.add(flowEntity); try {
private EntityChangeType getFlowPathIOChangeType(AtlasObjectId id) { final String guid = id.getGuid(); if (!isGuidAssigned(guid)) { return EntityChangeType.CREATED; } else { if (TYPE_NIFI_QUEUE.equals(id.getTypeName()) && queues.containsKey(id)) { // If an input/output is a queue, and it is owned by this NiFiFlow, then check if it's still needed. NiFiFlow knows active queues. if (stillExistingEntityGuids.contains(guid)) { return EntityChangeType.AS_IS; } else { return EntityChangeType.DELETED; } } else { // Otherwise, do not need to delete. return EntityChangeType.AS_IS; } } }
/** * Set input or output DataSet ids for a NiFiFlowPath. * The updated ids only containing active ids. * @return True if there is any changed IO reference (create, update, delete). */ private boolean setChangedIOIds(NiFiFlowPath path, AtlasEntity pathEntity, boolean isInput) { Set<AtlasObjectId> ids = isInput ? path.getInputs() : path.getOutputs(); String targetAttribute = isInput ? ATTR_INPUTS : ATTR_OUTPUTS; final Map<EntityChangeType, List<AtlasObjectId>> changedIOIds = ids.stream().collect(Collectors.groupingBy(this::getFlowPathIOChangeType)); // Remove DELETED references. final Set<AtlasObjectId> remainingFlowPathIOIds = toRemainingFlowPathIOIds(changedIOIds); // If references are changed, update it. if (path.isDataSetReferenceChanged(remainingFlowPathIOIds, isInput)) { pathEntity.setAttribute(targetAttribute, remainingFlowPathIOIds); return true; } return false; }
private AtlasObjectId createAtlasObjectId() { return new AtlasObjectId(atlasGuid, TYPE_NIFI_FLOW, Collections.singletonMap(ATTR_QUALIFIED_NAME, getQualifiedName())); }
public static Optional<AtlasObjectId> findIdByQualifiedName(Set<AtlasObjectId> ids, String qualifiedName) { return ids.stream().filter(id -> qualifiedName.equals(id.getUniqueAttributes().get(ATTR_QUALIFIED_NAME))).findFirst(); }
final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Created DataSet entities mutation response={}", mutationResponse); final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments(); for (AtlasEntity entity : createdEntities) { final String guid = guidAssignments.get(entity.getGuid()); final String qualifiedName = toStr(entity.getAttribute(ATTR_QUALIFIED_NAME)); if (StringUtils.isEmpty(guid)) { logger.warn("GUID was not assigned for {}::{} for some reason.", entity.getTypeName(), qualifiedName); continue; switch (entity.getTypeName()) { case TYPE_NIFI_INPUT_PORT: entityMap = nifiFlow.getRootInputPortEntities(); break; default: throw new RuntimeException(entity.getTypeName() + " is not expected."); entity.setGuid(guid); final AtlasObjectId idWithGuid = new AtlasObjectId(guid, entity.getTypeName(), Collections.singletonMap(ATTR_QUALIFIED_NAME, qualifiedName)); entityMap.put(idWithGuid, entity); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities); final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities); logger.debug("Updated DataSet entities mutation response={}", mutationResponse);
public AtlasEntity.AtlasEntityWithExtInfo searchEntityDef(AtlasObjectId id) throws AtlasServiceException { final String guid = id.getGuid(); if (!StringUtils.isEmpty(guid)) { return atlasClient.getEntityByGuid(guid); } final Map<String, String> attributes = new HashMap<>(); id.getUniqueAttributes().entrySet().stream().filter(entry -> entry.getValue() != null) .forEach(entry -> attributes.put(entry.getKey(), entry.getValue().toString())); return atlasClient.getEntityByAttribute(id.getTypeName(), attributes); }
public void setExEntity(AtlasEntity exEntity) { this.exEntity = exEntity; this.setAtlasGuid(exEntity.getGuid()); }
@SuppressWarnings("unchecked") private List<AtlasObjectId> toAtlasObjectIds(Object _references) { if (_references == null) { return Collections.emptyList(); } List<Map<String, Object>> references = (List<Map<String, Object>>) _references; return references.stream() .map(ref -> new AtlasObjectId(toStr(ref.get(ATTR_GUID)), toStr(ref.get(ATTR_TYPENAME)), ref)) .collect(Collectors.toList()); }
public Map<EntityChangeType, List<AtlasEntity>> getChangedDataSetEntities() { final Map<EntityChangeType, List<AtlasEntity>> changedEntities = Stream .of(rootInputPortEntities.values().stream(), rootOutputPortEntities.values().stream(), queues.values().stream()) .flatMap(Function.identity()) .collect(Collectors.groupingBy(entity -> getEntityChangeType(entity.getGuid()))); updateAudit.add("CREATED DataSet entities=" + changedEntities.get(EntityChangeType.CREATED)); updateAudit.add("UPDATED DataSet entities=" + changedEntities.get(EntityChangeType.UPDATED)); updateAudit.add("DELETED DataSet entities=" + changedEntities.get(EntityChangeType.DELETED)); return changedEntities; }