private AtlasEntity createOrUpdateRootGroupPortEntity(boolean isInput, String qualifiedName, String portName) { final Map<AtlasObjectId, AtlasEntity> ports = isInput ? rootInputPortEntities : rootOutputPortEntities; final Optional<AtlasObjectId> existingPortId = findIdByQualifiedName(ports.keySet(), qualifiedName); final String typeName = isInput ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT; if (existingPortId.isPresent()) { final AtlasEntity entity = ports.get(existingPortId.get()); final String portGuid = entity.getGuid(); stillExistingEntityGuids.add(portGuid); final Object currentName = entity.getAttribute(ATTR_NAME); if (isUpdated(currentName, portName)) { // Update port name and set updated flag. entity.setAttribute(ATTR_NAME, portName); updatedEntityGuids.add(portGuid); updateAudit.add(String.format("Name of %s %s changed from %s to %s", entity.getTypeName(), portGuid, currentName, portName)); } return entity; } else { final AtlasEntity entity = new AtlasEntity(typeName); entity.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); entity.setAttribute(ATTR_NAME, portName); entity.setAttribute(ATTR_QUALIFIED_NAME, qualifiedName); final AtlasObjectId portId = new AtlasObjectId(typeName, ATTR_QUALIFIED_NAME, qualifiedName); ports.put(portId, entity); return entity; } }
final Map<String, Object> attributes = nifiFlowEntity.getAttributes(); final NiFiFlow nifiFlow = new NiFiFlow(rootProcessGroupId); nifiFlow.setExEntity(nifiFlowEntity); nifiFlow.setDescription(toStr(attributes.get(ATTR_DESCRIPTION))); nifiFlow.getQueues().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_QUEUES)))); nifiFlow.getRootInputPortEntities().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_INPUT_PORTS)))); nifiFlow.getRootOutputPortEntities().putAll(toQualifiedNameIds(toAtlasObjectIds(nifiFlowEntity.getAttribute(ATTR_OUTPUT_PORTS)))); final String pathQualifiedName = toStr(flowPathEntity.getAttribute(ATTR_QUALIFIED_NAME)); final NiFiFlowPath flowPath = new NiFiFlowPath(getComponentIdFromQualifiedName(pathQualifiedName)); if (flowPathEntity.hasAttribute(ATTR_URL)) { final Matcher urlMatcher = FLOW_PATH_URL_PATTERN.matcher(toStr(flowPathEntity.getAttribute(ATTR_URL))); if (urlMatcher.matches()) { flowPath.setGroupId(urlMatcher.group(1)); flowPath.setName(toStr(flowPathEntity.getAttribute(ATTR_NAME))); flowPath.getInputs().addAll(toQualifiedNameIds(toAtlasObjectIds(flowPathEntity.getAttribute(ATTR_INPUTS))).keySet()); flowPath.getOutputs().addAll(toQualifiedNameIds(toAtlasObjectIds(flowPathEntity.getAttribute(ATTR_OUTPUTS))).keySet()); flowPath.startTrackingChanges(nifiFlow);
private AtlasEntity registerNiFiFlowEntity(final NiFiFlow nifiFlow) throws AtlasServiceException { final List<AtlasEntity> entities = new ArrayList<>(); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(entities); if (!nifiFlow.isMetadataUpdated()) { // Nothing has been changed, return existing entity. return nifiFlow.getExEntity(); } // Create parent flow entity using existing NiFiFlow entity if available, so that common properties are taken over. final AtlasEntity flowEntity = nifiFlow.getExEntity() != null ? new AtlasEntity(nifiFlow.getExEntity()) : new AtlasEntity(); flowEntity.setTypeName(TYPE_NIFI_FLOW); flowEntity.setVersion(1L); flowEntity.setAttribute(ATTR_NAME, nifiFlow.getFlowName()); flowEntity.setAttribute(ATTR_QUALIFIED_NAME, nifiFlow.toQualifiedName(nifiFlow.getRootProcessGroupId())); flowEntity.setAttribute(ATTR_URL, nifiFlow.getUrl()); flowEntity.setAttribute(ATTR_DESCRIPTION, nifiFlow.getDescription()); // If flowEntity is not persisted yet, then store nifi_flow entity to make nifiFlowId available for other entities. if (flowEntity.getGuid().startsWith("-")) { entities.add(flowEntity); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Registered a new nifi_flow entity, mutation response={}", mutationResponse); final String assignedNiFiFlowGuid = mutationResponse.getGuidAssignments().get(flowEntity.getGuid()); flowEntity.setGuid(assignedNiFiFlowGuid); nifiFlow.setAtlasGuid(assignedNiFiFlowGuid); } return flowEntity; }
public Tuple<AtlasObjectId, AtlasEntity> getOrCreateQueue(String destinationComponentId) { final String qualifiedName = toQualifiedName(destinationComponentId); final Optional<AtlasObjectId> existingQueueId = findIdByQualifiedName(queues.keySet(), qualifiedName); if (existingQueueId.isPresent()) { final AtlasEntity entity = queues.get(existingQueueId.get()); stillExistingEntityGuids.add(entity.getGuid()); return new Tuple<>(existingQueueId.get(), entity); } else { final AtlasObjectId queueId = new AtlasObjectId(TYPE_NIFI_QUEUE, ATTR_QUALIFIED_NAME, qualifiedName); final AtlasEntity queue = new AtlasEntity(TYPE_NIFI_QUEUE); queue.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); queue.setAttribute(ATTR_QUALIFIED_NAME, qualifiedName); queue.setAttribute(ATTR_NAME, "queue"); queue.setAttribute(ATTR_DESCRIPTION, "Input queue for " + destinationComponentId); queues.put(queueId, queue); return new Tuple<>(queueId, queue); } }
public AtlasEntity(AtlasEntity other) { super(other); if (other != null) { setGuid(other.getGuid()); setStatus(other.getStatus()); setCreatedBy(other.getCreatedBy()); setUpdatedBy(other.getUpdatedBy()); setCreateTime(other.getCreateTime()); setUpdateTime(other.getUpdateTime()); setVersion(other.getVersion()); setClassifications(other.getClassifications()); } }
public static AtlasObjectId getObjectId(AtlasEntity entity) { String qualifiedName = (String) entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME); AtlasObjectId ret = new AtlasObjectId(entity.getGuid(), entity.getTypeName(), Collections.singletonMap(ATTRIBUTE_QUALIFIED_NAME, qualifiedName)); return ret; }
AtlasEntity createStorageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed) throws Exception { AtlasEntity entity = new AtlasEntity(STORAGE_DESC_TYPE); entity.setAttribute("location", location); entity.setAttribute("inputFormat", inputFormat); entity.setAttribute("outputFormat", outputFormat); entity.setAttribute("compressed", compressed); return createInstance(entity, null); }
@Test public void testSetObjectIdAttrToNull() throws Exception { final AtlasEntity dbEntity = TestUtilsV2.createDBEntity(); final AtlasEntity db2Entity = TestUtilsV2.createDBEntity(); entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); entityStore.createOrUpdate(new AtlasEntityStream(db2Entity), false); final AtlasEntity tableEntity = TestUtilsV2.createTableEntity(dbEntity); tableEntity.setAttribute("databaseComposite", AtlasTypeUtil.getAtlasObjectId(db2Entity)); final EntityMutationResponse tblCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(tableEntity), false); final AtlasEntityHeader createdTblHeader = tblCreationResponse.getCreatedEntityByTypeNameAndAttribute(TABLE_TYPE, NAME, (String) tableEntity.getAttribute(NAME)); final AtlasEntity createdTblEntity = getEntityFromStore(createdTblHeader); init(); createdTblEntity.setAttribute("databaseComposite", null); final EntityMutationResponse tblUpdateResponse = entityStore.createOrUpdate(new AtlasEntityStream(createdTblEntity), true); final AtlasEntityHeader updatedTblHeader = tblUpdateResponse.getFirstEntityPartialUpdated(); final AtlasEntity updatedTblEntity = getEntityFromStore(updatedTblHeader); final AtlasEntity deletedDb2Entity = getEntityFromStore(db2Entity.getGuid()); assertEquals(deletedDb2Entity.getStatus(), AtlasEntity.Status.DELETED); }
AtlasEntity createColumn(String name, String dataType, String comment, String... traitNames) throws Exception { AtlasEntity entity = new AtlasEntity(COLUMN_TYPE); entity.setClassifications(toAtlasClassifications(traitNames)); entity.setAttribute("name", name); entity.setAttribute("dataType", dataType); entity.setAttribute("comment", comment); return createInstance(entity, traitNames); }
for (AtlasEntity entity : createdEntities) { final String guid = guidAssignments.get(entity.getGuid()); final String qualifiedName = toStr(entity.getAttribute(ATTR_QUALIFIED_NAME)); if (StringUtils.isEmpty(guid)) { logger.warn("GUID was not assigned for {}::{} for some reason.", entity.getTypeName(), qualifiedName); continue; switch (entity.getTypeName()) { case TYPE_NIFI_INPUT_PORT: entityMap = nifiFlow.getRootInputPortEntities(); break; default: throw new RuntimeException(entity.getTypeName() + " is not expected."); entity.setGuid(guid); final AtlasObjectId idWithGuid = new AtlasObjectId(guid, entity.getTypeName(), Collections.singletonMap(ATTR_QUALIFIED_NAME, qualifiedName)); entityMap.put(idWithGuid, entity); .filter(entry -> !DELETED.equals(entry.getKey())) .flatMap(entry -> entry.getValue().stream()) .filter(entity -> changedTypeNames.contains(entity.getTypeName())) .collect(Collectors.groupingBy(AtlasEntity::getTypeName));
public AtlasEntityHeader(AtlasEntity entity) { super(entity.getTypeName(), entity.getAttributes()); setGuid(entity.getGuid()); setClassifications(entity.getClassifications()); if (CollectionUtils.isNotEmpty(entity.getClassifications())) { this.classificationNames = new ArrayList<>(entity.getClassifications().size()); for (AtlasClassification classification : entity.getClassifications()) { this.classificationNames.add(classification.getTypeName()); } } }
private void addReferredEntities(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, int maxEntries) { for (int i = 1; i <= maxEntries; i++) { AtlasEntity entity1 = new AtlasEntity(); entity1.setGuid(String.format(knownEntityGuidFormat, i)); entityWithExtInfo.addReferredEntity(entity1); } }
private Tuple<EntityChangeType, AtlasEntity> toAtlasEntity(EntityChangeType changeType, final NiFiFlowPath path) { final AtlasEntity entity = EntityChangeType.CREATED.equals(changeType) ? new AtlasEntity() : new AtlasEntity(path.getExEntity()); entity.setTypeName(TYPE_NIFI_FLOW_PATH); entity.setVersion(1L); entity.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); final StringBuilder name = new StringBuilder(); final StringBuilder description = new StringBuilder(); path.getProcessComponentIds().forEach(pid -> { final String componentName = getProcessComponentName(pid); if (name.length() > 0) { name.append(", "); description.append(", "); } name.append(componentName); description.append(String.format("%s::%s", componentName, pid)); }); path.setName(name.toString()); entity.setAttribute(ATTR_NAME, name.toString()); entity.setAttribute(ATTR_DESCRIPTION, description.toString()); // Use first processor's id as qualifiedName. entity.setAttribute(ATTR_QUALIFIED_NAME, toQualifiedName(path.getId())); entity.setAttribute(ATTR_URL, path.createDeepLinkURL(getUrl())); final boolean inputsChanged = setChangedIOIds(path, entity, true); final boolean outputsChanged = setChangedIOIds(path, entity, false); // Even iff there's no flow path metadata changed, if any IO is changed then the pass should be updated. EntityChangeType finalChangeType = EntityChangeType.AS_IS.equals(changeType) ? (path.isMetadataUpdated() || inputsChanged || outputsChanged ? EntityChangeType.UPDATED : EntityChangeType.AS_IS) : changeType; return new Tuple<>(finalChangeType, entity); }
private void verifyExportForHrDataForConnected(ZipSource zipSource) throws IOException, AtlasBaseException { assertNotNull(zipSource.getCreationOrder()); assertTrue(zipSource.getCreationOrder().size() == 2); assertTrue(zipSource.hasNext()); AtlasEntity entity = zipSource.next(); assertNotNull(entity); assertTrue(entity.getTypeName().equals("Department")); assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE); verifyTypeDefs(zipSource); }
private AtlasEntity mapSystemAttributes(AtlasVertex entityVertex, AtlasEntity entity) { if (LOG.isDebugEnabled()) { LOG.debug("Mapping system attributes for type {}", entity.getTypeName()); } entity.setGuid(getGuid(entityVertex)); entity.setTypeName(getTypeName(entityVertex)); entity.setStatus(GraphHelper.getStatus(entityVertex)); entity.setVersion(GraphHelper.getVersion(entityVertex)); entity.setCreatedBy(GraphHelper.getCreatedByAsString(entityVertex)); entity.setUpdatedBy(GraphHelper.getModifiedByAsString(entityVertex)); entity.setCreateTime(new Date(GraphHelper.getCreatedTime(entityVertex))); entity.setUpdateTime(new Date(GraphHelper.getModifiedTime(entityVertex))); entity.setHomeId(GraphHelper.getHomeId(entityVertex)); entity.setIsProxy(GraphHelper.isProxy(entityVertex)); return entity; }
final String guid = entity.getGuid(); entity.setGuid(guidAssignments.get(guid)); final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME))); final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId); path.setExEntity(entity); logger.debug("Updated FlowPath entities mutation response={}", mutationResponse); updatedEntities.forEach(entity -> { final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME))); final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId); path.setExEntity(entity); return changedEntities.entrySet().stream() .filter(entry -> !DELETED.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()) .map(path -> new AtlasObjectId(path.getGuid(), TYPE_NIFI_FLOW_PATH, Collections.singletonMap(ATTR_QUALIFIED_NAME, path.getAttribute(ATTR_QUALIFIED_NAME)))) .collect(Collectors.toSet());
private AtlasEntity toHiveTableEntity(AtlasEntity entHiveDb, String tableName) { AtlasEntity entHiveTable = new AtlasEntity(HiveDataTypes.HIVE_TABLE.getName()); String qualifiedName = HiveMetaStoreBridge.getTableQualifiedName((String)entHiveDb.getAttribute(AtlasConstants.CLUSTER_NAME_ATTRIBUTE), (String)entHiveDb.getAttribute(AtlasClient.NAME), tableName); entHiveTable.setAttribute(AtlasClient.NAME, tableName.toLowerCase()); entHiveTable.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, qualifiedName); entHiveTable.setAttribute(ATTRIBUTE_DB, AtlasTypeUtil.getAtlasObjectId(entHiveDb)); return entHiveTable; }
private void renameColumns(List<AtlasObjectId> columns, AtlasEntityExtInfo oldEntityExtInfo, String newTableQualifiedName, List<HookNotification> notifications) { if (CollectionUtils.isNotEmpty(columns)) { for (AtlasObjectId columnId : columns) { AtlasEntity oldColumn = oldEntityExtInfo.getEntity(columnId.getGuid()); AtlasObjectId oldColumnId = new AtlasObjectId(oldColumn.getTypeName(), ATTRIBUTE_QUALIFIED_NAME, oldColumn.getAttribute(ATTRIBUTE_QUALIFIED_NAME)); AtlasEntity newColumn = new AtlasEntity(oldColumn.getTypeName(), ATTRIBUTE_QUALIFIED_NAME, getColumnQualifiedName(newTableQualifiedName, (String) oldColumn.getAttribute(ATTRIBUTE_NAME))); notifications.add(new EntityPartialUpdateRequestV2(getUserName(), oldColumnId, new AtlasEntityWithExtInfo(newColumn))); } } }
= atlasClient.searchEntityDef(new AtlasObjectId(TYPE_NIFI_FLOW_PATH, ATTR_QUALIFIED_NAME, flowPathQualifiedName)); final AtlasEntity flowPathEntity = flowPathExt.getEntity(); flowPathGuid = flowPathEntity.getGuid(); distinctInputs = toReferenceables(flowPathEntity.getAttribute(ATTR_INPUTS), metrics); distinctOutputs = toReferenceables(flowPathEntity.getAttribute(ATTR_OUTPUTS), metrics);
final AtlasEntity.AtlasEntityWithExtInfo entityExt = searchEntityDef(id); final AtlasEntity entity = entityExt.getEntity(); if (AtlasEntity.Status.DELETED.equals(entity.getStatus())) { return null; final Map<String, Object> uniqueAttrs = Collections.singletonMap(ATTR_QUALIFIED_NAME, entity.getAttribute(ATTR_QUALIFIED_NAME)); return new Tuple<>(new AtlasObjectId(id.getGuid(), id.getTypeName(), uniqueAttrs), entity); } catch (AtlasServiceException e) {