public Tuple<AtlasObjectId, AtlasEntity> getOrCreateQueue(String destinationComponentId) { final String qualifiedName = toQualifiedName(destinationComponentId); final Optional<AtlasObjectId> existingQueueId = findIdByQualifiedName(queues.keySet(), qualifiedName); if (existingQueueId.isPresent()) { final AtlasEntity entity = queues.get(existingQueueId.get()); stillExistingEntityGuids.add(entity.getGuid()); return new Tuple<>(existingQueueId.get(), entity); } else { final AtlasObjectId queueId = new AtlasObjectId(TYPE_NIFI_QUEUE, ATTR_QUALIFIED_NAME, qualifiedName); final AtlasEntity queue = new AtlasEntity(TYPE_NIFI_QUEUE); queue.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); queue.setAttribute(ATTR_QUALIFIED_NAME, qualifiedName); queue.setAttribute(ATTR_NAME, "queue"); queue.setAttribute(ATTR_DESCRIPTION, "Input queue for " + destinationComponentId); queues.put(queueId, queue); return new Tuple<>(queueId, queue); } }
private AtlasEntity createOrUpdateRootGroupPortEntity(boolean isInput, String qualifiedName, String portName) { final Map<AtlasObjectId, AtlasEntity> ports = isInput ? rootInputPortEntities : rootOutputPortEntities; final Optional<AtlasObjectId> existingPortId = findIdByQualifiedName(ports.keySet(), qualifiedName); final String typeName = isInput ? TYPE_NIFI_INPUT_PORT : TYPE_NIFI_OUTPUT_PORT; if (existingPortId.isPresent()) { final AtlasEntity entity = ports.get(existingPortId.get()); final String portGuid = entity.getGuid(); stillExistingEntityGuids.add(portGuid); final Object currentName = entity.getAttribute(ATTR_NAME); if (isUpdated(currentName, portName)) { // Update port name and set updated flag. entity.setAttribute(ATTR_NAME, portName); updatedEntityGuids.add(portGuid); updateAudit.add(String.format("Name of %s %s changed from %s to %s", entity.getTypeName(), portGuid, currentName, portName)); } return entity; } else { final AtlasEntity entity = new AtlasEntity(typeName); entity.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); entity.setAttribute(ATTR_NAME, portName); entity.setAttribute(ATTR_QUALIFIED_NAME, qualifiedName); final AtlasObjectId portId = new AtlasObjectId(typeName, ATTR_QUALIFIED_NAME, qualifiedName); ports.put(portId, entity); return entity; } }
/** * Set input or output DataSet ids for a NiFiFlowPath. * The updated ids only containing active ids. * @return True if there is any changed IO reference (create, update, delete). */ private boolean setChangedIOIds(NiFiFlowPath path, AtlasEntity pathEntity, boolean isInput) { Set<AtlasObjectId> ids = isInput ? path.getInputs() : path.getOutputs(); String targetAttribute = isInput ? ATTR_INPUTS : ATTR_OUTPUTS; final Map<EntityChangeType, List<AtlasObjectId>> changedIOIds = ids.stream().collect(Collectors.groupingBy(this::getFlowPathIOChangeType)); // Remove DELETED references. final Set<AtlasObjectId> remainingFlowPathIOIds = toRemainingFlowPathIOIds(changedIOIds); // If references are changed, update it. if (path.isDataSetReferenceChanged(remainingFlowPathIOIds, isInput)) { pathEntity.setAttribute(targetAttribute, remainingFlowPathIOIds); return true; } return false; }
flowEntity.setAttribute(ATTR_FLOW_PATHS, remainingPathIds); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_QUEUES, updatedDataSetEntities.get(TYPE_NIFI_QUEUE)); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_INPUT_PORTS, updatedDataSetEntities.get(TYPE_NIFI_INPUT_PORT)); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_OUTPUT_PORTS, updatedDataSetEntities.get(TYPE_NIFI_OUTPUT_PORT)); shouldUpdateNiFiFlow = true;
private Tuple<EntityChangeType, AtlasEntity> toAtlasEntity(EntityChangeType changeType, final NiFiFlowPath path) { final AtlasEntity entity = EntityChangeType.CREATED.equals(changeType) ? new AtlasEntity() : new AtlasEntity(path.getExEntity()); entity.setTypeName(TYPE_NIFI_FLOW_PATH); entity.setVersion(1L); entity.setAttribute(ATTR_NIFI_FLOW, getAtlasObjectId()); final StringBuilder name = new StringBuilder(); final StringBuilder description = new StringBuilder(); path.getProcessComponentIds().forEach(pid -> { final String componentName = getProcessComponentName(pid); if (name.length() > 0) { name.append(", "); description.append(", "); } name.append(componentName); description.append(String.format("%s::%s", componentName, pid)); }); path.setName(name.toString()); entity.setAttribute(ATTR_NAME, name.toString()); entity.setAttribute(ATTR_DESCRIPTION, description.toString()); // Use first processor's id as qualifiedName. entity.setAttribute(ATTR_QUALIFIED_NAME, toQualifiedName(path.getId())); entity.setAttribute(ATTR_URL, path.createDeepLinkURL(getUrl())); final boolean inputsChanged = setChangedIOIds(path, entity, true); final boolean outputsChanged = setChangedIOIds(path, entity, false); // Even iff there's no flow path metadata changed, if any IO is changed then the pass should be updated. EntityChangeType finalChangeType = EntityChangeType.AS_IS.equals(changeType) ? (path.isMetadataUpdated() || inputsChanged || outputsChanged ? EntityChangeType.UPDATED : EntityChangeType.AS_IS) : changeType; return new Tuple<>(finalChangeType, entity); }
private AtlasEntity registerNiFiFlowEntity(final NiFiFlow nifiFlow) throws AtlasServiceException { final List<AtlasEntity> entities = new ArrayList<>(); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(entities); if (!nifiFlow.isMetadataUpdated()) { // Nothing has been changed, return existing entity. return nifiFlow.getExEntity(); } // Create parent flow entity using existing NiFiFlow entity if available, so that common properties are taken over. final AtlasEntity flowEntity = nifiFlow.getExEntity() != null ? new AtlasEntity(nifiFlow.getExEntity()) : new AtlasEntity(); flowEntity.setTypeName(TYPE_NIFI_FLOW); flowEntity.setVersion(1L); flowEntity.setAttribute(ATTR_NAME, nifiFlow.getFlowName()); flowEntity.setAttribute(ATTR_QUALIFIED_NAME, nifiFlow.toQualifiedName(nifiFlow.getRootProcessGroupId())); flowEntity.setAttribute(ATTR_URL, nifiFlow.getUrl()); flowEntity.setAttribute(ATTR_DESCRIPTION, nifiFlow.getDescription()); // If flowEntity is not persisted yet, then store nifi_flow entity to make nifiFlowId available for other entities. if (flowEntity.getGuid().startsWith("-")) { entities.add(flowEntity); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Registered a new nifi_flow entity, mutation response={}", mutationResponse); final String assignedNiFiFlowGuid = mutationResponse.getGuidAssignments().get(flowEntity.getGuid()); flowEntity.setGuid(assignedNiFiFlowGuid); nifiFlow.setAtlasGuid(assignedNiFiFlowGuid); } return flowEntity; }
private AtlasEntity createHDFSPathEntity(AtlasEntityType hdfsPathEntityType, String path, String clusterName) { AtlasEntity entity = hdfsPathEntityType.createDefaultValue(); entity.setAttribute(HDFS_PATH_ATTRIBUTE_QUALIFIED_NAME, getQualifiedName(path, clusterName)); entity.setAttribute(HDFS_PATH_ATTRIBUTE_NAME_PATH, path); entity.setAttribute(HDFS_PATH_ATTRIBUTE_NAME_NAME, path); entity.setAttribute(HDFS_PATH_ATTRIBUTE_NAME_CLUSTER_NAME, clusterName); return entity; }
@Override public void transformComplete() { if (isCustomAttributeUpdated) { entity.setAttribute(CLUSTER_NAME_ATTRIBUTE, clusterName); entity.setAttribute(NAME_ATTRIBUTE, name); entity.setAttribute(PATH_ATTRIBUTE, toPath()); entity.setAttribute(QUALIFIED_NAME_ATTRIBUTE, toQualifiedName()); } }
private void updateEntityMapAttributes(AtlasEntity attrEntity, Map<String, String> map1, Map<String, Integer> map2, Map<String, Boolean> map3, Map<String, Float> map4, Map<String, Date> map5) { attrEntity.setAttribute("mapAttr1", map1); attrEntity.setAttribute("mapAttr2", map2); attrEntity.setAttribute("mapAttr3", map3); attrEntity.setAttribute("mapAttr4", map4); attrEntity.setAttribute("mapAttr5", map5); }
private void updateEntityMapAttributes(AtlasEntity attrEntity, Map<String, String> map1, Map<String, Integer> map2, Map<String, Boolean> map3, Map<String, Float> map4, Map<String, Date> map5) { attrEntity.setAttribute("mapAttr1", map1); attrEntity.setAttribute("mapAttr2", map2); attrEntity.setAttribute("mapAttr3", map3); attrEntity.setAttribute("mapAttr4", map4); attrEntity.setAttribute("mapAttr5", map5); }
@Override public AtlasEntity toEntity(AtlasServer obj) { AtlasEntity entity = getDefaultAtlasEntity(obj); entity.setAttribute(PROPERTY_NAME, obj.getName()); entity.setAttribute(PROPERTY_DISPLAY_NAME, obj.getDisplayName()); entity.setAttribute(PROPERTY_FULL_NAME, obj.getFullName()); entity.setAttribute(PROPERTY_ADDITIONAL_INFO, obj.getAdditionalInfo()); entity.setAttribute(PROPERTY_URLS, obj.getUrls()); return entity; }
private AtlasEntity getHiveDbEntity(String clusterName, String dbName) { AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_DATABASE); entity.setAttribute("name", dbName); entity.setAttribute("qualifiedName", dbName + "@" + clusterName); entity.setAttribute("location", "hdfs://localhost.localdomain:8020/warehouse/tablespace/managed/hive/" + dbName + ".db"); entity.setAttribute("clusterName", clusterName); entity.setAttribute("owner", "hive"); entity.setAttribute("ownerType", "USER"); return entity; }
protected AtlasEntity storageDescriptor(String location, String inputFormat, String outputFormat, boolean compressed, List<AtlasEntity> columns) { AtlasEntity storageDescriptor = new AtlasEntity(STORAGE_DESC_TYPE); storageDescriptor.setAttribute("location", location); storageDescriptor.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + location); storageDescriptor.setAttribute("inputFormat", inputFormat); storageDescriptor.setAttribute("outputFormat", outputFormat); storageDescriptor.setAttribute("compressed", compressed); storageDescriptor.setAttribute("cols", getAtlasObjectIds(columns)); return storageDescriptor; }
private AtlasEntity toHiveTableEntity(AtlasEntity entHiveDb, String tableName) { AtlasEntity entHiveTable = new AtlasEntity(HiveDataTypes.HIVE_TABLE.getName()); String qualifiedName = HiveMetaStoreBridge.getTableQualifiedName((String)entHiveDb.getAttribute(AtlasConstants.CLUSTER_NAME_ATTRIBUTE), (String)entHiveDb.getAttribute(AtlasClient.NAME), tableName); entHiveTable.setAttribute(AtlasClient.NAME, tableName.toLowerCase()); entHiveTable.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, qualifiedName); entHiveTable.setAttribute(ATTRIBUTE_DB, AtlasTypeUtil.getAtlasObjectId(entHiveDb)); return entHiveTable; }
AtlasEntity database(String name, String description, String owner, String locationUri, String... traitNames) { AtlasEntity database = new AtlasEntity(DATABASE_TYPE); database.setAttribute("name", name); database.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + name); database.setAttribute("description", description); database.setAttribute("owner", owner); database.setAttribute("locationUri", locationUri); database.setAttribute("createTime", System.currentTimeMillis()); database.setAttribute("clusterName", "cl1"); database.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList())); return database; }
protected AtlasEntity column(String name, String dataType, String comment, String... traitNames) { AtlasEntity column = new AtlasEntity(COLUMN_TYPE); column.setAttribute("name", name); column.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, "qualified:" + name); column.setAttribute("type", dataType); column.setAttribute("comment", comment); column.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList())); return column; }
public static AtlasEntity createProcessEntity(List<AtlasObjectId> inputs, List<AtlasObjectId> outputs) { AtlasEntity entity = new AtlasEntity(PROCESS_TYPE); entity.setAttribute(NAME, RandomStringUtils.randomAlphanumeric(10)); entity.setAttribute("inputs", inputs); entity.setAttribute("outputs", outputs); return entity; }
public void setAttribute(EntityAttribute attribute, String attributeValue) { if (attribute.appliesToEntityType(entity.getTypeName())) { entity.setAttribute(attribute.getAttributeName(), attributeValue); } }
private AtlasEntity getDefaultTableEntity(String name) throws AtlasBaseException { AtlasEntityType type = (AtlasEntityType) typeRegistry.getType(TYPE_RDBMS_TABLES); AtlasEntity ret = type.createDefaultValue(); ret.setAttribute("name", name); return ret; }
@Test(expectedExceptions = AtlasBaseException.class) public void testCreateRequiredAttrNull() throws Exception { //Update required attribute Map<String, AtlasEntity> tableCloneMap = new HashMap<>(); AtlasEntity tableEntity = new AtlasEntity(TABLE_TYPE); tableEntity.setAttribute(TestUtilsV2.NAME, "table_" + TestUtilsV2.randomString()); tableCloneMap.put(tableEntity.getGuid(), tableEntity); entityStore.createOrUpdate(new InMemoryMapEntityStream(tableCloneMap), false); Assert.fail("Expected exception while creating with required attribute null"); }