@Override public AtlasClientV2 createClient(String[] baseUrls) { return new AtlasClientV2(baseUrls, new String[]{user, password}); } }
private AtlasTypesDef getTypeDefs(String ... typeNames) throws AtlasServiceException { final AtlasTypesDef typeDefs = new AtlasTypesDef(); for (int i = 0; i < typeNames.length; i++) { final MultivaluedMap<String, String> searchParams = new MultivaluedMapImpl(); searchParams.add(SearchFilter.PARAM_NAME, typeNames[i]); final AtlasTypesDef typeDef = atlasClient.getAllTypeDefs(new SearchFilter(searchParams)); typeDefs.getEntityDefs().addAll(typeDef.getEntityDefs()); } logger.debug("typeDefs={}", typeDefs); return typeDefs; }
public AtlasEntity.AtlasEntityWithExtInfo searchEntityDef(AtlasObjectId id) throws AtlasServiceException { final String guid = id.getGuid(); if (!StringUtils.isEmpty(guid)) { return atlasClient.getEntityByGuid(guid); } final Map<String, String> attributes = new HashMap<>(); id.getUniqueAttributes().entrySet().stream().filter(entry -> entry.getValue() != null) .forEach(entry -> attributes.put(entry.getKey(), entry.getValue().toString())); return atlasClient.getEntityByAttribute(id.getTypeName(), attributes); }
final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Created FlowPath entities mutation response={}", mutationResponse); final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments(); createdEntities.forEach(entity -> { final String guid = entity.getGuid(); entity.setGuid(guidAssignments.get(guid)); final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME))); final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId); path.setExEntity(entity); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities); final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities); logger.debug("Updated FlowPath entities mutation response={}", mutationResponse); updatedEntities.forEach(entity -> { final String pathId = getComponentIdFromQualifiedName(toStr(entity.getAttribute(ATTR_QUALIFIED_NAME))); final NiFiFlowPath path = nifiFlow.getFlowPaths().get(pathId); path.setExEntity(entity); return changedEntities.entrySet().stream() .filter(entry -> !DELETED.equals(entry.getKey())).flatMap(entry -> entry.getValue().stream()) .map(path -> new AtlasObjectId(path.getGuid(), TYPE_NIFI_FLOW_PATH, Collections.singletonMap(ATTR_QUALIFIED_NAME, path.getAttribute(ATTR_QUALIFIED_NAME)))) .collect(Collectors.toSet());
final Map<Boolean, List<HookNotification.HookNotificationMessage>> createAndOthers = messages.stream().collect(groupingBy(msg -> ENTITY_CREATE.equals(msg.getType()))); .flatMap(msg -> ((HookNotification.EntityCreateRequest) msg).getEntities().stream()) .collect(groupingBy(ref -> TYPE_NIFI_FLOW_PATH.equals(ref.typeName))); .collect(toMap(ref -> ref.get(ATTR_QUALIFIED_NAME), ref -> ref, (r1, r2) -> { newEntities.addAll(newFlowPaths); if (!newEntities.isEmpty()) { notifier.accept(Collections.singletonList(new HookNotification.EntityCreateRequest(NIFI_USER, newEntities))); -> ENTITY_PARTIAL_UPDATE.equals(msg.getType()) && TYPE_NIFI_FLOW_PATH.equals(((HookNotification.EntityPartialUpdateRequest)msg).getTypeName()) && ATTR_QUALIFIED_NAME.equals(((HookNotification.EntityPartialUpdateRequest)msg).getAttribute()) )); = atlasClient.searchEntityDef(new AtlasObjectId(TYPE_NIFI_FLOW_PATH, ATTR_QUALIFIED_NAME, flowPathQualifiedName)); final AtlasEntity flowPathEntity = flowPathExt.getEntity(); flowPathGuid = flowPathEntity.getGuid(); distinctInputs = toReferenceables(flowPathEntity.getAttribute(ATTR_INPUTS), metrics); distinctOutputs = toReferenceables(flowPathEntity.getAttribute(ATTR_OUTPUTS), metrics); if (ClientResponse.Status.NOT_FOUND.equals(e.getStatus())) { logger.debug("nifi_flow_path was not found for qualifiedName {}", flowPathQualifiedName); } else { fromReferenceable(msg.getEntity().get(ATTR_INPUTS), metrics) .entrySet().stream().filter(ref -> !distinctInputs.containsKey(ref.getKey())) .forEach(ref -> distinctInputs.put(ref.getKey(), ref.getValue()));
private AtlasEntity registerNiFiFlowEntity(final NiFiFlow nifiFlow) throws AtlasServiceException { final List<AtlasEntity> entities = new ArrayList<>(); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(entities); if (!nifiFlow.isMetadataUpdated()) { // Nothing has been changed, return existing entity. return nifiFlow.getExEntity(); } // Create parent flow entity using existing NiFiFlow entity if available, so that common properties are taken over. final AtlasEntity flowEntity = nifiFlow.getExEntity() != null ? new AtlasEntity(nifiFlow.getExEntity()) : new AtlasEntity(); flowEntity.setTypeName(TYPE_NIFI_FLOW); flowEntity.setVersion(1L); flowEntity.setAttribute(ATTR_NAME, nifiFlow.getFlowName()); flowEntity.setAttribute(ATTR_QUALIFIED_NAME, nifiFlow.toQualifiedName(nifiFlow.getRootProcessGroupId())); flowEntity.setAttribute(ATTR_URL, nifiFlow.getUrl()); flowEntity.setAttribute(ATTR_DESCRIPTION, nifiFlow.getDescription()); // If flowEntity is not persisted yet, then store nifi_flow entity to make nifiFlowId available for other entities. if (flowEntity.getGuid().startsWith("-")) { entities.add(flowEntity); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Registered a new nifi_flow entity, mutation response={}", mutationResponse); final String assignedNiFiFlowGuid = mutationResponse.getGuidAssignments().get(flowEntity.getGuid()); flowEntity.setGuid(assignedNiFiFlowGuid); nifiFlow.setAtlasGuid(assignedNiFiFlowGuid); } return flowEntity; }
flowEntity.setAttribute(ATTR_FLOW_PATHS, remainingPathIds); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_QUEUES, updatedDataSetEntities.get(TYPE_NIFI_QUEUE)); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_INPUT_PORTS, updatedDataSetEntities.get(TYPE_NIFI_INPUT_PORT)); shouldUpdateNiFiFlow = true; flowEntity.setAttribute(ATTR_OUTPUT_PORTS, updatedDataSetEntities.get(TYPE_NIFI_OUTPUT_PORT)); shouldUpdateNiFiFlow = true; final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(entities); entities.add(flowEntity); try { final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("mutation response={}", mutationResponse); } catch (AtlasServiceException e) { if (e.getStatus().getStatusCode() == AtlasErrorCode.INSTANCE_NOT_FOUND.getHttpCode().getStatusCode() && e.getMessage().contains(AtlasErrorCode.INSTANCE_NOT_FOUND.getErrorCode())) {
try { metrics.dataSetSearched++; final AtlasEntity.AtlasEntityWithExtInfo refExt = atlasClient.searchEntityDef(new AtlasObjectId(guid, typeName)); final String qualifiedName = (String) refExt.getEntity().getAttribute(ATTR_QUALIFIED_NAME); typedQualifiedNameToRef.put(toTypedQualifiedName(typeName, qualifiedName), new Referenceable(guid, typeName, Collections.EMPTY_MAP)); return qualifiedName; } catch (AtlasServiceException e) { if (ClientResponse.Status.NOT_FOUND.equals(e.getStatus())) { logger.warn("{} entity was not found for guid {}", typeName, guid); } else {
final Map<String, AtlasEntityDef> existingDefs = getTypeDefs(typeNames.toArray(new String[typeNames.size()])).getEntityDefs().stream() .collect(Collectors.toMap(AtlasEntityDef::getName, Function.identity())); final AtlasTypesDef type = new AtlasTypesDef(); final NiFiTypes.EntityDefinition def = ENTITIES.get(typeName); final AtlasEntityDef entity = new AtlasEntityDef(); type.getEntityDefs().add(entity); entity.setName(typeName); entity.setSuperTypes(superTypes); entity.setAttributeDefs(attributes); }); ? atlasClient.updateAtlasTypeDefs(type) : atlasClient.createAtlasTypeDefs(type); logger.debug("Result={}", atlasTypeDefsResult);
@Override public StructType defineStructType(String name, boolean errorIfExists, AttributeDefinition... attrDefs) throws AtlasException { throw new AtlasException("Internal Error: define type called on TransientTypeSystem"); }
/** * List traits for a given entity identified by its GUID * @param guid GUID of the entity * @return List<String> - traitnames associated with entity * @throws AtlasServiceException */ public List<String> listTraits(final String guid) throws AtlasServiceException { JSONObject jsonResponse = callAPIWithBodyAndParams(API.LIST_TRAITS, null, guid, URI_TRAITS); return extractResults(jsonResponse, AtlasClient.RESULTS, new ExtractOperation<String, String>()); }
/** * Returns all type names in the system * @return list of type names * @throws AtlasServiceException */ public List<String> listTypes() throws AtlasServiceException { final JSONObject jsonObject = callAPIWithQueryParams(API.LIST_TYPES, null); return extractResults(jsonObject, AtlasClient.RESULTS, new ExtractOperation<String, String>()); }
if (ClientResponse.Status.NOT_FOUND.equals(e.getStatus())){ getLogger().debug("Existing flow was not found for {}@{}", new Object[]{rootProcessGroup.getId(), clusterName}); } else {
public AtlasBaseException(AtlasErrorCode errorCode, Throwable cause, boolean enableSuppression, boolean writableStackTrace, String ... params) { super(errorCode.getFormattedErrorMessage(params), cause, enableSuppression, writableStackTrace); this.atlasErrorCode = AtlasErrorCode.INTERNAL_ERROR; }
/** * Create a new Atlas client. * @param ugi UserGroupInformation * @param doAsUser * @param baseUrls A list of URLs that point to an ensemble of Atlas servers working in * High Availability mode. The client will automatically determine the * active instance on startup and also when there is a scenario of * failover. */ public AtlasClient(UserGroupInformation ugi, String doAsUser, String... baseUrls) { initializeState(baseUrls, ugi, doAsUser); }
public boolean typeWithNameExists(String name) { try { callAPI(API_V2.GET_TYPEDEF_BY_NAME, String.class, null, name); } catch (AtlasServiceException e) { return false; } return true; }
/** * This is an utility method to delete unused types. * Should be used during development or testing only. * @param typeNames to delete */ void deleteTypeDefs(String ... typeNames) throws AtlasServiceException { final AtlasTypesDef existingTypeDef = getTypeDefs(typeNames); try { atlasClient.deleteAtlasTypeDefs(existingTypeDef); } catch (UniformInterfaceException e) { if (e.getResponse().getStatus() == 204) { // 204 is a successful response. // NOTE: However after executing this, Atlas should be restarted to work properly. logger.info("Deleted type defs: {}", existingTypeDef); } else { throw e; } } }
ApplicationProperties.forceReload();
final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(createdEntities); final EntityMutationResponse mutationResponse = atlasClient.createEntities(atlasEntities); logger.debug("Created DataSet entities mutation response={}", mutationResponse); final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments(); for (AtlasEntity entity : createdEntities) { final String guid = guidAssignments.get(entity.getGuid()); final String qualifiedName = toStr(entity.getAttribute(ATTR_QUALIFIED_NAME)); if (StringUtils.isEmpty(guid)) { logger.warn("GUID was not assigned for {}::{} for some reason.", entity.getTypeName(), qualifiedName); continue; switch (entity.getTypeName()) { case TYPE_NIFI_INPUT_PORT: entityMap = nifiFlow.getRootInputPortEntities(); break; default: throw new RuntimeException(entity.getTypeName() + " is not expected."); entity.setGuid(guid); final AtlasObjectId idWithGuid = new AtlasObjectId(guid, entity.getTypeName(), Collections.singletonMap(ATTR_QUALIFIED_NAME, qualifiedName)); entityMap.put(idWithGuid, entity); final AtlasEntity.AtlasEntitiesWithExtInfo atlasEntities = new AtlasEntity.AtlasEntitiesWithExtInfo(updatedEntities); final EntityMutationResponse mutationResponse = atlasClient.updateEntities(atlasEntities); logger.debug("Updated DataSet entities mutation response={}", mutationResponse);
@Override public AtlasClientV2 createClient(String[] baseUrls) { final Configuration hadoopConf = new Configuration(); hadoopConf.set("hadoop.security.authentication", "kerberos"); UserGroupInformation.setConfiguration(hadoopConf); final UserGroupInformation ugi; try { UserGroupInformation.loginUserFromKeytab(principal, keytab); ugi = UserGroupInformation.getCurrentUser(); } catch (IOException e) { throw new RuntimeException("Failed to login with Kerberos due to: " + e, e); } return new AtlasClientV2(ugi, null, baseUrls); } }