logger.debug("Created FlowPath entities mutation response={}", mutationResponse); final Map<String, String> guidAssignments = mutationResponse.getGuidAssignments(); createdEntities.forEach(entity -> { final String guid = entity.getGuid();
@VisibleForTesting AtlasEntityWithExtInfo createEntityInAtlas(AtlasEntityWithExtInfo entity) throws Exception { AtlasEntityWithExtInfo ret = null; EntityMutationResponse response = atlasClientV2.createEntity(entity); List<AtlasEntityHeader> entities = response.getCreatedEntities(); if (CollectionUtils.isNotEmpty(entities)) { AtlasEntityWithExtInfo getByGuidResponse = atlasClientV2.getEntityByGuid(entities.get(0).getGuid()); ret = getByGuidResponse; LOG.info("Created {} entity: name={}, guid={}", ret.getEntity().getTypeName(), ret.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME), ret.getEntity().getGuid()); } return ret; }
public void onEntitiesMutated(EntityMutationResponse entityMutationResponse, boolean isImport) throws AtlasBaseException { if (CollectionUtils.isEmpty(entityChangeListeners) || instanceConverter == null) { return; } List<AtlasEntityHeader> createdEntities = entityMutationResponse.getCreatedEntities(); List<AtlasEntityHeader> updatedEntities = entityMutationResponse.getUpdatedEntities(); List<AtlasEntityHeader> partiallyUpdatedEntities = entityMutationResponse.getPartialUpdatedEntities(); List<AtlasEntityHeader> deletedEntities = entityMutationResponse.getDeletedEntities(); // complete full text mapping before calling toReferenceables(), from notifyListners(), to // include all vertex updates in the current graph-transaction doFullTextMapping(createdEntities); doFullTextMapping(updatedEntities); doFullTextMapping(partiallyUpdatedEntities); notifyListeners(createdEntities, EntityOperation.CREATE, isImport); notifyListeners(updatedEntities, EntityOperation.UPDATE, isImport); notifyListeners(partiallyUpdatedEntities, EntityOperation.PARTIAL_UPDATE, isImport); notifyListeners(deletedEntities, EntityOperation.DELETE, isImport); notifyPropagatedEntities(); }
private float updateImportMetrics(AtlasEntity.AtlasEntityWithExtInfo currentEntity, EntityMutationResponse resp, AtlasImportResult importResult, Set<String> processedGuids, int currentIndex, int streamSize, float currentPercent) { updateImportMetrics("entity:%s:created", resp.getCreatedEntities(), processedGuids, importResult); updateImportMetrics("entity:%s:updated", resp.getUpdatedEntities(), processedGuids, importResult); updateImportMetrics("entity:%s:deleted", resp.getDeletedEntities(), processedGuids, importResult); String lastEntityImported = String.format("entity:last-imported:%s:[%s]:(%s)", currentEntity.getEntity().getTypeName(), currentIndex, currentEntity.getEntity().getGuid()); return updateImportProgress(LOG, currentIndex + 1, streamSize, currentPercent, lastEntityImported); }
private boolean noEntityMutation(EntityMutationResponse er) { return er == null || (CollectionUtils.isEmpty(er.getCreatedEntities()) && CollectionUtils.isEmpty(er.getUpdatedEntities())); } }
validateMutationResponse(response, EntityOperation.CREATE, 5); AtlasEntityHeader dept1 = response.getFirstCreatedEntityByTypeName(TestUtilsV2.DEPARTMENT_TYPE); validateEntity(deptEntity, getEntityFromStore(dept1), deptEntity.getEntities().get(0)); final Map<EntityOperation, List<AtlasEntityHeader>> entitiesMutated = response.getMutatedEntities(); List<AtlasEntityHeader> entitiesCreated = entitiesMutated.get(EntityOperation.CREATE); EntityMutationResponse dbCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); validateMutationResponse(dbCreationResponse, EntityOperation.CREATE, 1); dbEntityGuid = dbCreationResponse.getCreatedEntities().get(0).getGuid(); AtlasEntityHeader db1 = dbCreationResponse.getFirstCreatedEntityByTypeName(TestUtilsV2.DATABASE_TYPE); validateEntity(dbEntity, getEntityFromStore(db1)); EntityMutationResponse tableCreationResponse = entityStore.createOrUpdate(new AtlasEntityStream(tblEntity), false); validateMutationResponse(tableCreationResponse, EntityOperation.CREATE, 1); tblEntityGuid = tableCreationResponse.getCreatedEntities().get(0).getGuid(); AtlasEntityHeader tableEntity = tableCreationResponse.getFirstCreatedEntityByTypeName(TABLE_TYPE); validateEntity(tblEntity, getEntityFromStore(tableEntity)); validateMutationResponse(entityMutationResponse, EntityOperation.CREATE, 1); AtlasEntityHeader createdEntity = entityMutationResponse.getFirstCreatedEntityByTypeName(TestUtilsV2.ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR); validateEntity(nestedCollectionAttrEntity, getEntityFromStore(createdEntity));
@JsonIgnore public List<AtlasEntityHeader> getDeletedEntitiesByTypeName(String typeName) { return getEntitiesByType(getEntitiesByOperation(EntityOperation.DELETE), typeName); }
public <T extends AtlasBaseModelObject> void saveNoLoad(T obj) throws AtlasBaseException { Objects.requireNonNull(obj, "Can't save a null object"); AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataAccess.save()"); } DataTransferObject<T> dto = (DataTransferObject<T>) dtoRegistry.get(obj.getClass()); AtlasEntityWithExtInfo entityWithExtInfo = dto.toEntityWithExtInfo(obj); EntityMutationResponse entityMutationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), false); if (noEntityMutation(entityMutationResponse)) { throw new AtlasBaseException(AtlasErrorCode.DATA_ACCESS_SAVE_FAILED, obj.toString()); } // Update GUID assignment for newly created entity if (CollectionUtils.isNotEmpty(entityMutationResponse.getCreatedEntities())) { String assignedGuid = entityMutationResponse.getGuidAssignments().get(obj.getGuid()); if (!obj.getGuid().equals(assignedGuid)) { obj.setGuid(assignedGuid); } } } finally { AtlasPerfTracer.log(perf); } }
@JsonIgnore public void addEntity(EntityOperation op, AtlasEntityHeader header) { // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { if (entityHeaderExists(getCreatedEntities(), header.getGuid())) { return; } } if (mutatedEntities == null) { mutatedEntities = new HashMap<>(); } List<AtlasEntityHeader> opEntities = mutatedEntities.get(op); if (opEntities == null) { opEntities = new ArrayList<>(); mutatedEntities.put(op, opEntities); } if (!entityHeaderExists(opEntities, header.getGuid())) { opEntities.add(header); } }
@JsonIgnore public AtlasEntityHeader getFirstUpdatedEntityByTypeName(String typeName) { return getFirstEntityByType(getEntitiesByOperation(EntityOperation.UPDATE), typeName); }
@JsonIgnore public AtlasEntityHeader getFirstEntityPartialUpdated() { final List<AtlasEntityHeader> entitiesByOperation = getEntitiesByOperation(EntityOperation.PARTIAL_UPDATE); if ( entitiesByOperation != null && entitiesByOperation.size() > 0) { return entitiesByOperation.get(0); } return null; }
@Test(dependsOnMethods = "deletetingCollections") public void addingCollections() throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid); assertNotNull(entityWithExtInfo); addNewTables(entityWithExtInfo); addNewRegions(entityWithExtInfo); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), true); assertNotNull(response); assertTrue(response.getPartialUpdatedEntities().size() > 0); assertAttribute(dbGuid, storageGuid, 3, 3); }
@JsonIgnore public AtlasEntityHeader getUpdatedEntityByTypeNameAndAttribute(String typeName, String attrName, String attrVal) { return getEntityByTypeAndUniqueAttribute(getEntitiesByOperation(EntityOperation.UPDATE), typeName, attrName, attrVal); }
@Test public void testDefaultValueForPrimitiveTypes() throws Exception { init(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(primitiveEntity), false); List<AtlasEntityHeader> entitiesCreatedResponse = response.getEntitiesByOperation(EntityOperation.CREATE); List<AtlasEntityHeader> entitiesCreatedwithdefault = response.getMutatedEntities().get(EntityOperation.CREATE); AtlasEntity entityCreated = getEntityFromStore(entitiesCreatedResponse.get(0)); Map attributesMap = entityCreated.getAttributes(); String description = (String) attributesMap.get("description"); String check = (String) attributesMap.get("check"); String sourceCode = (String) attributesMap.get("sourcecode"); float diskUsage = (float) attributesMap.get("diskUsage"); boolean isstoreUse = (boolean) attributesMap.get("isstoreUse"); int cost = (int) attributesMap.get("Cost"); assertEquals(description,"test"); assertEquals(check,"check"); //defaultValue assertEquals(diskUsage,70.5f); assertEquals(isstoreUse,true); assertEquals(sourceCode,"Hello World"); assertEquals(cost,30); }
EntityMutationResponse ret = new EntityMutationResponse(); ret.setGuidAssignments(new HashMap<String, String>()); EntityMutationResponse resp = entityStore.createOrUpdateForImport(oneEntityStream); if (resp.getGuidAssignments() != null) { ret.getGuidAssignments().putAll(resp.getGuidAssignments());
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); AtlasEntityHeader updatedTableHeader = response.getFirstUpdatedEntityByTypeName(tableEntity.getTypeName()); validateEntity(entitiesInfo, getEntityFromStore(updatedTableHeader)); response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); updatedTableHeader = response.getFirstUpdatedEntityByTypeName(tableEntity.getTypeName()); validateEntity(entitiesInfo, getEntityFromStore(updatedTableHeader)); response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); updatedTableHeader = response.getFirstUpdatedEntityByTypeName(tableEntity.getTypeName()); AtlasEntity updatedEntity = getEntityFromStore(updatedTableHeader); assertEquals(response.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE).size(), 2); // col1, col2 are deleted response = entityStore.createOrUpdate(new AtlasEntityStream(entitiesInfo), false); updatedTableHeader = response.getFirstUpdatedEntityByTypeName(tableEntity.getTypeName()); validateEntity(entitiesInfo, getEntityFromStore(updatedTableHeader)); assertEquals(response.getEntitiesByOperation(EntityMutations.EntityOperation.DELETE).size(), 2);
Map<EntityOperation, List<AtlasEntityHeader>> mutatedEntities = reponse.getMutatedEntities(); Map<String, String> guidAssignments = reponse.getGuidAssignments();
@VisibleForTesting AtlasEntityWithExtInfo updateEntityInAtlas(AtlasEntityWithExtInfo entity) throws Exception { AtlasEntityWithExtInfo ret = null; EntityMutationResponse response = atlasClientV2.updateEntity(entity); if (response != null) { List<AtlasEntityHeader> entities = response.getUpdatedEntities(); if (CollectionUtils.isNotEmpty(entities)) { AtlasEntityWithExtInfo getByGuidResponse = atlasClientV2.getEntityByGuid(entities.get(0).getGuid()); ret = getByGuidResponse; LOG.info("Updated {} entity: name={}, guid={} ", ret.getEntity().getTypeName(), ret.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME), ret.getEntity().getGuid()); } else { LOG.info("Entity: name={} ", entity.toString() + " not updated as it is unchanged from what is in Atlas" ); ret = entity; } } else { LOG.info("Entity: name={} ", entity.toString() + " not updated as it is unchanged from what is in Atlas" ); ret = entity; } return ret; }
private boolean deleteEntity(String entityName) throws AtlasBaseException { String entityGuid = entitiesMap.get(entityName); EntityMutationResponse response = entityStore.deleteById(entityGuid); return CollectionUtils.isNotEmpty(response.getDeletedEntities()); }
@Test(dependsOnMethods = "typeCreationFromFile") public void entityCreationUsingSoftRef() throws IOException, AtlasBaseException { final int EXPECTED_ENTITY_COUNT = 6; AtlasEntity.AtlasEntityWithExtInfo dbEntity = AtlasType.fromJson( TestResourceFileUtils.getJson(RDBMS_DB_FILE), AtlasEntity.AtlasEntityWithExtInfo.class); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); assertNotNull(response); assertTrue(response.getCreatedEntities().size() == EXPECTED_ENTITY_COUNT); assertGraphStructure(response.getCreatedEntities().get(0).getGuid(), response.getCreatedEntities().get(1).getGuid(), RDBMS_SD_PROPERTY); dbGuid = response.getCreatedEntities().get(0).getGuid(); storageGuid = response.getCreatedEntities().get(1).getGuid(); }