private boolean noEntityMutation(EntityMutationResponse er) { return er == null || (CollectionUtils.isEmpty(er.getCreatedEntities()) && CollectionUtils.isEmpty(er.getUpdatedEntities())); } }
private boolean noEntityMutation(EntityMutationResponse er) { return er == null || (CollectionUtils.isEmpty(er.getCreatedEntities()) && CollectionUtils.isEmpty(er.getUpdatedEntities())); } }
@JsonIgnore public void addEntity(EntityOperation op, AtlasEntityHeader header) { // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { if (entityHeaderExists(getCreatedEntities(), header.getGuid())) { return; } } if (mutatedEntities == null) { mutatedEntities = new HashMap<>(); } List<AtlasEntityHeader> opEntities = mutatedEntities.get(op); if (opEntities == null) { opEntities = new ArrayList<>(); mutatedEntities.put(op, opEntities); } if (!entityHeaderExists(opEntities, header.getGuid())) { opEntities.add(header); } }
@JsonIgnore public void addEntity(EntityOperation op, AtlasObjectId entity) { if (mutatedEntities == null) { mutatedEntities = new HashMap<>(); } else { // if an entity is already included in CREATE, ignore subsequent UPDATE, PARTIAL_UPDATE if (op == EntityOperation.UPDATE || op == EntityOperation.PARTIAL_UPDATE) { if (entityHeaderExists(getCreatedEntities(), entity.getGuid())) { return; } } } List<AtlasEntityHeader> opEntities = mutatedEntities.get(op); if (opEntities == null) { opEntities = new ArrayList<>(); mutatedEntities.put(op, opEntities); } if (!entityHeaderExists(opEntities, entity.getGuid())) { opEntities.add(new AtlasEntityHeader(entity.getTypeName(), entity.getGuid(), entity.getUniqueAttributes())); } }
@VisibleForTesting AtlasEntityWithExtInfo createEntityInAtlas(AtlasEntityWithExtInfo entity) throws Exception { AtlasEntityWithExtInfo ret = null; EntityMutationResponse response = atlasClientV2.createEntity(entity); List<AtlasEntityHeader> entities = response.getCreatedEntities(); if (CollectionUtils.isNotEmpty(entities)) { AtlasEntityWithExtInfo getByGuidResponse = atlasClientV2.getEntityByGuid(entities.get(0).getGuid()); ret = getByGuidResponse; LOG.info("Created {} entity: name={}, guid={}", ret.getEntity().getTypeName(), ret.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME), ret.getEntity().getGuid()); } return ret; }
@Test(dependsOnMethods = "typeCreationFromFile") public void entityCreationUsingSoftRef() throws IOException, AtlasBaseException { final int EXPECTED_ENTITY_COUNT = 6; AtlasEntity.AtlasEntityWithExtInfo dbEntity = AtlasType.fromJson( TestResourceFileUtils.getJson(RDBMS_DB_FILE), AtlasEntity.AtlasEntityWithExtInfo.class); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false); assertNotNull(response); assertTrue(response.getCreatedEntities().size() == EXPECTED_ENTITY_COUNT); assertGraphStructure(response.getCreatedEntities().get(0).getGuid(), response.getCreatedEntities().get(1).getGuid(), RDBMS_SD_PROPERTY); dbGuid = response.getCreatedEntities().get(0).getGuid(); storageGuid = response.getCreatedEntities().get(1).getGuid(); }
private AtlasEntityWithExtInfo createEntityInAtlas(AtlasEntityWithExtInfo entity) throws Exception { AtlasEntityWithExtInfo ret = null; EntityMutationResponse response = atlasClientV2.createEntity(entity); List<AtlasEntityHeader> entities = response.getCreatedEntities(); if (CollectionUtils.isNotEmpty(entities)) { AtlasEntityWithExtInfo getByGuidResponse = atlasClientV2.getEntityByGuid(entities.get(0).getGuid()); ret = getByGuidResponse; LOG.info("Created {} entity: name={}, guid={}", ret.getEntity().getTypeName(), ret.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME), ret.getEntity().getGuid()); } return ret; }
private AtlasEntityWithExtInfo createEntityInAtlas(AtlasEntityWithExtInfo entity) throws Exception { AtlasEntityWithExtInfo ret = null; EntityMutationResponse response = atlasClientV2.createEntity(entity); List<AtlasEntityHeader> entities = response.getCreatedEntities(); if (CollectionUtils.isNotEmpty(entities)) { AtlasEntityWithExtInfo getByGuidResponse = atlasClientV2.getEntityByGuid(entities.get(0).getGuid()); ret = getByGuidResponse; LOG.info("Created {} entity: name={}, guid={}", ret.getEntity().getTypeName(), ret.getEntity().getAttribute(ATTRIBUTE_QUALIFIED_NAME), ret.getEntity().getGuid()); } return ret; }
public AtlasEntity.AtlasEntityWithExtInfo getCreateEntity(String path, String clusterName) throws AtlasBaseException { String pathWithTrailingSeparator = getPathWithTrailingSeparator(path); AtlasEntityType hdfsPathEntityType = getHdfsPathEntityType(); AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = getHDFSPathEntity(hdfsPathEntityType, pathWithTrailingSeparator, clusterName); if(entityWithExtInfo != null) { return entityWithExtInfo; } AtlasEntity entity = createHDFSPathEntity(hdfsPathEntityType, pathWithTrailingSeparator, clusterName); AtlasEntityStream entityStream = new AtlasEntityStream(entity); EntityMutationResponse entityMutationResponse = entityStore.createOrUpdate(entityStream, false); if(entityMutationResponse.getCreatedEntities().size() == 0) { return null; } return getHDFSPathEntity(hdfsPathEntityType, pathWithTrailingSeparator, clusterName); }
@Test public void testCreateWithDuplicateGuids() throws Exception { init(); AtlasEntityWithExtInfo tblEntity2 = TestUtilsV2.createTableEntityDuplicatesV2(dbEntity.getEntity()); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(tblEntity2), false); List<AtlasEntityHeader> createdEntities = response.getCreatedEntities(); assertTrue(CollectionUtils.isNotEmpty(createdEntities)); assertEquals(createdEntities.size(), 2); String tableGuid = createdEntities.get(0).getGuid(); AtlasEntityWithExtInfo tableEntity = entityStore.getById(tableGuid); assertEquals(tableEntity.getReferredEntities().size(), 1); List<AtlasObjectId> columns = (List<AtlasObjectId>) tableEntity.getEntity().getAttribute("columns"); assertEquals(columns.size(), 1); Set<AtlasObjectId> uniqueColumns = new HashSet<>(columns); assertTrue(uniqueColumns.size() == 1); }
public <T extends AtlasBaseModelObject> void saveNoLoad(T obj) throws AtlasBaseException { Objects.requireNonNull(obj, "Can't save a null object"); AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataAccess.save()"); } DataTransferObject<T> dto = (DataTransferObject<T>) dtoRegistry.get(obj.getClass()); AtlasEntityWithExtInfo entityWithExtInfo = dto.toEntityWithExtInfo(obj); EntityMutationResponse entityMutationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), false); if (noEntityMutation(entityMutationResponse)) { throw new AtlasBaseException(AtlasErrorCode.DATA_ACCESS_SAVE_FAILED, obj.toString()); } // Update GUID assignment for newly created entity if (CollectionUtils.isNotEmpty(entityMutationResponse.getCreatedEntities())) { String assignedGuid = entityMutationResponse.getGuidAssignments().get(obj.getGuid()); if (!obj.getGuid().equals(assignedGuid)) { obj.setGuid(assignedGuid); } } } finally { AtlasPerfTracer.log(perf); } }
public <T extends AtlasBaseModelObject> T save(T obj) throws AtlasBaseException { Objects.requireNonNull(obj, "Can't save a null object"); AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DataAccess.save()"); } DataTransferObject<T> dto = (DataTransferObject<T>) dtoRegistry.get(obj.getClass()); AtlasEntityWithExtInfo entityWithExtInfo = dto.toEntityWithExtInfo(obj); EntityMutationResponse entityMutationResponse = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), false); if (noEntityMutation(entityMutationResponse)) { throw new AtlasBaseException(AtlasErrorCode.DATA_ACCESS_SAVE_FAILED, obj.toString()); } // Update GUID assignment for newly created entity if (CollectionUtils.isNotEmpty(entityMutationResponse.getCreatedEntities())) { String assignedGuid = entityMutationResponse.getGuidAssignments().get(obj.getGuid()); if (!obj.getGuid().equals(assignedGuid)) { obj.setGuid(assignedGuid); } } return this.load(obj); } finally { AtlasPerfTracer.log(perf); } }
private float updateImportMetrics(AtlasEntity.AtlasEntityWithExtInfo currentEntity, EntityMutationResponse resp, AtlasImportResult importResult, Set<String> processedGuids, int currentIndex, int streamSize, float currentPercent) { updateImportMetrics("entity:%s:created", resp.getCreatedEntities(), processedGuids, importResult); updateImportMetrics("entity:%s:updated", resp.getUpdatedEntities(), processedGuids, importResult); updateImportMetrics("entity:%s:deleted", resp.getDeletedEntities(), processedGuids, importResult); String lastEntityImported = String.format("entity:last-imported:%s:[%s]:(%s)", currentEntity.getEntity().getTypeName(), currentIndex, currentEntity.getEntity().getGuid()); return updateImportProgress(LOG, currentIndex, streamSize, currentPercent, lastEntityImported); }
private float updateImportMetrics(AtlasEntity.AtlasEntityWithExtInfo currentEntity, EntityMutationResponse resp, AtlasImportResult importResult, Set<String> processedGuids, int currentIndex, int streamSize, float currentPercent) { updateImportMetrics("entity:%s:created", resp.getCreatedEntities(), processedGuids, importResult); updateImportMetrics("entity:%s:updated", resp.getUpdatedEntities(), processedGuids, importResult); updateImportMetrics("entity:%s:deleted", resp.getDeletedEntities(), processedGuids, importResult); String lastEntityImported = String.format("entity:last-imported:%s:[%s]:(%s)", currentEntity.getEntity().getTypeName(), currentIndex, currentEntity.getEntity().getGuid()); return updateImportProgress(LOG, currentIndex + 1, streamSize, currentPercent, lastEntityImported); }
public void onEntitiesMutated(EntityMutationResponse entityMutationResponse, boolean isImport) throws AtlasBaseException { if (CollectionUtils.isEmpty(entityChangeListeners) || instanceConverter == null) { return; } List<AtlasEntityHeader> createdEntities = entityMutationResponse.getCreatedEntities(); List<AtlasEntityHeader> updatedEntities = entityMutationResponse.getUpdatedEntities(); List<AtlasEntityHeader> partiallyUpdatedEntities = entityMutationResponse.getPartialUpdatedEntities(); List<AtlasEntityHeader> deletedEntities = entityMutationResponse.getDeletedEntities(); // complete full text mapping before calling toReferenceables(), from notifyListners(), to // include all vertex updates in the current graph-transaction doFullTextMapping(createdEntities); doFullTextMapping(updatedEntities); doFullTextMapping(partiallyUpdatedEntities); notifyListeners(createdEntities, EntityOperation.CREATE, isImport); notifyListeners(updatedEntities, EntityOperation.UPDATE, isImport); notifyListeners(partiallyUpdatedEntities, EntityOperation.PARTIAL_UPDATE, isImport); notifyListeners(deletedEntities, EntityOperation.DELETE, isImport); notifyPropagatedEntities(); }
public void onEntitiesMutated(EntityMutationResponse entityMutationResponse, boolean isImport) throws AtlasBaseException { if (CollectionUtils.isEmpty(entityChangeListeners) || instanceConverter == null) { return; } List<AtlasEntityHeader> createdEntities = entityMutationResponse.getCreatedEntities(); List<AtlasEntityHeader> updatedEntities = entityMutationResponse.getUpdatedEntities(); List<AtlasEntityHeader> partiallyUpdatedEntities = entityMutationResponse.getPartialUpdatedEntities(); List<AtlasEntityHeader> deletedEntities = entityMutationResponse.getDeletedEntities(); // complete full text mapping before calling toReferenceables(), from notifyListners(), to // include all vertex updates in the current graph-transaction doFullTextMapping(createdEntities); doFullTextMapping(updatedEntities); doFullTextMapping(partiallyUpdatedEntities); notifyListeners(createdEntities, EntityOperation.CREATE, isImport); notifyListeners(updatedEntities, EntityOperation.UPDATE, isImport); notifyListeners(partiallyUpdatedEntities, EntityOperation.PARTIAL_UPDATE, isImport); notifyListeners(deletedEntities, EntityOperation.DELETE, isImport); notifyPropagatedEntities(); }
@BeforeClass public void setUp() throws Exception { new GraphBackedSearchIndexer(typeRegistry); // create employee relationship types AtlasTypesDef employeeTypes = getDepartmentEmployeeTypes(); typeDefStore.createTypesDef(employeeTypes); AtlasEntitiesWithExtInfo employeeInstances = getDepartmentEmployeeInstances(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(employeeInstances), false); for (AtlasEntityHeader entityHeader : response.getCreatedEntities()) { employeeNameIdMap.put((String) entityHeader.getAttribute(NAME), getAtlasObjectId(entityHeader)); } init(); AtlasTypesDef typesDef = getInverseReferenceTestTypes(); AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry); if (!typesToCreate.isEmpty()) { typeDefStore.createTypesDef(typesToCreate); } }
@BeforeClass public void setUp() throws Exception { new GraphBackedSearchIndexer(typeRegistry); // create employee relationship types AtlasTypesDef employeeTypes = getDepartmentEmployeeTypes(); typeDefStore.createTypesDef(employeeTypes); AtlasEntitiesWithExtInfo employeeInstances = getDepartmentEmployeeInstances(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(employeeInstances), false); for (AtlasEntityHeader entityHeader : response.getCreatedEntities()) { employeeNameIdMap.put((String) entityHeader.getAttribute(NAME), getAtlasObjectId(entityHeader)); } init(); AtlasTypesDef typesDef = getInverseReferenceTestTypes(); AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry); if (!typesToCreate.isEmpty()) { typeDefStore.createTypesDef(typesToCreate); } }
@BeforeClass public void setUp() throws Exception { RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(), TestUtilsV2.defineInverseReferenceTestTypes() }; for (AtlasTypesDef typesDef : testTypesDefs) { AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry); if (!typesToCreate.isEmpty()) { typeDefStore.createTypesDef(typesToCreate); } } deptEntity = TestUtilsV2.createDeptEg2(); init(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(deptEntity), false); for (AtlasEntityHeader entityHeader : response.getCreatedEntities()) { nameIdMap.put((String)entityHeader.getAttribute(NAME), AtlasTypeUtil.getAtlasObjectId(entityHeader)); } }
@BeforeClass public void setUp() throws Exception { RequestContext.clear(); RequestContext.get().setUser(TestUtilsV2.TEST_USER, null); AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(), TestUtilsV2.defineInverseReferenceTestTypes() }; for (AtlasTypesDef typesDef : testTypesDefs) { AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesDef, typeRegistry); if (!typesToCreate.isEmpty()) { typeDefStore.createTypesDef(typesToCreate); } } deptEntity = TestUtilsV2.createDeptEg2(); init(); EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(deptEntity), false); for (AtlasEntityHeader entityHeader : response.getCreatedEntities()) { nameIdMap.put((String)entityHeader.getAttribute(NAME), AtlasTypeUtil.getAtlasObjectId(entityHeader)); } }