private AtlasTypesDef getTypeDefs(String ... typeNames) throws AtlasServiceException { final AtlasTypesDef typeDefs = new AtlasTypesDef(); for (int i = 0; i < typeNames.length; i++) { final MultivaluedMap<String, String> searchParams = new MultivaluedMapImpl(); searchParams.add(SearchFilter.PARAM_NAME, typeNames[i]); final AtlasTypesDef typeDef = atlasClient.getAllTypeDefs(new SearchFilter(searchParams)); typeDefs.getEntityDefs().addAll(typeDef.getEntityDefs()); } logger.debug("typeDefs={}", typeDefs); return typeDefs; }
public void removeTypesDef(AtlasTypesDef typesDef) throws AtlasBaseException { if (null != typesDef && !typesDef.isEmpty()) { removeTypesWithNoRefResolve(typesDef.getEnumDefs()); removeTypesWithNoRefResolve(typesDef.getStructDefs()); removeTypesWithNoRefResolve(typesDef.getClassificationDefs()); removeTypesWithNoRefResolve(typesDef.getEntityDefs()); removeTypesWithNoRefResolve(typesDef.getRelationshipDefs()); resolveReferences(); } }
public AtlasTypesDef scrub(AtlasTypesDef typesDef) { this.typesDef = typesDef; display("incoming: ", typesDef); createClassificationNameIndexMap(typesDef.getClassificationDefs()); for (AtlasStructDef structDef : new ArrayList<>(typesDef.getStructDefs())) { // work on copy of typesDef.getStructDefs(), as the list is modified by checkAndUpdate() checkAndUpdate(structDef); } for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) { checkAndUpdate(entityDef); } display("scrubbed: ", typesDef); return typesDef; }
@Deprecated public AtlasStructDef createStructDef(AtlasStructDef structDef) throws AtlasServiceException { AtlasTypesDef atlasTypesDef = new AtlasTypesDef(); atlasTypesDef.getStructDefs().add(structDef); AtlasTypesDef created = createAtlasTypeDefs(atlasTypesDef); assert created != null; assert created.getStructDefs() != null; return created.getStructDefs().get(0); }
@Deprecated public AtlasClassificationDef createClassificationDef(AtlasClassificationDef classificationDef) throws AtlasServiceException { AtlasTypesDef atlasTypesDef = new AtlasTypesDef(); atlasTypesDef.getClassificationDefs().add(classificationDef); AtlasTypesDef created = createAtlasTypeDefs(atlasTypesDef); assert created != null; assert created.getClassificationDefs() != null; return created.getClassificationDefs().get(0); }
if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) { for (AtlasClassificationDef classifiDef : typesDef.getClassificationDefs()) { ret.getClassificationDefs().add(classifiDefStore.update(classifiDef)); if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) { for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) { ret.getEntityDefs().add(entityDefStore.update(entityDef)); if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) { for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) { ret.getRelationshipDefs().add(relationDefStore.update(relationshipDef));
private String getCreateTag(String classificationName) throws AtlasBaseException { AtlasClassificationDef classificationDef = typeRegistry.getClassificationDefByName(classificationName); if(classificationDef != null) { return classificationName; } classificationDef = new AtlasClassificationDef(classificationName); AtlasTypesDef typesDef = new AtlasTypesDef(); typesDef.setClassificationDefs(Collections.singletonList(classificationDef)); typeDefStore.createTypesDef(typesDef); LOG.info("created classification: {}", classificationName); return classificationName; } }
@Deprecated public AtlasEnumDef createEnumDef(AtlasEnumDef enumDef) throws AtlasServiceException { AtlasTypesDef atlasTypesDef = new AtlasTypesDef(); atlasTypesDef.getEnumDefs().add(enumDef); AtlasTypesDef created = createAtlasTypeDefs(atlasTypesDef); assert created != null; assert created.getEnumDefs() != null; return created.getEnumDefs().get(0); }
public static AtlasTypesDef toAtlasTypesDef(String typeDefinition, AtlasTypeRegistry registry) throws AtlasBaseException { AtlasTypesDef ret = new AtlasTypesDef(); try { if (StringUtils.isEmpty(typeDefinition)) { throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition); } TypesDef typesDef = TypesSerialization.fromJson(typeDefinition); if (CollectionUtils.isNotEmpty(typesDef.enumTypesAsJavaList())) { List<AtlasEnumDef> enumDefs = toAtlasEnumDefs(typesDef.enumTypesAsJavaList()); ret.setEnumDefs(enumDefs); } if (CollectionUtils.isNotEmpty(typesDef.structTypesAsJavaList())) { List<AtlasStructDef> structDefs = toAtlasStructDefs(typesDef.structTypesAsJavaList()); ret.setStructDefs(structDefs); } if (CollectionUtils.isNotEmpty(typesDef.classTypesAsJavaList())) { List<AtlasEntityDef> entityDefs = toAtlasEntityDefs(typesDef.classTypesAsJavaList(), registry); ret.setEntityDefs(entityDefs); } if (CollectionUtils.isNotEmpty(typesDef.traitTypesAsJavaList())) { List<AtlasClassificationDef> classificationDefs = toAtlasClassificationDefs(typesDef.traitTypesAsJavaList()); ret.setClassificationDefs(classificationDefs); } } catch (Exception e) { LOG.error("Invalid type definition = {}", typeDefinition, e); throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition); } return ret; }
private void assertTraitMap(TypesDefScrubber typesDefScrubber, AtlasTypesDef td, int classificationIndex, String expectedStructName, int attrIndex) { String label = typesDefScrubber.getEdgeLabel(td.getEntityDefs().get(0).getName(), td.getEntityDefs().get(0).getAttributeDefs().get(attrIndex).getName()); assertTrue(typesDefScrubber.getTraitToTypeMap().containsKey(label)); assertEquals(typesDefScrubber.getTraitToTypeMap().get(label).getTypeName(), td.getClassificationDefs().get(classificationIndex).getName()); assertEquals(typesDefScrubber.getTraitToTypeMap().get(label).getLegacyTypeName(), expectedStructName); }
/** * @return True when required NiFi types are already created. */ public boolean isNiFiTypeDefsRegistered() throws AtlasServiceException { final Set<String> typeNames = ENTITIES.keySet(); final Map<String, AtlasEntityDef> existingDefs = getTypeDefs(typeNames.toArray(new String[typeNames.size()])).getEntityDefs().stream() .collect(Collectors.toMap(AtlasEntityDef::getName, Function.identity())); return typeNames.stream().allMatch(existingDefs::containsKey); }
public static AtlasTypesDef getTypesDef(List<AtlasEnumDef> enums, List<AtlasStructDef> structs, List<AtlasClassificationDef> traits, List<AtlasEntityDef> classes) { return new AtlasTypesDef(enums, structs, traits, classes); }
@BeforeClass public void setUp() throws Exception { super.setUp(); AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineDeptEmployeeTypes(), TestUtilsV2.defineHiveTypes(), TestUtilsV2.defineTypeWithNestedCollectionAttributes(), }; createTypesDef(testTypesDefs); deptEntity = TestUtilsV2.createDeptEg2(); dbEntity = TestUtilsV2.createDBEntityV2(); tblEntity = TestUtilsV2.createTableEntityV2(dbEntity.getEntity()); nestedCollectionAttrEntity = TestUtilsV2.createNestedCollectionAttrEntity(); primitiveEntity = TestUtilsV2.createprimitiveEntityV2(); AtlasTypesDef typesDef11 = new AtlasTypesDef(); List primitiveEntityDef = new ArrayList<AtlasEntityDef>(); primitiveEntityDef.add(TestUtilsV2.createPrimitiveEntityDef()); typesDef11.setEntityDefs(primitiveEntityDef); typeDefStore.createTypesDef(typesDef11); }
AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class); if (typesDef == null || typesDef.isEmpty()) { LOG.info("No type in file {}", typeDefFile.getAbsolutePath()); AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, atlasTypeRegistry, true); if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) { atlasTypeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate);
private AtlasClassificationDef getClassificationDefByName(String name) { if (classificationIndexMap.containsKey(name)) { return typesDef.getClassificationDefs().get(classificationIndexMap.get(name)); } return null; }
typesToUpdate.getEntityDefs().add(newEntityDef); if (CollectionUtils.isNotEmpty(typesDef.getRelationshipDefs())) { for (AtlasRelationshipDef relationshipDef : typesDef.getRelationshipDefs()) { AtlasRelationshipDef oldRelationshipDef = typeRegistry.getRelationshipDefByName(relationshipDef.getName());
private void addStructDefToTypesDef(AtlasStructDef structDef) { for (AtlasStructDef sDef : typesDef.getStructDefs()) { if (StringUtils.equals(sDef.getName(), structDef.getName())) { return; } } typesDef.getStructDefs().add(structDef); }
private void updateEnumDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) throws AtlasBaseException { for (AtlasEnumDef def : typeDefinitionMap.getEnumDefs()) { AtlasEnumDef existing = typeRegistry.getEnumDefByName(def.getName()); if (existing != null && addElements(existing, def)) { typeDefStore.updateEnumDefByName(existing.getName(), existing); result.incrementMeticsCounter("typedef:enum:update"); } } }
protected AtlasClassificationDef getTagWithName(AtlasTypesDef typesDef, String tagName, String attributeType) { AtlasClassificationDef aTag = new AtlasClassificationDef(tagName); AtlasAttributeDef attributeDef = new AtlasAttributeDef("testAttribute", attributeType, true, AtlasAttributeDef.Cardinality.SINGLE, 0, 1, false, true, false, Collections.emptyList()); aTag.addAttribute(attributeDef); typesDef.setClassificationDefs(Arrays.asList(aTag)); return aTag; }
private void updateMetricsForTypesDef(AtlasTypesDef typeDefinitionMap, AtlasImportResult result) { result.incrementMeticsCounter("typedef:classification", typeDefinitionMap.getClassificationDefs().size()); result.incrementMeticsCounter("typedef:enum", typeDefinitionMap.getEnumDefs().size()); result.incrementMeticsCounter("typedef:entitydef", typeDefinitionMap.getEntityDefs().size()); result.incrementMeticsCounter("typedef:struct", typeDefinitionMap.getStructDefs().size()); } }