private void markAsBuilt(Set<Integer> built, DataTypeCollection typeCollection) { for (DataType type : typeCollection.getTypes()) { built.add(type.getId()); } }
public DataTypeRepo add(DataType type) { if (typeByName.containsKey(type.getName()) || typeById.containsKey(type.getId())) { throw new IllegalStateException("Data type '" + type.getName() + "', id '" + type.getId() + "' is already registered."); } typeByName.put(type.getName(), type); typeById.put(type.getId(), type); return this; }
public DataTypeRepo replace(DataType type) { if (!typeByName.containsKey(type.getName()) || !typeById.containsKey(type.getId())) { throw new IllegalStateException("Data type '" + type.getName() + "' is not registered."); } typeByName.remove(type.getName()); typeByName.put(type.getName(), type); typeById.remove(type.getId()); typeById.put(type.getId(), type); return this; }
public SDDocumentTypeOrderer(List<SDDocumentType> sdTypes, DeployLogger deployLogger) { this.deployLogger = deployLogger; for (SDDocumentType type : sdTypes) { createdSDTypes.put(type.getDocumentName(), type); } DocumentTypeManager dtm = new DocumentTypeManager(); for (DataType type : dtm.getDataTypes()) { seenTypes.add(type.getId()); } }
private void buildConfig(DataTypeCollection type, DocumentmanagerConfig.Builder documentConfigBuilder, Set<DataType> built) { for (DataType dataType : type.getTypes()) { if (built.contains(dataType)) continue; built.add(dataType); if (dataType instanceof TemporaryStructuredDataType) continue; if ((dataType.getId() < 0) || (dataType.getId()> DataType.lastPredefinedDataTypeId())) { Datatype.Builder dataTypeBuilder = new Datatype.Builder(); documentConfigBuilder.datatype(dataTypeBuilder); buildConfig(dataType, dataTypeBuilder); } } }
public DataTypeCollection getAllTypes() { DataTypeRepo repo = new DataTypeRepo(); Set<Name> seen = new HashSet<>(); Deque<NewDocumentType> stack = new LinkedList<>(); stack.push(this); while (!stack.isEmpty()) { NewDocumentType docType = stack.pop(); if (seen.contains(docType.name)) { continue; // base type } seen.add(docType.name); for (DataType dataType : docType.getTypes()) { if (repo.getDataType(dataType.getId()) == null) { repo.add(dataType); } } stack.addAll(docType.inherits.values()); } return repo; }
/** * The field id must be unique within a document type, and also * within a (unknown at this time) hierarchy of document types. * In addition it should be as resilient to doctype content changes * and inheritance hierarchy changes as possible. * All of this is enforced for names, so id's should follow names. * Therefore we hash on name. */ protected int calculateIdV7(DocumentType owner) { String combined = getName() + dataType.getId(); int newId = BobHash.hash(combined); // Using a portfriendly hash if (newId < 0) newId = -newId; // Highest bit is reserved to tell 7-bit id's from 31-bit ones validateId(newId, owner, Document.SERIALIZED_VERSION); return newId; }
private void visit(SDDocumentType docOrStruct) { int id; if (docOrStruct.isStruct()) { id = new StructDataType(docOrStruct.getName()).getId(); } else { id = new DocumentType(docOrStruct.getName()).getId(); } if (seenTypes.contains(id)) { return; } else { seenTypes.add((new StructDataType(docOrStruct.getName()).getId())); } for (Field field : docOrStruct.fieldSet()) { if (!seenTypes.contains(field.getDataType().getId())) { //we haven't seen this before, do it visit(field.getDataType()); } } processingOrder.add(docOrStruct); }
private static boolean internalAddType(NewDocumentType dt, DataType type, boolean dryRun) { DataType oldType = dt.getDataTypeRecursive(type.getId()); if (oldType == null) { if ( ! dryRun) { dt.add(type); } return true; } else if ((type instanceof StructDataType) && (oldType instanceof StructDataType)) { StructDataType s = (StructDataType) type; StructDataType os = (StructDataType) oldType; if ((os.getFieldCount() == 0) && (s.getFieldCount() > os.getFieldCount())) { if ( ! dryRun) { dt.replace(type); } return true; } } return false; }
private void buildConfig(MapDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) { dataTypeBuilder. type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.MAP). map(new DocumenttypesConfig.Documenttype.Datatype.Map.Builder(). key(new DocumenttypesConfig.Documenttype.Datatype.Map.Key.Builder(). id(type.getKeyType().getId())). value(new DocumenttypesConfig.Documenttype.Datatype.Map.Value.Builder(). id(type.getValueType().getId()))); buildConfig(type.getKeyType(), documentBuilder, built); buildConfig(type.getValueType(), documentBuilder, built); }
private void buildConfig(ArrayDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) { dataTypeBuilder. type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.ARRAY). array(new DocumenttypesConfig.Documenttype.Datatype.Array.Builder(). element(new DocumenttypesConfig.Documenttype.Datatype.Array.Element.Builder().id(type.getNestedType().getId()))); buildConfig(type.getNestedType(), documentBuilder, built); }
private void buildConfig(AnnotationType type, DocumentmanagerConfig.Annotationtype.Builder atb) { atb. id(type.getId()). name(type.getName()); if (type.getDataType() != null) { atb.datatype(type.getDataType().getId()); } if ( ! type.getInheritedTypes().isEmpty()) { for (AnnotationType inherited : type.getInheritedTypes()) { atb.inherits(new DocumentmanagerConfig.Annotationtype.Inherits.Builder().id(inherited.getId())); } } } private void buildConfig(Collection<AnnotationType> types, DocumentmanagerConfig.Builder builder) {
private void buildConfig(AnnotationType annotation, DocumenttypesConfig.Documenttype.Annotationtype.Builder builder) { builder. id(annotation.getId()). name(annotation.getName()); DataType dt = annotation.getDataType(); if (dt != null) { builder.datatype(dt.getId()); } for (AnnotationType inherited : annotation.getInheritedTypes()) { builder.inherits(new DocumenttypesConfig.Documenttype.Annotationtype.Inherits.Builder().id(inherited.getId())); } }
/** * Write out the value of weighted set field * * @param field - field description (name and data type) * @param ws - field value */ public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> ws) { WeightedSetDataType type = ws.getDataType(); putInt(null, type.getNestedType().getId()); putInt(null, ws.size()); Iterator<T> it = ws.fieldValueIterator(); while (it.hasNext()) { FieldValue key = it.next(); java.lang.Integer value = ws.get(key); int sizePos = buf.position(); putInt(null, 0); int startPos = buf.position(); key.serialize(this); putInt(null, value); int finalPos = buf.position(); int size = finalPos - startPos; buf.position(sizePos); putInt(null, size); buf.position(finalPos); } }
/** * Write out the value of weighted set field * * @param field - field description (name and data type) * @param ws - field value */ public <T extends FieldValue> void write(FieldBase field, WeightedSet<T> ws) { WeightedSetDataType type = ws.getDataType(); putInt(null, type.getNestedType().getId()); putInt(null, ws.size()); Iterator<T> it = ws.fieldValueIterator(); while (it.hasNext()) { FieldValue key = it.next(); java.lang.Integer value = ws.get(key); int sizePos = buf.position(); putInt(null, 0); int startPos = buf.position(); key.serialize(this); putInt(null, value); int finalPos = buf.position(); int size = finalPos - startPos; buf.position(sizePos); putInt(null, size); buf.position(finalPos); } }
private void buildConfig(WeightedSetDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) { dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.WSET). wset(new DocumenttypesConfig.Documenttype.Datatype.Wset.Builder(). key(new DocumenttypesConfig.Documenttype.Datatype.Wset.Key.Builder(). id(type.getNestedType().getId())). createifnonexistent(type.createIfNonExistent()). removeifzero(type.removeIfZero())); buildConfig(type.getNestedType(), documentBuilder, built); }
private void buildConfig(NewDocumentType documentType, DocumenttypesConfig.Builder builder) { if (documentType == VespaDocumentType.INSTANCE) { return; } DocumenttypesConfig.Documenttype.Builder db = new DocumenttypesConfig.Documenttype.Builder(); db. id(documentType.getId()). name(documentType.getName()). headerstruct(documentType.getHeader().getId()). bodystruct(documentType.getBody().getId()); Set<Integer> built = new HashSet<>(); for (NewDocumentType inherited : documentType.getInherited()) { db.inherits(new DocumenttypesConfig.Documenttype.Inherits.Builder().id(inherited.getId())); markAsBuilt(built, inherited.getAllTypes()); } for (DataType dt : documentType.getTypes()) { buildConfig(dt, db, built); } for (AnnotationType annotation : documentType.getAnnotations()) { DocumenttypesConfig.Documenttype.Annotationtype.Builder atb = new DocumenttypesConfig.Documenttype.Annotationtype.Builder(); db.annotationtype(atb); buildConfig(annotation, atb); } buildConfig(documentType.getFieldSets(), db); builder.documenttype(db); }
buf.putInt(annotation.getType().getDataType().getId()); annotation.getFieldValue().serialize(this);
buf.putInt(annotation.getType().getDataType().getId()); annotation.getFieldValue().serialize(this);
private void buildConfig(StructDataType type, DocumenttypesConfig.Documenttype.Datatype.Builder dataTypeBuilder, DocumenttypesConfig.Documenttype.Builder documentBuilder, Set<Integer> built) { dataTypeBuilder.type(DocumenttypesConfig.Documenttype.Datatype.Type.Enum.STRUCT); DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder structBuilder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Builder(); dataTypeBuilder.sstruct(structBuilder); structBuilder.name(type.getName()); if (type.getCompressionConfig().type.getCode() != 0) { structBuilder.compression(new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Builder(). type(DocumenttypesConfig.Documenttype.Datatype.Sstruct.Compression.Type.Enum.valueOf(type.getCompressionConfig().type.toString())). level(type.getCompressionConfig().compressionLevel). threshold((int) type.getCompressionConfig().threshold). minsize((int) type.getCompressionConfig().minsize)); } for (com.yahoo.document.Field field : type.getFields()) { DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder builder = new DocumenttypesConfig.Documenttype.Datatype.Sstruct.Field.Builder(); builder.name(field.getName()). id(field.getId()). id_v6(field.getIdV6()). datatype(field.getDataType().getId()); if (field.getDataType() instanceof TensorDataType) { builder.detailedtype(((TensorDataType) field.getDataType()).getTensorType().toString()); } structBuilder.field(builder); buildConfig(field.getDataType(), documentBuilder, built); } }