private static String getColumnValue(FieldMapping fm) { switch (fm.getMappingType()) { case COLUMN: case COUNTER: return fm.getFamilyAsString() + ":" + fm.getQualifierAsString(); case KEY_AS_COLUMN: return fm.getFamilyAsString() + ":" + (fm.getPrefix() == null ? "" : fm.getPrefix()); default: return null; } }
/** * Deserialize an entity field from the HBase Result. * * @param fieldMapping * The FieldMapping that specifies this field's mapping type and * field name. * @param result * The HBase Result that represents a row in HBase. * @return The field Object we deserialized from the Result. */ public Object deserialize(FieldMapping fieldMapping, Result result) { String fieldName = fieldMapping.getFieldName(); MappingType mappingType = fieldMapping.getMappingType(); if (mappingType == MappingType.COLUMN || mappingType == MappingType.COUNTER) { return deserializeColumn(fieldMapping.getFieldName(), fieldMapping.getFamily(), fieldMapping.getQualifier(), result); } else if (mappingType == MappingType.KEY_AS_COLUMN) { return deserializeKeyAsColumn(fieldMapping.getFieldName(), fieldMapping.getFamily(), fieldMapping.getPrefix(), result); } else if (mappingType == MappingType.OCC_VERSION) { return deserializeOCCColumn(result); } else { throw new ValidationException( "Invalid field mapping for field with name: " + fieldName); } }
/** * Adds a mapping to store the record field {@code name} in the record key. * * The underlying dataset's {@link PartitionStrategy} must have an identity * partitioner for the record field {@code name} so that the value can be * recovered. * * @param name The name of a record field * @return This Builder for method chaining */ public Builder key(String name) { addField(FieldMapping.key(name)); return this; }
/** * Get the column families required by this schema. * * @return The set of column families. */ public Set<String> getRequiredColumnFamilies() { Set<String> set = new HashSet<String>(); for (FieldMapping mapping : fieldMappings) { if (FieldMapping.MappingType.KEY != mapping.getMappingType()) set.add(mapping.getFamilyAsString()); } return set; }
/** * Get the columns required by this schema. * * @return The set of columns */ public Set<String> getRequiredColumns() { Set<String> set = new HashSet<String>(); for (FieldMapping fieldMapping : fieldMappings) { if (FieldMapping.MappingType.KEY == fieldMapping.getMappingType()) { continue; } else if (FieldMapping.MappingType.KEY_AS_COLUMN == fieldMapping.getMappingType()) { set.add(fieldMapping.getFamilyAsString() + ":"); } else { set.add(fieldMapping.getFamilyAsString() + ":" + fieldMapping.getQualifierAsString()); } } return set; }
/** * Initialize the AvroRecordBuilderFactories for all keyAsColumn mapped fields * that are record types. We need to be able to get record builders for these * since the records are broken across many columns, and need to be * constructed by the composer. */ private void initRecordBuilderFactories() { for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY_AS_COLUMN) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName) .schema(); Schema.Type fieldSchemaType = fieldSchema.getType(); if (fieldSchemaType == Schema.Type.RECORD) { AvroRecordBuilderFactory<E> factory = buildAvroRecordBuilderFactory(fieldSchema); kacRecordBuilderFactories.put(fieldName, factory); } } } }
public SingleFieldEntityFilter(EntitySchema entitySchema, EntitySerDe<?> entitySerDe, String fieldName, Object filterValue, CompareFilter.CompareOp equalityOperator) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping.getMappingType() != MappingType.COLUMN) { throw new DatasetException( "SingleColumnValueFilter only compatible with COLUMN mapping types."); } byte[] family = fieldMapping.getFamily(); byte[] qualifier = fieldMapping.getQualifier(); byte[] comparisonBytes = entitySerDe.serializeColumnValueToBytes(fieldName, filterValue); this.filter = new SingleColumnValueFilter(family, qualifier, equalityOperator, comparisonBytes); }
/** * Validate that a {@link FieldMapping} is compatible with this builder's * current set of mappings and add it to the set of mappings. * * A mapping is not compatible if it results in: * <pre> * 1. Multiple occVersion mappings in the mapping set * 2. Both a counter and an occVersion mapping in the mapping set * </pre> * * @param fm a {@code FieldMapping} to add to this builder */ private void addField(FieldMapping fm) { // validate! if (fm.getMappingType() == FieldMapping.MappingType.OCC_VERSION) { ValidationException.check(!hasOCCVersion, "Cannot use multiple occVersion fields"); ValidationException.check(!hasCounter, "Cannot use both counter and occVersion fields"); hasOCCVersion = true; } else if (fm.getMappingType() == FieldMapping.MappingType.COUNTER) { ValidationException.check(!hasOCCVersion, "Cannot use both counter and occVersion fields"); hasCounter = true; } fieldMappings.add(fm); } }
return FieldMapping.version(source); } else if ("key".equals(type)) { return FieldMapping.key(source); ValidationException.check(qualifier != null && !qualifier.isEmpty(), "Column mapping %s must have a %s", source, QUALIFIER); return FieldMapping.column(source, family, qualifier); return FieldMapping.keyAsColumn(source, family, prefix); ValidationException.check(qualifier != null && !qualifier.isEmpty(), "Counter mapping %s must have a %s", source, QUALIFIER); return FieldMapping.counter(source, family, qualifier);
/** * Ensure that the column mappings for the shared fields between the old and * new schema haven't changed. * * @param oldSchema * @param newSchema * @return true if the mappings are compatible, false if not. */ private static boolean mappingCompatible(EntitySchema oldSchema, EntitySchema newSchema) { for (FieldMapping oldFieldMapping : oldSchema.getColumnMappingDescriptor() .getFieldMappings()) { FieldMapping newFieldMapping = newSchema.getColumnMappingDescriptor() .getFieldMapping(oldFieldMapping.getFieldName()); if (newFieldMapping != null) { if (!oldFieldMapping.equals(newFieldMapping)) { return false; } } } return true; }
public FieldMapping getFieldMapping(String fieldName) { for (FieldMapping fm : fieldMappings) { if (fm.getFieldName().equals(fieldName)) { return fm; } } return null; }
public static FieldMapping counter(String name, String family, String qualifier) { return new FieldMapping(name, MappingType.COUNTER, family, qualifier, null); }
/** * Adds a mapping to store the record field {@code name} in a column using * the {@code family} and {@code qualifier}. * * @param name The name of a record field * @param family The column family for storing the record field value * @param qualifier The column qualifier for storing the record field value * @return This Builder for method chaining */ public Builder column(String name, String family, String qualifier) { addField(FieldMapping.column(name, family, qualifier)); return this; }
/** * Adds a counter mapping to store record field {@code name} in a column * using the {@code family} and {@code qualifier}. The record field can be * updated atomically using * {@link RandomAccessDataset#increment(Key, String, long)} * * The record field must be an int or a long. * * Counters cannot be used in combination with optimistic concurrency (OCC). * * @param name The name of a record field * @param family The column family for storing the record counter value * @param qualifier The column qualifier for storing the record field value * @return This Builder for method chaining */ public Builder counter(String name, String family, String qualifier) { addField(FieldMapping.counter(name, family, qualifier)); return this; }
/** * Initialize the AvroRecordBuilderFactories for all keyAsColumn mapped fields * that are record types. We need to be able to get record builders for these * since the records are broken across many columns, and need to be * constructed by the composer. */ private void initRecordBuilderFactories() { for (FieldMapping fieldMapping : avroSchema.getColumnMappingDescriptor().getFieldMappings()) { if (fieldMapping.getMappingType() == MappingType.KEY_AS_COLUMN) { String fieldName = fieldMapping.getFieldName(); Schema fieldSchema = avroSchema.getAvroSchema().getField(fieldName) .schema(); Schema.Type fieldSchemaType = fieldSchema.getType(); if (fieldSchemaType == Schema.Type.RECORD) { AvroRecordBuilderFactory<E> factory = buildAvroRecordBuilderFactory(fieldSchema); kacRecordBuilderFactories.put(fieldName, factory); } } } }
public SingleFieldEntityFilter(EntitySchema entitySchema, EntitySerDe<?> entitySerDe, String fieldName, Object filterValue, CompareFilter.CompareOp equalityOperator) { FieldMapping fieldMapping = entitySchema.getColumnMappingDescriptor() .getFieldMapping(fieldName); if (fieldMapping.getMappingType() != MappingType.COLUMN) { throw new DatasetException( "SingleColumnValueFilter only compatible with COLUMN mapping types."); } byte[] family = fieldMapping.getFamily(); byte[] qualifier = fieldMapping.getQualifier(); byte[] comparisonBytes = entitySerDe.serializeColumnValueToBytes(fieldName, filterValue); this.filter = new SingleColumnValueFilter(family, qualifier, equalityOperator, comparisonBytes); }
public static Map<Integer, FieldMapping> parseKeyMappingsFromSchemaFields( Schema schema) { Map<Integer, FieldMapping> keyMappings = Maps.newHashMap(); if (Schema.Type.RECORD == schema.getType()) { for (Schema.Field field : schema.getFields()) { if (field.getJsonProp(MAPPING) != null) { // parse the String because Avro uses com.codehaus.jackson JsonNode mappingNode = JsonUtil.parse( field.getJsonProp(MAPPING).toString()); FieldMapping fm = parseFieldMapping(field.name(), mappingNode); if (FieldMapping.MappingType.KEY == fm.getMappingType() && mappingNode.has(VALUE)) { Integer index = mappingNode.get(VALUE).asInt(); keyMappings.put(index, fm); } } } return keyMappings; } throw new IllegalArgumentException( "Cannot parse field-level mappings from non-Record"); }
/** * Ensure that the column mappings for the shared fields between the old and * new schema haven't changed. * * @param oldSchema * @param newSchema * @return true if the mappings are compatible, false if not. */ private static boolean mappingCompatible(EntitySchema oldSchema, EntitySchema newSchema) { for (FieldMapping oldFieldMapping : oldSchema.getColumnMappingDescriptor() .getFieldMappings()) { FieldMapping newFieldMapping = newSchema.getColumnMappingDescriptor() .getFieldMapping(oldFieldMapping.getFieldName()); if (newFieldMapping != null) { if (!oldFieldMapping.equals(newFieldMapping)) { return false; } } } return true; }
private static PartitionStrategy buildPartitionStrategyForKeyMappings( Map<Integer, FieldMapping> keyMappings) { PartitionStrategy.Builder builder = new PartitionStrategy.Builder(); for (Integer index : new TreeSet<Integer>(keyMappings.keySet())) { builder.identity(keyMappings.get(index).getFieldName()); } return builder.build(); }
public static FieldMapping keyAsColumn(String name, String family) { return new FieldMapping( name, MappingType.KEY_AS_COLUMN, family, null, null); }