private static Schema merge(Schema left, Schema right) { if (left == null) { return right; } else if (right == null) { return left; } else { return SchemaUtil.merge(left, right); } } }
public AvroKeySchema(Schema entitySchema, PartitionStrategy partitionStrategy) { super(null, partitionStrategy); this.schema = SchemaUtil.keySchema(entitySchema, partitionStrategy); }
@VisibleForTesting static TypeInfo convert(Schema schema) { return SchemaUtil.visit(schema, new Converter()); }
@SuppressWarnings("unchecked") public static <S> Class<? extends S> getSourceType(FieldPartitioner<S, ?> fp, Schema schema) { return (Class<S>) getClassForType(fieldSchema(schema, fp.getSourceName()).getType()); }
return union(left, right); case RECORD: if (left.getName() == null && right.getName() == null && fieldSimilarity(left, right) < SIMILARITY_THRESH) { return null; } else if (!Objects.equal(left.getName(), right.getName())) { coalesce(left.getName(), right.getName()), coalesce(left.getDoc(), right.getDoc()), coalesce(left.getNamespace(), right.getNamespace()), false ); combinedRecord.setFields(mergeFields(left, right)); mergeOrUnion(left.getValueType(), right.getValueType())); mergeOrUnion(left.getElementType(), right.getElementType())); return Schema.createEnum( left.getName(), coalesce(left.getDoc(), right.getDoc()), coalesce(left.getNamespace(), right.getNamespace()), ImmutableList.copyOf(symbols) );
fields.add(new Schema.Field( leftField.name(), mergeOrUnion(leftField.schema(), rightField.schema()), coalesce(leftField.doc(), rightField.doc()), coalesce(leftField.defaultValue(), rightField.defaultValue()) )); } else { if (leftField.defaultValue() != null) { fields.add(copy(leftField)); } else { fields.add(new Schema.Field( leftField.name(), nullableForDefault(leftField.schema()), leftField.doc(), NULL_DEFAULT )); if (left.getField(rightField.name()) == null) { if (rightField.defaultValue() != null) { fields.add(copy(rightField)); } else { fields.add(new Schema.Field( rightField.name(), nullableForDefault(rightField.schema()), rightField.doc(), NULL_DEFAULT ));
@SuppressWarnings("unchecked") private <S, T> T valueFor(FieldPartitioner<S, T> fp) { if (values.containsKey(fp.getName())) { return Conversions.convert(values.get(fp.getName()), SchemaUtil.getPartitionType(fp, schema)); } else if (values.containsKey(fp.getSourceName())) { return fp.apply(Conversions.convert(values.get(fp.getSourceName()), SchemaUtil.getSourceType(fp, schema))); } else { throw new IllegalStateException( "Cannot create Key, missing data for field:" + fp.getName()); } } }
public static <T> T valueForDirname(FieldPartitioner<?, T> field, Schema schema, String name) { return Conversions.convert(dirnameToValueString(name), SchemaUtil.getPartitionType(field, schema)); }
/** * Builds a Schema for the FieldPartitioner using the given Schema to * determine types not fixed by the FieldPartitioner. * * @param fp a FieldPartitioner * @param schema an entity Schema that will be partitioned * @return a Schema for the field partitioner */ public static Schema partitionFieldSchema(FieldPartitioner<?, ?> fp, Schema schema) { if (fp instanceof IdentityFieldPartitioner) { // copy the schema directly from the entity to preserve annotations return fieldSchema(schema, fp.getSourceName()); } else { Class<?> fieldType = getPartitionType(fp, schema); if (fieldType == Integer.class) { return Schema.create(Schema.Type.INT); } else if (fieldType == Long.class) { return Schema.create(Schema.Type.LONG); } else if (fieldType == String.class) { return Schema.create(Schema.Type.STRING); } else { throw new ValidationException( "Cannot encode partition " + fp.getName() + " with type " + fp.getSourceType() ); } } }
/** * Returns a {@link Schema} for the given field name, which could be either a * schema field or a partition field. * * @param schema an entity Schema that will be partitioned * @param strategy a {@code PartitionStrategy} used to partition entities * @param name a schema or partition field name * @return a Schema for the partition or schema field */ public static Schema fieldSchema(Schema schema, PartitionStrategy strategy, String name) { if (strategy != null && Accessor.getDefault().hasPartitioner(strategy, name)) { return partitionFieldSchema(Accessor.getDefault().getPartitioner(strategy, name), schema); } Schema nested = fieldSchema(schema, name); if (nested != null) { return nested; } throw new IllegalArgumentException( "Not a schema or partition field: " + name); }
public static Constraints fromQueryMap(Schema schema, PartitionStrategy strategy, Map<String, String> query) { Map<String, Predicate> constraints = Maps.newLinkedHashMap(); Map<String, Object> provided = Maps.newHashMap(); for (Map.Entry<String, String> entry : query.entrySet()) { String name = entry.getKey(); if (SchemaUtil.isField(schema, strategy, name)) { Schema fieldSchema = SchemaUtil.fieldSchema(schema, strategy, name); Predicate predicate = Predicates.fromString( entry.getValue(), fieldSchema); constraints.put(name, predicate); if (predicate instanceof In) { Set values = Predicates.asSet((In) predicate); if (values.size() == 1) { provided.put(name, Iterables.getOnlyElement(values)); } } } } return new Constraints(schema, strategy, constraints, provided); }
private static void checkPartitionStrategy( Schema schema, @Nullable PartitionStrategy strategy) { if (strategy == null) { return; } for (FieldPartitioner fp : strategy.getFieldPartitioners()) { if (fp instanceof ProvidedFieldPartitioner) { // provided partitioners are not based on the entity fields continue; } // check the entity is a record if there is a non-provided partitioner ValidationException.check(schema.getType() == Schema.Type.RECORD, "Cannot partition non-records: %s", schema); // the source name should be a field in the schema, but not necessarily // the record. Schema fieldSchema; try { fieldSchema = SchemaUtil.fieldSchema(schema, fp.getSourceName()); } catch (IllegalArgumentException e) { throw new ValidationException( "Cannot partition on " + fp.getSourceName(), e); } ValidationException.check( SchemaUtil.isConsistentWithExpectedType( fieldSchema.getType(), fp.getSourceType()), "Field type %s does not match partitioner %s", fieldSchema.getType(), fp); } } }
Set<String> keyMappedFields = Sets.newHashSet(); for (FieldMapping fm : mappings.getFieldMappings()) { Schema fieldSchema = SchemaUtil.fieldSchema(schema, fm.getFieldName()); ValidationException.check( SchemaUtil.isConsistentWithMappingType( fieldSchema.getType(), fm.getMappingType()), "Field type %s is not compatible with mapping %s",
/** * Checks that the type of each of {@code values} is consistent with the type of * field {@code fieldName} declared in the Avro schema (from {@code descriptor}). */ public static void checkTypeConsistency(Schema schema, String fieldName, Object... values) { checkTypeConsistency(schema, null, fieldName, values); }
private Map<String, String> toQueryMap(Map<String, String> queryMap, boolean normalized) { for (Map.Entry<String, Predicate> entry : constraints.entrySet()) { String name = entry.getKey(); Schema fieldSchema = SchemaUtil.fieldSchema(schema, strategy, name); if(normalized) { queryMap.put(name, Predicates.toNormalizedString(entry.getValue(), fieldSchema)); } else { queryMap.put(name, Predicates.toString(entry.getValue(), fieldSchema)); } } return queryMap; }
@SuppressWarnings("deprecation") static List<FieldSchema> partitionColumns(PartitionStrategy strategy, Schema schema) { List<FieldSchema> columns = Lists.newArrayList(); for (FieldPartitioner<?, ?> fp : Accessor.getDefault().getFieldPartitioners(strategy)) { columns.add(new FieldSchema(fp.getName(), getHiveType(SchemaUtil.getPartitionType(fp, schema)), "Partition column derived from '" + fp.getSourceName() + "' column, " + "generated by Kite.")); } return columns; }
@SuppressWarnings("unchecked") public static <S, T> Class<? extends T> getPartitionType(FieldPartitioner<S, T> fp, Schema schema) { if (fp instanceof ProvidedFieldPartitioner) { // provided partitioners have no source field schema return fp.getType(); } Class<? extends S> inputType = (Class<S>) getClassForType( fieldSchema(schema, fp.getSourceName()).getType()); return fp.getType(inputType); }
public Constraints fromAfter(String name, Comparable value) { SchemaUtil.checkTypeConsistency(schema, strategy, name, value); checkContained(name, value); Range added = Ranges.greaterThan(value); return new Constraints(this, name, combine(constraints.get(name), added)); }
/** * Checks that the type of each of {@code values} is consistent with the type of * field {@code fieldName} declared in the Avro schema (from {@code descriptor}). */ public static void checkTypeConsistency(Schema schema, PartitionStrategy strategy, String fieldName, Object... values) { Schema fieldSchema = fieldSchema(schema, strategy, fieldName); for (Object value : values) { // SpecificData#validate checks consistency for generic, reflect, // and specific models. Preconditions.checkArgument( SpecificData.get().validate(fieldSchema, value), "Value '%s' of type '%s' inconsistent with field schema %s.", value, value.getClass(), fieldSchema); } }
private static Schema merge(@Nullable Schema left, @Nullable Schema right) { if (left == null) { return right; } else if (right == null) { return left; } else { return SchemaUtil.merge(left, right); } }