final PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo<?>) typeInfo; HashMap<String, TaggedValue> containerMapping = new HashMap<String, TaggedValue>(); for (int i = 0; i < pojoTypeInfo.getArity(); i++) { final String fieldName = pojoTypeInfo.getPojoFieldAt(i).getField().getName(); containerMapping.put(fieldName, convertTypeInfoToTaggedValue(input, pojoTypeInfo.getTypeAt(i), (flatFieldExpr.length() > 0 ? flatFieldExpr + "." : "") + fieldName, pojoTypeInfo.getFlatFields(fieldName), groupedKeys));
private void configure(String lineDelimiter, String fieldDelimiter, PojoTypeInfo<OUT> pojoTypeInfo, String[] fieldNames, boolean[] includedFieldsMask) { if (includedFieldsMask == null) { includedFieldsMask = createDefaultMask(fieldNames.length); } for (String name : fieldNames) { if (name == null) { throw new NullPointerException("Field name must not be null."); } if (pojoTypeInfo.getFieldIndex(name) < 0) { throw new IllegalArgumentException("Field \"" + name + "\" not part of POJO type " + pojoTypeInfo.getTypeClass().getCanonicalName()); } } setDelimiter(lineDelimiter); setFieldDelimiter(fieldDelimiter); Class<?>[] classes = new Class<?>[fieldNames.length]; for (int i = 0; i < fieldNames.length; i++) { try { classes[i] = pojoTypeInfo.getTypeAt(pojoTypeInfo.getFieldIndex(fieldNames[i])).getTypeClass(); } catch (IndexOutOfBoundsException e) { throw new IllegalArgumentException("Invalid field name: " + fieldNames[i]); } } this.pojoTypeClass = pojoTypeInfo.getTypeClass(); this.pojoTypeInfo = pojoTypeInfo; setFieldsGeneric(includedFieldsMask, classes); setOrderOfPOJOFields(fieldNames); }
return new PojoTypeInfo<>(pojoClass, pojoFields);
public PojoCsvInputFormat(Path filePath, String lineDelimiter, String fieldDelimiter, PojoTypeInfo<OUT> pojoTypeInfo) { this(filePath, lineDelimiter, fieldDelimiter, pojoTypeInfo, pojoTypeInfo.getFieldNames(), createDefaultMask(pojoTypeInfo.getArity())); }
@Override @PublicEvolving @SuppressWarnings("unchecked") public TypeSerializer<T> createSerializer(ExecutionConfig config) { if (config.isForceKryoEnabled()) { return new KryoSerializer<>(getTypeClass(), config); } if (config.isForceAvroEnabled()) { return AvroUtils.getAvroUtils().createAvroSerializer(getTypeClass()); } return createPojoSerializer(config); }
PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo) typeInfo; int fieldIndex = pojoTypeInfo.getFieldIndex(decomp.head); "Unable to find field \"" + decomp.head + "\" in type " + typeInfo + "."); } else { PojoField pojoField = pojoTypeInfo.getPojoFieldAt(fieldIndex); TypeInformation<?> fieldType = pojoTypeInfo.getTypeAt(fieldIndex); if (decomp.tail == null) { @SuppressWarnings("unchecked")
private void checkFromTuplePojo(TypeInformation<?> typeInformation) { Assert.assertTrue(typeInformation instanceof PojoTypeInfo<?>); Assert.assertEquals(4, typeInformation.getTotalFields()); PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeInformation; for(int i = 0; i < pojoTypeForClass.getArity(); i++) { PojoField field = pojoTypeForClass.getPojoFieldAt(i); String name = field.getField().getName(); if(name.equals("special")) { Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.getTypeInformation()); } else if(name.equals("f0") || name.equals("f1")) { Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.getTypeInformation()); } else if(name.equals("f2")) { Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.getTypeInformation()); } else { Assert.fail("unexpected field"); } } }
@SuppressWarnings("unchecked") @Internal private static <T extends SpecificRecordBase> List<PojoField> generateFieldsFromAvroSchema(Class<T> typeClass) { PojoTypeExtractor pte = new PojoTypeExtractor(); ArrayList<Type> typeHierarchy = new ArrayList<>(); typeHierarchy.add(typeClass); TypeInformation ti = pte.analyzePojo(typeClass, typeHierarchy, null, null, null); if (!(ti instanceof PojoTypeInfo)) { throw new IllegalStateException("Expecting type to be a PojoTypeInfo"); } PojoTypeInfo pti = (PojoTypeInfo) ti; List<PojoField> newFields = new ArrayList<>(pti.getTotalFields()); for (int i = 0; i < pti.getArity(); i++) { PojoField f = pti.getPojoFieldAt(i); TypeInformation newType = f.getTypeInformation(); // check if type is a CharSequence if (newType instanceof GenericTypeInfo) { if ((newType).getTypeClass().equals(CharSequence.class)) { // replace the type by a org.apache.avro.util.Utf8 newType = new GenericTypeInfo(org.apache.avro.util.Utf8.class); } } PojoField newField = new PojoField(f.getField(), newType); newFields.add(newField); } return newFields; }
private static TypeInformation<?> getTypeOfPojoField(TypeInformation<?> pojoInfo, Field field) { for (int j = 0; j < pojoInfo.getArity(); j++) { PojoField pf = ((PojoTypeInfo<?>) pojoInfo).getPojoFieldAt(j); if (pf.getField().getName().equals(field.getName())) { return pf.getTypeInformation(); } } return null; }
/** * Extracts the {@link InternalType}s from {@link PojoTypeInfo} corresponding to the given fieldNames. */ private static <OUT> InternalType[] extractTypeInfo(PojoTypeInfo<OUT> pojoTypeInfo, String[] fieldNames) { Preconditions.checkNotNull(pojoTypeInfo); Preconditions.checkNotNull(fieldNames); Preconditions.checkArgument(pojoTypeInfo.getArity() >= fieldNames.length); RowType rowType = (RowType) TypeConverters.createInternalTypeFromTypeInfo(pojoTypeInfo); InternalType[] fieldTypes = new InternalType[fieldNames.length]; for (int i = 0; i < fieldNames.length; ++i) { String fieldName = fieldNames[i]; Preconditions.checkNotNull(fieldName, "The field name cannot be null."); int fieldPos = pojoTypeInfo.getFieldIndex(fieldName); Preconditions.checkArgument(fieldPos >= 0, "Field \"" + fieldName + "\" is not a member of POJO class " + pojoTypeInfo.getTypeClass().getName()); fieldTypes[i] = rowType.getInternalTypeAt(fieldPos).toInternalType(); } return fieldTypes; }
private void setOrderOfPOJOFields(String[] fieldNames) { Preconditions.checkNotNull(fieldNames); int includedCount = 0; for (boolean isIncluded : fieldIncluded) { if (isIncluded) { includedCount++; } } Preconditions.checkArgument(includedCount == fieldNames.length, includedCount + " CSV fields and " + fieldNames.length + " POJO fields selected. The number of selected CSV and POJO fields must be equal."); for (String field : fieldNames) { Preconditions.checkNotNull(field, "The field name cannot be null."); Preconditions.checkArgument(pojoTypeInfo.getFieldIndex(field) != -1, "Field \"" + field + "\" is not a member of POJO class " + pojoTypeClass.getName()); } pojoFieldNames = Arrays.copyOfRange(fieldNames, 0, fieldNames.length); }
@Override public String toString() { List<String> fieldStrings = new ArrayList<String>(); for (PojoField field : fields) { fieldStrings.add(field.getField().getName() + ": " + field.getTypeInformation().toString()); } return "PojoType<" + getTypeClass().getName() + ", fields = [" + StringUtils.join(fieldStrings, ", ") + "]" + ">"; }
public PojoCsvInputFormat(Path filePath, PojoTypeInfo<OUT> pojoTypeInfo, String[] fieldNames) { this(filePath, DEFAULT_LINE_DELIMITER, DEFAULT_FIELD_DELIMITER, pojoTypeInfo, fieldNames, createDefaultMask(pojoTypeInfo.getArity())); }
public PojoCsvInputFormat(Path filePath, String lineDelimiter, String fieldDelimiter, PojoTypeInfo<OUT> pojoTypeInfo, int[] includedFieldsMask) { this(filePath, lineDelimiter, fieldDelimiter, pojoTypeInfo, pojoTypeInfo.getFieldNames(), includedFieldsMask); }
public PojoParquetInputFormat(Path filePath, PojoTypeInfo<OUT> pojoTypeInfo, String[] fieldNames, ExecutionConfig config) { super(filePath, extractTypeInfo(pojoTypeInfo, fieldNames), fieldNames); this.pojoSerializer = pojoTypeInfo.createSerializer(config); this.pojoTypeClass = pojoTypeInfo.getTypeClass(); }
pType.getFlatFields("nestedClass.dumm2", 0, result); int[] fields = new int[1]; // see below fields[0] = result.get(0).getPosition(); TypeComparator<TestUserClass> pojoComp = pType.createComparator( fields, new boolean[]{true}, 0, new ExecutionConfig()); Assert.fail("Keys must be compatible: "+e.getMessage()); TypeComparator<TestUserClass> multiPojoComp = pType.createComparator( expressKey.computeLogicalKeyPositions(), new boolean[]{true, true, true}, 0, new ExecutionConfig()); int multiPojoHash = multiPojoComp.hash(pojoTestRecord);
assertEquals(4, deepNestedTupleTypeInfo.getFlatFields("*").get(4).getPosition()); assertEquals(0, pojoTypeInfo.getFlatFields("a").get(0).getPosition()); assertEquals(1, pojoTypeInfo.getFlatFields("b").get(0).getPosition()); assertEquals(2, pojoTypeInfo.getFlatFields("*").size()); assertEquals(0, pojoTypeInfo.getFlatFields("*").get(0).getPosition()); assertEquals(1, pojoTypeInfo.getFlatFields("*").get(1).getPosition());
PojoTypeInfo<?> pojoTypeInfo = (PojoTypeInfo) typeInfo; int fieldIndex = pojoTypeInfo.getFieldIndex(decomp.head); "Unable to find field \"" + decomp.head + "\" in type " + typeInfo + "."); } else { PojoField pojoField = pojoTypeInfo.getPojoFieldAt(fieldIndex); TypeInformation<?> fieldType = pojoTypeInfo.getTypeAt(fieldIndex); if (decomp.tail == null) { @SuppressWarnings("unchecked")
@Test public void testPojoWithGenerics() { TypeInformation<?> typeForClass = TypeExtractor.createTypeInfo(ParentSettingGenerics.class); Assert.assertTrue(typeForClass instanceof PojoTypeInfo<?>); PojoTypeInfo<?> pojoTypeForClass = (PojoTypeInfo<?>) typeForClass; for(int i = 0; i < pojoTypeForClass.getArity(); i++) { PojoField field = pojoTypeForClass.getPojoFieldAt(i); String name = field.getField().getName(); if(name.equals("field1")) { Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.getTypeInformation()); } else if (name.equals("field2")) { Assert.assertEquals(BasicTypeInfo.LONG_TYPE_INFO, field.getTypeInformation()); } else if (name.equals("field3")) { Assert.assertEquals(BasicTypeInfo.STRING_TYPE_INFO, field.getTypeInformation()); } else if (name.equals("key")) { Assert.assertEquals(BasicTypeInfo.INT_TYPE_INFO, field.getTypeInformation()); } else { Assert.fail("Unexpected field "+field); } } }