private Object getColTypes(ARecordType record) throws HyracksDataException { int n = record.getFieldTypes().length; if (n < 1) { throw new RuntimeDataException(ErrorCode.PARSER_HIVE_GET_COLUMNS); } //First Column String cols = getHiveTypeString(record.getFieldTypes(), 0); for (int i = 1; i < n; i++) { cols = cols + "," + getHiveTypeString(record.getFieldTypes(), i); } return cols; }
/** * Check if the defined type contains ADM special types. * if it contains unsupported types. * * @param recordType * @throws AsterixException */ private void checkRecordTypeCompatibility(ARecordType recordType) throws AsterixException { final IAType[] fieldTypes = recordType.getFieldTypes(); for (IAType type : fieldTypes) { checkTypeCompatibility(type); } }
protected void checkOptionalConstraints(ARecordType recordType, BitSet nullBitmap) throws RuntimeDataException { for (int i = 0; i < recordType.getFieldTypes().length; i++) { if (!nullBitmap.get(i) && !isMissableType(recordType.getFieldTypes()[i])) { throw new RuntimeDataException(ErrorCode.PARSER_TWEET_PARSER_CLOSED_FIELD_NULL, recordType.getFieldNames()[i]); } } }
public static final boolean hasOptionalField(ARecordType recType) { for (int i = 0; i < recType.getFieldTypes().length; i++) { IAType type = recType.getFieldTypes()[i]; if (type != null) { ATypeTag tag = type.getTypeTag(); if (tag == ATypeTag.NULL || tag == ATypeTag.MISSING) { return true; } if (tag != ATypeTag.UNION) { continue; } // union AUnionType unionType = (AUnionType) type; if (unionType.isUnknownableType()) { return true; } } } return false; }
/** * Creates a map from name to type for fields in the passed type * * @param recordType the type to be mapped * @return a map mapping all fields to their types */ private static Map<String, IAType> createRecordNameTypeMap(ARecordType recordType) { LinkedHashMap<String, IAType> recordNameTypesMap = new LinkedHashMap<>(); for (int j = 0; j < recordType.getFieldNames().length; j++) { recordNameTypesMap.put(recordType.getFieldNames()[j], recordType.getFieldTypes()[j]); } return recordNameTypesMap; }
public IAType getClosedFieldType(ARecordType recordType, int fieldId) { IAType aType = recordType.getFieldTypes()[fieldId]; if (NonTaggedFormatUtil.isOptional(aType)) { // optional field: add the embedded non-null type tag aType = ((AUnionType) aType).getActualType(); } return aType; }
public static ARecordType mergeRecordTypes(ARecordType recType1, ARecordType recType2) { String[] fieldNames = new String[recType1.getFieldNames().length + recType2.getFieldNames().length]; IAType[] fieldTypes = new IAType[recType1.getFieldTypes().length + recType2.getFieldTypes().length]; int i = 0; for (; i < recType1.getFieldNames().length; i++) { fieldNames[i] = recType1.getFieldNames()[i]; fieldTypes[i] = recType1.getFieldTypes()[i]; } for (int j = 0; j < recType2.getFieldNames().length; i++, j++) { fieldNames[i] = recType2.getFieldNames()[j]; fieldTypes[i] = recType2.getFieldTypes()[j]; } return new ARecordType(null, fieldNames, fieldTypes, true); }
@Override protected IAType getResultType(ILogicalExpression expr, IAType... strippedInputTypes) throws AlgebricksException { IAType firstArgType = strippedInputTypes[0]; if (firstArgType.getTypeTag() != ATypeTag.OBJECT) { return BuiltinType.ANY; } AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr; Integer pos = ConstantExpressionUtil.getIntArgument(funcExpr, 1); if (pos == null) { return BuiltinType.ANY; } ARecordType recType = (ARecordType) firstArgType; return recType.getFieldTypes()[pos]; }
private int checkOptionalConstraints(ARecordType recType, BitSet nulls) { for (int i = 0; i < recType.getFieldTypes().length; i++) { if (nulls.get(i) == false) { IAType type = recType.getFieldTypes()[i]; if ((type.getTypeTag() != ATypeTag.NULL) && (type.getTypeTag() != ATypeTag.UNION)) { return i; } if (type.getTypeTag() != ATypeTag.UNION) { continue; } // union AUnionType unionType = (AUnionType) type; if (!unionType.isUnknownableType()) { return i; } } } return -1; }
public CSVToRecordWithMetadataAndPKConverter(final int valueIndex, final char delimiter, final ARecordType metaType, final ARecordType recordType, final int[] keyIndicator, final int[] keyIndexes, final IAType[] keyTypes) { this.cursor = new FieldCursorForDelimitedDataParser(null, delimiter, ExternalDataConstants.QUOTE); this.record = new CharArrayRecord(); this.valueIndex = valueIndex; this.recordWithMetadata = new RecordWithMetadataAndPK<char[]>(record, metaType.getFieldTypes(), recordType, keyIndicator, keyIndexes, keyTypes); }
private boolean validateKey(int i, ARecordType type, boolean unique) { if (type.getFieldNames().length <= i) { throw new IllegalArgumentException("key index must be less than number of fields"); } return unique && type.getFieldTypes()[i].getTypeTag() != ATypeTag.BOOLEAN; }
private String getCompressionScheme(ARecord datasetRecord) { final ARecordType datasetType = datasetRecord.getType(); final int compressionIndex = datasetType .getFieldIndex(MetadataRecordTypes.DATASET_ARECORD_BLOCK_LEVEL_STORAGE_COMPRESSION_FIELD_NAME); if (compressionIndex >= 0) { final ARecordType compressionType = (ARecordType) datasetType.getFieldTypes()[compressionIndex]; final int schemeIndex = compressionType .getFieldIndex(MetadataRecordTypes.DATASET_ARECORD_DATASET_COMPRESSION_SCHEME_FIELD_NAME); return ((AString) datasetRecord.getValueByPos(schemeIndex)).getStringValue(); } return CompressionManager.NONE; }
public static int checkOptionalConstraints(ARecordType recType, BitSet nulls) { for (int i = 0; i < recType.getFieldTypes().length; i++) { if (nulls.get(i) == false) { IAType type = recType.getFieldTypes()[i]; if (type.getTypeTag() != ATypeTag.MISSING && type.getTypeTag() != ATypeTag.UNION) { return i; } if (type.getTypeTag() == ATypeTag.UNION) { // union AUnionType unionType = (AUnionType) type; if (!unionType.isUnknownableType()) { return i; } } } } return -1; }
private void writeFeedAdaptorField(IARecordBuilder recordBuilder, Feed feed, ArrayBackedValueStorage fieldValueBuffer) throws HyracksDataException { UnorderedListBuilder listBuilder = new UnorderedListBuilder(); ArrayBackedValueStorage listEleBuffer = new ArrayBackedValueStorage(); listBuilder.reset((AUnorderedListType) MetadataRecordTypes.FEED_RECORDTYPE .getFieldTypes()[MetadataRecordTypes.FEED_ARECORD_ADAPTOR_CONFIG_INDEX]); for (Map.Entry<String, String> property : feed.getConfiguration().entrySet()) { String name = property.getKey(); String value = property.getValue(); listEleBuffer.reset(); writePropertyTypeRecord(name, value, listEleBuffer.getDataOutput()); listBuilder.addItem(listEleBuffer); } listBuilder.write(fieldValueBuffer.getDataOutput(), true); recordBuilder.addField(MetadataRecordTypes.FEED_ARECORD_ADAPTOR_CONFIG_INDEX, fieldValueBuffer); }
RecordUnwrapEvaluator(IScalarEvaluator eval0, ARecordType recordType) { this.eval0 = eval0; openRecordVisitablePointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE); if (recordType != null) { inputRecordVisitable = new ARecordVisitablePointable(recordType); if (hasDerivedType(recordType.getFieldTypes())) { requiresCast = true; castVisitor = new ACastVisitor(); castVisitorArg = new Triple<>(openRecordVisitablePointable, openRecordVisitablePointable.getInputRecordType(), Boolean.FALSE); } } }
public ARecordSerializerDeserializer(ARecordType recordType) { if (recordType != null) { this.recordType = recordType; this.numberOfSchemaFields = recordType.getFieldNames().length; serializers = new ISerializerDeserializer[numberOfSchemaFields]; deserializers = new ISerializerDeserializer[numberOfSchemaFields]; for (int i = 0; i < numberOfSchemaFields; i++) { IAType t = recordType.getFieldTypes()[i]; IAType t2 = (t.getTypeTag() == ATypeTag.UNION) ? ((AUnionType) t).getActualType() : t; serializers[i] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(t2); deserializers[i] = SerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(t2); } } else { this.recordType = null; this.numberOfSchemaFields = 0; this.serializers = null; this.deserializers = null; } }
RecordRemoveEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, ARecordType recordType) { this.eval0 = eval0; this.eval1 = eval1; openRecordPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE); if (recordType != null) { inputRecordVisitable = new ARecordVisitablePointable(recordType); if (hasDerivedType(recordType.getFieldTypes())) { requiresCast = true; castVisitor = new ACastVisitor(); castVisitorArg = new Triple<>(openRecordPointable, openRecordPointable.getInputRecordType(), Boolean.FALSE); } } }
RecordRenameEvaluator(IScalarEvaluator eval0, IScalarEvaluator eval1, IScalarEvaluator eval2, ARecordType recordType) { this.eval0 = eval0; this.eval1 = eval1; this.eval2 = eval2; openRecordPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE); if (recordType != null) { inputRecordVisitable = new ARecordVisitablePointable(recordType); if (hasDerivedType(recordType.getFieldTypes())) { requiresCast = true; castVisitor = new ACastVisitor(); castVisitorArg = new Triple<>(openRecordPointable, openRecordPointable.getInputRecordType(), Boolean.FALSE); } } }
private void writeAppliedFunctionsField(IARecordBuilder rb, FeedConnection fc, ArrayBackedValueStorage buffer) throws HyracksDataException { UnorderedListBuilder listBuilder = new UnorderedListBuilder(); ArrayBackedValueStorage listEleBuffer = new ArrayBackedValueStorage(); listBuilder.reset((AUnorderedListType) MetadataRecordTypes.FEED_CONNECTION_RECORDTYPE .getFieldTypes()[MetadataRecordTypes.FEED_CONN_APPLIED_FUNCTIONS_FIELD_INDEX]); if (fc.getAppliedFunctions() != null) { List<FunctionSignature> appliedFunctions = fc.getAppliedFunctions(); for (FunctionSignature af : appliedFunctions) { listEleBuffer.reset(); aString.setValue(af.getNamespace() + "." + af.getName()); stringSerde.serialize(aString, listEleBuffer.getDataOutput()); listBuilder.addItem(listEleBuffer); } } listBuilder.write(buffer.getDataOutput(), true); rb.addField(MetadataRecordTypes.FEED_CONN_APPLIED_FUNCTIONS_FIELD_INDEX, buffer); } }
@SuppressWarnings({ "unchecked", "rawtypes" }) public static void serializeSchemalessRecord(ARecord record, DataOutput dataOutput, boolean writeTypeTag) throws HyracksDataException { ISerializerDeserializer<AString> stringSerde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING); RecordBuilder confRecordBuilder = new RecordBuilder(); confRecordBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE); ArrayBackedValueStorage fieldNameBytes = new ArrayBackedValueStorage(); ArrayBackedValueStorage fieldValueBytes = new ArrayBackedValueStorage(); for (int i = 0; i < record.getType().getFieldNames().length; i++) { String fieldName = record.getType().getFieldNames()[i]; fieldValueBytes.reset(); fieldNameBytes.reset(); stringSerde.serialize(new AString(fieldName), fieldNameBytes.getDataOutput()); ISerializerDeserializer valueSerde = SerializerDeserializerProvider.INSTANCE .getSerializerDeserializer(record.getType().getFieldTypes()[i]); valueSerde.serialize(record.getValueByPos(i), fieldValueBytes.getDataOutput()); confRecordBuilder.addField(fieldNameBytes, fieldValueBytes); } confRecordBuilder.write(dataOutput, writeTypeTag); }