private static ARecordType getStorageBlockCompressionType() { final String[] schemeName = { STORAGE_BLOCK_COMPRESSION_SCHEME_PARAMETER_NAME }; final IAType[] schemeType = { BuiltinType.ASTRING }; return new ARecordType(STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME, schemeName, schemeType, false); } }
/** * Create a fully open record type with the passed name * * @param name * @return */ public static ARecordType createOpenRecordType(String name) { return new ARecordType(name, new String[0], new IAType[0], true); }
@Override public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException { return new ARecordType(null, new String[] { "sum", "count" }, new IAType[] { AUnionType.createNullableType(BuiltinType.ADOUBLE, "OptionalDouble"), BuiltinType.AINT64 }, false); } }
public static ARecordType createRecordType(String recordTypeName, String[] fieldNames, IAType[] fieldTypes, boolean isOpen) { ARecordType recordType = new ARecordType(recordTypeName, fieldNames, fieldTypes, isOpen); if (recordTypeName != null) { recordType.generateNestedDerivedTypeNames(); } return recordType; }
public AbstractSingleVarStatisticsAggregateFunction(IScalarEvaluatorFactory[] args, IHyracksTaskContext context, SourceLocation sourceLoc) throws HyracksDataException { super(sourceLoc); eval = args[0].createScalarEvaluator(context); recType = new ARecordType(null, new String[] { "m1", "m2", "count" }, new IAType[] { BuiltinType.ADOUBLE, BuiltinType.ADOUBLE, BuiltinType.AINT64 }, false); recordEval = new ClosedRecordConstructorEval(recType, new IScalarEvaluator[] { evalM1, evalM2, evalCount }); }
public AbstractAvgAggregateFunction(IScalarEvaluatorFactory[] args, IHyracksTaskContext context, SourceLocation sourceLoc) throws HyracksDataException { super(sourceLoc); eval = args[0].createScalarEvaluator(context); recType = new ARecordType(null, new String[] { "sum", "count" }, new IAType[] { BuiltinType.ADOUBLE, BuiltinType.AINT64 }, false); recordEval = new ClosedRecordConstructorEval(recType, new IScalarEvaluator[] { evalSum, evalCount }); }
public ARecordType deepCopy(ARecordType type) { IAType[] newTypes = new IAType[type.fieldNames.length]; for (int i = 0; i < type.fieldTypes.length; i++) { if (type.fieldTypes[i].getTypeTag() == ATypeTag.OBJECT) { newTypes[i] = deepCopy((ARecordType) type.fieldTypes[i]); } else { newTypes[i] = type.fieldTypes[i]; } } return new ARecordType(type.typeName, type.fieldNames, newTypes, type.isOpen); }
@Override public IAType computeType(ILogicalExpression expression, IVariableTypeEnvironment env, IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException { return new ARecordType(null, new String[] { "m1", "m2", "count" }, new IAType[] { AUnionType.createNullableType(BuiltinType.ADOUBLE, "OptionalDouble"), AUnionType.createNullableType(BuiltinType.ADOUBLE, "OptionalDouble"), BuiltinType.AINT64 }, false); } }
private static ARecordType getMergePolicyType() { //merge-policy.parameters final String[] parameterNames = { MERGE_POLICY_MERGABLE_SIZE_PARAMETER_NAME, MERGE_POLICY_TOLERANCE_COUNT_PARAMETER_NAME, MERGE_POLICY_NUMBER_COMPONENTS_PARAMETER_NAME }; final IAType[] parametersTypes = { AUnionType.createUnknownableType(BuiltinType.AINT64), AUnionType.createUnknownableType(BuiltinType.AINT64), AUnionType.createUnknownableType(BuiltinType.AINT64) }; final ARecordType parameters = new ARecordType(MERGE_POLICY_PARAMETERS_PARAMETER_NAME, parameterNames, parametersTypes, false); //merge-policy final String[] mergePolicyNames = { MERGE_POLICY_NAME_PARAMETER_NAME, MERGE_POLICY_PARAMETERS_PARAMETER_NAME }; final IAType[] mergePolicyTypes = { BuiltinType.ASTRING, AUnionType.createUnknownableType(parameters) }; return new ARecordType(MERGE_POLICY_PARAMETER_NAME, mergePolicyNames, mergePolicyTypes, false); }
private IAType buildOutputType(Deque<String> fieldPathStack, ARecordType inputRecordType, Set<String> fieldNameSet, List<List<String>> pathList) throws AlgebricksException { List<String> resultFieldNames = new ArrayList<>(); List<IAType> resultFieldTypes = new ArrayList<>(); String[] fieldNames = inputRecordType.getFieldNames(); IAType[] fieldTypes = inputRecordType.getFieldTypes(); for (int i = 0; i < fieldNames.length; i++) { if (!fieldNameSet.contains(fieldNames[i])) { // The main field is to be kept addField(inputRecordType, fieldNames[i], resultFieldNames, resultFieldTypes); } else if (!pathList.isEmpty() && fieldTypes[i].getTypeTag() == ATypeTag.OBJECT) { ARecordType subRecord = (ARecordType) fieldTypes[i]; fieldPathStack.push(fieldNames[i]); subRecord = deepCheckAndCopy(fieldPathStack, subRecord, pathList, inputRecordType.isOpen()); fieldPathStack.pop(); if (subRecord != null) { resultFieldNames.add(fieldNames[i]); resultFieldTypes.add(subRecord); } } } int n = resultFieldNames.size(); String resultTypeName = "result-record(" + inputRecordType.getTypeName() + ")"; return new ARecordType(resultTypeName, resultFieldNames.toArray(new String[n]), resultFieldTypes.toArray(new IAType[n]), true); // Make the output type open always }
private static ARecordType getWithObjectType() { final String[] withNames = { MERGE_POLICY_PARAMETER_NAME, STORAGE_BLOCK_COMPRESSION_PARAMETER_NAME }; final IAType[] withTypes = { AUnionType.createUnknownableType(getMergePolicyType()), AUnionType.createUnknownableType(getStorageBlockCompressionType()) }; return new ARecordType("withObject", withNames, withTypes, false); }
@Override public IAObject getIAObject() { // As the open part can be changed any time, we cannot pre-allocate the arrays. int numberOfOpenFields = openFields.size(); String[] openFieldNames = new String[numberOfOpenFields]; IAType[] openFieldTypes = new IAType[numberOfOpenFields]; IAObject[] openFieldValues = new IAObject[numberOfOpenFields]; IAObject[] closedFieldValues = new IAObject[fields.length]; int idx = 0; for (Map.Entry<String, IJObject> entry : openFields.entrySet()) { openFieldNames[idx] = entry.getKey(); openFieldTypes[idx] = entry.getValue().getIAObject().getType(); openFieldValues[idx] = entry.getValue().getIAObject(); } for (int iter1 = 0; iter1 < fields.length; iter1++) { closedFieldValues[iter1] = fields[iter1].getIAObject(); } ARecordType openPartRecType = new ARecordType(null, openFieldNames, openFieldTypes, true); ARecordType mergedRecordType = ARecordSerializerDeserializer.mergeRecordTypes(recordType, openPartRecType); IAObject[] mergedFields = ARecordSerializerDeserializer.mergeFields(closedFieldValues, openFieldValues); return new ARecord(mergedRecordType, mergedFields); }
private static ARecordType generalizeRecordTypes(ARecordType leftType, ARecordType rightType) { boolean knowsAdditonalFieldNames = true; Set<String> allPossibleAdditionalFieldNames = new HashSet<>(); if (leftType.isOpen() && !leftType.knowsAllPossibleAdditonalFieldNames()) { knowsAdditonalFieldNames = false; } else if (leftType.isOpen()) { allPossibleAdditionalFieldNames.addAll(leftType.getAllPossibleAdditonalFieldNames()); } if (rightType.isOpen() && !rightType.knowsAllPossibleAdditonalFieldNames()) { knowsAdditonalFieldNames = false; } else if (rightType.isOpen()) { allPossibleAdditionalFieldNames.addAll(rightType.getAllPossibleAdditonalFieldNames()); } boolean canBeClosed = !leftType.isOpen() && !rightType.isOpen(); List<String> fieldNames = new ArrayList<>(); List<IAType> fieldTypes = new ArrayList<>(); boolean leftAllMatched = generalizeRecordFields(leftType, rightType, allPossibleAdditionalFieldNames, fieldNames, fieldTypes); boolean rightAllMatched = generalizeRecordFields(rightType, leftType, allPossibleAdditionalFieldNames, fieldNames, fieldTypes); return new ARecordType("generalized-record-type", fieldNames.toArray(new String[fieldNames.size()]), fieldTypes.toArray(new IAType[fieldTypes.size()]), !(canBeClosed && leftAllMatched && rightAllMatched), knowsAdditonalFieldNames ? allPossibleAdditionalFieldNames : null); }
private ARecordType getStrippedPKType(List<String> partitioningKeys, ARecordType recType) throws AlgebricksException, HyracksDataException { List<String> fieldNames = new LinkedList<>(); List<IAType> fieldTypes = new LinkedList<>(); int j = 0; for (int i = 0; i < recType.getFieldNames().length; i++) { IAType fieldType; if (partitioningKeys.get(0).equals(recType.getFieldNames()[j])) { if (recType.getFieldTypes()[j].getTypeTag() == ATypeTag.OBJECT) { if (j != 0) { throw new AsterixException("Autogenerated key " + StringUtils.join(partitioningKeys, '.') + " should be a first field of the type " + recType.getTypeName()); } partitioningKeys.remove(0); fieldType = getStrippedPKType(partitioningKeys, (ARecordType) recType.getFieldTypes()[j]); } else { j++; continue; } } else { fieldType = recType.getFieldTypes()[j]; } fieldTypes.add(fieldType); fieldNames.add(recType.getFieldNames()[j]); j++; } return new ARecordType(recType.getTypeName(), fieldNames.toArray(new String[0]), fieldTypes.toArray(new IAType[0]), recType.isOpen()); }
@Test public void closedRecordTypeTest() throws IOException, URISyntaxException { // contruct type IAType geoFieldType = new ARecordType("GeoType", new String[] { "coordinates" }, new IAType[] { new AOrderedListType(AFLOAT, "point") }, true); ARecordType tweetRecordType = new ARecordType("TweetType", new String[] { "id", "geo" }, new IAType[] { AINT64, geoFieldType }, true); TweetParser parser = new TweetParser(tweetRecordType); List<String> lines = Files.readAllLines(Paths.get(getClass().getResource("/test_tweets.txt").toURI())); ByteArrayOutputStream is = new ByteArrayOutputStream(); DataOutput output = new DataOutputStream(is); CharArrayRecord record = new CharArrayRecord(); int regularCount = 0; for (int iter1 = 0; iter1 < lines.size(); iter1++) { record.set(lines.get(iter1)); try { parser.parse(record, output); regularCount++; } catch (HyracksDataException e) { Assert.assertTrue(e.toString().contains("Non-null") && (iter1 == 0 || iter1 == 1)); } } Assert.assertTrue(regularCount == 4); } }
@Test public void openRecordTypeTest() throws IOException, URISyntaxException { String[] ids = { "720549057849114629", "668950503552864257", "668945640186101761", "263602997047730177", "668948268605403136", "741701526859567104" }; // contruct type IAType geoFieldType = new ARecordType("GeoType", new String[] { "coordinates" }, new IAType[] { new AOrderedListType(AFLOAT, "point") }, true); List<IAType> unionTypeList = new ArrayList<>(); unionTypeList.add(geoFieldType); unionTypeList.add(ANULL); unionTypeList.add(AMISSING); IAType geoUnionType = new AUnionType(unionTypeList, "GeoType?"); ARecordType tweetRecordType = new ARecordType("TweetType", new String[] { "id", "geo" }, new IAType[] { AINT64, geoUnionType }, true); TweetParser parser = new TweetParser(tweetRecordType); CharArrayRecord record = new CharArrayRecord(); List<String> lines = Files.readAllLines(Paths.get(getClass().getResource("/test_tweets.txt").toURI())); ByteArrayOutputStream is = new ByteArrayOutputStream(); DataOutput output = new DataOutputStream(is); for (int iter1 = 0; iter1 < lines.size(); iter1++) { record.set(lines.get(iter1)); try { parser.parse(record, output); } catch (HyracksDataException e) { e.printStackTrace(); Assert.fail("Unexpected failure in parser."); } Assert.assertTrue((PA.getValue(parser, "aInt64")).toString().equals(ids[iter1])); } }
fieldTypes[i] = openFields[i].getType(); ARecordType openPartRecType = new ARecordType(null, fieldNames, fieldTypes, true); if (numberOfSchemaFields > 0) { ARecordType mergedRecordType = mergeRecordTypes(this.recordType, openPartRecType);
ATypeTag aggType = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(state[start + AGG_TYPE_OFFSET]); if (recordEval == null) { ARecordType recType = new ARecordType(null, new String[] { "sum", "count" }, new IAType[] { BuiltinType.ADOUBLE, BuiltinType.AINT64 }, false); recordEval = new ClosedRecordConstructorEval(recType, new IScalarEvaluator[] { evalSum, evalCount });
public static ARecordType mergeRecordTypes(ARecordType recType1, ARecordType recType2) { String[] fieldNames = new String[recType1.getFieldNames().length + recType2.getFieldNames().length]; IAType[] fieldTypes = new IAType[recType1.getFieldTypes().length + recType2.getFieldTypes().length]; int i = 0; for (; i < recType1.getFieldNames().length; i++) { fieldNames[i] = recType1.getFieldNames()[i]; fieldTypes[i] = recType1.getFieldTypes()[i]; } for (int j = 0; j < recType2.getFieldNames().length; i++, j++) { fieldNames[i] = recType2.getFieldNames()[j]; fieldTypes[i] = recType2.getFieldTypes()[j]; } return new ARecordType(null, fieldNames, fieldTypes, true); }
oldTypes[pos] = mergedNestedType(fname, fieldType1Copy.getFieldTypes()[i], resultType.getFieldTypes()[pos], sourceLoc); resultType = new ARecordType(resultType.getTypeName(), resultType.getFieldNames(), oldTypes, resultType.isOpen()); IAType[] combinedFieldTypes = ArrayUtils.addAll(resultType.getFieldTypes().clone(), fieldType1Copy.getFieldTypes()[i]); resultType = new ARecordType(resultType.getTypeName(), ArrayUtils.addAll(resultType.getFieldNames(), fieldType1Copy.getFieldNames()[i]), combinedFieldTypes, resultType.isOpen());