@Override protected int[] createFieldPermutationForBulkLoadOp() { /** * after tokenization, the field layout becomes * [token, num?, tag, primary key, filter value] * we need to restore it back to * [tag, token, num?, primary key, filter value] */ int[] fieldPermutation = new int[NUM_TAG_FIELDS + numTokenKeyPairFields + numFilterFields]; int numSecondaryKeys = getNumSecondaryKeys(); int idx = 0; // set tag fields for (int i = 0; i < NUM_TAG_FIELDS; i++) { fieldPermutation[idx++] = i + numSecondaryKeys; } // set secondary keys for (int i = 0; i < numSecondaryKeys; i++) { fieldPermutation[idx++] = i; } // set primary key + filter for (int i = 0; i < numPrimaryKeys + numFilterFields; i++) { fieldPermutation[idx++] = i + NUM_TAG_FIELDS + numSecondaryKeys; } return fieldPermutation; }
IOperatorDescriptor primaryScanOp = createPrimaryIndexScanDiskComponentsOp(spec, metadataProvider, getTaggedRecordDescriptor(dataset.getPrimaryRecordDescriptor(metadataProvider))); int numSecondaryKeys = index.getKeyFieldNames().size(); if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) { sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced()); spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0); RecordDescriptor taggedSecondaryRecDesc = getTaggedRecordDescriptor(secondaryRecDesc); RecordDescriptor taggedTokenKeyPairRecDesc = getTaggedRecordDescriptor(tokenKeyPairRecDesc); createAssignOp(spec, numSecondaryKeys, taggedSecondaryRecDesc); createTupleProcessorOp(spec, taggedSecondaryRecDesc, numSecondaryKeys, numPrimaryKeys, true); AbstractOperatorDescriptor tokenizerOp = createTokenizerOp(spec); ExternalSortOperatorDescriptor sortOp = createSortOp(spec, getTaggedSecondaryComparatorFactories(tokenKeyPairComparatorFactories), taggedTokenKeyPairRecDesc); createTreeIndexBulkLoadOp(spec, metadataProvider, taggedTokenKeyPairRecDesc, createFieldPermutationForBulkLoadOp(), getNumSecondaryKeys(), numPrimaryKeys, true);
public static SecondaryIndexOperationsHelper createIndexOperationsHelper(Dataset dataset, Index index, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper indexOperationsHelper; switch (index.getIndexType()) { case BTREE: indexOperationsHelper = new SecondaryCorrelatedBTreeOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; case RTREE: indexOperationsHelper = new SecondaryCorrelatedRTreeOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; case SINGLE_PARTITION_WORD_INVIX: case SINGLE_PARTITION_NGRAM_INVIX: case LENGTH_PARTITIONED_WORD_INVIX: case LENGTH_PARTITIONED_NGRAM_INVIX: indexOperationsHelper = new SecondaryCorrelatedInvertedIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; default: throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, sourceLoc, index.getIndexType()); } indexOperationsHelper.init(); return indexOperationsHelper; }
@Override protected ExternalSortOperatorDescriptor createSortOp(JobSpecification spec, IBinaryComparatorFactory[] taggedSecondaryComparatorFactories, RecordDescriptor taggedSecondaryRecDesc) { /** * after tokenization, the field layout becomes * [token, num?, tag, primary key, filter value] * we need to sort on * [tag, token, num?, primary key] */ int[] taggedSortFields = new int[taggedSecondaryComparatorFactories.length]; int numSecondaryKeys = getNumSecondaryKeys(); int idx = 0; // set component pos fields taggedSortFields[idx++] = numSecondaryKeys; // set secondary keys for (int i = 0; i < numSecondaryKeys; i++) { taggedSortFields[idx++] = i; } // set primary keys for (int i = 0; i < numPrimaryKeys; i++) { taggedSortFields[idx++] = i + numSecondaryKeys + NUM_TAG_FIELDS; } ExternalSortOperatorDescriptor sortOp = new ExternalSortOperatorDescriptor(spec, sortNumFrames, taggedSortFields, taggedSecondaryComparatorFactories, taggedSecondaryRecDesc); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, sortOp, primaryPartitionConstraint); return sortOp; }