public static JobSpecification buildSecondaryIndexCreationJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); return secondaryIndexHelper.buildCreationJobSpec(); }
protected int[] createFieldPermutationForBulkLoadOp() { int[] fieldPermutation = new int[NUM_TAG_FIELDS + getNumSecondaryKeys() + numPrimaryKeys + numFilterFields]; for (int i = 0; i < fieldPermutation.length; i++) { fieldPermutation[i] = i; } return fieldPermutation; }
public static JobSpecification buildDropIndexJobSpec(Index index, MetadataProvider metadataProvider, Dataset dataset, Set<DropOption> options, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); return secondaryIndexHelper.buildDropJobSpec(options); }
public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider, List<ExternalFile> files, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper; if (dataset.isCorrelated()) { secondaryIndexHelper = SecondaryCorrelatedTreeIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); } else { secondaryIndexHelper = SecondaryTreeIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); } if (files != null) { secondaryIndexHelper.setExternalFiles(files); } return secondaryIndexHelper.buildLoadingJobSpec(); }
public static SecondaryIndexOperationsHelper createIndexOperationsHelper(Dataset dataset, Index index, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper indexOperationsHelper; switch (index.getIndexType()) { case BTREE: indexOperationsHelper = new SecondaryBTreeOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; case RTREE: indexOperationsHelper = new SecondaryRTreeOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; case SINGLE_PARTITION_WORD_INVIX: case SINGLE_PARTITION_NGRAM_INVIX: case LENGTH_PARTITIONED_WORD_INVIX: case LENGTH_PARTITIONED_NGRAM_INVIX: indexOperationsHelper = new SecondaryInvertedIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; default: throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, sourceLoc, index.getIndexType()); } indexOperationsHelper.init(); return indexOperationsHelper; }
public static SecondaryIndexOperationsHelper createIndexOperationsHelper(Dataset dataset, Index index, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper indexOperationsHelper; switch (index.getIndexType()) { case BTREE: indexOperationsHelper = new SecondaryCorrelatedBTreeOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; case RTREE: indexOperationsHelper = new SecondaryCorrelatedRTreeOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; case SINGLE_PARTITION_WORD_INVIX: case SINGLE_PARTITION_NGRAM_INVIX: case LENGTH_PARTITIONED_WORD_INVIX: case LENGTH_PARTITIONED_NGRAM_INVIX: indexOperationsHelper = new SecondaryCorrelatedInvertedIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); break; default: throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, sourceLoc, index.getIndexType()); } indexOperationsHelper.init(); return indexOperationsHelper; }
/*** * Creates a node group that is associated with a new dataset. * * @param dataverseName, * the dataverse name of the dataset. * @param datasetName, * the name of the dataset. * @param ncNames, * the set of node names. * @param metadataProvider, * the metadata provider. * @return the name of the created node group. * @throws Exception */ public static String createNodeGroupForNewDataset(String dataverseName, String datasetName, Set<String> ncNames, MetadataProvider metadataProvider) throws Exception { return createNodeGroupForNewDataset(dataverseName, datasetName, 0L, ncNames, metadataProvider); }
protected SecondaryIndexOperationsHelper(Dataset dataset, Index index, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { this.dataset = dataset; this.index = index; this.metadataProvider = metadataProvider; this.itemType = (ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); this.metaType = DatasetUtil.getMetaType(metadataProvider, dataset); Pair<ARecordType, ARecordType> enforcedTypes = getEnforcedType(index, itemType, metaType); this.enforcedItemType = enforcedTypes.first; this.enforcedMetaType = enforcedTypes.second; this.sourceLoc = sourceLoc; this.sortNumFrames = getSortNumFrames(metadataProvider, sourceLoc); }
public static JobSpecification buildSecondaryIndexCompactJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); return secondaryIndexHelper.buildCompactJobSpec(); }
public static int[] getFilterFields(Dataset dataset, Index index, ITypeTraits[] filterTypeTraits) throws AlgebricksException { if (index.isPrimaryIndex()) { return DatasetUtil.createFilterFields(dataset); } return secondaryFilterFields(dataset, index, filterTypeTraits); }
public FileSplit[] splitsForIndex(MetadataTransactionContext mdTxnCtx, Dataset dataset, String indexName) throws AlgebricksException { return SplitsAndConstraintsUtil.getIndexSplits(dataset, indexName, mdTxnCtx, appCtx.getClusterStateManager()); }
public void refreshBegin(Dataset dataset) { String key = dataset.getDataverseName() + "." + dataset.getDatasetName(); ExternalDatasetAccessManager datasetAccessMgr = globalRegister.get(key); if (datasetAccessMgr == null) { datasetAccessMgr = globalRegister.put(key, new ExternalDatasetAccessManager()); } // aquire the correct lock datasetAccessMgr.refreshBegin(); }
@Override protected int[] createFieldPermutationForBulkLoadOp() { if (isPointMBR) { int[] fieldPermutation = new int[NUM_TAG_FIELDS + numNestedSecondaryKeyFields + numPrimaryKeys + numFilterFields]; int idx = 0; int numSecondaryKeyFieldsForPointMBR = numNestedSecondaryKeyFields / 2; for (int i = 0; i < NUM_TAG_FIELDS + numSecondaryKeyFieldsForPointMBR; i++) { fieldPermutation[idx++] = i; } //add the rest of the sk fields for pointMBR for (int i = 0; i < numSecondaryKeyFieldsForPointMBR; i++) { fieldPermutation[idx++] = NUM_TAG_FIELDS + i; } //add the pk and filter fields int end = numSecondaryKeyFieldsForPointMBR + numPrimaryKeys + numFilterFields; for (int i = numSecondaryKeyFieldsForPointMBR; i < end; i++) { fieldPermutation[idx++] = NUM_TAG_FIELDS + i; } return fieldPermutation; } else { return super.createFieldPermutationForBulkLoadOp(); } } }
@Override protected int[] createFieldPermutationForBulkLoadOp() { /** * after tokenization, the field layout becomes * [token, num?, tag, primary key, filter value] * we need to restore it back to * [tag, token, num?, primary key, filter value] */ int[] fieldPermutation = new int[NUM_TAG_FIELDS + numTokenKeyPairFields + numFilterFields]; int numSecondaryKeys = getNumSecondaryKeys(); int idx = 0; // set tag fields for (int i = 0; i < NUM_TAG_FIELDS; i++) { fieldPermutation[idx++] = i + numSecondaryKeys; } // set secondary keys for (int i = 0; i < numSecondaryKeys; i++) { fieldPermutation[idx++] = i; } // set primary key + filter for (int i = 0; i < numPrimaryKeys + numFilterFields; i++) { fieldPermutation[idx++] = i + NUM_TAG_FIELDS + numSecondaryKeys; } return fieldPermutation; }
protected void prepareCompactJobsForExternalDataset(List<Index> indexes, Dataset ds, List<JobSpecification> jobsToExecute, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { for (int j = 0; j < indexes.size(); j++) { jobsToExecute .add(IndexUtil.buildSecondaryIndexCompactJobSpec(ds, indexes.get(j), metadataProvider, sourceLoc)); } }
public static Pair<IFileSplitProvider, AlgebricksPartitionConstraint> getDataverseSplitProviderAndConstraints( IClusterStateManager clusterStateManager, String dataverse) { FileSplit[] splits = getDataverseSplits(clusterStateManager, dataverse); return StoragePathUtil.splitProviderAndPartitionConstraints(splits); }
private static void createRebalanceTarget(Dataset target, MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { JobSpecification spec = DatasetUtil.createDatasetJobSpec(target, metadataProvider); JobUtils.runJob(hcc, spec, true); }
public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitAndConstraints(String dataverse) { return SplitsAndConstraintsUtil.getDataverseSplitProviderAndConstraints(appCtx.getClusterStateManager(), dataverse); }
public static void insertDeleteUpsertBegin(IMetadataLockManager lockMgr, LockList locks, String datasetFullyQualifiedName) throws AlgebricksException { lockMgr.acquireDataverseReadLock(locks, MetadataUtil.getDataverseFromFullyQualifiedName(datasetFullyQualifiedName)); lockMgr.acquireDatasetModifyLock(locks, datasetFullyQualifiedName); }
public static JobSpecification buildDropIndexJobSpec(Index index, MetadataProvider metadataProvider, Dataset dataset, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper = SecondaryIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); return secondaryIndexHelper.buildDropJobSpec(EnumSet.noneOf(DropOption.class)); }