public static JobSpecification buildIndexUpdateOp(Dataset ds, Index index, List<ExternalFile> metadataFiles, List<ExternalFile> addedFiles, List<ExternalFile> appendedFiles, MetadataProvider metadataProvider, SourceLocation sourceLoc) throws AlgebricksException { // Create files list ArrayList<ExternalFile> files = new ArrayList<>(); for (ExternalFile metadataFile : metadataFiles) { if (metadataFile.getPendingOp() != ExternalFilePendingOp.APPEND_OP) { files.add(metadataFile); } else { metadataFile.setPendingOp(ExternalFilePendingOp.NO_OP); files.add(metadataFile); } } // add new files for (ExternalFile file : addedFiles) { files.add(file); } // add appended files for (ExternalFile file : appendedFiles) { files.add(file); } return IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, metadataProvider, files, sourceLoc); }
public static JobSpecification buildSecondaryIndexLoadingJobSpec(Dataset dataset, Index index, MetadataProvider metadataProvider, List<ExternalFile> files, SourceLocation sourceLoc) throws AlgebricksException { SecondaryIndexOperationsHelper secondaryIndexHelper; if (dataset.isCorrelated()) { secondaryIndexHelper = SecondaryCorrelatedTreeIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); } else { secondaryIndexHelper = SecondaryTreeIndexOperationsHelper.createIndexOperationsHelper(dataset, index, metadataProvider, sourceLoc); } if (files != null) { secondaryIndexHelper.setExternalFiles(files); } return secondaryIndexHelper.buildLoadingJobSpec(); }
private static void createAndLoadSecondaryIndexesForTarget(Dataset source, Dataset target, MetadataProvider metadataProvider, IHyracksClientConnection hcc) throws Exception { for (Index index : metadataProvider.getDatasetIndexes(source.getDataverseName(), source.getDatasetName())) { if (!index.isSecondaryIndex()) { continue; } // Creates the secondary index. JobSpecification indexCreationJobSpec = IndexUtil.buildSecondaryIndexCreationJobSpec(target, index, metadataProvider, null); JobUtils.runJob(hcc, indexCreationJobSpec, true); // Loads the secondary index. JobSpecification indexLoadingJobSpec = IndexUtil.buildSecondaryIndexLoadingJobSpec(target, index, metadataProvider, null); JobUtils.runJob(hcc, indexLoadingJobSpec, true); } }
primaryIndexName); Assert.assertNotNull(index); jobSpecification = IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider, null); jobSpecification.getOperatorMap().values().forEach(iOperatorDescriptor -> { Assert.assertFalse(iOperatorDescriptor instanceof AbstractSorterOperatorDescriptor); secondaryIndexName); Assert.assertNotNull(index); jobSpecification = IndexUtil.buildSecondaryIndexLoadingJobSpec(dataset, index, metadataProvider, null); final long numOfSortOperators = jobSpecification.getOperatorMap().values().stream() .filter(op -> op instanceof AbstractSorterOperatorDescriptor).count();
spec = IndexUtil.buildSecondaryIndexLoadingJobSpec(ds, index, metadataProvider, sourceLoc); MetadataManager.INSTANCE.commitTransaction(mdTxnCtx); bActiveTxn = false;