ILSMMergePolicyFactory mergePolicyFactory = (ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance(); if (isExternalDataset && mergePolicyFactory.getName().compareTo("correlated-prefix") == 0) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "The correlated-prefix merge policy cannot be used with external dataset."); if (mergePolicyFactory.getName().compareTo("no-merge") != 0) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Compaction policy properties are missing."); if (!mergePolicyFactory.getPropertiesNames().contains(entry.getKey())) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "Invalid compaction policy property: " + entry.getKey()); for (String p : mergePolicyFactory.getPropertiesNames()) { if (!compactionPolicyProperties.containsKey(p)) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
json.set("mergePolicyFactory", mergePolicyFactory.toJson(registry)); json.putPOJO("mergePolicyProperties", mergePolicyProperties); json.put("durable", durable);
@Override public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException { IIOManager ioManager = ncServiceCtx.getIoManager(); FileReference fileRef = ioManager.resolve(path); ioOpCallbackFactory.initialize(ncServiceCtx, this); return LSMRTreeUtils.createExternalRTree(ioManager, fileRef, storageManager.getBufferCache(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx), opTrackerProvider.getOperationTracker(ncServiceCtx, this), ioSchedulerProvider.getIoScheduler(ncServiceCtx), ioOpCallbackFactory, linearizeCmpFactory, buddyBTreeFields, durable, isPointMBR, metadataPageManagerFactory, ncServiceCtx.getTracer()); }
private static CompactionPolicy getCompactionPolicyEntity(String compactionPolicyClassName) throws AlgebricksException { try { String policyName = ((ILSMMergePolicyFactory) (Class.forName(compactionPolicyClassName).newInstance())).getName(); return new CompactionPolicy(MetadataConstants.METADATA_DATAVERSE_NAME, policyName, compactionPolicyClassName); } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { throw new MetadataException("Unable to instantiate builtin Merge Policy Factory", e); } }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); ioOpCallbackFactory.initialize(serviceCtx, this); return LSMBTreeUtil.createExternalBTreeWithBuddy(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, bloomFilterKeyFields, durable, metadataPageManagerFactory, serviceCtx.getTracer()); }
public static Pair<ILSMMergePolicyFactory, Map<String, String>> getMergePolicyFactory(Dataset dataset, MetadataTransactionContext mdTxnCtx) throws AlgebricksException { String policyName = dataset.getCompactionPolicy(); CompactionPolicy compactionPolicy = MetadataManager.INSTANCE.getCompactionPolicy(mdTxnCtx, MetadataConstants.METADATA_DATAVERSE_NAME, policyName); String compactionPolicyFactoryClassName = compactionPolicy.getClassName(); ILSMMergePolicyFactory mergePolicyFactory; Map<String, String> properties = dataset.getCompactionPolicyProperties(); try { mergePolicyFactory = (ILSMMergePolicyFactory) Class.forName(compactionPolicyFactoryClassName).newInstance(); if (mergePolicyFactory.getName().compareTo(CorrelatedPrefixMergePolicyFactory.NAME) == 0) { properties.put(CorrelatedPrefixMergePolicyFactory.KEY_DATASET_ID, Integer.toString(dataset.getDatasetId())); } } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { throw new AlgebricksException(e); } return new Pair<>(mergePolicyFactory, properties); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); ioOpCallbackFactory.initialize(serviceCtx, this); return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, durable, metadataPageManagerFactory, serviceCtx.getTracer()); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file); ioOpCallbackFactory.initialize(serviceCtx, this); return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR, metadataPageManagerFactory); }
@Override public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException { IIOManager ioManager = ncServiceCtx.getIoManager(); FileReference fileRef = ioManager.resolve(path); List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(ncServiceCtx, fileRef); ioOpCallbackFactory.initialize(ncServiceCtx, this); return LSMRTreeUtils.createLSMTree(ioManager, virtualBufferCaches, fileRef, storageManager.getBufferCache(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx), opTrackerProvider.getOperationTracker(ncServiceCtx, this), ioSchedulerProvider.getIoScheduler(ncServiceCtx), ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, buddyBTreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR, metadataPageManagerFactory); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file); ioOpCallbackFactory.initialize(serviceCtx, this); //TODO: enable updateAwareness for secondary LSMBTree indexes boolean updateAware = false; return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, isPrimary, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, durable, metadataPageManagerFactory, updateAware, serviceCtx.getTracer(), compressorDecompressorFactory); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file); for (int i = 0; i < vbcs.size(); i++) { IVirtualBufferCache vbc = vbcs.get(i); if (!(vbc instanceof TestVirtualBufferCache)) { vbcs.remove(i); vbcs.add(i, new TestVirtualBufferCache(vbc)); } } ioOpCallbackFactory.initialize(serviceCtx, this); return TestLsmBtreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, isPrimary, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, durable, metadataPageManagerFactory, false, serviceCtx.getTracer()); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file); IBufferCache bufferCache = storageManager.getBufferCache(serviceCtx); ILSMMergePolicy mergePolicy = mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx); ILSMIOOperationScheduler ioScheduler = ioSchedulerProvider.getIoScheduler(serviceCtx); ioOpCallbackFactory.initialize(serviceCtx, this); if (isPartitioned) { return InvertedIndexUtils.createPartitionedLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory, serviceCtx.getTracer()); } else { return InvertedIndexUtils.createLSMInvertedIndex(ioManager, virtualBufferCaches, typeTraits, cmpFactories, tokenTypeTraits, tokenCmpFactories, tokenizerFactory, bufferCache, file.getAbsolutePath(), bloomFilterFalsePositiveRate, mergePolicy, opTrackerProvider.getOperationTracker(serviceCtx, this), ioScheduler, ioOpCallbackFactory, invertedIndexFields, filterTypeTraits, filterCmpFactories, filterFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps, durable, metadataPageManagerFactory, serviceCtx.getTracer()); } }