public void replace(ILSMIndexOperationContext ctx) { // Called after exit and enter has been completed try { for (int i = 0; i < count; i++) { ILSMComponent removed = ctx.getComponentHolder().remove(swapIndexes[i]); if (removed.getType() == LSMComponentType.MEMORY) { LOGGER.info("Removed a memory component from the search operation"); } else { throw new IllegalStateException("Disk components can't be removed from the search operation"); } ctx.getComponentHolder().add(swapIndexes[i], diskComponents.get(i)); } } catch (Exception e) { LOGGER.warn("Failure replacing memory components with disk components", e); throw e; } }
public List<ILSMComponent> getMergingComponents() { return accessor.getOpContext().getComponentHolder(); }
public ILSMComponent getFlushingComponent() { return accessor.getOpContext().getComponentHolder().get(0); }
private ILSMComponent getFirstMemoryComponent(ILSMIndexOperationContext opCtx) { List<ILSMComponent> components = opCtx.getComponentHolder(); // backward for (int i = components.size() - 1; i >= 0; i--) { ILSMComponent next = components.get(i); if (next.getType() == LSMComponentType.MEMORY) { return next; } } return null; }
@Override public void getOperationalComponents(ILSMIndexOperationContext ctx) { List<ILSMComponent> operationalComponents = ctx.getComponentHolder(); List<ILSMDiskComponent> immutableComponents;
@Override protected boolean getAndEnterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean isTryOperation) throws HyracksDataException { validateOperationEnterComponentsState(ctx); synchronized (opTracker) { while (true) { lsmIndex.getOperationalComponents(ctx); // Before entering the components, prune those corner cases that indeed should not proceed. switch (opType) { case MERGE: if (ctx.getComponentHolder().size() < 2) { // There is only a single component. There is nothing to merge. return false; } default: break; } if (enterComponents(ctx, opType)) { return true; } else if (isTryOperation) { return false; } } } }
ExternalBTreeOpContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, -1); opCtx.setOperation(IndexOperation.MERGE); List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); boolean returnDeletedTuples = false; if (version == 0) { if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != diskComponents .get(diskComponents.size() - 1)) { returnDeletedTuples = true; if (ctx.getComponentHolder().get(ctx.getComponentHolder().size() - 1) != secondDiskComponents .get(secondDiskComponents.size() - 1)) { returnDeletedTuples = true;
@Override public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred) throws HyracksDataException { ExternalRTreeOpContext ctx = (ExternalRTreeOpContext) ictx; List<ILSMComponent> operationalComponents = ictx.getComponentHolder(); ctx.getInitialState().setOperationalComponents(operationalComponents); cursor.open(ctx.getInitialState(), pred); }
@Override public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred) throws HyracksDataException { ExternalBTreeWithBuddyOpContext ctx = (ExternalBTreeWithBuddyOpContext) ictx; List<ILSMComponent> operationalComponents = ictx.getComponentHolder(); ctx.getSearchInitialState().setOperationalComponents(operationalComponents); cursor.open(ctx.getSearchInitialState(), pred); }
lsmIndex.subsumeMergedComponents(newComponent, ctx.getComponentHolder()); if (replicationEnabled && newComponent != EmptyComponent.INSTANCE) { componentsToBeReplicated.clear();
@Override public void search(ILSMIndexOperationContext ictx, IIndexCursor cursor, ISearchPredicate pred) throws HyracksDataException { LSMInvertedIndexOpContext ctx = (LSMInvertedIndexOpContext) ictx; List<ILSMComponent> operationalComponents = ictx.getComponentHolder(); int numComponents = operationalComponents.size(); boolean includeMutableComponent = false; ArrayList<IIndexAccessor> indexAccessors = new ArrayList<>(numComponents); ArrayList<IIndexAccessor> deletedKeysBTreeAccessors = new ArrayList<>(numComponents); for (int i = 0; i < operationalComponents.size(); i++) { ILSMComponent component = operationalComponents.get(i); if (component.getType() == LSMComponentType.MEMORY) { includeMutableComponent = true; IIndexAccessor invIndexAccessor = component.getIndex().createAccessor(ctx.getIndexAccessParameters()); indexAccessors.add(invIndexAccessor); IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexMemoryComponent) component).getBuddyIndex() .createAccessor(NoOpIndexAccessParameters.INSTANCE); deletedKeysBTreeAccessors.add(deletedKeysAccessor); } else { IIndexAccessor invIndexAccessor = component.getIndex().createAccessor(ctx.getIndexAccessParameters()); indexAccessors.add(invIndexAccessor); IIndexAccessor deletedKeysAccessor = ((LSMInvertedIndexDiskComponent) component).getBuddyIndex() .createAccessor(NoOpIndexAccessParameters.INSTANCE); deletedKeysBTreeAccessors.add(deletedKeysAccessor); } } ICursorInitialState initState = createCursorInitialState(pred, ictx, includeMutableComponent, indexAccessors, deletedKeysBTreeAccessors, operationalComponents); cursor.open(initState, pred); }
synchronized (opTracker) { componentReplacementCtx.reset(); for (int i = 0; i < ctx.getComponentHolder().size(); i++) { if (i >= startIndex) { ILSMComponent next = ctx.getComponentHolder().get(i); if (next.getType() == LSMComponentType.MEMORY && next.getState() == ComponentState.UNREADABLE_UNWRITABLE) {
@Override public LSMBTreeWithBuddyMergeOperation createMergeOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMIndexOperationContext bctx = createOpContext(NoOpOperationCallback.INSTANCE, 0); bctx.setOperation(IndexOperation.MERGE); List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); LSMBTreeWithBuddySortedCursor cursor = new LSMBTreeWithBuddySortedCursor(bctx, buddyBTreeFields); LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents); ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getHarness(), bctx, opCtx -> new LSMBTreeWithBuddySearchCursor(opCtx, buddyBTreeFields)); // Since we have two lists of components, to tell whether we need to // keep deleted tuples, we need to know // which list to check against and we need to synchronize for this boolean keepDeleteTuples = false; if (version == 0) { keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != diskComponents .get(diskComponents.size() - 1); } else { keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != secondDiskComponents .get(secondDiskComponents.size() - 1); } LSMBTreeWithBuddyMergeOperation mergeOp = new LSMBTreeWithBuddyMergeOperation(accessor, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), ioOpCallback, fileManager.getBaseDir().getAbsolutePath(), keepDeleteTuples); ioOpCallback.scheduled(mergeOp); return mergeOp; }
@Override public void updateMeta(ILSMIndexOperationContext ctx, IValueReference key, IValueReference value) throws HyracksDataException { if (!lsmIndex.isMemoryComponentsAllocated()) { lsmIndex.allocateMemoryComponents(); } getAndEnterComponents(ctx, LSMOperationType.MODIFICATION, false); try { AbstractLSMMemoryComponent c = (AbstractLSMMemoryComponent) ctx.getComponentHolder().get(0); c.getMetadata().put(key, value); c.setModified(); } finally { exitAndComplete(ctx, LSMOperationType.MODIFICATION); } }
@Override public void forceUpdateMeta(ILSMIndexOperationContext ctx, IValueReference key, IValueReference value) throws HyracksDataException { if (!lsmIndex.isMemoryComponentsAllocated()) { lsmIndex.allocateMemoryComponents(); } getAndEnterComponents(ctx, LSMOperationType.FORCE_MODIFICATION, false); try { AbstractLSMMemoryComponent c = (AbstractLSMMemoryComponent) ctx.getComponentHolder().get(0); c.getMetadata().put(key, value); c.setModified(); } finally { exitAndComplete(ctx, LSMOperationType.FORCE_MODIFICATION); } }
private boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple, LSMOperationType opType) throws HyracksDataException { if (!lsmIndex.isMemoryComponentsAllocated()) { lsmIndex.allocateMemoryComponents(); } boolean failedOperation = false; if (!getAndEnterComponents(ctx, opType, tryOperation)) { return false; } try { lsmIndex.modify(ctx, tuple); // The mutable component is always in the first index. AbstractLSMMemoryComponent mutableComponent = (AbstractLSMMemoryComponent) ctx.getComponentHolder().get(0); mutableComponent.setModified(); } catch (Exception e) { failedOperation = true; throw e; } finally { exitComponents(ctx, opType, null, failedOperation); } return true; }
@Override public final ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { ILSMIndexAccessor accessor = operation.getAccessor(); ILSMIndexOperationContext opCtx = accessor.getOpContext(); ILSMMemoryComponent memoryComponent = (ILSMMemoryComponent) opCtx.getComponentHolder().get(0); if (memoryComponent != getOldestReadableMemoryComponent()) { throw new IllegalStateException("An attempt to flush a memory component that is not the oldest"); } if (!memoryComponent.isModified() || opCtx.getOperation() == IndexOperation.DELETE_COMPONENTS) { return EmptyComponent.INSTANCE; } if (LOGGER.isInfoEnabled()) { FlushOperation flushOp = (FlushOperation) operation; LOGGER.log(Level.INFO, "Flushing component with id: " + flushOp.getFlushingComponent().getId() + " in the index " + this); } return doFlush(operation); }
@Override public ILSMIOOperation createMergeOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, -1); rctx.setOperation(IndexOperation.MERGE); List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); LSMRTreeSortedCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields); LSMComponentFileReferences relMergeFileRefs = getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1), (ILSMDiskComponent) mergingComponents.get(0)); ILSMIndexAccessor accessor = new LSMRTreeAccessor(getHarness(), rctx, buddyBTreeFields); // create the merge operation. LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), ioOpCallback, fileManager.getBaseDir().getAbsolutePath()); ioOpCallback.scheduled(mergeOp); return mergeOp; }
throws HyracksDataException { validateOperationEnterComponentsState(ctx); List<ILSMComponent> components = ctx.getComponentHolder(); int numEntered = 0; boolean entranceSuccessful = false;
@Override public ILSMIOOperation createFlushOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMMemoryComponent flushingComponent = getCurrentMemoryComponent(); if (flushingComponent.getWriterCount() > 0) { throw new IllegalStateException( "createFlushOperation is called on a component with writers: " + flushingComponent); } // take care of the flush cycling ILSMIOOperation flushOp = TracedIOOperation.wrap(createFlushOperation(createOpContext(NoOpIndexAccessParameters.INSTANCE), fileManager.getRelFlushFileReference(), ioOpCallback), tracer); // Changing the flush status should *always* precede changing the mutable component. flushingComponent.schedule(LSMIOOperationType.FLUSH); numScheduledFlushes++; changeFlushStatusForCurrentMutableCompoent(false); changeMutableComponent(); ILSMIndexAccessor accessor = flushOp.getAccessor(); ILSMIndexOperationContext flushCtx = accessor.getOpContext(); flushCtx.setOperation(ctx.getOperation()); // Could be component delete flushCtx.getComponentHolder().add(flushingComponent); flushCtx.setIoOperation(flushOp); propagateMap(ctx, flushCtx); ioOpCallback.scheduled(flushOp); return flushOp; }