@Override public ILSMIOOperation scheduleFullMerge() throws HyracksDataException { ctx.setOperation(IndexOperation.FULL_MERGE); return lsmHarness.scheduleFullMerge(ctx); }
@Override public void getOperationalComponents(ILSMIndexOperationContext ctx) { List<ILSMComponent> operationalComponents = ctx.getComponentHolder(); List<ILSMDiskComponent> immutableComponents; switch (ctx.getOperation()) { case SEARCH: if (opCtx.getTargetIndexVersion() == 0) { operationalComponents.addAll(ctx.getComponentsToBeMerged()); break; case FULL_MERGE: break; case REPLICATE: operationalComponents.addAll(ctx.getComponentsToBeReplicated()); break; default: throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
private static void propagateMap(ILSMIndexOperationContext src, ILSMIndexOperationContext destination) { Map<String, Object> map = src.getParameters(); if (map != null && !map.isEmpty()) { destination.setParameters(new HashMap<>(map)); } }
@Override public void scheduleReplication(List<ILSMDiskComponent> lsmComponents, LSMOperationType opType) throws HyracksDataException { ctx.setOperation(IndexOperation.REPLICATE); ctx.getComponentsToBeReplicated().clear(); ctx.getComponentsToBeReplicated().addAll(lsmComponents); lsmHarness.scheduleReplication(ctx, lsmComponents, opType); }
@Override public ILSMIOOperation scheduleMerge(List<ILSMDiskComponent> components) throws HyracksDataException { ctx.setOperation(IndexOperation.MERGE); ctx.getComponentsToBeMerged().clear(); ctx.getComponentsToBeMerged().addAll(components); return lsmHarness.scheduleMerge(ctx); }
@Override public void getOperationalComponents(ILSMIndexOperationContext ctx) throws HyracksDataException { List<ILSMComponent> operationalComponents = ctx.getComponentHolder(); int cmc = currentMutableComponentId.get(); ctx.setCurrentMutableComponentId(cmc); operationalComponents.clear(); switch (ctx.getOperation()) { case UPDATE: case PHYSICALDELETE: ILSMComponent c = diskComponents.get(i); if (c.getLSMComponentFilter().satisfy( ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMinFilterTuple(), ((AbstractSearchPredicate) ctx.getSearchPredicate()).getMaxFilterTuple(), ctx.getFilterCmp())) { operationalComponents.add(c); operationalComponents.addAll(ctx.getComponentsToBeReplicated()); break; case DISK_COMPONENT_SCAN: break; default: throw new UnsupportedOperationException("Operation " + ctx.getOperation() + " not supported.");
@Override public ILSMIOOperation createFlushOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMMemoryComponent flushingComponent = getCurrentMemoryComponent(); if (flushingComponent.getWriterCount() > 0) { throw new IllegalStateException( "createFlushOperation is called on a component with writers: " + flushingComponent); } // take care of the flush cycling ILSMIOOperation flushOp = TracedIOOperation.wrap(createFlushOperation(createOpContext(NoOpIndexAccessParameters.INSTANCE), fileManager.getRelFlushFileReference(), ioOpCallback), tracer); // Changing the flush status should *always* precede changing the mutable component. flushingComponent.schedule(LSMIOOperationType.FLUSH); numScheduledFlushes++; changeFlushStatusForCurrentMutableCompoent(false); changeMutableComponent(); ILSMIndexAccessor accessor = flushOp.getAccessor(); ILSMIndexOperationContext flushCtx = accessor.getOpContext(); flushCtx.setOperation(ctx.getOperation()); // Could be component delete flushCtx.getComponentHolder().add(flushingComponent); flushCtx.setIoOperation(flushOp); propagateMap(ctx, flushCtx); ioOpCallback.scheduled(flushOp); return flushOp; }
public void replace(ILSMIndexOperationContext ctx) { // Called after exit and enter has been completed try { for (int i = 0; i < count; i++) { ILSMComponent removed = ctx.getComponentHolder().remove(swapIndexes[i]); if (removed.getType() == LSMComponentType.MEMORY) { LOGGER.info("Removed a memory component from the search operation"); } else { throw new IllegalStateException("Disk components can't be removed from the search operation"); } ctx.getComponentHolder().add(swapIndexes[i], diskComponents.get(i)); } } catch (Exception e) { LOGGER.warn("Failure replacing memory components with disk components", e); throw e; } }
if (predicate.test(memComponent)) { ctx.reset(); ctx.setOperation(IndexOperation.DELETE_COMPONENTS); ioOperation = scheduleFlush(ctx); } else { throw HyracksDataException.create(ioOperation.getFailure()); ctx.reset(); ctx.setOperation(IndexOperation.DELETE_COMPONENTS); List<ILSMDiskComponent> toBeDeleted; synchronized (opTracker) { for (ILSMDiskComponent component : diskComponents) { if (predicate.test(component)) { ctx.getComponentsToBeMerged().add(component); } else { if (ctx.getComponentsToBeMerged().isEmpty()) { return; toBeDeleted = new ArrayList<>(ctx.getComponentsToBeMerged());
throws HyracksDataException { validateOperationEnterComponentsState(ctx); List<ILSMComponent> components = ctx.getComponentHolder(); int numEntered = 0; boolean entranceSuccessful = false; ctx.setAccessingComponents(true); opTracker.beforeOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback()); return true;
@Override public LSMBTreeWithBuddyMergeOperation createMergeOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMIndexOperationContext bctx = createOpContext(NoOpOperationCallback.INSTANCE, 0); bctx.setOperation(IndexOperation.MERGE); List<ILSMComponent> mergingComponents = ctx.getComponentHolder(); LSMBTreeWithBuddySortedCursor cursor = new LSMBTreeWithBuddySortedCursor(bctx, buddyBTreeFields); LSMComponentFileReferences relMergeFileRefs = getMergeTargetFileName(mergingComponents); ILSMIndexAccessor accessor = new LSMTreeIndexAccessor(getHarness(), bctx, opCtx -> new LSMBTreeWithBuddySearchCursor(opCtx, buddyBTreeFields)); // Since we have two lists of components, to tell whether we need to // keep deleted tuples, we need to know // which list to check against and we need to synchronize for this boolean keepDeleteTuples = false; if (version == 0) { keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != diskComponents .get(diskComponents.size() - 1); } else { keepDeleteTuples = mergingComponents.get(mergingComponents.size() - 1) != secondDiskComponents .get(secondDiskComponents.size() - 1); } LSMBTreeWithBuddyMergeOperation mergeOp = new LSMBTreeWithBuddyMergeOperation(accessor, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), ioOpCallback, fileManager.getBaseDir().getAbsolutePath(), keepDeleteTuples); ioOpCallback.scheduled(mergeOp); return mergeOp; }
for (ILSMIndex lsmIndex : dsInfo.getDatasetPartitionOpenIndexes(partition)) { ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.getOpContext().setParameters(flushMap); ILSMIOOperation flush = accessor.scheduleFlush(); lastFlushTime = System.nanoTime();
@Override public ILSMIOOperation createMergeOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { List<ILSMDiskComponent> mergingComponents = ctx.getComponentsToBeMerged(); || (mergingComponents.size() < 2 && ctx.getOperation() != IndexOperation.DELETE_COMPONENTS)) { return NoOpIoOperation.INSTANCE; mergeCtx.setOperation(ctx.getOperation()); mergeCtx.getComponentHolder().addAll(mergingComponents); propagateMap(ctx, mergeCtx);
@Override public final ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { ILSMIndexAccessor accessor = operation.getAccessor(); ILSMIndexOperationContext opCtx = accessor.getOpContext(); ILSMMemoryComponent memoryComponent = (ILSMMemoryComponent) opCtx.getComponentHolder().get(0); if (memoryComponent != getOldestReadableMemoryComponent()) { throw new IllegalStateException("An attempt to flush a memory component that is not the oldest"); } if (!memoryComponent.isModified() || opCtx.getOperation() == IndexOperation.DELETE_COMPONENTS) { return EmptyComponent.INSTANCE; } if (LOGGER.isInfoEnabled()) { FlushOperation flushOp = (FlushOperation) operation; LOGGER.log(Level.INFO, "Flushing component with id: " + flushOp.getFlushingComponent().getId() + " in the index " + this); } return doFlush(operation); }
private void assertCorrectSearchComponents(ILSMIndexOperationContext opCtx, ILSMIndex index, int numSuccesfullyCompletedFlushes) throws HyracksDataException { opCtx.reset(); opCtx.setOperation(IndexOperation.SEARCH); index.getOperationalComponents(opCtx); List<ILSMMemoryComponent> memComponents = index.getMemoryComponents(); int first = numSuccesfullyCompletedFlushes % memComponents.size(); Assert.assertEquals(memComponents.get(first), getFirstMemoryComponent(opCtx)); }
@Override public void endScanDiskComponents(ILSMIndexOperationContext ctx) throws HyracksDataException { if (ctx.getOperation() == IndexOperation.DISK_COMPONENT_SCAN) { try { exitComponents(ctx, LSMOperationType.DISK_COMPONENT_SCAN, null, false); } catch (Exception e) { throw HyracksDataException.create(e); } } }
@Override public ILSMIOOperation scheduleFullMerge(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMIOOperation operation; synchronized (opTracker) { fullMergeIsRequested.set(true); ctx.getComponentsToBeMerged().addAll(lsmIndex.getDiskComponents()); operation = lsmIndex.createMergeOperation(ctx); if (operation != NoOpIoOperation.INSTANCE) { fullMergeIsRequested.set(false); } // If the merge cannot be scheduled because there is already an ongoing merge on // subset/all of the components, then whenever the current merge has finished, // it will schedule the full merge again. } ioScheduler.scheduleOperation(operation); return operation; }
private long getReplicatedComponentId() throws HyracksDataException { final ILSMIndexReplicationJob indexReplJob = (ILSMIndexReplicationJob) job; if (indexReplJob.getLSMOpType() != LSMOperationType.FLUSH) { return -1L; } final ILSMIndexOperationContext ctx = indexReplJob.getLSMIndexOperationContext(); LSMComponentId id = (LSMComponentId) ctx.getComponentsToBeReplicated().get(0).getId(); return id.getMinId(); } }
@Override public Map<String, Object> getParameters() { return accessor.getOpContext().getParameters(); }
public LSMIndexSearchCursor(ILSMIndexOperationContext opCtx, boolean returnDeletedTuples) { this.opCtx = opCtx; this.returnDeletedTuples = returnDeletedTuples; outputElement = null; needPushElementIntoQueue = false; switchComponentTupleBuilders = new ArrayTupleBuilder[opCtx.getIndex().getNumberOfAllMemoryComponents()]; switchRequest = new boolean[switchComponentTupleBuilders.length]; switchedElements = new PriorityQueueElement[switchComponentTupleBuilders.length]; }