/** * A helper method to ensure disk components have proper Ids (non-decreasing) * We may get rid of this method once component Id is stablized * * @throws HyracksDataException */ private void validateComponentIds() throws HyracksDataException { for (int i = 0; i < diskComponents.size() - 1; i++) { ILSMComponentId id1 = diskComponents.get(i).getId(); ILSMComponentId id2 = diskComponents.get(i + 1).getId(); IdCompareResult cmp = id1.compareTo(id2); if (cmp != IdCompareResult.UNKNOWN && cmp != IdCompareResult.GREATER_THAN) { throw new IllegalStateException( "found non-decreasing component ids (" + id1 + " -> " + id2 + ") on index " + this); } } }
public long getComponentLSN(List<ILSMDiskComponent> diskComponents) throws HyracksDataException { if (diskComponents.isEmpty()) { throw new IllegalArgumentException("Can't get LSN from an empty list of disk components"); } // Get max LSN from the diskComponents. Implies a merge IO operation or Recovery operation. long maxLSN = -1L; for (ILSMDiskComponent c : diskComponents) { DiskComponentMetadata md = c.getMetadata(); maxLSN = Math.max(getTreeIndexLSN(md), maxLSN); } return maxLSN; }
public LSMIndexDiskComponentBulkLoader(AbstractLSMIndex lsmIndex, ILSMIndexOperationContext opCtx, float fillFactor, boolean verifyInput, long numElementsHint) throws HyracksDataException { this.lsmIndex = lsmIndex; this.opCtx = opCtx; this.componentBulkLoader = opCtx.getIoOperation().getNewComponent().createBulkLoader(opCtx.getIoOperation(), fillFactor, verifyInput, numElementsHint, false, true, true); }
private void exitComponent(ILSMDiskComponent diskComponent) throws HyracksDataException { diskComponent.threadExit(LSMOperationType.SEARCH, false, false); if (diskComponent.getState() == ILSMComponent.ComponentState.INACTIVE) { if (replicationEnabled) { componentsToBeReplicated.clear(); componentsToBeReplicated.add(diskComponent); lsmIndex.scheduleReplication(null, componentsToBeReplicated, ReplicationOperation.DELETE, null); } diskComponent.deactivateAndDestroy(); } }
int numComponents = immutableComponents.size(); for (int i = 0; i < numComponents; i++) { if (immutableComponents.get(i).getComponentSize() > maxMergableComponentSize || immutableComponents.get(i).getState() != ComponentState.READABLE_UNWRITABLE) { continue; long startComponentSize = immutableComponents.get(i).getComponentSize(); boolean mergeable = true; for (; j < numComponents; j++) { long componentSize = immutableComponents.get(j).getComponentSize(); if (componentSize > maxMergableComponentSize || immutableComponents.get(j).getState() != ComponentState.READABLE_UNWRITABLE) {
component = createDiskComponent(componentFactory, flushOp.getTarget(), flushOp.getBTreeTarget(), flushOp.getBloomFilterTarget(), true); componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, numBTreeTuples.longValue(), false, false, false); flushLoadRTree(isEmpty, rTreeTupleSorter, componentBulkLoader); if (component.getLSMComponentFilter() != null) { List<ITupleReference> filterTuples = new ArrayList<>(); filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples, NoOpOperationCallback.INSTANCE); getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); flushingComponent.getMetadata().copy(component.getMetadata()); abort = false; componentBulkLoader.end();
component.createBulkLoader(operation, 1.0f, false, 0L, false, false, false); try { while (cursor.hasNext()) { cursor.close(); if (component.getLSMComponentFilter() != null) { List<ITupleReference> filterTuples = new ArrayList<>(); for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple()); getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples, NoOpOperationCallback.INSTANCE); getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
@Override public void end() throws HyracksDataException { try { ioOpCallback.afterOperation(loadOp); componentBulkLoader.end(); if (component.getComponentSize() > 0) { if (isTransaction) { // Since this is a transaction component, validate and // deactivate. it could later be added or deleted try { component.markAsValid(durable, loadOp); } finally { ioOpCallback.afterFinalize(loadOp); } component.deactivate(); } else { ioOpCallback.afterFinalize(loadOp); getHarness().addBulkLoadedComponent(loadOp); } } } finally { ioOpCallback.completed(loadOp); } }
for (ILSMComponentId id : componentIDs) { ILSMDiskComponent component = Mockito.mock(ILSMDiskComponent.class); Mockito.when(component.getId()).thenReturn(id); Mockito.when(component.getComponentSize()).thenReturn(DEFAULT_COMPONENT_SIZE); Mockito.when(component.getState()).thenReturn(ComponentState.READABLE_UNWRITABLE); components.add(component);
mergeOp.getBloomFilterTarget(), true); componentBulkLoader = loadMergeBulkLoader(mergeOp, cursor, mergedComponent); if (mergedComponent.getLSMComponentFilter() != null) { List<ITupleReference> filterTuples = new ArrayList<>(); for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple()); getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples, NoOpOperationCallback.INSTANCE); getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getMetadataHolder());
private void deactivateAndDestroyDiskComponents() throws HyracksDataException { for (ILSMDiskComponent c : diskComponents) { c.deactivateAndDestroy(); } diskComponents.clear(); }
private void deactivateDiskComponents() throws HyracksDataException { for (ILSMDiskComponent c : diskComponents) { c.deactivateAndPurge(); } }
private void destroyDiskComponents() throws HyracksDataException { for (ILSMDiskComponent c : diskComponents) { c.destroy(); } }
private boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException { Optional<Long> latestSeq = ((AbstractLSMIndex) index).getLatestDiskComponentSequence(); if (!latestSeq.isPresent()) { return false; } // sequence number starts from 0, and thus latestSeq + 1 gives the number of flushes int numFlushes = latestSeq.get().intValue() + 1; List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); Collections.reverse(immutableComponents); int size = immutableComponents.size(); int depth = 0; while (treeDepth(depth) < numFlushes) { depth++; } int mergedIndex = binomialIndex(depth, Math.min(depth, numComponents) - 1, numFlushes - treeDepth(depth - 1) - 1); if (mergedIndex == size - 1) { return false; } long mergeSize = 0; List<ILSMDiskComponent> mergableComponents = new ArrayList<ILSMDiskComponent>(); for (int i = mergedIndex; i < immutableComponents.size(); i++) { mergeSize = mergeSize + immutableComponents.get(i).getComponentSize(); mergableComponents.add(immutableComponents.get(i)); } Collections.reverse(mergableComponents); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleMerge(mergableComponents); return true; }
@Override protected LSMComponentFileReferences getMergeFileReferences(ILSMDiskComponent firstComponent, ILSMDiskComponent lastComponent) throws HyracksDataException { RTree firstTree = (RTree) firstComponent.getIndex(); RTree lastTree = (RTree) lastComponent.getIndex(); FileReference firstFile = firstTree.getFileReference(); FileReference lastFile = lastTree.getFileReference(); return fileManager.getRelMergeFileReference(firstFile.getFile().getName(), lastFile.getFile().getName()); } }
if (!inactiveDiskComponents.isEmpty()) { for (ILSMDiskComponent inactiveComp : inactiveDiskComponents) { if (inactiveComp.getFileReferenceCount() == 1) { inactiveDiskComponentsToBeDeleted = inactiveDiskComponentsToBeDeleted == null ? new LinkedList<>() : inactiveDiskComponentsToBeDeleted; c.deactivateAndDestroy();
filterToMinMax(flush.getNewComponent().getLSMComponentFilter()); Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getLeft(), minMax.getLeft(), comp)); MutablePair<ITupleReference, ITupleReference> expectedMergeMinMax = null; for (ILSMDiskComponent f : flushedComponents) { Pair<ITupleReference, ITupleReference> componentMinMax = filterToMinMax(f.getLSMComponentFilter()); if (expectedMergeMinMax == null) { expectedMergeMinMax = MutablePair.of(componentMinMax.getLeft(), componentMinMax.getRight()); filterToMinMax(flushedComponents.get(0).getLSMComponentFilter()); Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(expectedMergeMinMax.getLeft(), mergedMinMax.getLeft(), comp));
componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, numElements, false, false, false); IIndexCursor scanCursor = accessor.createSearchCursor(false); accessor.search(scanCursor, nullPred); accessor.destroy(); if (component.getLSMComponentFilter() != null) { List<ITupleReference> filterTuples = new ArrayList<>(); filterTuples.add(flushingComponent.getLSMComponentFilter().getMinTuple()); filterTuples.add(flushingComponent.getLSMComponentFilter().getMaxTuple()); getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples, NoOpOperationCallback.INSTANCE); getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder()); flushingComponent.getMetadata().copy(component.getMetadata());
component.createBulkLoader(operation, 1.0f, false, numElements, false, false, false); loadDeleteTuples(opCtx, btreeCursor, mergePred, componentBulkLoader); } finally { componentBulkLoader = component.createBulkLoader(operation, 1.0f, false, 0L, false, false, false); if (component.getLSMComponentFilter() != null) { List<ITupleReference> filterTuples = new ArrayList<>(); for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) { getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples, NoOpOperationCallback.INSTANCE); getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getMetadataHolder());
@Override public void end() throws HyracksDataException { try { ioOpCallback.afterOperation(loadOp); componentBulkLoader.end(); if (component.getComponentSize() > 0) { if (isTransaction) { // Since this is a transaction component, validate and // deactivate. it could later be added or deleted try { component.markAsValid(durable, loadOp); } finally { ioOpCallback.afterFinalize(loadOp); } component.deactivate(); } else { ioOpCallback.afterFinalize(loadOp); getHarness().addBulkLoadedComponent(loadOp); } } } finally { ioOpCallback.completed(loadOp); } }