@Override public boolean isMergeLagging(ILSMIndex index) throws HyracksDataException { // TODO: for now, we simply block the ingestion when there is an ongoing merge List<ILSMDiskComponent> immutableComponents = index.getDiskComponents(); return isMergeOngoing(immutableComponents); }
/** * schedule a merge operation according to this prefix merge policy * * @param index * @return true if merge is scheduled, false otherwise. * @throws HyracksDataException * @throws IndexException */ protected boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); // Reverse the components order so that we look at components from oldest to newest. Collections.reverse(immutableComponents); Pair<Integer, Integer> mergeableIndexes = getMergableComponentsIndex(immutableComponents); if (mergeableIndexes != null) { triggerScheduleMerge(index, immutableComponents, mergeableIndexes.getLeft(), mergeableIndexes.getRight()); return true; } else { return false; } }
private long lsnFromDiskComponents() { List<ILSMDiskComponent> diskComponents = index.getDiskComponents(); for (ILSMDiskComponent c : diskComponents) { try { long lsn = ComponentUtils.getLong(c.getMetadata(), ComponentUtils.MARKER_LSN_KEY, ComponentUtils.NOT_FOUND, buffer); if (lsn != ComponentUtils.NOT_FOUND) { return lsn; } } catch (HyracksDataException e) { throw new IllegalStateException("Unable to read metadata page. Disk Error?", e); } } return ComponentUtils.NOT_FOUND; }
private void loadNewComponent(int componentPos) throws HyracksDataException { endCurrentComponent(); int numTuples = getNumDeletedTuples(componentPos); ILSMDiskComponent primaryComponent = primaryIndex.getDiskComponents().get(componentPos); Map<String, Object> parameters = new HashMap<>(); parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, primaryComponent.getId()); componentBulkLoader = (LSMIndexDiskComponentBulkLoader) secondaryIndex.createBulkLoader(1.0f, false, numTuples, false, parameters); }
@Override public ILSMIOOperation scheduleFullMerge(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMIOOperation operation; synchronized (opTracker) { fullMergeIsRequested.set(true); ctx.getComponentsToBeMerged().addAll(lsmIndex.getDiskComponents()); operation = lsmIndex.createMergeOperation(ctx); if (operation != NoOpIoOperation.INSTANCE) { fullMergeIsRequested.set(false); } // If the merge cannot be scheduled because there is already an ongoing merge on // subset/all of the components, then whenever the current merge has finished, // it will schedule the full merge again. } ioScheduler.scheduleOperation(operation); return operation; }
private LSMComponentId getMostRecentComponentId(Collection<ILSMDiskComponent> deletedComponents) throws HyracksDataException { // must sync on opTracker to ensure list of components doesn't change synchronized (lsmIndex.getOperationTracker()) { List<ILSMDiskComponent> diskComponents = lsmIndex.getDiskComponents(); if (diskComponents.isEmpty()) { LOGGER.log(Level.INFO, "There are no disk components"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } if (deletedComponents.contains(diskComponents.get(diskComponents.size() - 1))) { LOGGER.log(Level.INFO, "All disk components have been deleted"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } int mostRecentComponentIndex = 0; for (int i = 0; i < diskComponents.size(); i++) { if (!deletedComponents.contains(diskComponents.get(i))) { break; } mostRecentComponentIndex++; } ILSMDiskComponent mostRecentDiskComponent = diskComponents.get(mostRecentComponentIndex); return (LSMComponentId) mostRecentDiskComponent.getId(); } }
@Override public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); if (!areComponentsReadableWritableState(immutableComponents)) { return; } if (fullMergeIsRequested) { index.createAccessor(NoOpIndexAccessParameters.INSTANCE).scheduleFullMerge(); return; } scheduleMerge(index); }
private boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException { Optional<Long> latestSeq = ((AbstractLSMIndex) index).getLatestDiskComponentSequence(); if (!latestSeq.isPresent()) { return false; } // sequence number starts from 0, and thus latestSeq + 1 gives the number of flushes int numFlushes = latestSeq.get().intValue() + 1; List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); Collections.reverse(immutableComponents); int size = immutableComponents.size(); int depth = 0; while (treeDepth(depth) < numFlushes) { depth++; } int mergedIndex = binomialIndex(depth, Math.min(depth, numComponents) - 1, numFlushes - treeDepth(depth - 1) - 1); if (mergedIndex == size - 1) { return false; } long mergeSize = 0; List<ILSMDiskComponent> mergableComponents = new ArrayList<ILSMDiskComponent>(); for (int i = mergedIndex; i < immutableComponents.size(); i++) { mergeSize = mergeSize + immutableComponents.get(i).getComponentSize(); mergableComponents.add(immutableComponents.get(i)); } Collections.reverse(mergableComponents); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleMerge(mergableComponents); return true; }
@Override public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); if (!areComponentsReadableWritableState(immutableComponents)) { return; } if (fullMergeIsRequested) { ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleFullMerge(); return; } scheduleMerge(index); }
List<ILSMDiskComponent> diskComponents = index.getDiskComponents(); for (int i = diskComponents.size() - 1; i >= 0; i--) { if (i < diskComponents.size() - 1) {
private boolean flushingOrMerging() { // check if flushes are taking place for (ILSMMemoryComponent memComponent : lsmIndex.getMemoryComponents()) { if (memComponent.getState() == ComponentState.READABLE_UNWRITABLE_FLUSHING) { return true; } } // check if merges are taking place for (ILSMDiskComponent diskComponent : lsmIndex.getDiskComponents()) { if (diskComponent.getState() == ComponentState.READABLE_MERGING) { return true; } } return false; }
@Override protected void initializeBulkLoader() throws HyracksDataException { ILSMIndex targetIndex = (ILSMIndex) index; Map<String, Object> parameters = new HashMap<>(); parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, LSMComponentId.DEFAULT_COMPONENT_ID); if (usage.equals(BulkLoadUsage.LOAD)) { bulkLoader = targetIndex.createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, parameters); } else { primaryIndexHelper.open(); primaryIndex = (ILSMIndex) primaryIndexHelper.getIndexInstance(); List<ILSMDiskComponent> primaryComponents = primaryIndex.getDiskComponents(); if (!primaryComponents.isEmpty()) { ILSMComponentId bulkloadId = LSMComponentIdUtils.union(primaryComponents.get(0).getId(), primaryComponents.get(primaryComponents.size() - 1).getId()); parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, bulkloadId); } else { parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID); } bulkLoader = targetIndex.createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, parameters); } }
@Override protected boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); // Reverse the components order so that we look at components from oldest to newest. Collections.reverse(immutableComponents); Pair<Integer, Integer> mergeableIndexes = getMergableComponentsIndex(immutableComponents); if (mergeableIndexes == null) { //nothing to merge return false; } ILSMComponent leftComponent = immutableComponents.get(mergeableIndexes.getLeft()); ILSMComponent rightComponent = immutableComponents.get(mergeableIndexes.getRight()); ILSMComponentId targetId = LSMComponentIdUtils.union(leftComponent.getId(), rightComponent.getId()); int partition = ((PrimaryIndexOperationTracker) index.getOperationTracker()).getPartition(); Set<ILSMIndex> indexes = datasetLifecycleManager.getDatasetInfo(datasetId).getDatasetPartitionOpenIndexes(partition); triggerScheduledMerge(targetId, indexes); return true; }
List<ILSMDiskComponent> immutableComponents = new ArrayList<>(lsmIndex.getDiskComponents()); if (isMergeOngoing(immutableComponents)) { continue;
private static void fromDiskComponents(ILSMIndex index, IValueReference key, ArrayBackedValueStorage value) throws HyracksDataException { boolean loggable = LOGGER.isDebugEnabled(); if (loggable) { LOGGER.log(Level.DEBUG, "Getting " + key + " from disk components of " + index); } for (ILSMDiskComponent c : index.getDiskComponents()) { if (loggable) { LOGGER.log(Level.DEBUG, "Getting " + key + " from disk components " + c); } c.getMetadata().get(key, value); if (value.getLength() != 0) { // Found return; } } }
ILSMComponentId id = new LSMComponentId(minId, maxId); flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, index.getCurrentMemoryComponent().getId()); if (!index.getDiskComponents().isEmpty()) { ILSMDiskComponent diskComponent = index.getDiskComponents().get(0); ILSMComponentId maxDiskComponentId = diskComponent.getId(); if (maxDiskComponentId.compareTo(id) != IdCompareResult.LESS_THAN) {
if (componentReplacementCtx.proceed(lsmIndex.getDiskComponents())) {
Mockito.when(index.getDiskComponents()).thenReturn(components);
flush(ctx); Assert.assertEquals(getExpectedMemoryComponentIndex(1), index.getCurrentMemoryComponentIndex()); Assert.assertEquals(0, index.getDiskComponents().size()); CountingIoOperationCallback ioCallback = (CountingIoOperationCallback) index.getIOOperationCallback(); Assert.assertEquals(1, index.getDiskComponents().size()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterOperationCount()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterFinalizeCount()); Assert.assertEquals(2, index.getDiskComponents().size()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterOperationCount()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterFinalizeCount());
flush(ctx); Assert.assertEquals(getExpectedMemoryComponentIndex(1), index.getCurrentMemoryComponentIndex()); Assert.assertEquals(1, index.getDiskComponents().size()); CountingIoOperationCallback ioCallback = (CountingIoOperationCallback) index.getIOOperationCallback(); Assert.assertEquals(2, index.getDiskComponents().size()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterOperationCount()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterFinalizeCount()); Assert.assertEquals(3, index.getDiskComponents().size()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterOperationCount()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterFinalizeCount());