@Override public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); if (!areComponentsReadableWritableState(immutableComponents)) { return; } if (fullMergeIsRequested) { index.createAccessor(NoOpIndexAccessParameters.INSTANCE).scheduleFullMerge(); return; } scheduleMerge(index); }
private LSMComponentId getMostRecentComponentId(Collection<ILSMDiskComponent> deletedComponents) throws HyracksDataException { // must sync on opTracker to ensure list of components doesn't change synchronized (lsmIndex.getOperationTracker()) { List<ILSMDiskComponent> diskComponents = lsmIndex.getDiskComponents(); if (diskComponents.isEmpty()) { LOGGER.log(Level.INFO, "There are no disk components"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } if (deletedComponents.contains(diskComponents.get(diskComponents.size() - 1))) { LOGGER.log(Level.INFO, "All disk components have been deleted"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } int mostRecentComponentIndex = 0; for (int i = 0; i < diskComponents.size(); i++) { if (!deletedComponents.contains(diskComponents.get(i))) { break; } mostRecentComponentIndex++; } ILSMDiskComponent mostRecentDiskComponent = diskComponents.get(mostRecentComponentIndex); return (LSMComponentId) mostRecentDiskComponent.getId(); } }
private Predicate<ILSMIndex> newLaggingDatasetPredicate(long checkpointTargetLSN) { return lsmIndex -> { final LSMIOOperationCallback ioCallback = (LSMIOOperationCallback) lsmIndex.getIOOperationCallback(); return ioCallback.getPersistenceLsn() < checkpointTargetLSN; }; } }
private Predicate<ILSMIndex> newIdleDatasetPredicate() { final long currentTime = System.nanoTime(); return lsmIndex -> { if (lsmIndex.isPrimaryIndex()) { PrimaryIndexOperationTracker opTracker = (PrimaryIndexOperationTracker) lsmIndex.getOperationTracker(); return currentTime - opTracker.getLastFlushTime() >= datasetCheckpointInterval; } return false; }; }
@Override public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback, IModificationOperationCallback modificationCallback) throws HyracksDataException { // Flush will only be handled by last exiting thread. if (opType == LSMOperationType.MODIFICATION && threadRefCount.decrementAndGet() == 0 && index.hasFlushRequestForCurrentMutableComponent()) { ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleFlush(); } } }
private boolean flushingOrMerging() { // check if flushes are taking place for (ILSMMemoryComponent memComponent : lsmIndex.getMemoryComponents()) { if (memComponent.getState() == ComponentState.READABLE_UNWRITABLE_FLUSHING) { return true; } } // check if merges are taking place for (ILSMDiskComponent diskComponent : lsmIndex.getDiskComponents()) { if (diskComponent.getState() == ComponentState.READABLE_MERGING) { return true; } } return false; }
OrderedIndexTestContext ctx = createTestContext(fieldSerdes, numKeys); ILSMIndex index = (ILSMIndex) ctx.getIndex(); index.create(); index.activate(); Assert.assertEquals(getExpectedMemoryComponentIndex(0), index.getCurrentMemoryComponentIndex()); flush(ctx); Assert.assertEquals(getExpectedMemoryComponentIndex(1), index.getCurrentMemoryComponentIndex()); Assert.assertEquals(0, index.getDiskComponents().size()); CountingIoOperationCallback ioCallback = (CountingIoOperationCallback) index.getIOOperationCallback(); Assert.assertEquals(getExpectedMemoryComponentIndex(0), index.getCurrentMemoryComponentIndex()); Assert.assertEquals(1, index.getDiskComponents().size()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterOperationCount()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterFinalizeCount()); Assert.assertEquals(getExpectedMemoryComponentIndex(1), index.getCurrentMemoryComponentIndex()); Assert.assertEquals(2, index.getDiskComponents().size()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterOperationCount()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterFinalizeCount());
index.create(); index.activate(); Assert.assertEquals(getExpectedMemoryComponentIndex(0), index.getCurrentMemoryComponentIndex()); int numMemoryComponents = index.getNumberOfAllMemoryComponents(); secondUser.step(); secondUser.step(); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); ILSMIndexOperationContext opCtx = accessor.getOpContext(); assertCorrectSearchComponents(opCtx, index, 0); for (int i = 0; i < numFlushes - index.getNumberOfAllMemoryComponents(); i++) { flushSemaphore.release(); firstUser.step(); int expectedMemoryComponent = numFlushes % numMemoryComponents; Assert.assertEquals(getExpectedMemoryComponentIndex(expectedMemoryComponent), index.getCurrentMemoryComponentIndex()); Assert.assertEquals(0, index.getDiskComponents().size()); EncapsulatingIoCallback encapsulating = (EncapsulatingIoCallback) index.getIOOperationCallback(); CountingIoOperationCallback ioCallback = (CountingIoOperationCallback) encapsulating.getEncapsulated(); flush(ctx); Assert.assertEquals(getExpectedMemoryComponentIndex((expectedMemoryComponent + 1) % numMemoryComponents), index.getCurrentMemoryComponentIndex()); Assert.assertEquals(1, index.getDiskComponents().size()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterOperationCount()); Assert.assertEquals(ioCallback.getBeforeOperationCount(), ioCallback.getAfterFinalizeCount());
Mockito.when(index.getDiskComponents()).thenReturn(components); Mockito.when(index.createAccessor(Mockito.any(IIndexAccessParameters.class))).thenReturn(accessor); Mockito.when(index.isPrimaryIndex()).thenReturn(isPrimary); if (isPrimary) { PrimaryIndexOperationTracker opTracker = Mockito.mock(PrimaryIndexOperationTracker.class); Mockito.when(opTracker.getPartition()).thenReturn(partition); Mockito.when(index.getOperationTracker()).thenReturn(opTracker);
if (lsmIndex.hasFlushRequestForCurrentMutableComponent()) { needsFlush = true; break; if (lsmIndex.isPrimaryIndex()) { if (lsmIndex.isCurrentMutableComponentEmpty()) { LOGGER.info("Primary index on dataset {} and partition {} is empty... skipping flush", dsInfo.getDatasetID(), partition); ILSMOperationTracker opTracker = lsmIndex.getOperationTracker(); synchronized (opTracker) { ILSMMemoryComponent memComponent = lsmIndex.getCurrentMemoryComponent(); if (memComponent.getWriterCount() > 0) { throw new IllegalStateException( memComponent.setUnwritable(); if (lsmIndex.isPrimaryIndex()) { primaryId = (LSMComponentId) memComponent.getId();
Map<String, Object> flushMap = new HashMap<>(); flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.getOpContext().setParameters(flushMap); long minId = logRecord.getFlushingComponentMinId(); long maxId = logRecord.getFlushingComponentMaxId(); ILSMComponentId id = new LSMComponentId(minId, maxId); flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, index.getCurrentMemoryComponent().getId()); if (!index.getDiskComponents().isEmpty()) { ILSMDiskComponent diskComponent = index.getDiskComponents().get(0); ILSMComponentId maxDiskComponentId = diskComponent.getId(); if (maxDiskComponentId.compareTo(id) != IdCompareResult.LESS_THAN) { index.getCurrentMemoryComponent().resetId(id, true); ILSMIOOperation flush = accessor.scheduleFlush(); try { throw HyracksDataException.create(flush.getFailure()); index.resetCurrentComponentIndex();
/** * Put LSM metadata state into the index's current memory component. * * @param index, * the LSM index. * @param key, * the key for the metadata state. * @param pointable, * the value for the metadata state. * @throws HyracksDataException */ public static void put(ILSMIndex index, IValueReference key, IPointable pointable) throws HyracksDataException { // write the opTracker to ensure the component layout don't change synchronized (index.getOperationTracker()) { index.getCurrentMemoryComponent().getMetadata().put(key, pointable); } }
protected void triggerReplication(List<ILSMDiskComponent> lsmComponents, LSMOperationType opType) throws HyracksDataException { ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleReplication(lsmComponents, opType); }
@Override public boolean isMergeLagging(ILSMIndex index) throws HyracksDataException { // TODO: for now, we simply block the ingestion when there is an ongoing merge List<ILSMDiskComponent> immutableComponents = index.getDiskComponents(); return isMergeOngoing(immutableComponents); }
if (!lsmIndex.isMemoryComponentsAllocated()) { lsmIndex.allocateMemoryComponents(); ILSMMemoryComponent memComponent = lsmIndex.getCurrentMemoryComponent(); if (predicate.test(memComponent)) { synchronized (opTracker) { waitForFlushesAndMerges(); List<ILSMDiskComponent> diskComponents = lsmIndex.getDiskComponents(); for (ILSMDiskComponent component : diskComponents) { if (predicate.test(component)) { if (lsmIndex.getDiskComponents().contains(component)) { throw HyracksDataException.create(ErrorCode.A_MERGE_OPERATION_HAS_FAILED, component.toString());
@Override public void register(long resourceId, int partition, ILSMIndex index, IModificationOperationCallback callback, boolean primaryIndex) { synchronized (txnOpTrackers) { if (!txnOpTrackers.containsKey(resourceId)) { final ITransactionOperationTracker txnOpTracker = (ITransactionOperationTracker) index.getOperationTracker(); txnOpTrackers.put(resourceId, txnOpTracker); txnOpTracker.beforeTransaction(resourceId); } } }
private void loadNewComponent(int componentPos) throws HyracksDataException { endCurrentComponent(); int numTuples = getNumDeletedTuples(componentPos); ILSMDiskComponent primaryComponent = primaryIndex.getDiskComponents().get(componentPos); Map<String, Object> parameters = new HashMap<>(); parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, primaryComponent.getId()); componentBulkLoader = (LSMIndexDiskComponentBulkLoader) secondaryIndex.createBulkLoader(1.0f, false, numTuples, false, parameters); }
@Test public void testAllocateComponentId() throws HyracksDataException { int numMemoryComponents = 2; DatasetInfo dsInfo = new DatasetInfo(101, null); ILSMComponentIdGenerator idGenerator = new LSMComponentIdGenerator(numMemoryComponents, MIN_VALID_COMPONENT_ID); ILSMIndex mockIndex = Mockito.mock(ILSMIndex.class); Mockito.when(mockIndex.getNumberOfAllMemoryComponents()).thenReturn(numMemoryComponents); ILSMMemoryComponent mockComponent = Mockito.mock(AbstractLSMMemoryComponent.class); Mockito.when(mockIndex.getCurrentMemoryComponent()).thenReturn(mockComponent); LSMIOOperationCallback callback = new LSMIOOperationCallback(dsInfo, mockIndex, idGenerator.getId(), mockIndexCheckpointManagerProvider()); ILSMComponentId initialId = idGenerator.getId(); // simulate a partition is flushed before allocated idGenerator.refresh(); long flushLsn = 1L; ILSMComponentId nextComponentId = idGenerator.getId(); callback.allocated(mockComponent); callback.recycled(mockComponent); checkMemoryComponent(initialId, mockComponent); }
ILSMComponentIdGenerator idGenerator = new LSMComponentIdGenerator(numMemoryComponents, MIN_VALID_COMPONENT_ID); ILSMIndex mockIndex = Mockito.mock(ILSMIndex.class); Mockito.when(mockIndex.getNumberOfAllMemoryComponents()).thenReturn(numMemoryComponents); ILSMMemoryComponent mockComponent = Mockito.mock(AbstractLSMMemoryComponent.class); Mockito.when(mockIndex.getCurrentMemoryComponent()).thenReturn(mockComponent); LSMIOOperationCallback callback = new LSMIOOperationCallback(dsInfo, mockIndex, idGenerator.getId(), mockIndexCheckpointManagerProvider()); checkMemoryComponent(id, mockComponent); Mockito.when(mockIndex.isMemoryComponentsAllocated()).thenReturn(true); for (int i = 0; i < 100; i++) {
@Override public ILSMIOOperation scheduleFullMerge(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMIOOperation operation; synchronized (opTracker) { fullMergeIsRequested.set(true); ctx.getComponentsToBeMerged().addAll(lsmIndex.getDiskComponents()); operation = lsmIndex.createMergeOperation(ctx); if (operation != NoOpIoOperation.INSTANCE) { fullMergeIsRequested.set(false); } // If the merge cannot be scheduled because there is already an ongoing merge on // subset/all of the components, then whenever the current merge has finished, // it will schedule the full merge again. } ioScheduler.scheduleOperation(operation); return operation; }