protected void triggerReplication(List<ILSMDiskComponent> lsmComponents, LSMOperationType opType) throws HyracksDataException { ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleReplication(lsmComponents, opType); }
private void triggerScheduleMerge(ILSMIndex index, List<ILSMDiskComponent> immutableComponents, int startIndex, int endIndex) throws HyracksDataException { List<ILSMDiskComponent> mergableComponents = new ArrayList<>(immutableComponents.subList(startIndex, endIndex + 1)); // Reverse the components order back to its original order Collections.reverse(mergableComponents); index.createAccessor(NoOpIndexAccessParameters.INSTANCE).scheduleMerge(mergableComponents); }
@Override public void completeOperation(ILSMIndex index, LSMOperationType opType, ISearchOperationCallback searchCallback, IModificationOperationCallback modificationCallback) throws HyracksDataException { // Flush will only be handled by last exiting thread. if (opType == LSMOperationType.MODIFICATION && threadRefCount.decrementAndGet() == 0 && index.hasFlushRequestForCurrentMutableComponent()) { ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleFlush(); } } }
@Override public void initialize() throws HyracksDataException { indexHelper.open(); ILSMIndex index = (ILSMIndex) indexHelper.getIndexInstance(); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleFullMerge(); }
private boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException { Optional<Long> latestSeq = ((AbstractLSMIndex) index).getLatestDiskComponentSequence(); if (!latestSeq.isPresent()) { return false; } // sequence number starts from 0, and thus latestSeq + 1 gives the number of flushes int numFlushes = latestSeq.get().intValue() + 1; List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); Collections.reverse(immutableComponents); int size = immutableComponents.size(); int depth = 0; while (treeDepth(depth) < numFlushes) { depth++; } int mergedIndex = binomialIndex(depth, Math.min(depth, numComponents) - 1, numFlushes - treeDepth(depth - 1) - 1); if (mergedIndex == size - 1) { return false; } long mergeSize = 0; List<ILSMDiskComponent> mergableComponents = new ArrayList<ILSMDiskComponent>(); for (int i = mergedIndex; i < immutableComponents.size(); i++) { mergeSize = mergeSize + immutableComponents.get(i).getComponentSize(); mergableComponents.add(immutableComponents.get(i)); } Collections.reverse(mergableComponents); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleMerge(mergableComponents); return true; }
@Override public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); if (!areComponentsReadableWritableState(immutableComponents)) { return; } if (fullMergeIsRequested) { index.createAccessor(NoOpIndexAccessParameters.INSTANCE).scheduleFullMerge(); return; } scheduleMerge(index); }
@Override public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); if (!areComponentsReadableWritableState(immutableComponents)) { return; } if (fullMergeIsRequested) { ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleFullMerge(); return; } scheduleMerge(index); }
ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.scheduleMerge(mergableComponents);
synchronized (scheduledFlushes) { for (ILSMIndex lsmIndex : dsInfo.getDatasetPartitionOpenIndexes(partition)) { ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.getOpContext().setParameters(flushMap); ILSMIOOperation flush = accessor.scheduleFlush();
Mockito.when(index.createAccessor(Mockito.any(IIndexAccessParameters.class))).thenReturn(accessor); Mockito.when(index.isPrimaryIndex()).thenReturn(isPrimary); if (isPrimary) {
Map<String, Object> flushMap = new HashMap<>(); flushMap.put(LSMIOOperationCallback.KEY_FLUSH_LOG_LSN, flushLsn); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.getOpContext().setParameters(flushMap); long minId = logRecord.getFlushingComponentMinId();
ILSMIndex index = (ILSMIndex) datasetLifecycleManager.getIndex(logRecord.getDatasetId(), logRecord.getResourceId()); ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); ILSMIndexOperationContext opCtx = indexAccessor.getOpContext(); opCtx.setFilterSkip(true);
private static void redo(ILogRecord logRecord, IDatasetLifecycleManager datasetLifecycleManager) { try { int datasetId = logRecord.getDatasetId(); long resourceId = logRecord.getResourceId(); ILSMIndex index = (ILSMIndex) datasetLifecycleManager.getIndex(datasetId, resourceId); ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); ILSMIndexOperationContext opCtx = indexAccessor.getOpContext(); opCtx.setFilterSkip(true); opCtx.setRecovery(true); if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.INSERT_BYTE) { indexAccessor.forceInsert(logRecord.getNewValue()); } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.DELETE_BYTE) { indexAccessor.forceDelete(logRecord.getNewValue()); } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.UPSERT_BYTE) { // redo, upsert the new value indexAccessor.forceUpsert(logRecord.getNewValue()); } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.FILTER_BYTE) { opCtx.setFilterSkip(false); indexAccessor.updateFilter(logRecord.getNewValue()); } else { throw new IllegalStateException("Unsupported OperationType: " + logRecord.getNewOp()); } } catch (Exception e) { throw new IllegalStateException("Failed to redo", e); } }
IModificationOperationCallback modCallback = createIndexModificationCallback(op, txnCtx, metadataIndex); IIndexAccessParameters iap = new IndexAccessParameters(modCallback, NoOpOperationCallback.INSTANCE); ILSMIndexAccessor indexAccessor = lsmIndex.createAccessor(iap); txnCtx.setWriteTxn(true); txnCtx.register(metadataIndex.getResourceId(),
secondUser.step(); secondUser.step(); ILSMIndexAccessor accessor = index.createAccessor(NoOpIndexAccessParameters.INSTANCE); ILSMIndexOperationContext opCtx = accessor.getOpContext(); assertCorrectSearchComponents(opCtx, index, 0);