private void after(ILSMIOOperation operation) { if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) { synchronized (runningFlushOperations) { runningFlushOperations.remove(operation.getIndexIdentifier()); if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { failedGroups.putIfAbsent(operation.getIndexIdentifier(), operation.getFailure()); } operation.complete(); runningFlushOperations.notifyAll(); } } else { operation.complete(); } }
@Override public void sync() throws InterruptedException { ioOp.sync(); }
@Override public ILSMDiskComponent getNewComponent() { return ioOp.getNewComponent(); }
private void run(ILSMIOOperation operation) { try { operation.call(); } catch (Throwable th) { // NOSONAR Must catch all LOGGER.log(Level.ERROR, "IO Operation failed", th); operation.setStatus(LSMIOOperationStatus.FAILURE); operation.setFailure(th); } if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { failureCallback.operationFailed(operation, operation.getFailure()); } }
@Override public void afterFinalize(ILSMIOOperation operation) throws HyracksDataException { if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { return; } if (operation.getIOOpertionType() != LSMIOOperationType.LOAD && operation.getAccessor().getOpContext().getOperation() == IndexOperation.DELETE_COMPONENTS) { deleteComponentsFromCheckpoint(operation); } else if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH || operation.getIOOpertionType() == LSMIOOperationType.LOAD) { addComponentToCheckpoint(operation); } }
private void flush(OrderedIndexTestContext ctx) throws HyracksDataException, InterruptedException { ILSMIOOperation flush = scheduleFlush(ctx); flush.sync(); if (flush.getStatus() == LSMIOOperationStatus.FAILURE) { throw HyracksDataException.create(flush.getFailure()); } }
@Override public void afterOperation(ILSMIOOperation operation) throws HyracksDataException { if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { return; } if (operation.getIOOpertionType() == LSMIOOperationType.LOAD) { Map<String, Object> map = operation.getParameters(); putComponentIdIntoMetadata(operation.getNewComponent(), (LSMComponentId) map.get(KEY_FLUSHED_COMPONENT_ID)); } else if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) { Map<String, Object> map = operation.getParameters(); putLSNIntoMetadata(operation.getNewComponent(), (Long) map.get(KEY_FLUSH_LOG_LSN)); putComponentIdIntoMetadata(operation.getNewComponent(), ((FlushOperation) operation).getFlushingComponent().getId()); } else if (operation.getIOOpertionType() == LSMIOOperationType.MERGE) { List<ILSMDiskComponent> mergedComponents = operation.getAccessor().getOpContext().getComponentsToBeMerged(); putLSNIntoMetadata(operation.getNewComponent(), mergedComponents); putComponentIdIntoMetadata(operation.getNewComponent(), mergedComponents); LongPointable markerLsn = LongPointable.FACTORY.createPointable(ComponentUtils.getLong(mergedComponents.get(0).getMetadata(), ComponentUtils.MARKER_LSN_KEY, ComponentUtils.NOT_FOUND, buffer)); operation.getNewComponent().getMetadata().put(ComponentUtils.MARKER_LSN_KEY, markerLsn); } }
flush.sync(); if (flush.getStatus() == LSMIOOperationStatus.FAILURE) { throw HyracksDataException.create(flush.getFailure()); filterToMinMax(flush.getNewComponent().getLSMComponentFilter()); Assert.assertEquals(0, TreeIndexTestUtils.compareFilterTuples(obsMinMax.getLeft(), minMax.getLeft(), comp));
public void doIo(ILSMIOOperation operation) { try { operation.getCallback().beforeOperation(operation); ILSMDiskComponent newComponent = operation.getIOOpertionType() == LSMIOOperationType.FLUSH ? lsmIndex.flush(operation) : lsmIndex.merge(operation); operation.setNewComponent(newComponent); operation.getCallback().afterOperation(operation); if (newComponent != null) { newComponent.markAsValid(lsmIndex.isDurable(), operation); operation.setStatus(LSMIOOperationStatus.FAILURE); operation.setFailure(e); if (LOGGER.isErrorEnabled()) { LOGGER.log(Level.ERROR, "{} operation failed on {}", operation.getIOOpertionType(), lsmIndex, e); operation.getCallback().afterFinalize(operation); } catch (Throwable th) {// NOSONAR Must catch all operation.setStatus(LSMIOOperationStatus.FAILURE); operation.setFailure(th); if (LOGGER.isErrorEnabled()) { LOGGER.log(Level.ERROR, "{} operation.afterFinalize failed on {}", operation.getIOOpertionType(), lsmIndex, th); if (operation.getStatus() == LSMIOOperationStatus.FAILURE) { operation.cleanup(lsmIndex.getBufferCache());
private void before(ILSMIOOperation operation) throws InterruptedException { String id = operation.getIndexIdentifier(); if (operation.getIOOpertionType() == LSMIOOperationType.FLUSH) { synchronized (runningFlushOperations) { while (true) { if (failedGroups.containsKey(id)) { operation.setStatus(LSMIOOperationStatus.FAILURE); operation.setFailure(new RuntimeException("Operation group " + id + " has permanently failed", failedGroups.get(id))); return; } if (runningFlushOperations.containsKey(id)) { runningFlushOperations.wait(); } else { runningFlushOperations.put(id, operation); break; } } } } } }
@Override public void merge(ILSMIOOperation operation) throws HyracksDataException { if (LOGGER.isDebugEnabled()) { LOGGER.debug("Started a merge operation for index: {}", lsmIndex); } synchronized (opTracker) { enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE); } try { doIo(operation); } finally { exitComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE, operation.getNewComponent(), operation.getStatus() == LSMIOOperationStatus.FAILURE); opTracker.completeOperation(lsmIndex, LSMOperationType.MERGE, operation.getAccessor().getOpContext().getSearchOperationCallback(), operation.getAccessor().getOpContext().getModificationCallback()); } if (LOGGER.isDebugEnabled()) { LOGGER.debug("Finished the merge operation for index: {}. Result: {}", lsmIndex, operation.getStatus()); } }
@SuppressWarnings("squid:S1181") @Override public void addBulkLoadedComponent(ILSMIOOperation ioOperation) throws HyracksDataException { ILSMDiskComponent c = ioOperation.getNewComponent(); try { c.markAsValid(lsmIndex.isDurable(), ioOperation); } catch (Throwable th) { ioOperation.setFailure(th); } if (ioOperation.hasFailed()) { throw HyracksDataException.create(ioOperation.getFailure()); } synchronized (opTracker) { lsmIndex.addDiskComponent(c); if (replicationEnabled) { componentsToBeReplicated.clear(); componentsToBeReplicated.add(c); triggerReplication(componentsToBeReplicated, LSMOperationType.LOAD); } mergePolicy.diskComponentAdded(lsmIndex, false); } }
private void presistComponentToDisk() throws HyracksDataException { try { lsmIndex.getIOOperationCallback().afterOperation(opCtx.getIoOperation()); componentBulkLoader.end(); } catch (Throwable th) { // NOSONAR Must not call afterFinalize without setting failure fail(th); throw th; } finally { lsmIndex.getIOOperationCallback().afterFinalize(opCtx.getIoOperation()); } if (opCtx.getIoOperation().getStatus() == LSMIOOperationStatus.SUCCESS && opCtx.getIoOperation().getNewComponent().getComponentSize() > 0) { lsmIndex.getHarness().addBulkLoadedComponent(opCtx.getIoOperation()); } }
private void deleteComponentsFromCheckpoint(ILSMIOOperation operation) throws HyracksDataException { // component was deleted... if a flush, do nothing.. if a merge, must update the checkpoint file if (operation.getIOOpertionType() == LSMIOOperationType.MERGE) { // Get component id of the last disk component LSMComponentId mostRecentComponentId = getMostRecentComponentId(operation.getAccessor().getOpContext().getComponentsToBeMerged()); // Update the checkpoint file FileReference target = operation.getTarget(); final ResourceReference ref = ResourceReference.of(target.getAbsolutePath()); indexCheckpointManagerProvider.get(ref).setLastComponentId(mostRecentComponentId.getMaxId()); } else if (operation.getIOOpertionType() != LSMIOOperationType.FLUSH) { throw new IllegalStateException("Unexpected IO operation: " + operation.getIOOpertionType()); } }
@Override public LSMIOOperationStatus call() throws HyracksDataException { final String name = getTarget().getRelativePath(); final long tid = tracer.durationB(name, traceCategory, null); try { return ioOp.call(); } finally { tracer.durationE(ioOp.getIOOpertionType().name().toLowerCase(), traceCategory, tid, "{\"size\":" + getTarget().getFile().length() + ", \"path\": \"" + ioOp.getTarget().getRelativePath() + "\"}"); } }
private void addComponentToCheckpoint(ILSMIOOperation operation) throws HyracksDataException { // will always update the checkpoint file even if no new component was created FileReference target = operation.getTarget(); Map<String, Object> map = operation.getParameters(); final Long lsn = operation.getIOOpertionType() == LSMIOOperationType.FLUSH ? (Long) map.get(KEY_FLUSH_LOG_LSN) : 0L; final LSMComponentId id = (LSMComponentId) map.get(KEY_FLUSHED_COMPONENT_ID); final ResourceReference ref = ResourceReference.of(target.getAbsolutePath()); final long componentSequence = IndexComponentFileReference.of(ref.getName()).getSequenceEnd(); indexCheckpointManagerProvider.get(ref).flushed(componentSequence, lsn, id.getMaxId()); }
@Override public void afterFinalize(ILSMIOOperation op) throws HyracksDataException { lsmBtree.afterIoFinalizeCalled(); super.afterFinalize(op); synchronized (TestLsmIoOpCallbackFactory.this) { if (op.getNewComponent() != null) { if (op.getNewComponent() == EmptyComponent.INSTANCE) { if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) { rollbackFlushes++; } else { rollbackMerges++; } } else { if (op.getIOOpertionType() == LSMIOOperationType.FLUSH) { completedFlushes++; } else { completedMerges++; } } } else { recordFailure(op.getIOOpertionType()); } TestLsmIoOpCallbackFactory.this.notifyAll(); } lsmBtree.afterIoFinalizeReturned(); }
protected TracedIOOperation(ILSMIOOperation ioOp, ITracer tracer, long traceCategory) { this.ioOp = ioOp; this.tracer = tracer; this.ioOpType = ioOp.getIOOpertionType(); this.traceCategory = traceCategory; }
@Override public LSMIOOperationStatus getStatus() { return ioOp.getStatus(); }