@Override public void resetCurrentComponentIndex() { synchronized (lsmHarness.getOperationTracker()) { // validate no reader in any of the memory components and that all of them are INVALID for (ILSMMemoryComponent c : memoryComponents) { if (c.getReaderCount() > 0) { throw new IllegalStateException( "Attempt to reset current component index while readers are inside the components. " + c); } if (c.getState() != ComponentState.INACTIVE) { throw new IllegalStateException( "Attempt to reset current component index while a component is not INACTIVE. " + c); } } currentMutableComponentId.set(0); memoryComponents.get(0); try { memoryComponents.get(0).resetId(null, true); } catch (HyracksDataException e) { throw new IllegalStateException(e); } } }
@Override public boolean isCurrentMutableComponentEmpty() throws HyracksDataException { synchronized (getOperationTracker()) { ILSMMemoryComponent cmc = getCurrentMemoryComponent(); ComponentState state = cmc.getState(); return state == ComponentState.READABLE_UNWRITABLE_FLUSHING || state == ComponentState.INACTIVE || state == ComponentState.UNREADABLE_UNWRITABLE || !cmc.isModified(); } }
@Override public void updateFilter(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException { if (ctx.getFilterTuple() != null && !ctx.isFilterSkipped()) { if (ctx.isRecovery()) { memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter().update(tuple, ctx.getFilterCmp(), ctx.getModificationCallback()); } else { ctx.getFilterTuple().reset(tuple); memoryComponents.get(currentMutableComponentId.get()).getLSMComponentFilter() .update(ctx.getFilterTuple(), ctx.getFilterCmp(), ctx.getModificationCallback()); } } }
private long lsnFromImmutableMemoryComponents() { List<ILSMMemoryComponent> memComponents = index.getMemoryComponents(); int numOtherMemComponents = memComponents.size() - 1; int next = index.getCurrentMemoryComponentIndex(); long lsn = ComponentUtils.NOT_FOUND; for (int i = 0; i < numOtherMemComponents; i++) { next = next - 1; if (next < 0) { next = memComponents.size() - 1; } ILSMMemoryComponent c = index.getMemoryComponents().get(next); if (c.isReadable()) { try { lsn = ComponentUtils.getLong(c.getMetadata(), ComponentUtils.MARKER_LSN_KEY, ComponentUtils.NOT_FOUND, buffer); } catch (HyracksDataException e) { // Should never happen since this is a memory component throw new IllegalStateException(e); } if (lsn != ComponentUtils.NOT_FOUND) { return lsn; } } } return lsn; }
synchronized (opTracker) { ILSMMemoryComponent memComponent = lsmIndex.getCurrentMemoryComponent(); if (memComponent.getWriterCount() > 0) { throw new IllegalStateException( "Can't request a flush on a component with writers inside: Index:" + lsmIndex + " Component:" + memComponent); if (memComponent.getState() == ComponentState.READABLE_WRITABLE && memComponent.isModified()) { memComponent.setUnwritable(); primaryId = (LSMComponentId) memComponent.getId();
@Override public final ILSMDiskComponent flush(ILSMIOOperation operation) throws HyracksDataException { ILSMIndexAccessor accessor = operation.getAccessor(); ILSMIndexOperationContext opCtx = accessor.getOpContext(); ILSMMemoryComponent memoryComponent = (ILSMMemoryComponent) opCtx.getComponentHolder().get(0); if (memoryComponent != getOldestReadableMemoryComponent()) { throw new IllegalStateException("An attempt to flush a memory component that is not the oldest"); } if (!memoryComponent.isModified() || opCtx.getOperation() == IndexOperation.DELETE_COMPONENTS) { return EmptyComponent.INSTANCE; } if (LOGGER.isInfoEnabled()) { FlushOperation flushOp = (FlushOperation) operation; LOGGER.log(Level.INFO, "Flushing component with id: " + flushOp.getFlushingComponent().getId() + " in the index " + this); } return doFlush(operation); }
long maxId = logRecord.getFlushingComponentMaxId(); ILSMComponentId id = new LSMComponentId(minId, maxId); flushMap.put(LSMIOOperationCallback.KEY_NEXT_COMPONENT_ID, index.getCurrentMemoryComponent().getId()); if (!index.getDiskComponents().isEmpty()) { ILSMDiskComponent diskComponent = index.getDiskComponents().get(0); index.getCurrentMemoryComponent().resetId(id, true); ILSMIOOperation flush = accessor.scheduleFlush(); try {
public TestFlushOperation(ILSMIndexAccessor accessor, FileReference target, ILSMIOOperationCallback callback, String indexIdentifier, LSMComponentFileReferences files, LSMComponentId componentId) throws HyracksDataException { super(accessor, target, callback, indexIdentifier); this.files = files; flushingComponent = accessor.getOpContext().getIndex().getCurrentMemoryComponent(); Mockito.when(flushingComponent.getId()).thenReturn(componentId); }
@Override public void recycled(ILSMMemoryComponent component) throws HyracksDataException { component.resetId(componentIds.poll(), false); }
@Override public void after(long lsn) { pointable.setLong(lsn); try { index.getCurrentMemoryComponent().getMetadata().put(ComponentUtils.MARKER_LSN_KEY, pointable); } catch (HyracksDataException e) { throw new IllegalStateException(e); } }
private void addImmutableMemoryComponents(List<ILSMComponent> operationalComponents) { int cmc = currentMutableComponentId.get(); int numImmutableMemoryComponents = Integer.min(numScheduledFlushes, memoryComponents.size()); int next = numScheduledFlushes < memoryComponents.size() ? cmc : getNextToBeFlushed(); for (int i = 0; i < numImmutableMemoryComponents; i++) { next--; if (next < 0) { next = memoryComponents.size() - 1; } //newer components first ILSMMemoryComponent c = memoryComponents.get(next); if (c.isReadable()) { operationalComponents.add(c); } } }
@Override public ILSMIOOperation createFlushOperation(ILSMIndexOperationContext ctx) throws HyracksDataException { ILSMMemoryComponent flushingComponent = getCurrentMemoryComponent(); if (flushingComponent.getWriterCount() > 0) { throw new IllegalStateException( "createFlushOperation is called on a component with writers: " + flushingComponent); } // take care of the flush cycling ILSMIOOperation flushOp = TracedIOOperation.wrap(createFlushOperation(createOpContext(NoOpIndexAccessParameters.INSTANCE), fileManager.getRelFlushFileReference(), ioOpCallback), tracer); // Changing the flush status should *always* precede changing the mutable component. flushingComponent.schedule(LSMIOOperationType.FLUSH); numScheduledFlushes++; changeFlushStatusForCurrentMutableCompoent(false); changeMutableComponent(); ILSMIndexAccessor accessor = flushOp.getAccessor(); ILSMIndexOperationContext flushCtx = accessor.getOpContext(); flushCtx.setOperation(ctx.getOperation()); // Could be component delete flushCtx.getComponentHolder().add(flushingComponent); flushCtx.setIoOperation(flushOp); propagateMap(ctx, flushCtx); ioOpCallback.scheduled(flushOp); return flushOp; }
private boolean flushingOrMerging() { // check if flushes are taking place for (ILSMMemoryComponent memComponent : lsmIndex.getMemoryComponents()) { if (memComponent.getState() == ComponentState.READABLE_UNWRITABLE_FLUSHING) { return true; } } // check if merges are taking place for (ILSMDiskComponent diskComponent : lsmIndex.getDiskComponents()) { if (diskComponent.getState() == ComponentState.READABLE_MERGING) { return true; } } return false; }
Assert.assertTrue(primaryIndexes[PARTITION_0].getCurrentMemoryComponent().isModified()); Assert.assertTrue(primaryIndexes[PARTITION_1].getCurrentMemoryComponent().isModified());
private void checkComponentIds(int partitionIndex) throws HyracksDataException { // check memory component if (primaryIndexes[partitionIndex].isMemoryComponentsAllocated()) { ILSMMemoryComponent primaryMemComponent = primaryIndexes[partitionIndex].getCurrentMemoryComponent(); ILSMMemoryComponent secondaryMemComponent = secondaryIndexes[partitionIndex].getCurrentMemoryComponent(); Assert.assertEquals(primaryMemComponent.getId(), secondaryMemComponent.getId()); Assert.assertEquals(primaryIndexes[partitionIndex].getCurrentMemoryComponentIndex(), secondaryIndexes[partitionIndex].getCurrentMemoryComponentIndex()); } List<ILSMDiskComponent> primaryDiskComponents = primaryIndexes[partitionIndex].getDiskComponents(); List<ILSMDiskComponent> secondaryDiskComponents = secondaryIndexes[partitionIndex].getDiskComponents(); Assert.assertEquals(primaryDiskComponents.size(), secondaryDiskComponents.size()); for (int i = 0; i < primaryDiskComponents.size(); i++) { Assert.assertEquals(primaryDiskComponents.get(i).getId(), secondaryDiskComponents.get(i).getId()); } }
if (c.isReadable()) { c.getMetadata().get(key, value); if (value.getLength() != 0) {
private void checkMemoryComponent(ILSMComponentId expected, ILSMMemoryComponent memoryComponent) throws HyracksDataException { ArgumentCaptor<ILSMComponentId> idArgument = ArgumentCaptor.forClass(ILSMComponentId.class); ArgumentCaptor<Boolean> forceArgument = ArgumentCaptor.forClass(Boolean.class); Mockito.verify(memoryComponent).resetId(idArgument.capture(), forceArgument.capture()); assertEquals(expected, idArgument.getValue()); assertEquals(false, forceArgument.getValue().booleanValue()); Mockito.reset(memoryComponent); }
/** * Put LSM metadata state into the index's current memory component. * * @param index, * the LSM index. * @param key, * the key for the metadata state. * @param pointable, * the value for the metadata state. * @throws HyracksDataException */ public static void put(ILSMIndex index, IValueReference key, IPointable pointable) throws HyracksDataException { // write the opTracker to ensure the component layout don't change synchronized (index.getOperationTracker()) { index.getCurrentMemoryComponent().getMetadata().put(key, pointable); } }
private void addOperationalMemoryComponents(List<ILSMComponent> operationalComponents, boolean modification) { // add current memory component first if needed if (numScheduledFlushes < memoryComponents.size()) { ILSMMemoryComponent c = memoryComponents.get(currentMutableComponentId.get()); // The current mutable component is added if modification or readable // This ensures that activation of new component only happens in case of modifications // and allow for controlling that without stopping search operations if (modification || c.isReadable()) { operationalComponents.add(c); } } if (modification && numScheduledFlushes >= memoryComponents.size()) { // will fail the enterComponent call and retry operationalComponents.add(memoryComponents.get(0)); return; } addImmutableMemoryComponents(operationalComponents); }
@Test public void testFlushMetadataOnlyComponent() throws Exception { // allow all operations StorageTestUtils.allowAllOps(lsmBtree); // ensure no disk component and memory component is empty Assert.assertEquals(0, lsmBtree.getDiskComponents().size()); Assert.assertFalse(lsmBtree.isMemoryComponentsAllocated()); MutableArrayValueReference key = new MutableArrayValueReference("FlushMetadataOnlyTestKey".getBytes()); MutableArrayValueReference value = new MutableArrayValueReference("FlushMetadataOnlyTestValue".getBytes()); indexDataflowHelper.open(); ILSMIndexAccessor accessor = lsmBtree.createAccessor(NoOpIndexAccessParameters.INSTANCE); accessor.updateMeta(key, value); Assert.assertTrue(lsmBtree.isMemoryComponentsAllocated()); Assert.assertTrue(lsmBtree.getCurrentMemoryComponent().isModified()); indexDataflowHelper.close(); // flush synchronously StorageTestUtils.flush(dsLifecycleMgr, lsmBtree, false); // assert one disk component Assert.assertEquals(1, lsmBtree.getDiskComponents().size()); ArrayBackedValueStorage pointable = new ArrayBackedValueStorage(); ComponentUtils.get(lsmBtree, key, pointable); Assert.assertTrue(DataUtils.equals(pointable, value)); // ensure that we can search this component StorageTestUtils.searchAndAssertCount(nc, PARTITION, 0); }