/** * A helper method to ensure disk components have proper Ids (non-decreasing) * We may get rid of this method once component Id is stablized * * @throws HyracksDataException */ private void validateComponentIds() throws HyracksDataException { for (int i = 0; i < diskComponents.size() - 1; i++) { ILSMComponentId id1 = diskComponents.get(i).getId(); ILSMComponentId id2 = diskComponents.get(i + 1).getId(); IdCompareResult cmp = id1.compareTo(id2); if (cmp != IdCompareResult.UNKNOWN && cmp != IdCompareResult.GREATER_THAN) { throw new IllegalStateException( "found non-decreasing component ids (" + id1 + " -> " + id2 + ") on index " + this); } } }
public boolean proceed(List<ILSMDiskComponent> allDiskComponents) throws HyracksDataException { for (int i = 0; i < components.size(); i++) { replacedComponentIds.add(components.get(i).getId()); // ensure that disk component exists boolean found = false; final ILSMComponentId replacedComponentId = replacedComponentIds.get(i); LOGGER.trace("looking for a component with the id: {}", replacedComponentId); for (ILSMDiskComponent dc : allDiskComponents) { ILSMComponentId diskComponentId = dc.getId(); LOGGER.trace("next disk component id: {}", diskComponentId); if (diskComponentId.equals(replacedComponentId)) { found = true; diskComponents.add(dc); break; } } if (!found) { // component has been merged? LOGGER.warn("memory component {} was flushed and merged before search cursor replaces it", replacedComponentId); return false; } } return true; }
private void loadNewComponent(int componentPos) throws HyracksDataException { endCurrentComponent(); int numTuples = getNumDeletedTuples(componentPos); ILSMDiskComponent primaryComponent = primaryIndex.getDiskComponents().get(componentPos); Map<String, Object> parameters = new HashMap<>(); parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, primaryComponent.getId()); componentBulkLoader = (LSMIndexDiskComponentBulkLoader) secondaryIndex.createBulkLoader(1.0f, false, numTuples, false, parameters); }
private LSMComponentId getMostRecentComponentId(Collection<ILSMDiskComponent> deletedComponents) throws HyracksDataException { // must sync on opTracker to ensure list of components doesn't change synchronized (lsmIndex.getOperationTracker()) { List<ILSMDiskComponent> diskComponents = lsmIndex.getDiskComponents(); if (diskComponents.isEmpty()) { LOGGER.log(Level.INFO, "There are no disk components"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } if (deletedComponents.contains(diskComponents.get(diskComponents.size() - 1))) { LOGGER.log(Level.INFO, "All disk components have been deleted"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } int mostRecentComponentIndex = 0; for (int i = 0; i < diskComponents.size(); i++) { if (!deletedComponents.contains(diskComponents.get(i))) { break; } mostRecentComponentIndex++; } ILSMDiskComponent mostRecentDiskComponent = diskComponents.get(mostRecentComponentIndex); return (LSMComponentId) mostRecentDiskComponent.getId(); } }
@Override protected void initializeBulkLoader() throws HyracksDataException { ILSMIndex targetIndex = (ILSMIndex) index; Map<String, Object> parameters = new HashMap<>(); parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, LSMComponentId.DEFAULT_COMPONENT_ID); if (usage.equals(BulkLoadUsage.LOAD)) { bulkLoader = targetIndex.createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, parameters); } else { primaryIndexHelper.open(); primaryIndex = (ILSMIndex) primaryIndexHelper.getIndexInstance(); List<ILSMDiskComponent> primaryComponents = primaryIndex.getDiskComponents(); if (!primaryComponents.isEmpty()) { ILSMComponentId bulkloadId = LSMComponentIdUtils.union(primaryComponents.get(0).getId(), primaryComponents.get(primaryComponents.size() - 1).getId()); parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, bulkloadId); } else { parameters.put(LSMIOOperationCallback.KEY_FLUSHED_COMPONENT_ID, LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID); } bulkLoader = targetIndex.createBulkLoader(fillFactor, verifyInput, numElementsHint, checkIfEmptyIndex, parameters); } }
private long getReplicatedComponentId() throws HyracksDataException { final ILSMIndexReplicationJob indexReplJob = (ILSMIndexReplicationJob) job; if (indexReplJob.getLSMOpType() != LSMOperationType.FLUSH) { return -1L; } final ILSMIndexOperationContext ctx = indexReplJob.getLSMIndexOperationContext(); LSMComponentId id = (LSMComponentId) ctx.getComponentsToBeReplicated().get(0).getId(); return id.getMinId(); } }
LSMComponentId id = (LSMComponentId) c.getId(); strBuilder.append('{'); strBuilder.append("\"min\":");
@Override public Void answer(InvocationOnMock invocation) throws Throwable { List<ILSMDiskComponent> mergedComponents = invocation.getArgumentAt(0, List.class); mergedComponents.forEach(component -> { try { resultComponentIDs.add(component.getId()); } catch (HyracksDataException e) { e.printStackTrace(); } }); return null; } }).when(accessor).scheduleMerge(Mockito.anyListOf(ILSMDiskComponent.class));
ILSMComponentId id = component.getId(); IdCompareResult cmp = targetId.compareTo(id); if (cmp == IdCompareResult.INCLUDE) {
for (ILSMComponentId id : componentIDs) { ILSMDiskComponent component = Mockito.mock(ILSMDiskComponent.class); Mockito.when(component.getId()).thenReturn(id); Mockito.when(component.getComponentSize()).thenReturn(DEFAULT_COMPONENT_SIZE); Mockito.when(component.getState()).thenReturn(ComponentState.READABLE_UNWRITABLE);
if (!index.getDiskComponents().isEmpty()) { ILSMDiskComponent diskComponent = index.getDiskComponents().get(0); ILSMComponentId maxDiskComponentId = diskComponent.getId(); if (maxDiskComponentId.compareTo(id) != IdCompareResult.LESS_THAN) { throw new IllegalStateException("Illegal state of component Id. Max disk component Id "
private void checkComponentIds(int partitionIndex) throws HyracksDataException { // check memory component if (primaryIndexes[partitionIndex].isMemoryComponentsAllocated()) { ILSMMemoryComponent primaryMemComponent = primaryIndexes[partitionIndex].getCurrentMemoryComponent(); ILSMMemoryComponent secondaryMemComponent = secondaryIndexes[partitionIndex].getCurrentMemoryComponent(); Assert.assertEquals(primaryMemComponent.getId(), secondaryMemComponent.getId()); Assert.assertEquals(primaryIndexes[partitionIndex].getCurrentMemoryComponentIndex(), secondaryIndexes[partitionIndex].getCurrentMemoryComponentIndex()); } List<ILSMDiskComponent> primaryDiskComponents = primaryIndexes[partitionIndex].getDiskComponents(); List<ILSMDiskComponent> secondaryDiskComponents = secondaryIndexes[partitionIndex].getDiskComponents(); Assert.assertEquals(primaryDiskComponents.size(), secondaryDiskComponents.size()); for (int i = 0; i < primaryDiskComponents.size(); i++) { Assert.assertEquals(primaryDiskComponents.get(i).getId(), secondaryDiskComponents.get(i).getId()); } }
@Test public void testFlushOneFullOneEmpty() { try { int totalNumOfComponents = TOTAL_NUM_OF_RECORDS / RECORDS_PER_COMPONENT; for (int j = 0; j < totalNumOfComponents; j++) { actors[0].add(new Request(Request.Action.INSERT_PATCH)); actors[0].add(new Request(Request.Action.FLUSH_DATASET)); } ensureDone(actors[0]); // search now and ensure partition 0 has all the records StorageTestUtils.searchAndAssertCount(nc, 0, dataset, storageManager, TOTAL_NUM_OF_RECORDS); // and that partition 1 has no records StorageTestUtils.searchAndAssertCount(nc, 1, dataset, storageManager, 0); // and that partition 0 has numFlushes disk components Assert.assertEquals(totalNumOfComponents, primaryLsmBtrees[0].getDiskComponents().size()); // and that partition 1 has no disk components Assert.assertEquals(0, primaryLsmBtrees[1].getDiskComponents().size()); // and that in partition 0, all secondary components has a corresponding primary List<ILSMDiskComponent> secondaryDiskComponents = secondaryLsmBtrees[0].getDiskComponents(); List<ILSMDiskComponent> primaryDiskComponents = primaryLsmBtrees[0].getDiskComponents(); for (int i = 0; i < secondaryDiskComponents.size(); i++) { Assert.assertEquals(secondaryDiskComponents.get(i).getId(), primaryDiskComponents.get(i).getId()); } } catch (Throwable e) { e.printStackTrace(); Assert.fail(e.getMessage()); } }
List<ILSMDiskComponent> primaryDiskComponents = primaryLsmBtrees[0].getDiskComponents(); for (int i = 0; i < secondaryDiskComponents.size(); i++) { Assert.assertEquals(secondaryDiskComponents.get(i).getId(), primaryDiskComponents.get(i).getId()); primaryDiskComponents = primaryLsmBtrees[1].getDiskComponents(); for (int i = 0; i < secondaryDiskComponents.size(); i++) { Assert.assertEquals(secondaryDiskComponents.get(i).getId(), primaryDiskComponents.get(i).getId());
LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId(); long min = id.getMinId();
LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId(); long min = id.getMinId();
LSMComponentId id = (LSMComponentId) primaryDiskComponent.getId(); long min = id.getMinId();