@Override public void register(long resourceId, int partition, ILSMIndex index, IModificationOperationCallback callback, boolean primaryIndex) { synchronized (txnOpTrackers) { if (!txnOpTrackers.containsKey(resourceId)) { final ITransactionOperationTracker txnOpTracker = (ITransactionOperationTracker) index.getOperationTracker(); txnOpTrackers.put(resourceId, txnOpTracker); txnOpTracker.beforeTransaction(resourceId); } } }
@Override public void register(long resourceId, int partition, ILSMIndex index, IModificationOperationCallback callback, boolean primaryIndex) { super.register(resourceId, partition, index, callback, primaryIndex); synchronized (opTrackers) { if (primaryIndex && !opTrackers.containsKey(resourceId)) { opTrackers.put(resourceId, index.getOperationTracker()); callbacks.put(resourceId, callback); indexPendingOps.put(resourceId, new AtomicInteger(0)); } } }
@Override public void register(long resourceId, int partition, ILSMIndex index, IModificationOperationCallback callback, boolean primaryIndex) { super.register(resourceId, partition, index, callback, primaryIndex); AtomicInteger pendingOps = partitionPendingOps.computeIfAbsent(partition, p -> new AtomicInteger(0)); resourcePendingOps.put(resourceId, pendingOps); if (primaryIndex) { Pair<PrimaryIndexOperationTracker, IModificationOperationCallback> pair = new Pair<>((PrimaryIndexOperationTracker) index.getOperationTracker(), callback); primaryIndexTrackers.put(partition, pair); } }
private Predicate<ILSMIndex> newIdleDatasetPredicate() { final long currentTime = System.nanoTime(); return lsmIndex -> { if (lsmIndex.isPrimaryIndex()) { PrimaryIndexOperationTracker opTracker = (PrimaryIndexOperationTracker) lsmIndex.getOperationTracker(); return currentTime - opTracker.getLastFlushTime() >= datasetCheckpointInterval; } return false; }; }
private LSMComponentId getMostRecentComponentId(Collection<ILSMDiskComponent> deletedComponents) throws HyracksDataException { // must sync on opTracker to ensure list of components doesn't change synchronized (lsmIndex.getOperationTracker()) { List<ILSMDiskComponent> diskComponents = lsmIndex.getDiskComponents(); if (diskComponents.isEmpty()) { LOGGER.log(Level.INFO, "There are no disk components"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } if (deletedComponents.contains(diskComponents.get(diskComponents.size() - 1))) { LOGGER.log(Level.INFO, "All disk components have been deleted"); return LSMComponentId.EMPTY_INDEX_LAST_COMPONENT_ID; } int mostRecentComponentIndex = 0; for (int i = 0; i < diskComponents.size(); i++) { if (!deletedComponents.contains(diskComponents.get(i))) { break; } mostRecentComponentIndex++; } ILSMDiskComponent mostRecentDiskComponent = diskComponents.get(mostRecentComponentIndex); return (LSMComponentId) mostRecentDiskComponent.getId(); } }
public static void waitForOperations(ILSMIndex index) throws InterruptedException { // wait until number of activeOperation reaches 0 PrimaryIndexOperationTracker opTracker = (PrimaryIndexOperationTracker) index.getOperationTracker(); long maxWaitTime = 60000L; // 1 minute long before = System.currentTimeMillis(); while (opTracker.getNumActiveOperations() > 0) { Thread.sleep(5); // NOSONAR: Test code with a timeout if (System.currentTimeMillis() - before > maxWaitTime) { throw new IllegalStateException( (System.currentTimeMillis() - before) + "ms passed without completing the frame operation"); } } }
strBuilder.append("\", \"components\":["); synchronized (index.getOperationTracker()) { List<ILSMDiskComponent> diskComponents = index.getDiskComponents(); for (int i = diskComponents.size() - 1; i >= 0; i--) {
/** * Put LSM metadata state into the index's current memory component. * * @param index, * the LSM index. * @param key, * the key for the metadata state. * @param pointable, * the value for the metadata state. * @throws HyracksDataException */ public static void put(ILSMIndex index, IValueReference key, IPointable pointable) throws HyracksDataException { // write the opTracker to ensure the component layout don't change synchronized (index.getOperationTracker()) { index.getCurrentMemoryComponent().getMetadata().put(key, pointable); } }
@Override protected boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException { List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getDiskComponents()); // Reverse the components order so that we look at components from oldest to newest. Collections.reverse(immutableComponents); Pair<Integer, Integer> mergeableIndexes = getMergableComponentsIndex(immutableComponents); if (mergeableIndexes == null) { //nothing to merge return false; } ILSMComponent leftComponent = immutableComponents.get(mergeableIndexes.getLeft()); ILSMComponent rightComponent = immutableComponents.get(mergeableIndexes.getRight()); ILSMComponentId targetId = LSMComponentIdUtils.union(leftComponent.getId(), rightComponent.getId()); int partition = ((PrimaryIndexOperationTracker) index.getOperationTracker()).getPartition(); Set<ILSMIndex> indexes = datasetLifecycleManager.getDatasetInfo(datasetId).getDatasetPartitionOpenIndexes(partition); triggerScheduledMerge(targetId, indexes); return true; }
private long getLsn() { long lsn; try { lsn = ComponentUtils.getLong(index.getCurrentMemoryComponent().getMetadata(), ComponentUtils.MARKER_LSN_KEY, ComponentUtils.NOT_FOUND, buffer); } catch (HyracksDataException e) { // Should never happen since this is a memory component throw new IllegalStateException(e); } if (lsn == ComponentUtils.NOT_FOUND) { synchronized (index.getOperationTracker()) { // look for it in previous memory component if exists lsn = lsnFromImmutableMemoryComponents(); if (lsn == ComponentUtils.NOT_FOUND) { // look for it in disk component lsn = lsnFromDiskComponents(); } } } return lsn; }
private void closeIndex(IndexInfo indexInfo) throws HyracksDataException { if (indexInfo.isOpen()) { ILSMOperationTracker opTracker = indexInfo.getIndex().getOperationTracker(); synchronized (opTracker) { indexInfo.getIndex().deactivate(false); } indexCheckpointManagerProvider.close(DatasetResourceReference.of(indexInfo.getLocalResource())); indexInfo.setOpen(false); } }
PrimaryIndexOperationTracker opTracker = Mockito.mock(PrimaryIndexOperationTracker.class); Mockito.when(opTracker.getPartition()).thenReturn(partition); Mockito.when(index.getOperationTracker()).thenReturn(opTracker);
ILSMOperationTracker opTracker = lsmIndex.getOperationTracker(); synchronized (opTracker) { ILSMMemoryComponent memComponent = lsmIndex.getCurrentMemoryComponent();
@Override public synchronized void open(String resourcePath) throws HyracksDataException { validateDatasetLifecycleManagerState(); int did = getDIDfromResourcePath(resourcePath); long resourceID = getResourceIDfromResourcePath(resourcePath); DatasetResource dsr = datasets.get(did); DatasetInfo dsInfo = dsr.getDatasetInfo(); if (dsInfo == null || !dsInfo.isRegistered()) { throw new HyracksDataException( "Failed to open index with resource ID " + resourceID + " since it does not exist."); } IndexInfo iInfo = dsInfo.getIndexes().get(resourceID); if (iInfo == null) { throw new HyracksDataException( "Failed to open index with resource ID " + resourceID + " since it does not exist."); } dsr.open(true); dsr.touch(); if (!iInfo.isOpen()) { ILSMOperationTracker opTracker = iInfo.getIndex().getOperationTracker(); synchronized (opTracker) { iInfo.getIndex().activate(); } iInfo.setOpen(true); } iInfo.touch(); }
if (index != null) { final PrimaryIndexOperationTracker opTracker = (PrimaryIndexOperationTracker) index.getOperationTracker(); Assert.assertEquals(0, opTracker.getNumActiveOperations());
synchronized (index.getOperationTracker()) { ILSMMemoryComponent cmc = index.getCurrentMemoryComponent(); if (cmc.isReadable()) {