private static FileReference getLocalResourceFileByName(IIOManager ioManager, String resourcePath) throws HyracksDataException { String fileName = resourcePath + File.separator + StorageConstants.METADATA_FILE_NAME; return ioManager.resolve(fileName); }
/** * Get the file reference for the split * * @param ioManager * @return * @throws HyracksDataException */ @Override public FileReference getFileReference(IIOManager ioManager) throws HyracksDataException { return ioManager.resolve(getPath()); } }
/** * Get the path of the index containing the passed reference * * @param ioManager * @param ref * @return * @throws HyracksDataException */ public static Path getIndexPath(IIOManager ioManager, ResourceReference ref) throws HyracksDataException { return ioManager.resolve(ref.getRelativePath().toString()).getFile().toPath(); } }
public static Path getComponentMaskPath(INcApplicationContext appCtx, String componentFile) throws IOException { final IIOManager ioManager = appCtx.getIoManager(); final FileReference localPath = ioManager.resolve(componentFile); final Path resourceDir = Files.createDirectories(localPath.getFile().getParentFile().toPath()); final String componentSequence = PersistentLocalResourceRepository.getComponentSequence(componentFile); return Paths.get(resourceDir.toString(), StorageConstants.COMPONENT_MASK_FILE_PREFIX + componentSequence); }
@Override public void perform(INcApplicationContext appCtx, IReplicationWorker worker) { try { final IIOManager ioManager = appCtx.getIoManager(); final File localFile = ioManager.resolve(file).getFile(); if (localFile.exists()) { Files.delete(localFile.toPath()); LOGGER.info(() -> "Deleted file: " + localFile.getAbsolutePath()); } else { LOGGER.warn(() -> "Requested to delete a non-existing file: " + localFile.getAbsolutePath()); } ReplicationProtocol.sendAck(worker.getChannel(), worker.getReusableBuffer()); } catch (IOException e) { throw new ReplicationException(e); } }
@Override public IIndex createInstance(INCServiceContext ctx) throws HyracksDataException { IIOManager ioManager = ctx.getIoManager(); FileReference resourceRef = ioManager.resolve(path); return RTreeUtils.createRTree(storageManager.getBufferCache(ctx), typeTraits, valueProviderFactories, comparatorFactories, rtreePolicyType, resourceRef, false, pageManagerFactory); }
@Override public void perform(INcApplicationContext appCtx, IReplicationWorker worker) { try { final IIOManager ioManager = appCtx.getIoManager(); final File indexFile = ioManager.resolve(file).getFile(); if (indexFile.exists()) { File indexDir = indexFile.getParentFile(); IoUtil.delete(indexDir); LOGGER.info(() -> "Deleted index: " + indexFile.getAbsolutePath()); } else { LOGGER.warning(() -> "Requested to delete a non-existing index: " + indexFile.getAbsolutePath()); } ReplicationProtocol.sendAck(worker.getChannel(), worker.getReusableBuffer()); } catch (IOException e) { throw new ReplicationException(e); } }
public List<String> getPartitionReplicatedFiles(int partition, IReplicationStrategy strategy) throws HyracksDataException { final List<String> partitionReplicatedFiles = new ArrayList<>(); final Set<File> replicatedIndexes = new HashSet<>(); final Map<Long, LocalResource> partitionResources = getPartitionResources(partition); for (LocalResource lr : partitionResources.values()) { DatasetLocalResource datasetLocalResource = (DatasetLocalResource) lr.getResource(); if (strategy.isMatch(datasetLocalResource.getDatasetId())) { replicatedIndexes.add(ioManager.resolve(lr.getPath()).getFile()); } } for (File indexDir : replicatedIndexes) { partitionReplicatedFiles.addAll(getIndexFiles(indexDir)); } return partitionReplicatedFiles; }
@Override protected void performOpOnIndex(IIndexDataflowHelper indexHelper, IHyracksTaskContext ctx) throws HyracksDataException { String path = indexHelper.getResource().getPath(); IIOManager ioManager = ctx.getIoManager(); FileReference file = ioManager.resolve(path); LOGGER.warn("performing the operation on " + file.getFile().getAbsolutePath()); // Get index IIndex index = indexHelper.getIndexInstance(); // commit transaction ((ITwoPCIndex) index).commitTransaction(); LOGGER.warn("operation on " + file.getFile().getAbsolutePath() + " Succeded"); } }
/** * Gets a set of files for the indexes in partition {@code partition}. Each file points * to where the index's files are stored. * * @param partition * @return The set of indexes files * @throws HyracksDataException */ public Set<File> getPartitionIndexes(int partition) throws HyracksDataException { final Map<Long, LocalResource> partitionResourcesMap = getResources(resource -> { DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource(); return dsResource.getPartition() == partition; }); Set<File> indexes = new HashSet<>(); for (LocalResource localResource : partitionResourcesMap.values()) { indexes.add(ioManager.resolve(localResource.getPath()).getFile()); } return indexes; }
@Override public IIndex createInstance(INCServiceContext ctx) throws HyracksDataException { IBufferCache bufferCache = storageManager.getBufferCache(ctx); IIOManager ioManager = ctx.getIoManager(); FileReference resourceRef = ioManager.resolve(path); return BTreeUtils.createBTree(bufferCache, typeTraits, comparatorFactories, BTreeLeafFrameType.REGULAR_NSM, resourceRef, pageManagerFactory.createPageManager(bufferCache), false); }
public void replicate(String file, boolean metadata) { try { final IIOManager ioManager = appCtx.getIoManager(); final ISocketChannel channel = replica.getChannel(); final FileReference filePath = ioManager.resolve(file); ReplicateFileTask task = new ReplicateFileTask(file, filePath.getFile().length(), metadata); ReplicationProtocol.sendTo(replica, task); // send the file itself try (RandomAccessFile fromFile = new RandomAccessFile(filePath.getFile(), "r"); FileChannel fileChannel = fromFile.getChannel()) { NetworkingUtil.sendFile(fileChannel, channel); } ReplicationProtocol.waitForAck(replica); } catch (IOException e) { throw new ReplicationException(e); } }
@Override protected void performOpOnIndex(IIndexDataflowHelper indexDataflowHelper, IHyracksTaskContext ctx) throws HyracksDataException { String path = indexDataflowHelper.getResource().getPath(); IIOManager ioManager = ctx.getIoManager(); FileReference file = ioManager.resolve(path); AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(ctx.getIoManager(), file); fileManager.recoverTransaction(); } }
@Override protected void performOpOnIndex(IIndexDataflowHelper indexDataflowHelper, IHyracksTaskContext ctx) throws HyracksDataException { String path = indexDataflowHelper.getResource().getPath(); IIOManager ioManager = ctx.getIoManager(); FileReference file = ioManager.resolve(path); AbortRecoverLSMIndexFileManager fileManager = new AbortRecoverLSMIndexFileManager(ctx.getIoManager(), file); fileManager.deleteTransactionFiles(); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); ioOpCallbackFactory.initialize(serviceCtx, this); return LSMBTreeUtil.createExternalBTree(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, durable, metadataPageManagerFactory, serviceCtx.getTracer()); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); ioOpCallbackFactory.initialize(serviceCtx, this); return LSMBTreeUtil.createExternalBTreeWithBuddy(ioManager, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, bloomFilterKeyFields, durable, metadataPageManagerFactory, serviceCtx.getTracer()); }
@Override public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException { IIOManager ioManager = ncServiceCtx.getIoManager(); FileReference fileRef = ioManager.resolve(path); ioOpCallbackFactory.initialize(ncServiceCtx, this); return LSMRTreeUtils.createExternalRTree(ioManager, fileRef, storageManager.getBufferCache(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx), opTrackerProvider.getOperationTracker(ncServiceCtx, this), ioSchedulerProvider.getIoScheduler(ncServiceCtx), ioOpCallbackFactory, linearizeCmpFactory, buddyBTreeFields, durable, isPointMBR, metadataPageManagerFactory, ncServiceCtx.getTracer()); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file); ioOpCallbackFactory.initialize(serviceCtx, this); return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR, metadataPageManagerFactory); }
@Override public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException { IIOManager ioManager = ncServiceCtx.getIoManager(); FileReference fileRef = ioManager.resolve(path); List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(ncServiceCtx, fileRef); ioOpCallbackFactory.initialize(ncServiceCtx, this); return LSMRTreeUtils.createLSMTree(ioManager, virtualBufferCaches, fileRef, storageManager.getBufferCache(ncServiceCtx), typeTraits, cmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, ncServiceCtx), opTrackerProvider.getOperationTracker(ncServiceCtx, this), ioSchedulerProvider.getIoScheduler(ncServiceCtx), ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, buddyBTreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR, metadataPageManagerFactory); }
@Override public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException { IIOManager ioManager = serviceCtx.getIoManager(); FileReference file = ioManager.resolve(path); List<IVirtualBufferCache> vbcs = vbcProvider.getVirtualBufferCaches(serviceCtx, file); ioOpCallbackFactory.initialize(serviceCtx, this); //TODO: enable updateAwareness for secondary LSMBTree indexes boolean updateAware = false; return LSMBTreeUtil.createLSMTree(ioManager, vbcs, file, storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, bloomFilterKeyFields, bloomFilterFalsePositiveRate, mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx), opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx), ioOpCallbackFactory, isPrimary, filterTypeTraits, filterCmpFactories, btreeFields, filterFields, durable, metadataPageManagerFactory, updateAware, serviceCtx.getTracer(), compressorDecompressorFactory); }