public FileReference(IODeviceHandle dev, String path) { file = new File(dev.getMount(), path); this.dev = dev; this.path = path; }
/** * Get handles for IO devices * * @param ioDevices * comma separated list of devices * @return */ public static List<IODeviceHandle> getDevices(String[] ioDevices) { List<IODeviceHandle> devices = new ArrayList<>(); for (String ioDevice : ioDevices) { String devPath = ioDevice.trim(); devices.add(new IODeviceHandle(new File(devPath), ".")); } return devices; }
@Override public synchronized FileReference createWorkspaceFile(String prefix) throws HyracksDataException { IODeviceHandle dev = workspaces.get(workspaceIndex); workspaceIndex = (workspaceIndex + 1) % workspaces.size(); String waPath = dev.getWorkspace(); File waf; try { waf = File.createTempFile(prefix, WORKSPACE_FILE_SUFFIX, new File(dev.getMount(), waPath)); } catch (IOException e) { throw HyracksDataException.create(e); } return dev.createFileRef(waPath + File.separator + waf.getName()); }
@Override public void deleteWorkspaceFiles() throws HyracksDataException { for (IODeviceHandle ioDevice : workspaces) { File workspaceFolder = new File(ioDevice.getMount(), ioDevice.getWorkspace()); if (workspaceFolder.exists() && workspaceFolder.isDirectory()) { File[] workspaceFiles = workspaceFolder.listFiles(WORKSPACE_FILES_FILTER); for (File workspaceFile : workspaceFiles) { IoUtil.delete(workspaceFile); } } } }
@Override public boolean equals(Object o) { if (!(o instanceof FileReference)) { return false; } return path.equals(((FileReference) o).path) && dev.equals(((FileReference) o).dev); }
@Override public IODeviceHandle resolve(String relPath, List<IODeviceHandle> devices) throws HyracksDataException { int numDevices = devices.size(); String path = relPath; // if number of devices is 1, we return the device if (numDevices == 1) { return devices.get(0); } // check if it exists already on a device int nextSeparator = path.lastIndexOf(File.separator); while (nextSeparator > 0) { for (IODeviceHandle dev : devices) { if (dev.contains(path)) { return dev; } } path = path.substring(0, nextSeparator); nextSeparator = path.lastIndexOf(File.separator); } // one last attempt for (IODeviceHandle dev : devices) { if (dev.contains(path)) { return dev; } } // not on any device, round robin assignment return devices.get(next.getAndIncrement() % numDevices); }
Thread.currentThread().setUncaughtExceptionHandler(getLifeCycleComponentManager()); ioManager = new IOManager(IODeviceHandle.getDevices(ncConfig.getIODevices()), application.getFileDeviceResolver()); try { workQueue = new WorkQueue(id, Thread.NORM_PRIORITY); // Reserves MAX_PRIORITY of the heartbeat thread.
public IOManager(List<IODeviceHandle> devices, IFileDeviceResolver deviceComputer) throws HyracksDataException { this.ioDevices = Collections.unmodifiableList(devices); checkDeviceValidity(devices); workspaces = new ArrayList<>(); for (IODeviceHandle d : ioDevices) { if (d.getWorkspace() != null) { try { FileUtil.forceMkdirs(new File(d.getMount(), d.getWorkspace())); } catch (IOException e) { throw HyracksDataException.create(e); } workspaces.add(d); } } if (workspaces.isEmpty()) { throw new HyracksDataException("No devices with workspace found"); } workspaceIndex = 0; this.deviceComputer = deviceComputer; submittedRequests = new ArrayBlockingQueue<>(IO_REQUEST_QUEUE_SIZE); freeRequests = new ArrayBlockingQueue<>(IO_REQUEST_QUEUE_SIZE); int numIoThreads = ioDevices.size() * 2; executor = Executors.newFixedThreadPool(numIoThreads); for (int i = 0; i < numIoThreads; i++) { executor.execute(new IoRequestHandler(i, submittedRequests)); } }
@Override public boolean equals(Object o) { if (o == this) { return true; } else if (o instanceof IODeviceHandle) { return mount.getAbsolutePath().equals(((IODeviceHandle) o).getMount().getAbsolutePath()); } return false; }
public synchronized static IOManager getIOManager() throws HyracksDataException { if (ioManager == null) { List<IODeviceHandle> devices = new ArrayList<>(); devices.add(new IODeviceHandle(new File(System.getProperty("user.dir") + File.separator + "target"), "iodev_test_wa")); ioManager = new IOManager(devices, new DefaultDeviceResolver()); } return ioManager; }
public IODeviceHandle getDevice(String fullPath) { Path full = Paths.get(fullPath); for (IODeviceHandle d : ioDevices) { if (full.startsWith(Paths.get(d.getMount().getAbsolutePath()))) { return d; } } return null; }
private static IOManager createIoManager() throws HyracksException { List<IODeviceHandle> devices = new ArrayList<>(); devices.add(new IODeviceHandle(new File(System.getProperty("java.io.tmpdir")), ".")); return new IOManager(devices, new DefaultDeviceResolver()); }
@Override public long getTotalDiskUsage() { long totalSize = 0; for (IODeviceHandle handle : ioDevices) { totalSize += FileUtils.sizeOfDirectory(handle.getMount()); } return totalSize; } }
private void checkDeviceValidity(List<IODeviceHandle> devices) throws HyracksDataException { for (IODeviceHandle d : devices) { Path p = Paths.get(d.getMount().toURI()); for (IODeviceHandle e : devices) { if (e != d) { Path q = Paths.get(e.getMount().toURI()); if (p.equals(q)) { throw HyracksDataException.create(ErrorCode.DUPLICATE_IODEVICE); } else if (p.startsWith(q)) { throw HyracksDataException.create(ErrorCode.NESTED_IODEVICES); } } } } }
public PersistentLocalResourceRepository(IIOManager ioManager, IIndexCheckpointManagerProvider indexCheckpointManagerProvider, IPersistedResourceRegistry persistedResourceRegistry) { this.ioManager = ioManager; this.indexCheckpointManagerProvider = indexCheckpointManagerProvider; this.persistedResourceRegistry = persistedResourceRegistry; storageRoots = new Path[ioManager.getIODevices().size()]; final List<IODeviceHandle> ioDevices = ioManager.getIODevices(); for (int i = 0; i < ioDevices.size(); i++) { storageRoots[i] = Paths.get(ioDevices.get(i).getMount().getAbsolutePath(), StorageConstants.STORAGE_ROOT_DIR_NAME); } createStorageRoots(); resourceCache = CacheBuilder.newBuilder().maximumSize(MAX_CACHED_RESOURCES).build(); }
public void tearDown() throws HyracksDataException { diskBufferCache.close(); IODeviceHandle dev = ioManager.getIODevices().get(ioDeviceId); File dir = new File(dev.getMount(), onDiskDir); FilenameFilter filter = new FilenameFilter() { @Override public boolean accept(File dir, String name) { return !name.startsWith("."); } }; String[] files = dir.list(filter); if (files != null) { for (String fileName : files) { File file = new File(dir.getPath() + File.separator + fileName); file.delete(); } } dir.delete(); }
private static void checkNcStore(NodeControllerService nc) throws Exception { List<IODeviceHandle> ioDevices = nc.getIoManager().getIODevices(); int expectedPartitionNum = -1; for (IODeviceHandle ioDevice : ioDevices) { File[] dataDirs = ioDevice.getMount().listFiles(); for (File dataDir : dataDirs) { String dirName = dataDir.getName(); if (!dirName.equals(StorageConstants.STORAGE_ROOT_DIR_NAME)) { // Skips non-storage directories. continue; } int numPartitions = getNumResidentPartitions(dataDir.listFiles()); if (expectedPartitionNum < 0) { // Sets the expected number of partitions to the number of partitions on the first io device. expectedPartitionNum = numPartitions; } else { // Checks whether the number of partitions of the current io device is expected. if (expectedPartitionNum != numPartitions) { throw new Exception("Non-uniform data distribution on io devices: " + dataDir.getAbsolutePath() + " number of partitions: " + numPartitions + " expected number of partitions: " + expectedPartitionNum); } } } } }
public void setUp() throws HyracksDataException { ioManager = TestStorageManagerComponentHolder.getIOManager(); ioDeviceId = 0; onDiskDir = ioManager.getIODevices().get(ioDeviceId).getMount() + sep + "lsm_btree_" + simpleDateFormat.format(new Date()) + sep; ctx = TestUtils.create(getHyracksFrameSize()); TestStorageManagerComponentHolder.init(diskPageSize, diskNumPages, diskMaxOpenFiles); file = ioManager.resolveAbsolutePath(onDiskDir); diskBufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx.getJobletContext().getServiceContext()); virtualBufferCaches = new ArrayList<>(); for (int i = 0; i < numMutableComponents; i++) { IVirtualBufferCache virtualBufferCache = new VirtualBufferCache(new HeapBufferAllocator(), memPageSize, memNumPages / numMutableComponents); virtualBufferCaches.add(virtualBufferCache); } rnd.setSeed(RANDOM_SEED); }