/** * Optimize buffer size. * * @param bufSize Requested buffer size. * @param fileInfo File info. * @return Optimized buffer size. */ @SuppressWarnings("IfMayBeConditional") private static int optimizeBufferSize(int bufSize, GridGgfsFileInfo fileInfo) { assert bufSize > 0; if (fileInfo == null) return bufSize; int blockSize = fileInfo.blockSize(); if (blockSize <= 0) return bufSize; if (bufSize <= blockSize) // Optimize minimum buffer size to be equal file's block size. return blockSize; int maxBufSize = blockSize * MAX_BLOCKS_CNT; if (bufSize > maxBufSize) // There is no profit or optimization from larger buffers. return maxBufSize; if (fileInfo.length() == 0) // Make buffer size multiple of block size (optimized for new files). return bufSize / blockSize * blockSize; return bufSize; }
/** * Optimize buffer size. * * @param bufSize Requested buffer size. * @param fileInfo File info. * @return Optimized buffer size. */ @SuppressWarnings("IfMayBeConditional") private static int optimizeBufferSize(int bufSize, GridGgfsFileInfo fileInfo) { assert bufSize > 0; if (fileInfo == null) return bufSize; int blockSize = fileInfo.blockSize(); if (blockSize <= 0) return bufSize; if (bufSize <= blockSize) // Optimize minimum buffer size to be equal file's block size. return blockSize; int maxBufSize = blockSize * MAX_BLOCKS_CNT; if (bufSize > maxBufSize) // There is no profit or optimization from larger buffers. return maxBufSize; if (fileInfo.length() == 0) // Make buffer size multiple of block size (optimized for new files). return bufSize / blockSize * blockSize; return bufSize; }
/** {@inheritDoc} */ @Override protected synchronized void storeDataBlock(ByteBuffer block) throws GridException, IOException { int writeLen = block.remaining(); preStoreDataBlocks(null, writeLen); int blockSize = fileInfo.blockSize(); // If data length is not enough to fill full block, fill the remainder and return. if (remainderDataLen + writeLen < blockSize) { if (remainder == null) remainder = new byte[blockSize]; else if (remainder.length != blockSize) { assert remainderDataLen == remainder.length; byte[] allocated = new byte[blockSize]; U.arrayCopy(remainder, 0, allocated, 0, remainder.length); remainder = allocated; } block.get(remainder, remainderDataLen, writeLen); remainderDataLen += writeLen; } else { remainder = data.storeDataBlocks(fileInfo, fileInfo.length() + space, remainder, remainderDataLen, block, false, streamRange, batch); remainderDataLen = remainder == null ? 0 : remainder.length; } }
/** {@inheritDoc} */ @Override protected synchronized void storeDataBlock(ByteBuffer block) throws GridException, IOException { int writeLen = block.remaining(); preStoreDataBlocks(null, writeLen); int blockSize = fileInfo.blockSize(); // If data length is not enough to fill full block, fill the remainder and return. if (remainderDataLen + writeLen < blockSize) { if (remainder == null) remainder = new byte[blockSize]; else if (remainder.length != blockSize) { assert remainderDataLen == remainder.length; byte[] allocated = new byte[blockSize]; U.arrayCopy(remainder, 0, allocated, 0, remainder.length); remainder = allocated; } block.get(remainder, remainderDataLen, writeLen); remainderDataLen += writeLen; } else { remainder = data.storeDataBlocks(fileInfo, fileInfo.length() + space, remainder, remainderDataLen, block, false, streamRange, batch); remainderDataLen = remainder == null ? 0 : remainder.length; } }
/** {@inheritDoc} */ @Override protected synchronized void storeDataBlocks(DataInput in, int len) throws GridException, IOException { preStoreDataBlocks(in, len); int blockSize = fileInfo.blockSize(); // If data length is not enough to fill full block, fill the remainder and return. if (remainderDataLen + len < blockSize) { if (remainder == null) remainder = new byte[blockSize]; else if (remainder.length != blockSize) { assert remainderDataLen == remainder.length; byte[] allocated = new byte[blockSize]; U.arrayCopy(remainder, 0, allocated, 0, remainder.length); remainder = allocated; } in.readFully(remainder, remainderDataLen, len); remainderDataLen += len; } else { remainder = data.storeDataBlocks(fileInfo, fileInfo.length() + space, remainder, remainderDataLen, in, len, false, streamRange, batch); remainderDataLen = remainder == null ? 0 : remainder.length; } }
/** {@inheritDoc} */ @Override protected synchronized void storeDataBlocks(DataInput in, int len) throws GridException, IOException { preStoreDataBlocks(in, len); int blockSize = fileInfo.blockSize(); // If data length is not enough to fill full block, fill the remainder and return. if (remainderDataLen + len < blockSize) { if (remainder == null) remainder = new byte[blockSize]; else if (remainder.length != blockSize) { assert remainderDataLen == remainder.length; byte[] allocated = new byte[blockSize]; U.arrayCopy(remainder, 0, allocated, 0, remainder.length); remainder = allocated; } in.readFully(remainder, remainderDataLen, len); remainderDataLen += len; } else { remainder = data.storeDataBlocks(fileInfo, fileInfo.length() + space, remainder, remainderDataLen, in, len, false, streamRange, batch); remainderDataLen = remainder == null ? 0 : remainder.length; } }
/** * Gets initial affinity range. This range will have 0 length and will start from first * non-occupied file block. * * @param fileInfo File info to build initial range for. * @return Affinity range. */ private GridGgfsFileAffinityRange initialStreamRange(GridGgfsFileInfo fileInfo) { if (!ggfsCtx.configuration().isFragmentizerEnabled()) return null; if (!Boolean.parseBoolean(fileInfo.properties().get(GridGgfs.PROP_PREFER_LOCAL_WRITES))) return null; int blockSize = fileInfo.blockSize(); // Find first non-occupied block offset. long off = ((fileInfo.length() + blockSize - 1) / blockSize) * blockSize; // Need to get last affinity key and reuse it if we are on the same node. long lastBlockOff = off - fileInfo.blockSize(); if (lastBlockOff < 0) lastBlockOff = 0; GridGgfsFileMap map = fileInfo.fileMap(); GridUuid prevAffKey = map == null ? null : map.affinityKey(lastBlockOff, false); GridUuid affKey = data.nextAffinityKey(prevAffKey); return affKey == null ? null : new GridGgfsFileAffinityRange(off, off, affKey); }
/** * Gets initial affinity range. This range will have 0 length and will start from first * non-occupied file block. * * @param fileInfo File info to build initial range for. * @return Affinity range. */ private GridGgfsFileAffinityRange initialStreamRange(GridGgfsFileInfo fileInfo) { if (!ggfsCtx.configuration().isFragmentizerEnabled()) return null; if (!Boolean.parseBoolean(fileInfo.properties().get(GridGgfs.PROP_PREFER_LOCAL_WRITES))) return null; int blockSize = fileInfo.blockSize(); // Find first non-occupied block offset. long off = ((fileInfo.length() + blockSize - 1) / blockSize) * blockSize; // Need to get last affinity key and reuse it if we are on the same node. long lastBlockOff = off - fileInfo.blockSize(); if (lastBlockOff < 0) lastBlockOff = 0; GridGgfsFileMap map = fileInfo.fileMap(); GridUuid prevAffKey = map == null ? null : map.affinityKey(lastBlockOff, false); GridUuid affKey = data.nextAffinityKey(prevAffKey); return affKey == null ? null : new GridGgfsFileAffinityRange(off, off, affKey); }
/** * @param fileInfo File info to construct listing entry from. */ public GridGgfsListingEntry(GridGgfsFileInfo fileInfo) { fileId = fileInfo.id(); affKey = fileInfo.affinityKey(); if (fileInfo.isFile()) { blockSize = fileInfo.blockSize(); len = fileInfo.length(); } props = fileInfo.properties(); accessTime = fileInfo.accessTime(); modificationTime = fileInfo.modificationTime(); }
/** * @param fileInfo File info to construct listing entry from. */ public GridGgfsListingEntry(GridGgfsFileInfo fileInfo) { fileId = fileInfo.id(); affKey = fileInfo.affinityKey(); if (fileInfo.isFile()) { blockSize = fileInfo.blockSize(); len = fileInfo.length(); } props = fileInfo.properties(); accessTime = fileInfo.accessTime(); modificationTime = fileInfo.modificationTime(); }
/** * Constructs directory info. * * @param path Path. */ public GridGgfsFileImpl(GridGgfsPath path, GridGgfsFileInfo info, long globalGrpBlockSize) { A.notNull(path, "path"); A.notNull(info, "info"); this.path = path; fileId = info.id(); if (info.isFile()) { blockSize = info.blockSize(); len = info.length(); grpBlockSize = info.affinityKey() == null ? globalGrpBlockSize : info.length() == 0 ? globalGrpBlockSize : info.length(); } props = info.properties(); if (props == null) props = Collections.emptyMap(); accessTime = info.accessTime(); modificationTime = info.modificationTime(); }
/** * Tries to remove blocks affected by fragmentizer. If {@code cleanNonColocated} is {@code true}, will remove * non-colocated blocks as well. * * @param fileInfo File info to clean up. * @param range Range to clean up. * @param cleanNonColocated {@code True} if all blocks should be cleaned. */ public void cleanBlocks(GridGgfsFileInfo fileInfo, GridGgfsFileAffinityRange range, boolean cleanNonColocated) { long startIdx = range.startOffset() / fileInfo.blockSize(); long endIdx = range.endOffset() / fileInfo.blockSize(); if (log.isDebugEnabled()) log.debug("Cleaning blocks [fileInfo=" + fileInfo + ", range=" + range + ", cleanNonColocated=" + cleanNonColocated + ", startIdx=" + startIdx + ", endIdx=" + endIdx + ']'); try { try (GridDataLoader<GridGgfsBlockKey, byte[]> ldr = dataLoader()) { for (long idx = startIdx; idx <= endIdx; idx++) { ldr.removeData(new GridGgfsBlockKey(fileInfo.id(), range.affinityKey(), fileInfo.evictExclude(), idx)); if (cleanNonColocated) ldr.removeData(new GridGgfsBlockKey(fileInfo.id(), null, fileInfo.evictExclude(), idx)); } } } catch (GridException e) { log.error("Failed to clean up file range [fileInfo=" + fileInfo + ", range=" + range + ']', e); } }
/** * Constructs directory info. * * @param path Path. */ public GridGgfsFileImpl(GridGgfsPath path, GridGgfsFileInfo info, long globalGrpBlockSize) { A.notNull(path, "path"); A.notNull(info, "info"); this.path = path; fileId = info.id(); if (info.isFile()) { blockSize = info.blockSize(); len = info.length(); grpBlockSize = info.affinityKey() == null ? globalGrpBlockSize : info.length() == 0 ? globalGrpBlockSize : info.length(); } props = info.properties(); if (props == null) props = Collections.emptyMap(); accessTime = info.accessTime(); modificationTime = info.modificationTime(); }
/** * Tries to remove blocks affected by fragmentizer. If {@code cleanNonColocated} is {@code true}, will remove * non-colocated blocks as well. * * @param fileInfo File info to clean up. * @param range Range to clean up. * @param cleanNonColocated {@code True} if all blocks should be cleaned. */ public void cleanBlocks(GridGgfsFileInfo fileInfo, GridGgfsFileAffinityRange range, boolean cleanNonColocated) { long startIdx = range.startOffset() / fileInfo.blockSize(); long endIdx = range.endOffset() / fileInfo.blockSize(); if (log.isDebugEnabled()) log.debug("Cleaning blocks [fileInfo=" + fileInfo + ", range=" + range + ", cleanNonColocated=" + cleanNonColocated + ", startIdx=" + startIdx + ", endIdx=" + endIdx + ']'); try { try (GridDataLoader<GridGgfsBlockKey, byte[]> ldr = dataLoader()) { for (long idx = startIdx; idx <= endIdx; idx++) { ldr.removeData(new GridGgfsBlockKey(fileInfo.id(), range.affinityKey(), fileInfo.evictExclude(), idx)); if (cleanNonColocated) ldr.removeData(new GridGgfsBlockKey(fileInfo.id(), null, fileInfo.evictExclude(), idx)); } } } catch (GridException e) { log.error("Failed to clean up file range [fileInfo=" + fileInfo + ", range=" + range + ']', e); } }
@Override public GridGgfsSecondaryOutputStreamDescriptor onSuccess(Map<GridGgfsPath, GridGgfsFileInfo> infos) throws Exception { GridGgfsFileInfo info = infos.get(path); if (info.isDirectory()) throw new GridGgfsException("Failed to open output stream to the file in the " + "secondary file system because the path points to a directory: " + path); out = fs.append(path, bufSize, false, null); // Synchronize file ending. long len = info.length(); int blockSize = info.blockSize(); int remainder = (int)(len % blockSize); if (remainder > 0) { int blockIdx = (int)(len / blockSize); GridGgfsReader reader = fs.open(path, bufSize); try { ggfsCtx.data().dataBlock(info, path, blockIdx, reader).get(); } finally { reader.close(); } } // Set lock and return. info = lockInfo(info); metaCache.putx(info.id(), info); return new GridGgfsSecondaryOutputStreamDescriptor(infos.get(path.parent()).id(), info, out); }
@Override public GridGgfsSecondaryOutputStreamDescriptor onSuccess(Map<GridGgfsPath, GridGgfsFileInfo> infos) throws Exception { GridGgfsFileInfo info = infos.get(path); if (info.isDirectory()) throw new GridGgfsException("Failed to open output stream to the file in the " + "secondary file system because the path points to a directory: " + path); out = fs.append(path, bufSize, false, null); // Synchronize file ending. long len = info.length(); int blockSize = info.blockSize(); int remainder = (int)(len % blockSize); if (remainder > 0) { int blockIdx = (int)(len / blockSize); GridGgfsReader reader = fs.open(path, bufSize); try { ggfsCtx.data().dataBlock(info, path, blockIdx, reader).get(); } finally { reader.close(); } } // Set lock and return. info = lockInfo(info); metaCache.putx(info.id(), info); return new GridGgfsSecondaryOutputStreamDescriptor(infos.get(path.parent()).id(), info, out); }
/** * @param blockIdx Block index. * @param fileInfo File info. * @return Block key. */ public GridGgfsBlockKey blockKey(long blockIdx, GridGgfsFileInfo fileInfo) { if (fileInfo.affinityKey() != null) return new GridGgfsBlockKey(fileInfo.id(), fileInfo.affinityKey(), fileInfo.evictExclude(), blockIdx); if (fileInfo.fileMap() != null) { GridUuid affKey = fileInfo.fileMap().affinityKey(blockIdx * fileInfo.blockSize(), false); return new GridGgfsBlockKey(fileInfo.id(), affKey, fileInfo.evictExclude(), blockIdx); } return new GridGgfsBlockKey(fileInfo.id(), null, fileInfo.evictExclude(), blockIdx); }
/** * @param blockIdx Block index. * @param fileInfo File info. * @return Block key. */ public GridGgfsBlockKey blockKey(long blockIdx, GridGgfsFileInfo fileInfo) { if (fileInfo.affinityKey() != null) return new GridGgfsBlockKey(fileInfo.id(), fileInfo.affinityKey(), fileInfo.evictExclude(), blockIdx); if (fileInfo.fileMap() != null) { GridUuid affKey = fileInfo.fileMap().affinityKey(blockIdx * fileInfo.blockSize(), false); return new GridGgfsBlockKey(fileInfo.id(), affKey, fileInfo.evictExclude(), blockIdx); } return new GridGgfsBlockKey(fileInfo.id(), null, fileInfo.evictExclude(), blockIdx); }