@Override public long getLength() { return this.blockLocation.getLength(); }
/** * Update blocksDistribution with blockLocations * @param blocksDistribution the hdfs blocks distribution * @param blockLocations an array containing block location */ static public void addToHDFSBlocksDistribution( HDFSBlocksDistribution blocksDistribution, BlockLocation[] blockLocations) throws IOException { for (BlockLocation bl : blockLocations) { String[] hosts = bl.getHosts(); long len = bl.getLength(); blocksDistribution.addHostsAndBlockWeight(hosts, len); } }
/** * Convert IGFS affinity block location into Hadoop affinity block location. * * @param block IGFS affinity block location. * @return Hadoop affinity block location. */ private IgfsBlockLocation convertBlockLocation(BlockLocation block) { try { String[] names = block.getNames(); String[] hosts = block.getHosts(); return new IgfsBlockLocationImpl( block.getOffset(), block.getLength(), Arrays.asList(names), Arrays.asList(hosts)); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed convert block location: " + block); } }
@Override public List<OrcSplit> getSplits() throws IOException { List<OrcSplit> splits = Lists.newArrayList(); for (HdfsFileStatusWithId file : fileStatuses) { FileStatus fileStatus = file.getFileStatus(); if (fileStatus.getLen() != 0) { Object fileKey = file.getFileId(); if (fileKey == null && allowSyntheticFileIds) { fileKey = new SyntheticFileId(fileStatus); } TreeMap<Long, BlockLocation> blockOffsets = SHIMS.getLocationsWithOffset(fs, fileStatus); for (Map.Entry<Long, BlockLocation> entry : blockOffsets.entrySet()) { OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), fileKey, entry.getKey(), entry.getValue().getLength(), entry.getValue().getHosts(), null, isOriginal, true, deltas, -1, fileStatus.getLen()); splits.add(orcSplit); } } } // add uncovered ACID delta splits splits.addAll(super.getSplits()); return splits; }
Map.Entry<Long, BlockLocation> startEntry = locations.floorEntry(offset); BlockLocation start = startEntry.getValue(); if (offset + length <= start.getOffset() + start.getLength()) { for (BlockLocation block : navigableMap.values()) { long overlap = getOverlap(offset, length, block.getOffset(), block.getLength()); if (overlap > 0) { for(String host: block.getHosts()) {
Map.Entry<Long, BlockLocation> startEntry = locations.floorEntry(offset); BlockLocation start = startEntry.getValue(); if (offset + length <= start.getOffset() + start.getLength()) { for (BlockLocation block : navigableMap.values()) { long overlap = getOverlap(offset, length, block.getOffset(), block.getLength()); if (overlap > 0) { for(String host: block.getHosts()) {
/** * Compute HDFS blocks distribution of a given file, or a portion of the file * @param fs file system * @param status file status of the file * @param start start position of the portion * @param length length of the portion * @return The HDFS blocks distribution */ static public HDFSBlocksDistribution computeHDFSBlocksDistribution( final FileSystem fs, FileStatus status, long start, long length) throws IOException { HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution(); BlockLocation [] blockLocations = fs.getFileBlockLocations(status, start, length); for(BlockLocation bl : blockLocations) { String [] hosts = bl.getHosts(); long len = bl.getLength(); blocksDistribution.addHostsAndBlockWeight(hosts, len); } return blocksDistribution; }
long blockEnd = Math.min(start + length, blockLocation.getOffset() + blockLocation.getLength()); if (blockStart > blockEnd) {
do { lbs = dfs.getFileBlockLocations(f, 0, 1); } while (lbs.length != 1 && lbs[0].getLength() != repCount); final String name = lbs[0].getNames()[0]; Assert.assertTrue(name.indexOf(':') > 0);
private static Map<String, Object> toJsonMap( final BlockLocation blockLocation) throws IOException { if (blockLocation == null) { return null; } final Map<String, Object> m = new HashMap<>(); m.put("length", blockLocation.getLength()); m.put("offset", blockLocation.getOffset()); m.put("corrupt", blockLocation.isCorrupt()); m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes())); m.put("cachedHosts", blockLocation.getCachedHosts()); m.put("hosts", blockLocation.getHosts()); m.put("names", blockLocation.getNames()); m.put("topologyPaths", blockLocation.getTopologyPaths()); return m; }
protected FileMetadata toMetadata(LocatedFileStatus fileStatus) { List<FileMetadata.BlockInfo> blocks = new ArrayList<>(); blocks.addAll(Arrays.stream(fileStatus.getBlockLocations()) .map(block -> new FileMetadata.BlockInfo(block.getOffset(), block.getLength(), block.isCorrupt())) .collect(Collectors.toList())); return new FileMetadata(fileStatus.getPath().toString(), fileStatus.getLen(), blocks); }
public BlockStorageLocation(BlockLocation loc, VolumeId[] volumeIds) throws IOException { // Initialize with data from passed in BlockLocation super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), loc .getOffset(), loc.getLength(), loc.isCorrupt()); this.volumeIds = volumeIds; }
public BlockStorageLocation(BlockLocation loc, VolumeId[] volumeIds) throws IOException { // Initialize with data from passed in BlockLocation super(loc.getNames(), loc.getHosts(), loc.getTopologyPaths(), loc .getOffset(), loc.getLength(), loc.isCorrupt()); this.volumeIds = volumeIds; }
void compareBLs(BlockLocation[] viewBL, BlockLocation[] targetBL) { Assert.assertEquals(targetBL.length, viewBL.length); int i = 0; for (BlockLocation vbl : viewBL) { Assert.assertEquals(vbl.toString(), targetBL[i].toString()); Assert.assertEquals(targetBL[i].getOffset(), vbl.getOffset()); Assert.assertEquals(targetBL[i].getLength(), vbl.getLength()); i++; } }
void compareBLs(BlockLocation[] viewBL, BlockLocation[] targetBL) { Assert.assertEquals(targetBL.length, viewBL.length); int i = 0; for (BlockLocation vbl : viewBL) { Assert.assertEquals(vbl.toString(), targetBL[i].toString()); Assert.assertEquals(targetBL[i].getOffset(), vbl.getOffset()); Assert.assertEquals(targetBL[i].getLength(), vbl.getLength()); i++; } }
void compareBLs(BlockLocation[] viewBL, BlockLocation[] targetBL) { Assert.assertEquals(targetBL.length, viewBL.length); int i = 0; for (BlockLocation vbl : viewBL) { Assert.assertEquals(vbl.toString(), targetBL[i].toString()); Assert.assertEquals(targetBL[i].getOffset(), vbl.getOffset()); Assert.assertEquals(targetBL[i].getLength(), vbl.getLength()); i++; } }
void compareBLs(BlockLocation[] viewBL, BlockLocation[] targetBL) { Assert.assertEquals(targetBL.length, viewBL.length); int i = 0; for (BlockLocation vbl : viewBL) { Assert.assertEquals(vbl.toString(), targetBL[i].toString()); Assert.assertEquals(targetBL[i].getOffset(), vbl.getOffset()); Assert.assertEquals(targetBL[i].getLength(), vbl.getLength()); i++; } }