Refine search
FileSystem fs = path.getFileSystem(hadoopConf); FileStatus stat = fs.getFileStatus(path); length += stat.getLen(); BlockLocation[] locs = fs.getFileBlockLocations(stat, 0, length); for (int i = 0; i < locs.length; i++) { String[] hosts = locs[i].getHosts(); for (int j = 0; j < hosts.length; j++) { locations.add(hosts[j]);
Map.Entry<Long, BlockLocation> startEntry = locations.floorEntry(offset); BlockLocation start = startEntry.getValue(); if (offset + length <= start.getOffset() + start.getLength()) { hosts = start.getHosts(); } else { Map.Entry<Long, BlockLocation> endEntry = locations.floorEntry(offset + length); long maxSize = 0; for (BlockLocation block : navigableMap.values()) { long overlap = getOverlap(offset, length, block.getOffset(), block.getLength()); if (overlap > 0) { for(String host: block.getHosts()) { LongWritable val = sizes.get(host); if (val == null) { throw new IOException("File " + file.getPath().toString() + " should have had overlap on block starting at " + block.getOffset()); for(String host: block.getHosts()) { if (sizes.containsKey(host)) { if (sizes.get(host).get() >= threshold) { fileKey = new SyntheticFileId(file); return new OrcSplit(file.getPath(), fileKey, offset, length, hosts, orcTail, isOriginal, hasBase, deltas, scaledProjSize, fileLen, rootDir, offsetAndBucket);
@Override public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len) throws IOException { // In some places (e.g. FileInputFormat) this BlockLocation is used to // figure out sizes/offsets and so a completely blank one will not work. String [] hosts = {"DUMMY_HOST"}; return new BlockLocation[]{new BlockLocation(null, hosts, 0, file.getLen())}; }
/** * Update blocksDistribution with blockLocations * @param blocksDistribution the hdfs blocks distribution * @param blockLocations an array containing block location */ static public void addToHDFSBlocksDistribution( HDFSBlocksDistribution blocksDistribution, BlockLocation[] blockLocations) throws IOException { for (BlockLocation bl : blockLocations) { String[] hosts = bl.getHosts(); long len = bl.getLength(); blocksDistribution.addHostsAndBlockWeight(hosts, len); } }
long harBlockStart = location.getOffset() - fileOffsetInHar; long harBlockEnd = harBlockStart + location.getLength(); location.setOffset(start); location.setLength(location.getLength() - (start - harBlockStart)); } else { location.setOffset(harBlockStart); location.setLength(location.getLength() - (harBlockEnd - end));
/** * Convert IGFS affinity block location into Hadoop affinity block location. * * @param block IGFS affinity block location. * @return Hadoop affinity block location. */ private IgfsBlockLocation convertBlockLocation(BlockLocation block) { try { String[] names = block.getNames(); String[] hosts = block.getHosts(); return new IgfsBlockLocationImpl( block.getOffset(), block.getLength(), Arrays.asList(names), Arrays.asList(hosts)); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed convert block location: " + block); } }
} else { if(locations.length == 0 && !stat.isDirectory()) { locations = new BlockLocation[] { new BlockLocation() }; fileSize = stat.getLen(); blocks[0] = new OneBlockInfo(stat.getPath(), 0, fileSize, locations[0].getHosts(), locations[0].getTopologyPaths()); } else { ArrayList<OneBlockInfo> blocksList = new ArrayList<OneBlockInfo>( locations.length); for (int i = 0; i < locations.length; i++) { fileSize += locations[i].getLength(); long left = locations[i].getLength(); long myOffset = locations[i].getOffset(); long myLength = 0; do { OneBlockInfo oneblock = new OneBlockInfo(stat.getPath(), myOffset, myLength, locations[i].getHosts(), locations[i].getTopologyPaths()); left -= myLength; myOffset += myLength;
Configuration conf = getConf(); short REPLICATION_FACTOR = 5; final Path filePath = new Path("/testFile"); FileSystem localFileSys = FileSystem.getLocal(conf); Path workingDir = localFileSys.getWorkingDirectory(); Path dir = new Path(workingDir, "build/test/data/temp/decommission"); Path excludeFile = new Path(dir, "exclude"); assertTrue(localFileSys.mkdirs(dir)); DFSTestUtil.writeFile(localFileSys, excludeFile, ""); conf.set("dfs.hosts.exclude", excludeFile.toUri().getPath()); BlockLocation locs[] = fs.getFileBlockLocations( fs.getFileStatus(filePath), 0, Long.MAX_VALUE); for (String top : locs[0].getTopologyPaths()) { if (!top.startsWith("/rack2")) { String name = top.substring("/rack1".length()+1);
private void addBlocks(VolumeManager fs, String host, ArrayList<String> files, Map<String,Long> totalBlocks, Map<String,Long> localBlocks) throws Exception { long allBlocks = 0; long matchingBlocks = 0; if (!totalBlocks.containsKey(host)) { totalBlocks.put(host, 0L); localBlocks.put(host, 0L); } for (String file : files) { Path filePath = new Path(file); FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem(); FileStatus fileStatus = ns.getFileStatus(filePath); BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); for (BlockLocation blockLocation : fileBlockLocations) { allBlocks++; for (String location : blockLocation.getHosts()) { HostAndPort hap = HostAndPort.fromParts(location, 0); if (hap.getHost().equals(host)) { matchingBlocks++; break; } } } } totalBlocks.put(host, allBlocks + totalBlocks.get(host)); localBlocks.put(host, matchingBlocks + localBlocks.get(host)); }
private Set<ConfigurationTuple> getConfigurationTuples_old() throws IOException { Path p = new Path(props.getProperty(ConfigFields.APP_INPUT_PATH)); FileSystem fs = FileSystem.get(conf); FileStatus f = fs.getFileStatus(p); BlockLocation[] bl = fs.getFileBlockLocations(p, 0, f.getLen()); Set<ConfigurationTuple> configTuples = new HashSet<ConfigurationTuple>(); int workerId = 0; for (BlockLocation b : bl) { FileSplit split = FileSplit.newBuilder().setPath(p.toString()) .setOffset(b.getOffset()).setLength(b.getLength()).build(); StartupConfiguration config = StartupConfiguration.newBuilder() .setBatchSize(batchSize).setIterations(iterationCount) .setOther(appConfig).setSplit(split).build(); String wid = "worker-" + workerId; ConfigurationTuple tuple = new ConfigurationTuple(b.getHosts()[0], wid, config); configTuples.add(tuple); workerId++; } return configTuples; }
FileSystem fs = path.getFileSystem(conf); FileStatus stat = fs.getFileStatus(path); BlockLocation[] locations = fs.getFileBlockLocations(stat, 0, stat.getLen()); for (int i = 0; i < locations.length; i++) { fileSize += locations[i].getLength(); OneBlockInfo oneblock = new OneBlockInfo(path, locations[i].getOffset(), locations[i].getLength(), locations[i].getHosts(), locations[i].getTopologyPaths()); blocks[i] = oneblock;
public void testDataNodeRedirect() throws Exception { if (hdfs.exists(TEST_FILE)) { hdfs.delete(TEST_FILE, true); } FSDataOutputStream out = hdfs.create(TEST_FILE, (short) 1); out.writeBytes("0123456789"); out.close(); BlockLocation[] locations = hdfs.getFileBlockLocations(TEST_FILE, 0, 10); String locationName = locations[0].getNames()[0]; URL u = hftpFs.getNamenodeFileURL(TEST_FILE); HttpURLConnection conn = (HttpURLConnection)u.openConnection(); conn.setFollowRedirects(true); conn.connect(); conn.getInputStream(); boolean checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port for (DataNode node : cluster.getDataNodes()) { if (node.getDatanodeRegistration().getName().equals(locationName)) { checked = true; assertEquals(node.getDatanodeRegistration().getInfoPort(), conn.getURL().getPort()); } } assertTrue("The test never checked that location of " + "the block and hftp desitnation are the same", checked); } /**
String tableName = regionPath.getParent().getName(); int totalBlkCount = 0; FileStatus[] cfList = fs.listStatus(regionPath, new FSUtils.FamilyDirFilter(fs)); if (null == cfList) { return; if (!cfStatus.isDirectory()) { FileStatus[] storeFileLists = fs.listStatus(cfStatus.getPath()); if (null == storeFileLists) { continue; fs.getFileBlockLocations(storeFile, 0, storeFile.getLen()); if (null == blkLocations) { continue; for (String host: blk.getHosts()) { AtomicInteger count = blockCountMap.get(host); if (count == null) { hostToRun = hostToRun.substring(0, hostToRun.length()-1); String name = tableName + ":" + regionPath.getName(); synchronized (regionToBestLocalityRSMapping) { regionToBestLocalityRSMapping.put(name, hostToRun);
MapWork mapWork = Utilities.getMapWork(jobConf); List<Path> paths = Utilities.getInputPathsTez(jobConf, mapWork); FileSystem fs = paths.get(0).getFileSystem(jobConf); FileStatus[] fileStatuses = fs.listStatus(paths.get(0)); if (fileStatuses.length == 0) { splits = new InputSplit[1]; FileStatus fileStatus = fileStatuses[0]; BlockLocation[] locations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); Set<String> hostsSet = new HashSet<>(); for (BlockLocation location : locations) { hostsSet.addAll(Lists.newArrayList(location.getHosts())); FileSplit fileSplit = new FileSplit(fileStatus.getPath(), 0, fileStatus.getLen(), hosts); String alias = mapWork.getAliases().get(0); PartitionDesc partDesc = mapWork.getAliasToPartnInfo().get(alias); final String fileStr = path.toString(); if (!files.contains(fileStr)) { files.add(fileStr);
for (String file : files) { FileStatus status = TEST_UTIL.getDFSCluster().getFileSystem(). getFileStatus(new Path(new URI(file).getPath())); BlockLocation[] lbks = ((DistributedFileSystem)TEST_UTIL.getDFSCluster().getFileSystem()) for (BlockLocation lbk : lbks) { locations: for (String info : lbk.getNames()) { for (int j = 0; j < FAVORED_NODES_NUM; j++) { if (info.equals(nodeNames[(i + j) % REGION_SERVERS])) {
public static <K extends BufferSerializable> void add(Job job, K key, Path path) throws IOException { log.info("add(%s)", path.toString()); FileSystem fs = job.getFS(); FileStatus file = fs.getFileStatus(path); path = file.getPath(); long length = file.getLen(); if (length != 0) { BlockLocation[] blkLocations; if (file instanceof LocatedFileStatus) { blkLocations = ((LocatedFileStatus) file).getBlockLocations(); } else { blkLocations = fs.getFileBlockLocations(file, 0, length); } splits.add(new Split(path, 0, length, blkLocations[0].getHosts(), key)); } else { //Create empty hosts array for zero length files splits.add(new Split(path, 0, length, new String[0], key)); } }
private static void reportHdfsBlockLocations(List<FileRef> files) throws Exception { VolumeManager fs = VolumeManagerImpl.get(); System.out.println("\t\tFile block report : "); for (FileRef file : files) { FileStatus status = fs.getFileStatus(file.path()); if (status.isDirectory()) { // assume it is a map file status = fs.getFileStatus(new Path(file + "/data")); } FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem(); BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen()); System.out.println("\t\t\tBlocks for : " + file); for (BlockLocation blockLocation : locs) { System.out.printf("\t\t\t\t offset : %,13d hosts :", blockLocation.getOffset()); for (String host : blockLocation.getHosts()) { System.out.print(" " + host); } System.out.println(); } } System.out.println(); }
String rootDir = new Path(FSUtils.getWALRootDir(conf) + "/" + HConstants.HREGION_LOGDIR_NAME + "/" + targetRs.getServerName().toString()).toUri().getPath(); LOG.info("Log file found: " + hf.getLocalName() + " in " + rootDir); String logFile = rootDir + "/" + hf.getLocalName(); FileStatus fsLog = rfs.getFileStatus(new Path(logFile)); BlockLocation bl = bls[0]; LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " "); for (int i = 0; i < bl.getHosts().length - 1; i++) { LOG.info(bl.getHosts()[i] + " " + logFile); Assert.assertNotSame(bl.getHosts()[i], host4); String last = bl.getHosts()[bl.getHosts().length - 1]; LOG.info(last + " " + logFile); if (host4.equals(last)) { nbTest++; LOG.info(logFile + " is on the new datanode and is ok"); if (bl.getHosts().length == 3) {
@Override public List<String> getFileLocations(String path, long offset) throws IOException { List<String> ret = new ArrayList<String>(); try { FileStatus fStatus = mFs.getFileStatus(new Path(path)); BlockLocation[] bLocations = mFs.getFileBlockLocations(fStatus, offset, 1); if (bLocations.length > 0) { String[] names = bLocations[0].getNames(); Collections.addAll(ret, names); } } catch (IOException e) { LOG.error("Unable to get file location for " + path, e); } return ret; }
@Override @Nullable public List<String> getFileLocations(String path, FileLocationOptions options) throws IOException { // If the user has hinted the underlying storage nodes are not co-located with Alluxio // workers, short circuit without querying the locations. if (Boolean.valueOf(mUfsConf.get(PropertyKey.UNDERFS_HDFS_REMOTE))) { return null; } FileSystem hdfs = getFs(); List<String> ret = new ArrayList<>(); try { // The only usage of fileStatus is to get the path in getFileBlockLocations. // In HDFS 2, there is an API getFileBlockLocation(Path path, long offset, long len), // but in HDFS 1, the only API is // getFileBlockLocation(FileStatus stat, long offset, long len). // By constructing the file status manually, we can save one RPC call to getFileStatus. FileStatus fileStatus = new FileStatus(0L, false, 0, 0L, 0L, 0L, null, null, null, new Path(path)); BlockLocation[] bLocations = hdfs.getFileBlockLocations(fileStatus, options.getOffset(), 1); if (bLocations.length > 0) { String[] names = bLocations[0].getHosts(); Collections.addAll(ret, names); } } catch (IOException e) { LOG.warn("Unable to get file location for {} : {}", path, e.getMessage()); } return ret; }