DatanodeInfo[] dnis = lb.getLocations(); if (dnis != null && dnis.length > 1) { boolean found = false;
@Override public void reorderBlocks(Configuration c, LocatedBlocks lbs, String src) { for (LocatedBlock lb : lbs.getLocatedBlocks()) { if (lb.getLocations().length > 1) { DatanodeInfo[] infos = lb.getLocations(); if (infos[0].getHostName().equals(lookup)) { LOG.info("HFileSystem bad host, inverting"); DatanodeInfo tmp = infos[0]; infos[0] = infos[1]; infos[1] = tmp; } } } } }));
FanOutOneBlockAsyncDFSOutput(Configuration conf, FSUtils fsUtils, DistributedFileSystem dfs, DFSClient client, ClientProtocol namenode, String clientName, String src, long fileId, LocatedBlock locatedBlock, Encryptor encryptor, List<Channel> datanodeList, DataChecksum summer, ByteBufAllocator alloc) { this.conf = conf; this.fsUtils = fsUtils; this.dfs = dfs; this.client = client; this.namenode = namenode; this.fileId = fileId; this.clientName = clientName; this.src = src; this.block = locatedBlock.getBlock(); this.locations = locatedBlock.getLocations(); this.encryptor = encryptor; this.datanodeList = datanodeList; this.summer = summer; this.maxDataLen = MAX_DATA_LEN - (MAX_DATA_LEN % summer.getBytesPerChecksum()); this.alloc = alloc; this.buf = alloc.directBuffer(sendBufSizePRedictor.initialSize()); this.state = State.STREAMING; setupReceiver(conf.getInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, READ_TIMEOUT)); }
} catch (Exception e) { excludesNodes = ArrayUtils.add(excludesNodes, locatedBlock.getLocations()[i]); throw e;
private void testFromDFS(DistributedFileSystem dfs, String src, int repCount, String localhost) throws Exception { // Multiple times as the order is random for (int i = 0; i < 10; i++) { LocatedBlocks l; // The NN gets the block list asynchronously, so we may need multiple tries to get the list final long max = System.currentTimeMillis() + 10000; boolean done; do { Assert.assertTrue("Can't get enouth replica.", System.currentTimeMillis() < max); l = getNamenode(dfs.getClient()).getBlockLocations(src, 0, 1); Assert.assertNotNull("Can't get block locations for " + src, l); Assert.assertNotNull(l.getLocatedBlocks()); Assert.assertTrue(l.getLocatedBlocks().size() > 0); done = true; for (int y = 0; y < l.getLocatedBlocks().size() && done; y++) { done = (l.get(y).getLocations().length == repCount); } } while (!done); for (int y = 0; y < l.getLocatedBlocks().size() && done; y++) { Assert.assertEquals(localhost, l.get(y).getLocations()[repCount - 1].getHostName()); } } }
Assert.assertNotNull(l.getLocatedBlocks()); Assert.assertEquals(1, l.getLocatedBlocks().size()); Assert.assertTrue("Expecting " + repCount + " , got " + l.get(0).getLocations().length, System.currentTimeMillis() < max); } while (l.get(0).getLocations().length != repCount); Assert.assertEquals(host1, l.get(0).getLocations()[2].getHostName()); Assert.assertEquals(host1, l.get(0).getLocations()[2].getHostName());
Class<? extends Channel> channelClass) { Enum<?>[] storageTypes = locatedBlock.getStorageTypes(); DatanodeInfo[] datanodeInfos = locatedBlock.getLocations(); boolean connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME, DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
static List<MLocation> toLocations(LocatedBlock lb) { final DatanodeInfo[] datanodeInfos = lb.getLocations(); final StorageType[] storageTypes = lb.getStorageTypes(); final long size = lb.getBlockSize(); final List<MLocation> locations = new LinkedList<MLocation>(); for(int i = 0; i < datanodeInfos.length; i++) { locations.add(new MLocation(datanodeInfos[i], storageTypes[i], size)); } return locations; } }
/** * Move decommissioned/stale datanodes to the bottom. After sorting it will * update block indices and block tokens respectively. * * @param lb located striped block * @param comparator dn comparator */ private void sortLocatedStripedBlock(final LocatedBlock lb, Comparator<DatanodeInfo> comparator) { DatanodeInfo[] di = lb.getLocations(); HashMap<DatanodeInfo, Byte> locToIndex = new HashMap<>(); HashMap<DatanodeInfo, Token<BlockTokenIdentifier>> locToToken = new HashMap<>(); LocatedStripedBlock lsb = (LocatedStripedBlock) lb; for (int i = 0; i < di.length; i++) { locToIndex.put(di[i], lsb.getBlockIndices()[i]); locToToken.put(di[i], lsb.getBlockTokens()[i]); } // Move decommissioned/stale datanodes to the bottom Arrays.sort(di, comparator); // must update cache since we modified locations array lb.updateCachedStorageInfo(); // must update block indices and block tokens respectively for (int i = 0; i < di.length; i++) { lsb.getBlockIndices()[i] = locToIndex.get(di[i]); lsb.getBlockTokens()[i] = locToToken.get(di[i]); } }
/** Convert a LocatedBlock to a Json map. */ private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock ) throws IOException { if (locatedblock == null) { return null; } final Map<String, Object> m = new TreeMap<String, Object>(); m.put("blockToken", toJsonMap(locatedblock.getBlockToken())); m.put("isCorrupt", locatedblock.isCorrupt()); m.put("startOffset", locatedblock.getStartOffset()); m.put("block", toJsonMap(locatedblock.getBlock())); m.put("storageTypes", toJsonArray(locatedblock.getStorageTypes())); m.put("locations", toJsonArray(locatedblock.getLocations())); m.put("cachedLocations", toJsonArray(locatedblock.getCachedLocations())); return m; }
DatanodeInfo[] di = lb.getLocations(); if(nonDatanodeReader) { networktopology.sortByDistanceUsingNetworkLocation(client, lb.getLocations(), activeLen); } else { networktopology.sortByDistance(client, lb.getLocations(), activeLen);
/** * The given block is considered as low redundancy when the block locations * length is less than expected replication factor. For EC blocks, redundancy * is the summation of data + parity blocks. * * @param blockInfo * block * @param replication * replication factor of the given file block * @param ecPolicy * erasure coding policy of the given file block * @return true if the given block is low redundant. */ private boolean isLowRedundancyBlock(LocatedBlock blockInfo, int replication, ErasureCodingPolicy ecPolicy) { boolean hasLowRedundancyBlock = false; if (blockInfo.isStriped()) { // For EC blocks, redundancy is the summation of data + parity blocks. replication = ecPolicy.getNumDataUnits() + ecPolicy.getNumParityUnits(); } // block is considered as low redundancy when the block locations length is // less than expected replication factor. hasLowRedundancyBlock = blockInfo.getLocations().length < replication ? true : false; return hasLowRedundancyBlock; }
for (LocatedBlock lBlk : blocks.getLocatedBlocks()) { LocatedBlock lblock = lBlk; DatanodeInfo[] locs = lblock.getLocations(); if (locs == null || locs.length == 0) { if (fos != null) {
final int count = locations.locatedBlockCount(); if (count > 0) { return bestNode(locations.get(0).getLocations(), excludes);
/** * Client is reporting some bad block locations. */ void reportBadBlocks(LocatedBlock[] blocks) throws IOException { checkOperation(OperationCategory.WRITE); writeLock(); try { checkOperation(OperationCategory.WRITE); for (int i = 0; i < blocks.length; i++) { ExtendedBlock blk = blocks[i].getBlock(); DatanodeInfo[] nodes = blocks[i].getLocations(); String[] storageIDs = blocks[i].getStorageIDs(); for (int j = 0; j < nodes.length; j++) { NameNode.stateChangeLog.info("*DIR* reportBadBlocks for block: {} on" + " datanode: {}", blk, nodes[j].getXferAddr()); blockManager.findAndMarkBlockAsCorrupt(blk, nodes[j], storageIDs == null ? null: storageIDs[j], "client machine reported it"); } } } finally { writeUnlock("reportBadBlocks"); } }
public static RecoveringBlock convert(RecoveringBlockProto b) { LocatedBlock lb = PBHelperClient.convertLocatedBlockProto(b.getBlock()); RecoveringBlock rBlock; if (b.hasTruncateBlock()) { rBlock = new RecoveringBlock(lb.getBlock(), lb.getLocations(), PBHelperClient.convert(b.getTruncateBlock())); } else { rBlock = new RecoveringBlock(lb.getBlock(), lb.getLocations(), b.getNewGenStamp()); } if (b.hasEcPolicy()) { assert b.hasBlockIndices(); byte[] indices = b.getBlockIndices().toByteArray(); rBlock = new RecoveringStripedBlock(rBlock, indices, PBHelperClient.convertErasureCodingPolicy(b.getEcPolicy())); } return rBlock; }
if (ErasureCodingPolicyManager .checkStoragePolicySuitableForECStripedMode(policyId)) { types = policy.chooseStorageTypes((short) lb.getLocations().length); } else {
private void setCachedLocations(LocatedBlock block) { CachedBlock cachedBlock = new CachedBlock(block.getBlock().getBlockId(), (short)0, false); cachedBlock = cachedBlocks.get(cachedBlock); if (cachedBlock == null) { return; } List<DatanodeDescriptor> cachedDNs = cachedBlock.getDatanodes(Type.CACHED); for (DatanodeDescriptor datanode : cachedDNs) { // Filter out cached blocks that do not have a backing replica. // // This should not happen since it means the CacheManager thinks // something is cached that does not exist, but it's a safety // measure. boolean found = false; for (DatanodeInfo loc : block.getLocations()) { if (loc.equals(datanode)) { block.addCachedLoc(loc); found = true; break; } } if (!found) { LOG.warn("Datanode {} is not a valid cache location for block {} " + "because that node does not have a backing replica!", datanode, block.getBlock().getBlockName()); } } }
FileState fileState = analyzeFileState(fsn, iip, fileId, clientName, previous, onRetryBlock); if (onRetryBlock[0] != null && onRetryBlock[0].getLocations().length > 0) {
if (onRetryBlock[0].getLocations().length > 0) {