private static NameNodeConnector getNameNodeConnector(Configuration conf) throws IOException, InterruptedException { final Collection<URI> namenodes = DFSUtil.getInternalNsRpcUris(conf); final Path externalSPSPathId = HdfsServerConstants.MOVER_ID_PATH; while (true) { try { final List<NameNodeConnector> nncs = NameNodeConnector .newNameNodeConnectors(namenodes, ExternalStoragePolicySatisfier.class.getSimpleName(), externalSPSPathId, conf, NameNodeConnector.DEFAULT_MAX_IDLE_ITERATIONS); return nncs.get(0); } catch (IOException e) { LOG.warn("Failed to connect with namenode", e); Thread.sleep(3000); // retry the connection after few secs } } } }
/** Create {@link NameNodeConnector} for the given namenodes. */ public static List<NameNodeConnector> newNameNodeConnectors( Collection<URI> namenodes, String name, Path idPath, Configuration conf, int maxIdleIterations) throws IOException { final List<NameNodeConnector> connectors = new ArrayList<NameNodeConnector>( namenodes.size()); for (URI uri : namenodes) { NameNodeConnector nnc = new NameNodeConnector(name, uri, idPath, null, conf, maxIdleIterations); nnc.getKeyManager().startBlockKeyUpdater(); connectors.add(nnc); } return connectors; }
long getBytesMoved() { return nnc.getBytesMoved().get(); }
/** * Constructs a DBNameNodeConnector. * * @param clusterURI - URL to connect to. */ public DBNameNodeConnector(URI clusterURI, Configuration conf) throws IOException, URISyntaxException { // we don't care how many instances of disk balancers run. // The admission is controlled at the data node, where we will // execute only one plan at a given time. NameNodeConnector.setWrite2IdFile(false); try { connector = new NameNodeConnector("DiskBalancer", clusterURI, DISKBALANCER_ID_PATH, null, conf, 1); } catch (IOException ex) { LOG.error("Unable to connect to NameNode " + ex.toString()); throw ex; } this.clusterURI = clusterURI; }
ExtendedBlock eb = new ExtendedBlock(nnc.getBlockpoolID(), reportedBlock.getBlock()); final KeyManager km = nnc.getKeyManager(); Token<BlockTokenIdentifier> accessToken = km.getAccessToken(eb, new StorageType[]{target.storageType}, new String[0]); nnc.getBytesMoved().addAndGet(reportedBlock.getNumBytes()); target.getDDatanode().setHasSuccess(); LOG.info("Successfully moved " + this);
public DistributedFileSystem getDistributedFileSystem() { return nnc.getDistributedFileSystem(); }
connectors = NameNodeConnector.newNameNodeConnectors(namenodes, Balancer.class.getSimpleName(), BALANCER_ID_PATH, conf, p.getMaxIdleIteration()); for(NameNodeConnector nnc : connectors) { if (p.getBlockPools().size() == 0 || p.getBlockPools().contains(nnc.getBlockpoolID())) { final Balancer b = new Balancer(nnc, p, conf); final Result r = b.runOneIteration(); LOG.info("Skipping blockpool " + nnc.getBlockpoolID());
private BlockMovementStatus moveBlock() { ExtendedBlock eb = new ExtendedBlock(nnc.getBlockpoolID(), blkMovingInfo.getBlock()); final KeyManager km = nnc.getKeyManager(); Token<BlockTokenIdentifier> accessToken; try { accessToken = km.getAccessToken(eb, new StorageType[]{blkMovingInfo.getTargetStorageType()}, new String[0]); } catch (IOException e) { // TODO: handle failure retries LOG.warn( "Failed to move block:{} from src:{} to destin:{} to satisfy " + "storageType:{}", blkMovingInfo.getBlock(), blkMovingInfo.getSource(), blkMovingInfo.getTarget(), blkMovingInfo.getTargetStorageType(), e); return BlockMovementStatus.DN_BLK_STORAGE_MOVEMENT_FAILURE; } return blkDispatcher.moveBlock(blkMovingInfo, saslClient, eb, new Socket(), km, accessToken); } }
@Override public DatanodeStorageReport[] getLiveDatanodeStorageReport() throws IOException { return nnc.getLiveDatanodeStorageReport(); }
Mover(NameNodeConnector nnc, Configuration conf, AtomicInteger retryCount, Map<Long, Set<DatanodeInfo>> excludedPinnedBlocks) { final long movedWinWidth = conf.getLong( DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY, DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_DEFAULT); final int moverThreads = conf.getInt( DFSConfigKeys.DFS_MOVER_MOVERTHREADS_KEY, DFSConfigKeys.DFS_MOVER_MOVERTHREADS_DEFAULT); final int maxConcurrentMovesPerNode = conf.getInt( DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT); final int maxNoMoveInterval = conf.getInt( DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY, DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT); this.retryMaxAttempts = conf.getInt( DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT); this.retryCount = retryCount; this.dispatcher = new Dispatcher(nnc, Collections.<String> emptySet(), Collections.<String> emptySet(), movedWinWidth, moverThreads, 0, maxConcurrentMovesPerNode, maxNoMoveInterval, conf); this.storages = new StorageMap(); this.targetPaths = nnc.getTargetPaths(); this.blockStoragePolicies = new BlockStoragePolicy[1 << BlockStoragePolicySuite.ID_BIT_LENGTH]; this.excludedPinnedBlocks = excludedPinnedBlocks; }
public NameNodeConnector(String name, URI nameNodeUri, Path idPath, List<Path> targetPaths, Configuration conf, int maxNotChangedIterations) throws IOException { this.nameNodeUri = nameNodeUri; this.idPath = idPath; this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays .asList(new Path("/")) : targetPaths; this.maxNotChangedIterations = maxNotChangedIterations; this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri, NamenodeProtocol.class).getProxy(); this.client = NameNodeProxies.createProxy(conf, nameNodeUri, ClientProtocol.class, fallbackToSimpleAuth).getProxy(); this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf); final NamespaceInfo namespaceinfo = namenode.versionRequest(); this.blockpoolID = namespaceinfo.getBlockPoolID(); final FsServerDefaults defaults = fs.getServerDefaults(new Path("/")); this.keyManager = new KeyManager(blockpoolID, namenode, defaults.getEncryptDataTransfer(), conf); // if it is for test, we do not create the id file if (checkOtherInstanceRunning) { out = checkAndMarkRunning(); if (out == null) { // Exit if there is another one running. throw new IOException("Another " + name + " is running."); } } }
final long size = Math.min(getBlocksSize, blocksToReceive); final BlocksWithLocations newBlksLocs = nnc.getBlocks(getDatanodeInfo(), size, getBlocksMinBlockSize);
ExtendedBlock eb = new ExtendedBlock(nnc.getBlockpoolID(), block.getBlock()); final KeyManager km = nnc.getKeyManager(); Token<BlockTokenIdentifier> accessToken = km.getAccessToken(eb); IOStreamPair saslStreams = saslClient.socketSend(sock, unbufOut, nnc.getBytesMoved().addAndGet(block.getNumBytes()); LOG.info("Successfully moved " + this); } catch (IOException e) {
@Override public int getNumLiveDataNodes() { try { return nnc.getDistributedFileSystem() .getDataNodeStats(DatanodeReportType.LIVE).length; } catch (IOException e) { LOG.warn("Exception while getting number of live datanodes.", e); } return 0; }
/** * getNodes function returns a list of DiskBalancerDataNodes. * * @return Array of DiskBalancerDataNodes */ @Override public List<DiskBalancerDataNode> getNodes() throws Exception { Preconditions.checkNotNull(this.connector); List<DiskBalancerDataNode> nodeList = new LinkedList<>(); DatanodeStorageReport[] reports = this.connector .getLiveDatanodeStorageReport(); for (DatanodeStorageReport report : reports) { DiskBalancerDataNode datanode = getBalancerNodeFromDataNode( report.getDatanodeInfo()); getVolumeInfoFromStorageReports(datanode, report.getStorageReports()); nodeList.add(datanode); } return nodeList; }
Mover(NameNodeConnector nnc, Configuration conf, AtomicInteger retryCount) { final long movedWinWidth = conf.getLong( DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY, DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_DEFAULT); final int moverThreads = conf.getInt( DFSConfigKeys.DFS_MOVER_MOVERTHREADS_KEY, DFSConfigKeys.DFS_MOVER_MOVERTHREADS_DEFAULT); final int maxConcurrentMovesPerNode = conf.getInt( DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY, DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT); final int maxNoMoveInterval = conf.getInt( DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_KEY, DFSConfigKeys.DFS_MOVER_MAX_NO_MOVE_INTERVAL_DEFAULT); this.retryMaxAttempts = conf.getInt( DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY, DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT); this.retryCount = retryCount; this.dispatcher = new Dispatcher(nnc, Collections.<String> emptySet(), Collections.<String> emptySet(), movedWinWidth, moverThreads, 0, maxConcurrentMovesPerNode, maxNoMoveInterval, conf); this.storages = new StorageMap(); this.targetPaths = nnc.getTargetPaths(); this.blockStoragePolicies = new BlockStoragePolicy[1 << BlockStoragePolicySuite.ID_BIT_LENGTH]; }
public NameNodeConnector(String name, URI nameNodeUri, Path idPath, List<Path> targetPaths, Configuration conf, int maxNotChangedIterations) throws IOException { this.nameNodeUri = nameNodeUri; this.idPath = idPath; this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays .asList(new Path("/")) : targetPaths; this.maxNotChangedIterations = maxNotChangedIterations; this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri, NamenodeProtocol.class).getProxy(); this.client = NameNodeProxies.createProxy(conf, nameNodeUri, ClientProtocol.class, fallbackToSimpleAuth).getProxy(); this.fs = (DistributedFileSystem)FileSystem.get(nameNodeUri, conf); final NamespaceInfo namespaceinfo = namenode.versionRequest(); this.blockpoolID = namespaceinfo.getBlockPoolID(); final FsServerDefaults defaults = fs.getServerDefaults(new Path("/")); this.keyManager = new KeyManager(blockpoolID, namenode, defaults.getEncryptDataTransfer(), conf); // if it is for test, we do not create the id file out = checkAndMarkRunning(); if (out == null) { // Exit if there is another one running. throw new IOException("Another " + name + " is running."); } }
final BlocksWithLocations newBlocks = nnc.getBlocks(getDatanodeInfo(), size);
ExtendedBlock eb = new ExtendedBlock(nnc.getBlockpoolID(), block.getBlock()); final KeyManager km = nnc.getKeyManager(); Token<BlockTokenIdentifier> accessToken = km.getAccessToken(eb); IOStreamPair saslStreams = saslClient.socketSend(sock, unbufOut, nnc.getBytesMoved().addAndGet(block.getNumBytes()); LOG.info("Successfully moved " + this); } catch (IOException e) {
public static List<NameNodeConnector> newNameNodeConnectors( Map<URI, List<Path>> namenodes, String name, Path idPath, Configuration conf, int maxIdleIterations) throws IOException { final List<NameNodeConnector> connectors = new ArrayList<NameNodeConnector>( namenodes.size()); for (Map.Entry<URI, List<Path>> entry : namenodes.entrySet()) { NameNodeConnector nnc = new NameNodeConnector(name, entry.getKey(), idPath, entry.getValue(), conf, maxIdleIterations); nnc.getKeyManager().startBlockKeyUpdater(); connectors.add(nnc); } return connectors; }