Dispatcher(NameNodeConnector nnc, Set<String> includedNodes, Set<String> excludedNodes, long movedWinWidth, int moverThreads, int dispatcherThreads, int maxConcurrentMovesPerNode, long getBlocksSize, long getBlocksMinBlockSize, int blockMoveTimeout, int maxNoMoveInterval, long maxIterationTime, Configuration conf) { this.nnc = nnc; this.excludedNodes = excludedNodes; this.includedNodes = includedNodes; this.movedBlocks = new MovedBlocks<StorageGroup>(movedWinWidth); this.cluster = NetworkTopology.getInstance(conf); this.dispatchExecutor = dispatcherThreads == 0? null : Executors.newFixedThreadPool(dispatcherThreads); this.moverThreadAllocator = new Allocator(moverThreads); this.maxMoverThreads = moverThreads; this.maxConcurrentMovesPerNode = maxConcurrentMovesPerNode; this.getBlocksSize = getBlocksSize; this.getBlocksMinBlockSize = getBlocksMinBlockSize; this.blockMoveTimeout = blockMoveTimeout; this.maxNoMoveInterval = maxNoMoveInterval; this.saslClient = new SaslDataTransferClient(conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf), TrustedChannelResolver.getInstance(conf), nnc.fallbackToSimpleAuth); this.ioFileBufferSize = DFSUtilClient.getIoFileBufferSize(conf); this.connectToDnViaHostname = conf.getBoolean( HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME, HdfsClientConfigKeys.DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT); placementPolicies = new BlockPlacementPolicies(conf, null, cluster, null); this.maxIterationTime = maxIterationTime; }
public void executePendingMove(final PendingMove p) { // move the reportedBlock final DDatanode targetDn = p.target.getDDatanode(); ExecutorService moveExecutor = targetDn.getMoveExecutor(); if (moveExecutor == null) { final int nThreads = moverThreadAllocator.allocate(); if (nThreads > 0) { moveExecutor = targetDn.initMoveExecutor(nThreads); } } if (moveExecutor == null) { LOG.warn("No mover threads available: skip moving " + p); targetDn.removePendingBlock(p); p.proxySource.removePendingBlock(p); return; } moveExecutor.execute(new Runnable() { @Override public void run() { p.dispatch(); } }); }
/** Reset all fields in order to prepare for the next iteration */ void reset(Configuration conf) { cluster = NetworkTopology.getInstance(conf); storageGroupMap.clear(); sources.clear(); moverThreadAllocator.reset(); for(StorageGroup t : targets) { t.getDDatanode().shutdownMoveExecutor(); } targets.clear(); globalBlocks.removeAllButRetain(movedBlocks); movedBlocks.cleanup(); }
moverThreadAllocator.setLotSize(1); LOG.warn(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY + "=" + maxMoverThreads + " is too small for moving blocks to " + LOG.info("Limiting threads per target to the specified max."); moverThreadAllocator.setLotSize(threadsPerTarget); LOG.info("Allocating " + threadsPerTarget + " threads per target.");
moverThreadAllocator.setLotSize(1); LOG.warn(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY + "=" + maxMoverThreads + " is too small for moving blocks to " + LOG.info("Limiting threads per target to the specified max."); moverThreadAllocator.setLotSize(threadsPerTarget); LOG.info("Allocating " + threadsPerTarget + " threads per target.");
public void executePendingMove(final PendingMove p) { // move the block final DDatanode targetDn = p.target.getDDatanode(); ExecutorService moveExecutor = targetDn.getMoveExecutor(); if (moveExecutor == null) { final int nThreads = moverThreadAllocator.allocate(); if (nThreads > 0) { moveExecutor = targetDn.initMoveExecutor(nThreads); } } if (moveExecutor == null) { LOG.warn("No mover threads available: skip moving " + p); targetDn.removePendingBlock(p); p.proxySource.removePendingBlock(p); return; } moveExecutor.execute(new Runnable() { @Override public void run() { p.dispatch(); } }); }
public void executePendingMove(final PendingMove p) { // move the block final DDatanode targetDn = p.target.getDDatanode(); ExecutorService moveExecutor = targetDn.getMoveExecutor(); if (moveExecutor == null) { final int nThreads = moverThreadAllocator.allocate(); if (nThreads > 0) { moveExecutor = targetDn.initMoveExecutor(nThreads); } } if (moveExecutor == null) { LOG.warn("No mover threads available: skip moving " + p); targetDn.removePendingBlock(p); p.proxySource.removePendingBlock(p); return; } moveExecutor.execute(new Runnable() { @Override public void run() { p.dispatch(); } }); }
Dispatcher(NameNodeConnector nnc, Set<String> includedNodes, Set<String> excludedNodes, long movedWinWidth, int moverThreads, int dispatcherThreads, int maxConcurrentMovesPerNode, long getBlocksSize, long getBlocksMinBlockSize, int blockMoveTimeout, int maxNoMoveInterval, Configuration conf) { this.nnc = nnc; this.excludedNodes = excludedNodes; this.includedNodes = includedNodes; this.movedBlocks = new MovedBlocks<StorageGroup>(movedWinWidth); this.cluster = NetworkTopology.getInstance(conf); this.dispatchExecutor = dispatcherThreads == 0? null : Executors.newFixedThreadPool(dispatcherThreads); this.moverThreadAllocator = new Allocator(moverThreads); this.maxMoverThreads = moverThreads; this.maxConcurrentMovesPerNode = maxConcurrentMovesPerNode; this.getBlocksSize = getBlocksSize; this.getBlocksMinBlockSize = getBlocksMinBlockSize; this.blockMoveTimeout = blockMoveTimeout; this.maxNoMoveInterval = maxNoMoveInterval; this.saslClient = new SaslDataTransferClient(conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf), TrustedChannelResolver.getInstance(conf), nnc.fallbackToSimpleAuth); }
Dispatcher(NameNodeConnector nnc, Set<String> includedNodes, Set<String> excludedNodes, long movedWinWidth, int moverThreads, int dispatcherThreads, int maxConcurrentMovesPerNode, long getBlocksSize, long getBlocksMinBlockSize, int blockMoveTimeout, int maxNoMoveInterval, Configuration conf) { this.nnc = nnc; this.excludedNodes = excludedNodes; this.includedNodes = includedNodes; this.movedBlocks = new MovedBlocks<StorageGroup>(movedWinWidth); this.cluster = NetworkTopology.getInstance(conf); this.dispatchExecutor = dispatcherThreads == 0? null : Executors.newFixedThreadPool(dispatcherThreads); this.moverThreadAllocator = new Allocator(moverThreads); this.maxMoverThreads = moverThreads; this.maxConcurrentMovesPerNode = maxConcurrentMovesPerNode; this.getBlocksSize = getBlocksSize; this.getBlocksMinBlockSize = getBlocksMinBlockSize; this.blockMoveTimeout = blockMoveTimeout; this.maxNoMoveInterval = maxNoMoveInterval; this.saslClient = new SaslDataTransferClient(conf, DataTransferSaslUtil.getSaslPropertiesResolver(conf), TrustedChannelResolver.getInstance(conf), nnc.fallbackToSimpleAuth); }
/** Reset all fields in order to prepare for the next iteration */ void reset(Configuration conf) { cluster = NetworkTopology.getInstance(conf); storageGroupMap.clear(); sources.clear(); moverThreadAllocator.reset(); for(StorageGroup t : targets) { t.getDDatanode().shutdownMoveExecutor(); } targets.clear(); globalBlocks.removeAllButRetain(movedBlocks); movedBlocks.cleanup(); }
/** Aloocate a single lot of items */ int allocate() { return allocate(lotSize); }
/** Reset all fields in order to prepare for the next iteration */ void reset(Configuration conf) { cluster = NetworkTopology.getInstance(conf); storageGroupMap.clear(); sources.clear(); moverThreadAllocator.reset(); for(StorageGroup t : targets) { t.getDDatanode().shutdownMoveExecutor(); } targets.clear(); globalBlocks.removeAllButRetain(movedBlocks); movedBlocks.cleanup(); }
/** Aloocate a single lot of items */ int allocate() { return allocate(lotSize); }
moverThreadAllocator.setLotSize(1); LOG.warn(DFSConfigKeys.DFS_BALANCER_MOVERTHREADS_KEY + "=" + maxMoverThreads + " is too small for moving blocks to " + LOG.info("Limiting threads per target to the specified max."); moverThreadAllocator.setLotSize(threadsPerTarget); LOG.info("Allocating " + threadsPerTarget + " threads per target.");
/** Aloocate a single lot of items */ int allocate() { return allocate(lotSize); }