boolean chooseTarget(DBlock db, Source source, List<StorageType> targetTypes, Matcher matcher) { final NetworkTopology cluster = dispatcher.getCluster(); for (StorageType t : targetTypes) { final List<StorageGroup> targets = storages.getTargetStorages(t); Collections.shuffle(targets); for (StorageGroup target : targets) { if (matcher.match(cluster, source.getDatanodeInfo(), target.getDatanodeInfo())) { final PendingMove pm = source.addPendingMove(db, target); if (pm != null) { dispatcher.executePendingMove(pm); return true; } } } } return false; } }
private void matchSourceWithTargetToMove(Source source, StorageGroup target) { long size = Math.min(source.availableSizeToMove(), target.availableSizeToMove()); final Task task = new Task(target, size); source.addTask(task); target.incScheduledSize(task.getSize()); dispatcher.add(source, target); LOG.info("Decided to move "+StringUtils.byteDesc(size)+" bytes from " + source.getDisplayName() + " to " + target.getDisplayName()); }
this.blocksToReceive = 2 * getScheduledSize(); long previousMoveTimestamp = Time.monotonicNow(); while (getScheduledSize() > 0 && !isIterationOver() && (!srcBlocks.isEmpty() || blocksToReceive > 0)) { if (LOG.isTraceEnabled()) { LOG.trace(this + " blocksToReceive=" + blocksToReceive + ", scheduledSize=" + getScheduledSize() + ", srcBlocks#=" + srcBlocks.size()); final PendingMove p = chooseNextMove(); if (p != null) { removeMovedBlocks(); // filter already moved blocks if (shouldFetchMoreBlocks()) { final long received = getBlockList(); if (received == 0) { return; LOG.info("Failed to find a pending move for " + noMoveInterval + " ms. Skipping " + this); resetScheduledSize(); if (isIterationOver()) { LOG.info("The maximum iteration time (" + maxIterationTime/1000 + " seconds) has been reached. Stopping " + this);
this.blocksToReceive = 2 * getScheduledSize(); long previousMoveTimestamp = Time.monotonicNow(); while (getScheduledSize() > 0 && !isIterationOver() && (!srcBlocks.isEmpty() || blocksToReceive > 0)) { if (LOG.isTraceEnabled()) { LOG.trace(this + " blocksToReceive=" + blocksToReceive + ", scheduledSize=" + getScheduledSize() + ", srcBlocks#=" + srcBlocks.size()); final PendingMove p = chooseNextMove(); if (p != null) { removeMovedBlocks(); // filter already moved blocks if (shouldFetchMoreBlocks()) { blocksToReceive -= getBlockList(); continue; } catch (InterruptedException ignored) { LOG.info("Failed to find a pending move for " + noMoveInterval + " ms. Skipping " + this); resetScheduledSize(); if (isIterationOver()) { LOG.info("The maximum iteration time (" + MAX_ITERATION_TIME/1000 + " seconds) has been reached. Stopping " + this);
this.blocksToReceive = 2 * getScheduledSize(); long previousMoveTimestamp = Time.monotonicNow(); while (getScheduledSize() > 0 && !isIterationOver() && (!srcBlocks.isEmpty() || blocksToReceive > 0)) { if (LOG.isTraceEnabled()) { LOG.trace(this + " blocksToReceive=" + blocksToReceive + ", scheduledSize=" + getScheduledSize() + ", srcBlocks#=" + srcBlocks.size()); final PendingMove p = chooseNextMove(); if (p != null) { removeMovedBlocks(); // filter already moved blocks if (shouldFetchMoreBlocks()) { blocksToReceive -= getBlockList(); continue; } catch (InterruptedException ignored) { LOG.info("Failed to find a pending move for " + noMoveInterval + " ms. Skipping " + this); resetScheduledSize(); if (isIterationOver()) { LOG.info("The maximum iteration time (" + MAX_ITERATION_TIME/1000 + " seconds) has been reached. Stopping " + this);
final BlocksWithLocations newBlocks = nnc.getBlocks(getDatanodeInfo(), size); LOG.trace("getBlocks(" + getDatanodeInfo() + ", " + StringUtils.TraditionalBinaryPrefix.long2String(size, "B", 2) + ") returns " + newBlocks.getBlocks().length + " blocks."); if (!srcBlocks.contains(block) && isGoodBlockCandidate(block)) { if (LOG.isTraceEnabled()) { LOG.trace("Add " + block + " to " + this);
final BlocksWithLocations newBlocks = nnc.getBlocks(getDatanodeInfo(), size); LOG.trace("getBlocks(" + getDatanodeInfo() + ", " + StringUtils.TraditionalBinaryPrefix.long2String(size, "B", 2) + ") returns " + newBlocks.getBlocks().length + " blocks."); if (!srcBlocks.contains(block) && isGoodBlockCandidate(block)) { if (LOG.isTraceEnabled()) { LOG.trace("Add " + block + " to " + this);
final DatanodeInfo targetDN = target.getDatanodeInfo(); if (source.getDatanodeInfo().equals(targetDN) && addTo(source)) { return true;
final DatanodeInfo targetDN = target.getDatanodeInfo(); if (source.getDatanodeInfo().equals(targetDN) && addTo(source)) { return true;
/** * Choose the target storage within same Datanode if possible. */ boolean chooseTargetInSameNode(DBlock db, Source source, List<StorageType> targetTypes) { for (StorageType t : targetTypes) { StorageGroup target = storages.getTarget(source.getDatanodeInfo() .getDatanodeUuid(), t); if (target == null) { continue; } final PendingMove pm = source.addPendingMove(db, target); if (pm != null) { dispatcher.executePendingMove(pm); return true; } } return false; }
/** * Choose a block & a proxy source for this pendingMove whose source & * target have already been chosen. * * @return true if a block and its proxy are chosen; false otherwise */ private boolean chooseBlockAndProxy() { // source and target must have the same storage type final StorageType t = source.getStorageType(); // iterate all source's blocks until find a good one for (Iterator<DBlock> i = source.getBlockIterator(); i.hasNext();) { if (markMovedIfGoodBlock(i.next(), t)) { i.remove(); return true; } } return false; }
boolean chooseTarget(DBlock db, Source source, List<StorageType> targetTypes, Matcher matcher) { final NetworkTopology cluster = dispatcher.getCluster(); for (StorageType t : targetTypes) { final List<StorageGroup> targets = storages.getTargetStorages(t); Collections.shuffle(targets); for (StorageGroup target : targets) { if (matcher.match(cluster, source.getDatanodeInfo(), target.getDatanodeInfo())) { final PendingMove pm = source.addPendingMove(db, target); if (pm != null) { dispatcher.executePendingMove(pm); return true; } } } } return false; } }
boolean chooseTarget(DBlock db, Source source, List<StorageType> targetTypes, Matcher matcher) { final NetworkTopology cluster = dispatcher.getCluster(); for (StorageType t : targetTypes) { final List<StorageGroup> targets = storages.getTargetStorages(t); Collections.shuffle(targets); for (StorageGroup target : targets) { if (matcher.match(cluster, source.getDatanodeInfo(), target.getDatanodeInfo())) { final PendingMove pm = source.addPendingMove(db, target); if (pm != null) { dispatcher.executePendingMove(pm); return true; } } } } return false; } }
/** * Choose the target storage within same Datanode if possible. */ boolean chooseTargetInSameNode(DBlock db, Source source, List<StorageType> targetTypes) { for (StorageType t : targetTypes) { StorageGroup target = storages.getTarget(source.getDatanodeInfo() .getDatanodeUuid(), t); if (target == null) { continue; } final PendingMove pm = source.addPendingMove(db, target); if (pm != null) { dispatcher.executePendingMove(pm); return true; } } return false; }
/** * Choose a block & a proxy source for this pendingMove whose source & * target have already been chosen. * * @return true if a block and its proxy are chosen; false otherwise */ private boolean chooseBlockAndProxy() { // source and target must have the same storage type final StorageType t = source.getStorageType(); // iterate all source's blocks until find a good one for (Iterator<DBlock> i = source.getBlockIterator(); i.hasNext();) { if (markMovedIfGoodBlock(i.next(), t)) { i.remove(); return true; } } return false; }
/** Send a block replace request to the output stream */ private void sendRequest(DataOutputStream out, ExtendedBlock eb, Token<BlockTokenIdentifier> accessToken) throws IOException { new Sender(out).replaceBlock(eb, target.storageType, accessToken, source.getDatanodeInfo().getDatanodeUuid(), proxySource.datanode); }
/** Send a block replace request to the output stream */ private void sendRequest(DataOutputStream out, ExtendedBlock eb, Token<BlockTokenIdentifier> accessToken) throws IOException { new Sender(out).replaceBlock(eb, target.storageType, accessToken, source.getDatanodeInfo().getDatanodeUuid(), proxySource.datanode); }
/** Add a task */ void addTask(Task task) { Preconditions.checkState(task.target != this, "Source and target are the same storage group " + getDisplayName()); incScheduledSize(task.size); tasks.add(task); }
long bytesToMove() { Preconditions.checkState( storageGroupMap.size() >= sources.size() + targets.size(), "Mismatched number of storage groups (" + storageGroupMap.size() + " < " + sources.size() + " sources + " + targets.size() + " targets)"); long b = 0L; for (Source src : sources) { b += src.getScheduledSize(); } return b; }
private void matchSourceWithTargetToMove(Source source, StorageGroup target) { long size = Math.min(source.availableSizeToMove(), target.availableSizeToMove()); final Task task = new Task(target, size); source.addTask(task); target.incScheduledSize(task.getSize()); dispatcher.add(source, target); LOG.info("Decided to move "+StringUtils.byteDesc(size)+" bytes from " + source.getDisplayName() + " to " + target.getDisplayName()); }