@Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return context.getSourceVertexNumTasks(); }
@Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return context.getSourceVertexNumTasks(); }
@Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return context.getSourceVertexNumTasks(); }
@Override public int getNumDestinationTaskPhysicalInputs(int destTaskId) { return getContext().getSourceVertexNumTasks(); }
@Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return getContext().getSourceVertexNumTasks(); }
@Override public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) { return getContext().getSourceVertexNumTasks(); }
private ArrayList<EventRouteMetadata> getOrCreateCommonRouteMeta() { ArrayList<EventRouteMetadata> metaData = commonRouteMeta.get(); if (metaData == null) { synchronized (commonRouteMetaLock) { metaData = commonRouteMeta.get(); if (metaData == null) { int numSourceTasks = getContext().getSourceVertexNumTasks(); ArrayList<EventRouteMetadata> localEventMeta = Lists .newArrayListWithCapacity(numSourceTasks); for (int i=0; i<numSourceTasks; ++i) { localEventMeta.add(EventRouteMetadata.create(1, new int[]{i}, new int[]{0})); } Preconditions.checkState(commonRouteMeta.compareAndSet(null, localEventMeta)); metaData = commonRouteMeta.get(); } } } return metaData; }
@Override public void prepareForRouting() throws Exception { int numSourceTasks = getContext().getSourceVertexNumTasks(); commonRouteMeta = new EventRouteMetadata[numSourceTasks]; for (int i=0; i<numSourceTasks; ++i) { commonRouteMeta[i] = EventRouteMetadata.create(1, new int[]{i}, new int[]{0}); } }
private void checkState() { if (stateChecked.get()) { return; } // by the time routing is initiated all task counts must be determined and stable Preconditions.checkState(getContext().getSourceVertexNumTasks() == getContext() .getDestinationVertexNumTasks(), "1-1 source and destination task counts must match." + " Destination: " + getContext().getDestinationVertexName() + " tasks: " + getContext().getDestinationVertexNumTasks() + " Source: " + getContext().getSourceVertexName() + " tasks: " + getContext().getSourceVertexNumTasks()); stateChecked.set(true); } }
private void checkState() { if (stateChecked.get()) { return; } // by the time routing is initiated all task counts must be determined and stable Preconditions.checkState(getContext().getSourceVertexNumTasks() == getContext() .getDestinationVertexNumTasks(), "1-1 source and destination task counts must match." + " Destination: " + getContext().getDestinationVertexName() + " tasks: " + getContext().getDestinationVertexNumTasks() + " Source: " + getContext().getSourceVertexName() + " tasks: " + getContext().getSourceVertexNumTasks()); stateChecked.set(true); }
private void createIndices() { // source indices derive from num dest tasks (==partitions) int numTargetTasks = getContext().getDestinationVertexNumTasks(); sourceIndices = new int[numTargetTasks][]; for (int i=0; i<numTargetTasks; ++i) { sourceIndices[i] = new int[]{i}; } // target indices derive from num src tasks int numSourceTasks = getContext().getSourceVertexNumTasks(); targetIndices = new int[numSourceTasks][]; for (int i=0; i<numSourceTasks; ++i) { targetIndices[i] = new int[]{i}; } }
@Override public void initialize() { // Nothing to do. This class isn't currently designed to be used at the DAG API level. UserPayload userPayload = getContext().getUserPayload(); if (userPayload == null || userPayload.getPayload() == null || userPayload.getPayload().limit() == 0) { throw new RuntimeException("Could not initialize CustomShuffleEdgeManager" + " from provided user payload"); } CustomShuffleEdgeManagerConfig config; try { config = CustomShuffleEdgeManagerConfig.fromUserPayload(userPayload); } catch (InvalidProtocolBufferException e) { throw new RuntimeException("Could not initialize CustomShuffleEdgeManager" + " from provided user payload", e); } this.numSourceTaskOutputs = config.numSourceTaskOutputs; this.numDestinationTasks = config.numDestinationTasks; this.basePartitionRange = config.basePartitionRange; this.remainderRangeForLastShuffler = config.remainderRangeForLastShuffler; this.numSourceTasks = getContext().getSourceVertexNumTasks(); Preconditions.checkState(this.numDestinationTasks == getContext().getDestinationVertexNumTasks()); }
@Override public void prepareForRouting() throws Exception { // target indices derive from num src tasks int numSourceTasks = getContext().getSourceVertexNumTasks(); targetIndices = new int[numSourceTasks][]; for (int srcTaskIndex=0; srcTaskIndex<numSourceTasks; ++srcTaskIndex) { targetIndices[srcTaskIndex] = createIndices(basePartitionRange, srcTaskIndex, basePartitionRange); } // source indices derive from num dest tasks (==partitions) int numTargetTasks = getContext().getDestinationVertexNumTasks(); sourceIndices = new int[numTargetTasks][]; for (int destTaskIndex=0; destTaskIndex<numTargetTasks; ++destTaskIndex) { int partitionRange = basePartitionRange; if (destTaskIndex == (numTargetTasks-1)) { partitionRange = remainderRangeForLastShuffler; } // skip the basePartitionRange per destination task sourceIndices[destTaskIndex] = createIndices(partitionRange, destTaskIndex, basePartitionRange); } }
@Override public void initialize(CartesianProductConfigProto config) { String groupName = getContext().getVertexGroupName(); String srcName = groupName != null ? groupName : getContext().getSourceVertexName(); this.positionInSrc = config.getSourcesList().indexOf(srcName); if (config.hasNumPartitionsForFairCase()) { this.numPartition = config.getNumPartitionsForFairCase(); } else { this.numPartition = (int) Math.pow(config.getMaxParallelism(), 1.0 / config.getSourcesCount()); } if (config.getNumChunksCount() > 0) { // initialize after reconfiguration this.numChunkPerSrc = Ints.toArray(config.getNumChunksList()); grouper.init(getContext().getSourceVertexNumTasks() * numPartition, numChunkPerSrc[positionInSrc]); this.numTaskPerSrcVertexInGroup = Ints.toArray(config.getNumTaskPerVertexInGroupList()); this.positionInGroup = config.getPositionInGroup(); numDestConsumerPerChunk = 1; for (int numChunk : numChunkPerSrc) { numDestConsumerPerChunk *= numChunk; } numDestConsumerPerChunk /= numChunkPerSrc[positionInSrc]; } }