if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES) || (vertexType == VertexType.INITIALIZED_EDGES)) { hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload);
if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES) || (vertexType == VertexType.INITIALIZED_EDGES)) { hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload);
.build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); case XPROD_EDGE: EdgeManagerPluginDescriptor edgeManagerDescriptor = EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); List<String> crossProductSources = new ArrayList<>(); for (BaseWork parentWork : tezWork.getParents(work)) {
.build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
public static EdgeManagerPluginDescriptor convertEdgeManagerPluginDescriptorFromDAGPlan( TezEntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertTezUserPayloadFromDAGPlan(proto); EdgeManagerPluginDescriptor empd = EdgeManagerPluginDescriptor.create(className); setUserPayload(empd, payload); return empd; }
public ReconfigVertexParams computeRouting() { int currentParallelism = pendingTasks.size(); int finalTaskParallelism = 0; long[] estimatedPartitionOutputSize = estimatePartitionSize(); for (Map.Entry<String, SourceVertexInfo> vInfo : getBipartiteInfo()) { FairSourceVertexInfo info = (FairSourceVertexInfo)vInfo.getValue(); computeParallelism(estimatedPartitionOutputSize, info); if (finalTaskParallelism != 0) { Preconditions.checkState( finalTaskParallelism == info.getDestinationInputsProperties().size(), "the parallelism shall be the same for source vertices"); } finalTaskParallelism = info.getDestinationInputsProperties().size(); FairEdgeConfiguration fairEdgeConfig = new FairEdgeConfiguration( currentParallelism, info.getDestinationInputsProperties()); EdgeManagerPluginDescriptor descriptor = EdgeManagerPluginDescriptor.create( FairShuffleEdgeManager.class.getName()); descriptor.setUserPayload(fairEdgeConfig.getBytePayload()); vInfo.getValue().newDescriptor = descriptor; } ReconfigVertexParams params = new ReconfigVertexParams( finalTaskParallelism, null); return params; }
if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES) || (vertexType == VertexType.INITIALIZED_EDGES)) { hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload);
remainderRangeForLastShuffler : basePartitionRange)); EdgeManagerPluginDescriptor descriptor = EdgeManagerPluginDescriptor.create(CustomShuffleEdgeManager.class.getName()); descriptor.setUserPayload(edgeManagerConfig.toUserPayload());
EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); edgeManagerDescriptor.setUserPayload(userPayload); UnorderedPartitionedKVEdgeConfig edgeConf =
.build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null);
EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); cpEdgeManager.setUserPayload(userPayload); EdgeProperty cpEdgeProperty;