hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload);
hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload);
edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE: edgeManagerDescriptor.setUserPayload(cpConfig.toUserPayload(new TezConfiguration(conf))); UnorderedPartitionedKVEdgeConfig cpEdgeConf = UnorderedPartitionedKVEdgeConfig.newBuilder(keyClass, valClass,
edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE:
public ReconfigVertexParams computeRouting() { int currentParallelism = pendingTasks.size(); int finalTaskParallelism = 0; long[] estimatedPartitionOutputSize = estimatePartitionSize(); for (Map.Entry<String, SourceVertexInfo> vInfo : getBipartiteInfo()) { FairSourceVertexInfo info = (FairSourceVertexInfo)vInfo.getValue(); computeParallelism(estimatedPartitionOutputSize, info); if (finalTaskParallelism != 0) { Preconditions.checkState( finalTaskParallelism == info.getDestinationInputsProperties().size(), "the parallelism shall be the same for source vertices"); } finalTaskParallelism = info.getDestinationInputsProperties().size(); FairEdgeConfiguration fairEdgeConfig = new FairEdgeConfiguration( currentParallelism, info.getDestinationInputsProperties()); EdgeManagerPluginDescriptor descriptor = EdgeManagerPluginDescriptor.create( FairShuffleEdgeManager.class.getName()); descriptor.setUserPayload(fairEdgeConfig.getBytePayload()); vInfo.getValue().newDescriptor = descriptor; } ReconfigVertexParams params = new ReconfigVertexParams( finalTaskParallelism, null); return params; }
hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload);
EdgeManagerPluginDescriptor descriptor = EdgeManagerPluginDescriptor.create(CustomShuffleEdgeManager.class.getName()); descriptor.setUserPayload(edgeManagerConfig.toUserPayload());
builder.setPositionInGroup(i); edgeProperties.get(srcV.name).getEdgeManagerDescriptor() .setUserPayload(UserPayload.create(ByteBuffer.wrap(builder.build().toByteArray()))); builder.addNumTaskPerVertexInGroup(srcV.numTask);
edgeManagerDescriptor.setUserPayload(userPayload); UnorderedPartitionedKVEdgeConfig edgeConf = UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(),
edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE:
cpEdgeManager.setUserPayload(userPayload); EdgeProperty cpEdgeProperty; if (isPartitioned) {