.build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE: case XPROD_EDGE: EdgeManagerPluginDescriptor edgeManagerDescriptor = EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); List<String> crossProductSources = new ArrayList<>(); for (BaseWork parentWork : tezWork.getParents(work)) { edgeManagerDescriptor.setUserPayload(cpConfig.toUserPayload(new TezConfiguration(conf))); UnorderedPartitionedKVEdgeConfig cpEdgeConf = UnorderedPartitionedKVEdgeConfig.newBuilder(keyClass, valClass,
if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES) || (vertexType == VertexType.INITIALIZED_EDGES)) { hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload); && edgeEntry.getValue().getEdgeManagerDescriptor().getClassName() .equals(CustomPartitionEdge.class.getName())) { emMap.put(edgeEntry.getKey(), hiveEdgeManagerDesc);
@Override public String toString() { return "{ " + dataMovementType + " : " + inputDescriptor.getClassName() + " >> " + dataSourceType + " >> " + outputDescriptor.getClassName() + " >> " + (edgeManagerDescriptor == null ? "NullEdgeManager" : edgeManagerDescriptor.getClassName()) + " }"; }
public static Map<String,Object> convertEdgeManagerPluginDescriptor( EdgeManagerPluginDescriptor descriptor) { Map<String, Object> jsonDescriptor = new HashMap<String, Object>(); jsonDescriptor.put(EDGE_MANAGER_CLASS_KEY, descriptor.getClassName()); if (descriptor.getHistoryText() != null && !descriptor.getHistoryText().isEmpty()) { jsonDescriptor.put(USER_PAYLOAD_AS_TEXT, descriptor.getHistoryText()); } return jsonDescriptor; }
public static EdgeManagerPluginDescriptor convertEdgeManagerPluginDescriptorFromDAGPlan( TezEntityDescriptorProto proto) { String className = proto.getClassName(); UserPayload payload = convertTezUserPayloadFromDAGPlan(proto); EdgeManagerPluginDescriptor empd = EdgeManagerPluginDescriptor.create(className); setUserPayload(empd, payload); return empd; }
if (edgeProperty.getEdgeManagerDescriptor() != null) { UserPayload payload = null; if (edgeProperty.getEdgeManagerDescriptor().getUserPayload() != null) { payload = edgeProperty.getEdgeManagerDescriptor().getUserPayload(); String edgeManagerClassName = edgeProperty.getEdgeManagerDescriptor().getClassName(); edgeManager = ReflectionUtils .createClazzInstance(edgeManagerClassName, new Class[]{EdgeManagerPluginContext.class},
builder.setPositionInGroup(i); edgeProperties.get(srcV.name).getEdgeManagerDescriptor() .setUserPayload(UserPayload.create(ByteBuffer.wrap(builder.build().toByteArray()))); builder.addNumTaskPerVertexInGroup(srcV.numTask);
public static EdgeManagerPluginDescriptor create(String edgeManagerPluginClassName) { return new EdgeManagerPluginDescriptor(edgeManagerPluginClassName); } }
EdgeManagerPluginDescriptor empDescriptor = edgeProperty.getEdgeManagerDescriptor(); if (empDescriptor != null && empDescriptor.getClassName().equals(CartesianProductEdgeManager.class.getName())) { Preconditions.checkArgument( sourceVerticesConfig.contains(vertex) || sourceVerticesConfig.contains(group), v + " is in CartesianProductVertexManagerConfig but not a source vertex in DAG"); Preconditions.checkArgument( edgePropertyMap.get(v).getEdgeManagerDescriptor().getClassName() .equals(CartesianProductEdgeManager.class.getName()), v + " is in CartesianProductVertexManagerConfig and a source vertex, but has no " +
public static Map<String,Object> convertEdgeProperty( EdgeProperty edge) { Map<String, Object> jsonDescriptor = new HashMap<String, Object>(); jsonDescriptor.put(DATA_MOVEMENT_TYPE_KEY, edge.getDataMovementType().name()); jsonDescriptor.put(DATA_SOURCE_TYPE_KEY, edge.getDataSourceType().name()); jsonDescriptor.put(SCHEDULING_TYPE_KEY, edge.getSchedulingType().name()); jsonDescriptor.put(EDGE_SOURCE_CLASS_KEY, edge.getEdgeSource().getClassName()); jsonDescriptor.put(EDGE_DESTINATION_CLASS_KEY, edge.getEdgeDestination().getClassName()); String history = edge.getEdgeSource().getHistoryText(); if (history != null) { jsonDescriptor.put(OUTPUT_USER_PAYLOAD_AS_TEXT, history); } history = edge.getEdgeDestination().getHistoryText(); if (history != null) { jsonDescriptor.put(INPUT_USER_PAYLOAD_AS_TEXT, history); } EdgeManagerPluginDescriptor descriptor = edge.getEdgeManagerDescriptor(); if (descriptor != null) { jsonDescriptor.put(EDGE_MANAGER_CLASS_KEY, descriptor.getClassName()); if (descriptor.getHistoryText() != null && !descriptor.getHistoryText().isEmpty()) { jsonDescriptor.put(USER_PAYLOAD_AS_TEXT, descriptor.getHistoryText()); } } return jsonDescriptor; }
.build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE:
if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES) || (vertexType == VertexType.INITIALIZED_EDGES)) { hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload); && edgeEntry.getValue().getEdgeManagerDescriptor().getClassName() .equals(CustomPartitionEdge.class.getName())) { emMap.put(edgeEntry.getKey(), hiveEdgeManagerDesc);
&& e.getValue().getEdgeManagerDescriptor().getClassName() .equals(CartesianProductEdgeManager.class.getName())) { srcVerticesByName.put(e.getKey(), new SrcVertex());
public ReconfigVertexParams computeRouting() { int currentParallelism = pendingTasks.size(); int finalTaskParallelism = 0; long[] estimatedPartitionOutputSize = estimatePartitionSize(); for (Map.Entry<String, SourceVertexInfo> vInfo : getBipartiteInfo()) { FairSourceVertexInfo info = (FairSourceVertexInfo)vInfo.getValue(); computeParallelism(estimatedPartitionOutputSize, info); if (finalTaskParallelism != 0) { Preconditions.checkState( finalTaskParallelism == info.getDestinationInputsProperties().size(), "the parallelism shall be the same for source vertices"); } finalTaskParallelism = info.getDestinationInputsProperties().size(); FairEdgeConfiguration fairEdgeConfig = new FairEdgeConfiguration( currentParallelism, info.getDestinationInputsProperties()); EdgeManagerPluginDescriptor descriptor = EdgeManagerPluginDescriptor.create( FairShuffleEdgeManager.class.getName()); descriptor.setUserPayload(fairEdgeConfig.getBytePayload()); vInfo.getValue().newDescriptor = descriptor; } ReconfigVertexParams params = new ReconfigVertexParams( finalTaskParallelism, null); return params; }
if ((vertexType == VertexType.MULTI_INPUT_INITIALIZED_EDGES) || (vertexType == VertexType.INITIALIZED_EDGES)) { hiveEdgeManagerDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); UserPayload payload = getBytePayload(bucketToTaskMap); hiveEdgeManagerDesc.setUserPayload(payload); && edgeEntry.getValue().getEdgeManagerDescriptor().getClassName() .equals(CustomPartitionEdge.class.getName())) { emMap.put(edgeEntry.getKey(), hiveEdgeManagerDesc);
remainderRangeForLastShuffler : basePartitionRange)); EdgeManagerPluginDescriptor descriptor = EdgeManagerPluginDescriptor.create(CustomShuffleEdgeManager.class.getName()); descriptor.setUserPayload(edgeManagerConfig.toUserPayload());
EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); edgeManagerDescriptor.setUserPayload(userPayload); UnorderedPartitionedKVEdgeConfig edgeConf = UnorderedPartitionedKVEdgeConfig.newBuilder(Text.class.getName(), IntWritable.class.getName(),
.build(); EdgeManagerPluginDescriptor edgeDesc = EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload))); return et2Conf.createDefaultCustomEdgeProperty(edgeDesc); case CUSTOM_SIMPLE_EDGE:
EdgeManagerPluginDescriptor.create(CartesianProductEdgeManager.class.getName()); cpEdgeManager.setUserPayload(userPayload); EdgeProperty cpEdgeProperty; if (isPartitioned) {