@Override public void initialize() { ByteBuffer payload = context.getUserPayload().getPayload(); LOG.info("Initializing the edge, payload: " + payload); if (payload == null) { throw new RuntimeException("Invalid payload"); } // De-serialization code DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.reset(payload); conf = new CustomEdgeConfiguration(); try { conf.readFields(dibb); } catch (IOException e) { throw new RuntimeException(e); } LOG.info("Routing table: " + conf.getRoutingTable() + " num Buckets: " + conf.getNumBuckets()); }
UserPayload getBytePayload(Multimap<Integer, Integer> routingTable) throws IOException { CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(numBuckets, routingTable); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] serialized = dob.getData(); return UserPayload.create(ByteBuffer.wrap(serialized)); }
@Test(timeout = 5000) public void testGetBytePayload() throws IOException { int numBuckets = 10; VertexManagerPluginContext context = mock(VertexManagerPluginContext.class); CustomVertexConfiguration vertexConf = new CustomVertexConfiguration(numBuckets, TezWork.VertexType.INITIALIZED_EDGES); DataOutputBuffer dob = new DataOutputBuffer(); vertexConf.write(dob); UserPayload payload = UserPayload.create(ByteBuffer.wrap(dob.getData())); when(context.getUserPayload()).thenReturn(payload); CustomPartitionVertex vm = new CustomPartitionVertex(context); vm.initialize(); // prepare empty routing table Multimap<Integer, Integer> routingTable = HashMultimap.<Integer, Integer> create(); payload = vm.getBytePayload(routingTable); // get conf from user payload CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(); DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.reset(payload.getPayload()); edgeConf.readFields(dibb); assertEquals(numBuckets, edgeConf.getNumBuckets()); } }
@Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return conf.getNumBuckets(); }
@Override public void routeDataMovementEventToDestination(DataMovementEvent event, int sourceTaskIndex, int sourceOutputIndex, Map<Integer, List<Integer>> mapDestTaskIndices) { if (conf.getRoutingTable().get(sourceOutputIndex).size() == 0) { // No task for given input, return empty list with -1 as index mapDestTaskIndices.put(-1, new ArrayList<>()); return; } // Normal case. List<Integer> outputIndices = Collections.singletonList(sourceTaskIndex); for (Integer destIndex : conf.getRoutingTable().get(sourceOutputIndex)) { mapDestTaskIndices.put(destIndex, outputIndices); } }
@Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return conf.getNumBuckets(); }
@Override public void routeDataMovementEventToDestination(DataMovementEvent event, int sourceTaskIndex, int sourceOutputIndex, Map<Integer, List<Integer>> mapDestTaskIndices) { List<Integer> outputIndices = Collections.singletonList(sourceTaskIndex); for (Integer destIndex : conf.getRoutingTable().get(sourceOutputIndex)) { mapDestTaskIndices.put(destIndex, outputIndices); } }
@Override public void initialize() { ByteBuffer payload = context.getUserPayload().getPayload(); LOG.info("Initializing the edge, payload: " + payload); if (payload == null) { throw new RuntimeException("Invalid payload"); } // De-serialization code DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.reset(payload); conf = new CustomEdgeConfiguration(); try { conf.readFields(dibb); } catch (IOException e) { throw new RuntimeException(e); } LOG.info("Routing table: " + conf.getRoutingTable() + " num Buckets: " + conf.getNumBuckets()); }
UserPayload getBytePayload(Multimap<Integer, Integer> routingTable) throws IOException { CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(numBuckets, routingTable); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] serialized = dob.getData(); return UserPayload.create(ByteBuffer.wrap(serialized)); }
@Override public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) { return conf.getNumBuckets(); }
@Override public void routeDataMovementEventToDestination(DataMovementEvent event, int sourceTaskIndex, int sourceOutputIndex, Map<Integer, List<Integer>> mapDestTaskIndices) { List<Integer> outputIndices = Collections.singletonList(sourceTaskIndex); for (Integer destIndex : conf.getRoutingTable().get(sourceOutputIndex)) { mapDestTaskIndices.put(destIndex, outputIndices); } }
@Override public void initialize() { ByteBuffer payload = context.getUserPayload().getPayload(); LOG.info("Initializing the edge, payload: " + payload); if (payload == null) { throw new RuntimeException("Invalid payload"); } // De-serialization code DataInputByteBuffer dibb = new DataInputByteBuffer(); dibb.reset(payload); conf = new CustomEdgeConfiguration(); try { conf.readFields(dibb); } catch (IOException e) { throw new RuntimeException(e); } LOG.info("Routing table: " + conf.getRoutingTable() + " num Buckets: " + conf.getNumBuckets()); }
EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload)));
EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload)));
UserPayload getBytePayload(Multimap<Integer, Integer> routingTable) throws IOException { CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(routingTable.keySet().size(), routingTable); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] serialized = dob.getData(); return UserPayload.create(ByteBuffer.wrap(serialized)); }
EdgeManagerPluginDescriptor.create(CustomPartitionEdge.class.getName()); CustomEdgeConfiguration edgeConf = new CustomEdgeConfiguration(edgeProp.getNumBuckets(), null); DataOutputBuffer dob = new DataOutputBuffer(); edgeConf.write(dob); byte[] userPayload = dob.getData(); edgeDesc.setUserPayload(UserPayload.create(ByteBuffer.wrap(userPayload)));