public void addConnector(IConnectorDescriptor connector) { connectors.put(connector.getConnectorId(), connector); }
public int getProducerOutputIndex(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getLeft().getRight(); }
public int getConsumerInputIndex(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getRight().getRight(); }
public IOperatorDescriptor getProducer(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getLeft().getLeft(); }
public IOperatorDescriptor getConsumer(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getRight().getLeft(); }
public RecordDescriptor getConnectorRecordDescriptor(IConnectorDescriptor conn) { Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn.getConnectorId()); return connInfo.getLeft().getLeft().getOutputRecordDescriptors()[connInfo.getLeft().getRight()]; }
public void addConnector(IConnectorDescriptor conn) { jag.getConnectorMap().put(conn.getConnectorId(), conn); jag.getConnectorRecordDescriptorMap().put(conn.getConnectorId(), jobSpec.getConnectorRecordDescriptor(conn)); }
@Override public RecordDescriptor getOutputRecordDescriptor(ActivityId aid, int outputIndex) { ActivityCluster ac = acg.getActivityMap().get(aid); IConnectorDescriptor conn = ac.getActivityOutputMap().get(aid).get(outputIndex); return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); }
@Override public void visit(IConnectorDescriptor conn) { conn.contributeSchedulingConstraints(acceptor, acg.getConnectorMap().get(conn.getConnectorId()), ccServiceCtx); } });
@Override public RecordDescriptor getInputRecordDescriptor(ActivityId aid, int inputIndex) { ActivityCluster ac = acg.getActivityMap().get(aid); IConnectorDescriptor conn = ac.getActivityInputMap().get(aid).get(inputIndex); return ac.getConnectorRecordDescriptorMap().get(conn.getConnectorId()); } };
@Override public String toString() { StringBuilder buffer = new StringBuilder(); opMap.forEach((key, value) -> { buffer.append(key.getId()).append(" : ").append(value.toString()).append("\n"); List<IConnectorDescriptor> inputs = opInputMap.get(key); if (inputs != null && !inputs.isEmpty()) { buffer.append(" Inputs:\n"); for (IConnectorDescriptor c : inputs) { buffer.append(" ").append(c.getConnectorId().getId()).append(" : ").append(c.toString()) .append("\n"); } } List<IConnectorDescriptor> outputs = opOutputMap.get(key); if (outputs != null && !outputs.isEmpty()) { buffer.append(" Outputs:\n"); for (IConnectorDescriptor c : outputs) { buffer.append(" ").append(c.getConnectorId().getId()).append(" : ").append(c.toString()) .append("\n"); } } }); buffer.append("\n").append("Constraints:\n").append(userConstraints); return buffer.toString(); }
public void connect(IConnectorDescriptor conn, IOperatorDescriptor producerOp, int producerPort, IOperatorDescriptor consumerOp, int consumerPort) { insertIntoIndexedMap(opInputMap, consumerOp.getOperatorId(), consumerPort, conn); insertIntoIndexedMap(opOutputMap, producerOp.getOperatorId(), producerPort, conn); connectorOpMap.put(conn.getConnectorId(), Pair.of(Pair.of(producerOp, producerPort), Pair.of(consumerOp, consumerPort))); }
public void connect(IConnectorDescriptor connector, IActivity producerActivity, int producerPort, IActivity consumerActivity, int consumerPort, RecordDescriptor recordDescriptor) { if (!activities.containsKey(producerActivity.getActivityId()) || !activities.containsKey(consumerActivity.getActivityId())) { throw new IllegalStateException("Connected Activities belong to different Activity Clusters: " + producerActivity.getActivityId() + " and " + consumerActivity.getActivityId()); } insertIntoIndexedMap(activityInputMap, consumerActivity.getActivityId(), consumerPort, connector); insertIntoIndexedMap(activityOutputMap, producerActivity.getActivityId(), producerPort, connector); connectorActivityMap.put(connector.getConnectorId(), Pair.of(Pair.of(producerActivity, producerPort), Pair.of(consumerActivity, consumerPort))); connectorRecordDescriptorMap.put(connector.getConnectorId(), recordDescriptor); }
private static Pair<ActivityId, ActivityId> findMergePair(JobActivityGraph jag, Set<Set<ActivityId>> eqSets) { for (Set<ActivityId> eqSet : eqSets) { for (ActivityId t : eqSet) { List<IConnectorDescriptor> inputList = jag.getActivityInputMap().get(t); if (inputList != null) { for (IConnectorDescriptor conn : inputList) { ActivityId inTask = jag.getProducerActivity(conn.getConnectorId()); if (!eqSet.contains(inTask)) { return Pair.of(t, inTask); } } } List<IConnectorDescriptor> outputList = jag.getActivityOutputMap().get(t); if (outputList != null) { for (IConnectorDescriptor conn : outputList) { ActivityId outTask = jag.getConsumerActivity(conn.getConnectorId()); if (!eqSet.contains(outTask)) { return Pair.of(t, outTask); } } } } } return null; }
@Override public void addSourceEdge(int operatorInputIndex, IActivity task, int taskInputIndex) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("Adding source edge: " + task.getActivityId() + ":" + operatorInputIndex + " -> " + task.getActivityId() + ":" + taskInputIndex); } IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId()); IConnectorDescriptor conn = jobSpec.getInputConnectorDescriptor(op, operatorInputIndex); insertIntoIndexedMap(jag.getActivityInputMap(), task.getActivityId(), taskInputIndex, conn); connectorConsumerMap.put(conn.getConnectorId(), Pair.of(task, taskInputIndex)); }
@Override public void addTargetEdge(int operatorOutputIndex, IActivity task, int taskOutputIndex) { if (LOGGER.isTraceEnabled()) { LOGGER.trace("Adding target edge: " + task.getActivityId() + ":" + operatorOutputIndex + " -> " + task.getActivityId() + ":" + taskOutputIndex); } IOperatorDescriptor op = activityOperatorMap.get(task.getActivityId()); IConnectorDescriptor conn = jobSpec.getOutputConnectorDescriptor(op, operatorOutputIndex); insertIntoIndexedMap(jag.getActivityOutputMap(), task.getActivityId(), taskOutputIndex, conn); connectorProducerMap.put(conn.getConnectorId(), Pair.of(task, taskOutputIndex)); }
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new PipelinedPartition(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId); } };
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new MaterializingPipelinedPartition(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor()); } };
@Override public void close() throws HyracksDataException { closeTime = System.currentTimeMillis(); try { ((Task) ctx).setPartitionSendProfile( new PartitionProfile(new PartitionId(ctx.getJobletContext().getJobId(), cd.getConnectorId(), senderIndex, receiverIndex), openTime, closeTime, mrep)); } finally { writer.close(); } }
@Override public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException { return new MaterializedPartitionWriter(ctx, ncs.getPartitionManager(), new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId, ncs.getExecutor()); } };