protected Edge(@Nonnull Vertex source, int sourceOrdinal, Vertex destination, int destOrdinal) { this.source = source; this.sourceName = source.getName(); this.sourceOrdinal = sourceOrdinal; this.destination = destination; this.destName = destination != null ? destination.getName() : null; this.destOrdinal = destOrdinal; }
private boolean containsVertexName(Vertex vertex) { return nameToVertex.containsKey(vertex.getName()); }
/** * Sets the destination vertex of this edge, with ordinal 0. */ @Nonnull public Edge to(@Nonnull Vertex destination) { this.destination = destination; this.destName = destination.getName(); return this; }
/** * Sets the destination vertex and ordinal of this edge. */ @Nonnull public Edge to(@Nonnull Vertex destination, int ordinal) { this.destination = destination; this.destName = destination.getName(); this.destOrdinal = ordinal; return this; }
private static Map<String, Integer> assignVertexIds(DAG dag) { Map<String, Integer> vertexIdMap = new LinkedHashMap<>(); final int[] vertexId = {0}; dag.forEach(v -> vertexIdMap.put(v.getName(), vertexId[0]++)); return vertexIdMap; }
/** * Returns a string representation of the DAG. * * @param defaultLocalParallelism the local parallelism that will be shown if * neither overridden on the vertex nor the * preferred parallelism is defined by * meta-supplier */ @Nonnull public String toString(int defaultLocalParallelism) { final StringBuilder b = new StringBuilder("dag\n"); for (Vertex v : this) { b.append(" .vertex(\"").append(v.getName()).append("\")"); int localParallelism = getLocalParallelism(defaultLocalParallelism, v); if (localParallelism != -1) { b.append(".localParallelism(").append(localParallelism).append(')'); } b.append('\n'); } for (Edge e : edges) { b.append(" .edge(").append(e).append(")\n"); } return b.toString(); }
private void rewriteDagWithSnapshotRestore(DAG dag, long snapshotId, String mapName) { IMap<Object, Object> snapshotMap = nodeEngine.getHazelcastInstance().getMap(mapName); snapshotId = SnapshotValidator.validateSnapshot(snapshotId, jobIdString(), snapshotMap); logger.info("State of " + jobIdString() + " will be restored from snapshot " + snapshotId + ", map=" + mapName); List<Vertex> originalVertices = new ArrayList<>(); dag.iterator().forEachRemaining(originalVertices::add); Map<String, Integer> vertexToOrdinal = new HashMap<>(); Vertex readSnapshotVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "read", readMapP(mapName)); long finalSnapshotId = snapshotId; Vertex explodeVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "explode", () -> new ExplodeSnapshotP(vertexToOrdinal, finalSnapshotId)); dag.edge(between(readSnapshotVertex, explodeVertex).isolated()); int index = 0; // add the edges for (Vertex userVertex : originalVertices) { vertexToOrdinal.put(userVertex.getName(), index); int destOrdinal = dag.getInboundEdges(userVertex.getName()).size(); dag.edge(new SnapshotRestoreEdge(explodeVertex, index, userVertex, destOrdinal)); index++; } }
metaSupplier.preferredLocalParallelism(), defaultParallelism); final int totalParallelism = localParallelism * clusterSize; final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed); final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed); final ILogger logger = nodeEngine.getLogger(String.format("%s.%s#ProcessorMetaSupplier", metaSupplier.getClass().getName(), vertex.getName())); try { metaSupplier.init(new MetaSupplierCtx(instance, jobId, executionId, jobConfig, logger, vertex.getName(), localParallelism, totalParallelism, clusterSize)); } catch (Exception e) { throw sneakyThrow(e); for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) { final ProcessorSupplier processorSupplier = procSupplierFn.apply(e.getKey().getAddress()); checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\''); final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, localParallelism); vertexDef.addInboundEdges(inbound); vertexDef.addOutboundEdges(outbound);
for (Vertex v : this) { JsonObject vertex = new JsonObject(); vertex.add("name", v.getName()); vertex.add("parallelism", getLocalParallelism(defaultLocalParallelism, v)); vertices.add(vertex);