/** * Determines the range of {@link ScheduleGroupProperty} value that will prevent collision * with the existing {@link ScheduleGroupProperty}. * @param irVertexCollection collection of {@link IRVertex} * @return the minimum value for the {@link ScheduleGroupProperty} that won't collide with the existing values */ private int getNextScheudleGroup(final Collection<IRVertex> irVertexCollection) { int nextScheduleGroup = 0; for (final IRVertex irVertex : irVertexCollection) { final Optional<Integer> scheduleGroup = irVertex.getPropertyValue(ScheduleGroupProperty.class); if (scheduleGroup.isPresent()) { nextScheduleGroup = Math.max(scheduleGroup.get() + 1, nextScheduleGroup); } } return nextScheduleGroup; }
/** * Determines the range of {@link ScheduleGroupProperty} value that will prevent collision * with the existing {@link ScheduleGroupProperty}. * @param irVertexCollection collection of {@link IRVertex} * @return the minimum value for the {@link ScheduleGroupProperty} that won't collide with the existing values */ private int getNextScheudleGroup(final Collection<IRVertex> irVertexCollection) { int nextScheduleGroup = 0; for (final IRVertex irVertex : irVertexCollection) { final Optional<Integer> scheduleGroup = irVertex.getPropertyValue(ScheduleGroupProperty.class); if (scheduleGroup.isPresent()) { nextScheduleGroup = Math.max(scheduleGroup.get() + 1, nextScheduleGroup); } } return nextScheduleGroup; }
/** * Returns the maximum parallelism of the vertices of a IR DAG. * @param dag dag to observe. * @return the maximum parallelism, or 1 by default. */ private static int getMaxParallelism(final DAG<IRVertex, IREdge> dag) { return dag.getVertices().stream() .mapToInt(vertex -> vertex.getPropertyValue(ParallelismProperty.class).orElse(1)) .max().orElse(1); }
/** * Returns the maximum parallelism of the vertices of a IR DAG. * @param dag dag to observe. * @return the maximum parallelism, or 1 by default. */ private static int getMaxParallelism(final DAG<IRVertex, IREdge> dag) { return dag.getVertices().stream() .mapToInt(vertex -> vertex.getPropertyValue(ParallelismProperty.class).orElse(1)) .max().orElse(1); }
/** * checks if the edge is from transient container to a reserved container. * @param irEdge edge to check. * @return whether or not the edge satisfies the condition. */ static boolean fromTransientToReserved(final IREdge irEdge) { return ResourcePriorityProperty.TRANSIENT .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get()) && ResourcePriorityProperty.RESERVED .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get()); }
/** * checks if the edge is from reserved container to a transient container. * @param irEdge edge to check. * @return whether or not the edge satisfies the condition. */ static boolean fromReservedToTransient(final IREdge irEdge) { return ResourcePriorityProperty.RESERVED .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get()) && ResourcePriorityProperty.TRANSIENT .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get()); } }
/** * checks if the edge is from reserved container to a transient container. * @param irEdge edge to check. * @return whether or not the edge satisfies the condition. */ static boolean fromReservedToTransient(final IREdge irEdge) { return ResourcePriorityProperty.RESERVED .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get()) && ResourcePriorityProperty.TRANSIENT .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get()); } }
/** * checks if the edge is from transient container to a reserved container. * @param irEdge edge to check. * @return whether or not the edge satisfies the condition. */ static boolean fromTransientToReserved(final IREdge irEdge) { return ResourcePriorityProperty.TRANSIENT .equals(irEdge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get()) && ResourcePriorityProperty.RESERVED .equals(irEdge.getDst().getPropertyValue(ResourcePriorityProperty.class).get()); }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> v.setProperty(ResourceSlotProperty.of(true))); return dag; } }
private int getNumOfPipeToWait(final RuntimeEdge runtimeEdge) { final int dstParallelism = ((StageEdge) runtimeEdge).getDstIRVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException()); final CommunicationPatternProperty.Value commPattern = ((StageEdge) runtimeEdge) .getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()); return commPattern.equals(CommunicationPatternProperty.Value.OneToOne) ? 1 : dstParallelism; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> v.setProperty(ResourceSlotProperty.of(true))); return dag; } }
/** * Checks whether the irEdges are all from reserved containers. * @param irEdges irEdges to check. * @return whether of not they are from reserved containers. */ private boolean allO2OFromReserved(final List<IREdge> irEdges) { return irEdges.stream() .allMatch(edge -> CommunicationPatternProperty.Value.OneToOne.equals( edge.getPropertyValue(CommunicationPatternProperty.class).get()) && edge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get().equals( ResourcePriorityProperty.RESERVED)); } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex, if ResourceLocalityProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceLocalityProperty.class).isPresent()) .forEach(v -> v.setProperty(ResourceLocalityProperty.of(true))); return dag; } }
/** * Checks whether the irEdges are all from reserved containers. * @param irEdges irEdges to check. * @return whether of not they are from reserved containers. */ private boolean allO2OFromReserved(final List<IREdge> irEdges) { return irEdges.stream() .allMatch(edge -> CommunicationPatternProperty.Value.OneToOne.equals( edge.getPropertyValue(CommunicationPatternProperty.class).get()) && edge.getSrc().getPropertyValue(ResourcePriorityProperty.class).get().equals( ResourcePriorityProperty.RESERVED)); } }
private int getNumOfPipeToWait(final RuntimeEdge runtimeEdge) { final int dstParallelism = ((StageEdge) runtimeEdge).getDstIRVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException()); final CommunicationPatternProperty.Value commPattern = ((StageEdge) runtimeEdge) .getPropertyValue(CommunicationPatternProperty.class) .orElseThrow(() -> new IllegalStateException()); return commPattern.equals(CommunicationPatternProperty.Value.OneToOne) ? 1 : dstParallelism; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex, if ResourceLocalityProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceLocalityProperty.class).isPresent()) .forEach(v -> v.setProperty(ResourceLocalityProperty.of(true))); return dag; } }
static int getSourceParallelism(final InputReader inputReader) { return inputReader.getSrcIrVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException(inputReader.getSrcIrVertex().getId())); } }
static int getSourceParallelism(final InputReader inputReader) { return inputReader.getSrcIrVertex().getPropertyValue(ParallelismProperty.class) .orElseThrow(() -> new IllegalStateException(inputReader.getSrcIrVertex().getId())); } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { // On every vertex that receive push edge, if ResourceSlotProperty is not set, put it as false. // For other vertices, if ResourceSlotProperty is not set, put it as true. dag.getVertices().stream() .filter(v -> !v.getPropertyValue(ResourceSlotProperty.class).isPresent()) .forEach(v -> { if (dag.getIncomingEdgesOf(v).stream().anyMatch( e -> e.getPropertyValue(DataFlowProperty.class) .orElseThrow(() -> new RuntimeException(String.format("DataFlowProperty for %s must be set", e.getId()))).equals(DataFlowProperty.Value.Push))) { v.setPropertyPermanently(ResourceSlotProperty.of(false)); } else { v.setPropertyPermanently(ResourceSlotProperty.of(true)); } }); return dag; } }
@Override public DAG<IRVertex, IREdge> apply(final DAG<IRVertex, IREdge> dag) { dag.topologicalDo(dst -> dag.getIncomingEdgesOf(dst).forEach(edge -> { if (CommunicationPatternProperty.Value.Shuffle .equals(edge.getPropertyValue(CommunicationPatternProperty.class).get())) { final int parallelism = dst.getPropertyValue(ParallelismProperty.class).get(); final Map<Integer, KeyRange> metric = new HashMap<>(); for (int i = 0; i < parallelism; i++) { metric.put(i, HashRange.of(i, i + 1, false)); } edge.setProperty(DataSkewMetricProperty.of(new DataSkewMetricFactory(metric))); } })); return dag; } }