public StageLinkage(PlanFragmentId fragmentId, ExchangeLocationsConsumer parent, Set<SqlStageExecution> children) { this.currentStageFragmentId = fragmentId; this.parent = parent; this.childOutputBufferManagers = children.stream() .map(childStage -> { PartitioningHandle partitioningHandle = childStage.getFragment().getPartitioningScheme().getPartitioning().getHandle(); if (partitioningHandle.equals(FIXED_BROADCAST_DISTRIBUTION)) { return new BroadcastOutputBufferManager(childStage::setOutputBuffers); } else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { return new ScaledOutputBufferManager(childStage::setOutputBuffers); } else { int partitionCount = Ints.max(childStage.getFragment().getPartitioningScheme().getBucketToPartition().get()) + 1; return new PartitionedOutputBufferManager(partitioningHandle, partitionCount, childStage::setOutputBuffers); } }) .collect(toImmutableSet()); this.childStageIds = children.stream() .map(SqlStageExecution::getStageId) .collect(toImmutableSet()); }
public StageLinkage(PlanFragmentId fragmentId, ExchangeLocationsConsumer parent, Set<SqlStageExecution> children) { this.currentStageFragmentId = fragmentId; this.parent = parent; this.childOutputBufferManagers = children.stream() .map(childStage -> { PartitioningHandle partitioningHandle = childStage.getFragment().getPartitioningScheme().getPartitioning().getHandle(); if (partitioningHandle.equals(FIXED_BROADCAST_DISTRIBUTION)) { return new BroadcastOutputBufferManager(childStage::setOutputBuffers); } else if (partitioningHandle.equals(SCALED_WRITER_DISTRIBUTION)) { return new ScaledOutputBufferManager(childStage::setOutputBuffers); } else { int partitionCount = Ints.max(childStage.getFragment().getPartitioningScheme().getBucketToPartition().get()) + 1; return new PartitionedOutputBufferManager(partitioningHandle, partitionCount, childStage::setOutputBuffers); } }) .collect(toImmutableSet()); this.childStageIds = children.stream() .map(SqlStageExecution::getStageId) .collect(toImmutableSet()); }
PartitioningScheme partitioningScheme = fragment.getPartitioningScheme(); builder.append(indentString(1)) .append(format("Output layout: [%s]\n",
private static PlanFragment createExchangePlanFragment(String name, PlanFragment... fragments) { PlanNode planNode = new RemoteSourceNode( new PlanNodeId(name + "_id"), Stream.of(fragments) .map(PlanFragment::getId) .collect(toImmutableList()), fragments[0].getPartitioningScheme().getOutputLayout(), Optional.empty(), REPARTITION); return createFragment(planNode); }
private static PlanFragment createExchangePlanFragment(String name, PlanFragment... fragments) { PlanNode planNode = new RemoteSourceNode( new PlanNodeId(name + "_id"), Stream.of(fragments) .map(PlanFragment::getId) .collect(toImmutableList()), fragments[0].getPartitioningScheme().getOutputLayout(), Optional.empty(), REPARTITION); return createFragment(planNode); }
PartitioningScheme partitioningScheme = fragment.getPartitioningScheme(); builder.append(indentString(1)) .append(format("Output layout: [%s]\n",
fragment.getRoot(), TypeProvider.copyOf(fragment.getSymbols()), fragment.getPartitioningScheme(), fragment.getStageExecutionStrategy(), fragment.getPartitionedSources(),
fragment.getRoot(), TypeProvider.copyOf(fragment.getSymbols()), fragment.getPartitioningScheme(), fragment.getStageExecutionDescriptor(), fragment.getPartitionedSources(),
private static PlanFragment createUnionPlanFragment(String name, PlanFragment... fragments) { PlanNode planNode = new UnionNode( new PlanNodeId(name + "_id"), Stream.of(fragments) .map(fragment -> new RemoteSourceNode(new PlanNodeId(fragment.getId().toString()), fragment.getId(), fragment.getPartitioningScheme().getOutputLayout(), Optional.empty(), REPARTITION)) .collect(toImmutableList()), ImmutableListMultimap.of(), ImmutableList.of()); return createFragment(planNode); }
private static PlanFragment createUnionPlanFragment(String name, PlanFragment... fragments) { PlanNode planNode = new UnionNode( new PlanNodeId(name + "_id"), Stream.of(fragments) .map(fragment -> new RemoteSourceNode(new PlanNodeId(fragment.getId().toString()), fragment.getId(), fragment.getPartitioningScheme().getOutputLayout(), Optional.empty(), REPARTITION)) .collect(toImmutableList()), ImmutableListMultimap.of(), ImmutableList.of()); return createFragment(planNode); }
stageExecutionDescriptor, subplan.getFragment().getRoot(), subplan.getFragment().getPartitioningScheme().getOutputLayout(), plan.getTypes(), subplan.getFragment().getPartitionedSources(),
stageExecutionStrategy, subplan.getFragment().getRoot(), subplan.getFragment().getPartitioningScheme().getOutputLayout(), plan.getTypes(), subplan.getFragment().getPartitionedSources(),
newRoot = SimplePlanRewriter.rewriteWith(partitioningHandleReassigner, newRoot); PartitioningScheme outputPartitioningScheme = fragment.getPartitioningScheme(); Partitioning newOutputPartitioning = outputPartitioningScheme.getPartitioning(); if (outputPartitioningScheme.getPartitioning().getHandle().getConnectorId().isPresent()) {
newRoot = SimplePlanRewriter.rewriteWith(partitioningHandleReassigner, newRoot); PartitioningScheme outputPartitioningScheme = fragment.getPartitioningScheme(); Partitioning newOutputPartitioning = outputPartitioningScheme.getPartitioning(); if (outputPartitioningScheme.getPartitioning().getHandle().getConnectorId().isPresent()) {
PartitioningHandle partitioningHandle = plan.getRoot().getFragment().getPartitioningScheme().getPartitioning().getHandle(); OutputBuffers rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle) .withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID)
PartitioningHandle partitioningHandle = plan.getRoot().getFragment().getPartitioningScheme().getPartitioning().getHandle(); OutputBuffers rootOutputBuffers = createInitialEmptyOutputBuffers(partitioningHandle) .withBuffer(OUTPUT_BUFFER_ID, BROADCAST_PARTITION_ID)