@Override public Void visitRemoteSource(RemoteSourceNode node, Void context) { node.getSourceFragmentIds() .forEach(this::processFragment); return null; }
private SqlStageExecution(StageStateMachine stateMachine, RemoteTaskFactory remoteTaskFactory, NodeTaskMap nodeTaskMap, boolean summarizeTaskInfo, Executor executor, FailureDetector failureDetector) { this.stateMachine = stateMachine; this.remoteTaskFactory = requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"); this.nodeTaskMap = requireNonNull(nodeTaskMap, "nodeTaskMap is null"); this.summarizeTaskInfo = summarizeTaskInfo; this.executor = requireNonNull(executor, "executor is null"); this.failureDetector = requireNonNull(failureDetector, "failureDetector is null"); ImmutableMap.Builder<PlanFragmentId, RemoteSourceNode> fragmentToExchangeSource = ImmutableMap.builder(); for (RemoteSourceNode remoteSourceNode : stateMachine.getFragment().getRemoteSourceNodes()) { for (PlanFragmentId planFragmentId : remoteSourceNode.getSourceFragmentIds()) { fragmentToExchangeSource.put(planFragmentId, remoteSourceNode); } } this.exchangeSources = fragmentToExchangeSource.build(); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Void context) { for (PlanFragmentId planFragmentId : node.getSourceFragmentIds()) { PlanFragment target = fragmentsById.get(planFragmentId); printEdge(node, target.getRoot()); } return null; }
@Override public Set<PlanFragmentId> visitRemoteSource(RemoteSourceNode node, PlanFragmentId currentFragmentId) { ImmutableSet.Builder<PlanFragmentId> sources = ImmutableSet.builder(); Set<PlanFragmentId> previousFragmentSources = ImmutableSet.of(); for (PlanFragmentId remoteFragment : node.getSourceFragmentIds()) { // this current fragment depends on the remote fragment graph.addEdge(currentFragmentId, remoteFragment); // get all sources for the remote fragment Set<PlanFragmentId> remoteFragmentSources = processFragment(remoteFragment); sources.addAll(remoteFragmentSources); // For UNION there can be multiple sources. // Link the previous source to the current source, so we only // schedule one at a time. addEdges(previousFragmentSources, remoteFragmentSources); previousFragmentSources = remoteFragmentSources; } return sources.build(); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Integer indent) { print(indent, "- Remote%s[%s] => [%s]", node.getOrderingScheme().isPresent() ? "Merge" : "Source", Joiner.on(',').join(node.getSourceFragmentIds()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); return null; }
public synchronized void addExchangeLocations(PlanFragmentId fragmentId, Set<RemoteTask> sourceTasks, boolean noMoreExchangeLocations) { requireNonNull(fragmentId, "fragmentId is null"); requireNonNull(sourceTasks, "sourceTasks is null"); RemoteSourceNode remoteSource = exchangeSources.get(fragmentId); checkArgument(remoteSource != null, "Unknown remote source %s. Known sources are %s", fragmentId, exchangeSources.keySet()); this.sourceTasks.putAll(remoteSource.getId(), sourceTasks); for (RemoteTask task : getAllTasks()) { ImmutableMultimap.Builder<PlanNodeId, Split> newSplits = ImmutableMultimap.builder(); for (RemoteTask sourceTask : sourceTasks) { URI exchangeLocation = sourceTask.getTaskStatus().getSelf(); newSplits.put(remoteSource.getId(), createRemoteSplitFor(task.getTaskId(), exchangeLocation)); } task.addSplits(newSplits.build()); } if (noMoreExchangeLocations) { completeSourceFragments.add(fragmentId); // is the source now complete? if (completeSourceFragments.containsAll(remoteSource.getSourceFragmentIds())) { completeSources.add(remoteSource.getId()); for (RemoteTask task : getAllTasks()) { task.noMoreSplits(remoteSource.getId()); } } } }
@Override public PlanNode visitRemoteSource(RemoteSourceNode node, RewriteContext<Void> context) { return new RemoteSourceNode( node.getId(), node.getSourceFragmentIds(), canonicalizeAndDistinct(node.getOutputSymbols()), node.getOrderingScheme().map(this::canonicalizeAndDistinct), node.getExchangeType()); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Void context) { node.getSourceFragmentIds() .forEach(this::processFragment); return null; }
public SqlStageExecution(StageStateMachine stateMachine, RemoteTaskFactory remoteTaskFactory, NodeTaskMap nodeTaskMap) { this.stateMachine = stateMachine; this.remoteTaskFactory = requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"); this.nodeTaskMap = requireNonNull(nodeTaskMap, "nodeTaskMap is null"); ImmutableMap.Builder<PlanFragmentId, RemoteSourceNode> fragmentToExchangeSource = ImmutableMap.builder(); for (RemoteSourceNode remoteSourceNode : stateMachine.getFragment().getRemoteSourceNodes()) { for (PlanFragmentId planFragmentId : remoteSourceNode.getSourceFragmentIds()) { fragmentToExchangeSource.put(planFragmentId, remoteSourceNode); } } this.exchangeSources = fragmentToExchangeSource.build(); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Integer indent) { print(indent, "- RemoteSource[%s] => [%s]", Joiner.on(',').join(node.getSourceFragmentIds()), formatOutputs(node.getOutputSymbols())); return null; }
@Override public Void visitRemoteSource(RemoteSourceNode node, Void context) { for (PlanFragmentId planFragmentId : node.getSourceFragmentIds()) { PlanFragment target = fragmentsById.get(planFragmentId); printEdge(node, target.getRoot()); } return null; }
@Override public Set<PlanFragmentId> visitRemoteSource(RemoteSourceNode node, PlanFragmentId currentFragmentId) { ImmutableSet.Builder<PlanFragmentId> sources = ImmutableSet.builder(); Set<PlanFragmentId> previousFragmentSources = ImmutableSet.of(); for (PlanFragmentId remoteFragment : node.getSourceFragmentIds()) { // this current fragment depends on the remote fragment graph.addEdge(currentFragmentId, remoteFragment); // get all sources for the remote fragment Set<PlanFragmentId> remoteFragmentSources = processFragment(remoteFragment); sources.addAll(remoteFragmentSources); // For UNION there can be multiple sources. // Link the previous source to the current source, so we only // schedule one at a time. addEdges(previousFragmentSources, remoteFragmentSources); previousFragmentSources = remoteFragmentSources; } return sources.build(); }
public synchronized void addExchangeLocations(PlanFragmentId fragmentId, Set<URI> exchangeLocations, boolean noMoreExchangeLocations) { requireNonNull(fragmentId, "fragmentId is null"); requireNonNull(exchangeLocations, "exchangeLocations is null"); RemoteSourceNode remoteSource = exchangeSources.get(fragmentId); checkArgument(remoteSource != null, "Unknown remote source %s. Known sources are %s", fragmentId, exchangeSources.keySet()); this.exchangeLocations.putAll(remoteSource.getId(), exchangeLocations); for (RemoteTask task : getAllTasks()) { ImmutableMultimap.Builder<PlanNodeId, Split> newSplits = ImmutableMultimap.builder(); for (URI exchangeLocation : exchangeLocations) { newSplits.put(remoteSource.getId(), createRemoteSplitFor(task.getTaskInfo().getTaskId(), exchangeLocation)); } task.addSplits(newSplits.build()); } if (noMoreExchangeLocations) { completeSourceFragments.add(fragmentId); // is the source now complete? if (completeSourceFragments.containsAll(remoteSource.getSourceFragmentIds())) { completeSources.add(remoteSource.getId()); for (RemoteTask task : getAllTasks()) { task.noMoreSplits(remoteSource.getId()); } } } }
@Override public PlanNode visitRemoteSource(RemoteSourceNode node, RewriteContext<Void> context) { return new RemoteSourceNode(node.getId(), node.getSourceFragmentIds(), canonicalizeAndDistinct(node.getOutputSymbols())); }