private synchronized void noMoreBuffers() { if (outputBuffers.isNoMoreBufferIds()) { // verify all created buffers have been declared SetView<OutputBufferId> undeclaredCreatedBuffers = Sets.difference(buffers.keySet(), outputBuffers.getBuffers().keySet()); checkState(undeclaredCreatedBuffers.isEmpty(), "Final output buffers does not contain all created buffer ids: %s", undeclaredCreatedBuffers); } }
private void noMoreBuffers() { checkState(!Thread.holdsLock(this), "Can not set no more buffers while holding a lock on this"); List<SerializedPageReference> pages; synchronized (this) { pages = ImmutableList.copyOf(initialPagesForNewBuffers); initialPagesForNewBuffers.clear(); if (outputBuffers.isNoMoreBufferIds()) { // verify all created buffers have been declared SetView<OutputBufferId> undeclaredCreatedBuffers = Sets.difference(buffers.keySet(), outputBuffers.getBuffers().keySet()); checkState(undeclaredCreatedBuffers.isEmpty(), "Final output buffers does not contain all created buffer ids: %s", undeclaredCreatedBuffers); } } // dereference outside of synchronized to avoid making a callback while holding a lock pages.forEach(SerializedPageReference::dereferencePage); }
public PartitionedOutputBufferManager(PartitioningHandle partitioningHandle, int partitionCount, Consumer<OutputBuffers> outputBufferTarget) { checkArgument(partitionCount >= 1, "partitionCount must be at least 1"); ImmutableMap.Builder<OutputBufferId, Integer> partitions = ImmutableMap.builder(); for (int partition = 0; partition < partitionCount; partition++) { partitions.put(new OutputBufferId(partition), partition); } OutputBuffers outputBuffers = createInitialEmptyOutputBuffers(requireNonNull(partitioningHandle, "partitioningHandle is null")) .withBuffers(partitions.build()) .withNoMoreBufferIds(); outputBufferTarget.accept(outputBuffers); this.outputBuffers = outputBuffers.getBuffers(); }
public PartitionedOutputBuffer( String taskInstanceId, StateMachine<BufferState> state, OutputBuffers outputBuffers, DataSize maxBufferSize, Supplier<LocalMemoryContext> systemMemoryContextSupplier, Executor notificationExecutor) { this.state = requireNonNull(state, "state is null"); requireNonNull(outputBuffers, "outputBuffers is null"); checkArgument(outputBuffers.getType() == PARTITIONED, "Expected a PARTITIONED output buffer descriptor"); checkArgument(outputBuffers.isNoMoreBufferIds(), "Expected a final output buffer descriptor"); this.outputBuffers = outputBuffers; this.memoryManager = new OutputBufferMemoryManager( requireNonNull(maxBufferSize, "maxBufferSize is null").toBytes(), requireNonNull(systemMemoryContextSupplier, "systemMemoryContextSupplier is null"), requireNonNull(notificationExecutor, "notificationExecutor is null")); ImmutableList.Builder<ClientBuffer> partitions = ImmutableList.builder(); for (OutputBufferId bufferId : outputBuffers.getBuffers().keySet()) { ClientBuffer partition = new ClientBuffer(taskInstanceId, bufferId); partitions.add(partition); } this.partitions = partitions.build(); state.compareAndSet(OPEN, NO_MORE_BUFFERS); state.compareAndSet(NO_MORE_PAGES, FLUSHING); checkFlushComplete(); }
OutputBufferId rootBufferId = Iterables.getOnlyElement(rootOutputBuffers.getBuffers().keySet()); List<SqlStageExecution> stages = createStages( (fragmentId, tasks, noMoreExchangeLocations) -> updateQueryOutputLocations(queryStateMachine, rootBufferId, tasks, noMoreExchangeLocations),
private static void assertOutputBuffers(OutputBuffers outputBuffers) { assertNotNull(outputBuffers); assertTrue(outputBuffers.getVersion() > 0); assertTrue(outputBuffers.isNoMoreBufferIds()); Map<OutputBufferId, Integer> buffers = outputBuffers.getBuffers(); assertEquals(buffers.size(), 4); for (int partition = 0; partition < 4; partition++) { assertEquals(buffers.get(new OutputBufferId(partition)), Integer.valueOf(partition)); } } }
.sum(); List<BufferInfo> bufferStates = outputBuffers.getBuffers() .keySet().stream() .map(outputId -> new BufferInfo(outputId, false, 0, 0, PageBufferInfo.empty()))
for (Entry<OutputBufferId, Integer> entry : outputBuffers.getBuffers().entrySet()) { if (!buffers.containsKey(entry.getKey())) { ClientBuffer buffer = getBuffer(entry.getKey());
@Override public void setOutputBuffers(OutputBuffers newOutputBuffers) { checkState(!Thread.holdsLock(this), "Can not set output buffers while holding a lock on this"); requireNonNull(newOutputBuffers, "newOutputBuffers is null"); synchronized (this) { // ignore buffers added after query finishes, which can happen when a query is canceled // also ignore old versions, which is normal BufferState state = this.state.get(); if (state.isTerminal() || outputBuffers.getVersion() >= newOutputBuffers.getVersion()) { return; } // verify this is valid state change outputBuffers.checkValidTransition(newOutputBuffers); outputBuffers = newOutputBuffers; // add the new buffers for (OutputBufferId outputBufferId : outputBuffers.getBuffers().keySet()) { getBuffer(outputBufferId); } // update state if no more buffers is set if (outputBuffers.isNoMoreBufferIds()) { this.state.compareAndSet(OPEN, NO_MORE_BUFFERS); this.state.compareAndSet(NO_MORE_PAGES, FLUSHING); } } if (!state.get().canAddBuffers()) { noMoreBuffers(); } checkFlushComplete(); }
for (Entry<TaskId, Integer> entry : outputBuffers.getBuffers().entrySet()) { TaskId bufferId = entry.getKey(); if (!namedBuffers.containsKey(bufferId)) {
List<BufferInfo> bufferStates = outputBuffers.getBuffers() .keySet().stream() .map(outputId -> new BufferInfo(outputId, false, 0, 0, PageBufferInfo.empty()))
assertTrue(outputBuffers.isNoMoreBufferIds()); Map<TaskId, Integer> buffers = outputBuffers.getBuffers(); assertEquals(buffers.size(), 4); assertEquals(buffers.get(new TaskId(STAGE_ID, "0")), Integer.valueOf(2));