private synchronized void noMoreBuffers() { if (outputBuffers.isNoMoreBufferIds()) { // verify all created buffers have been declared SetView<OutputBufferId> undeclaredCreatedBuffers = Sets.difference(buffers.keySet(), outputBuffers.getBuffers().keySet()); checkState(undeclaredCreatedBuffers.isEmpty(), "Final output buffers does not contain all created buffer ids: %s", undeclaredCreatedBuffers); } }
private void noMoreBuffers() { checkState(!Thread.holdsLock(this), "Can not set no more buffers while holding a lock on this"); List<SerializedPageReference> pages; synchronized (this) { pages = ImmutableList.copyOf(initialPagesForNewBuffers); initialPagesForNewBuffers.clear(); if (outputBuffers.isNoMoreBufferIds()) { // verify all created buffers have been declared SetView<OutputBufferId> undeclaredCreatedBuffers = Sets.difference(buffers.keySet(), outputBuffers.getBuffers().keySet()); checkState(undeclaredCreatedBuffers.isEmpty(), "Final output buffers does not contain all created buffer ids: %s", undeclaredCreatedBuffers); } } // dereference outside of synchronized to avoid making a callback while holding a lock pages.forEach(SerializedPageReference::dereferencePage); }
if (outputBuffers.isNoMoreBufferIds()) {
public PartitionedOutputBuffer( String taskInstanceId, StateMachine<BufferState> state, OutputBuffers outputBuffers, DataSize maxBufferSize, Supplier<LocalMemoryContext> systemMemoryContextSupplier, Executor notificationExecutor) { this.state = requireNonNull(state, "state is null"); requireNonNull(outputBuffers, "outputBuffers is null"); checkArgument(outputBuffers.getType() == PARTITIONED, "Expected a PARTITIONED output buffer descriptor"); checkArgument(outputBuffers.isNoMoreBufferIds(), "Expected a final output buffer descriptor"); this.outputBuffers = outputBuffers; this.memoryManager = new OutputBufferMemoryManager( requireNonNull(maxBufferSize, "maxBufferSize is null").toBytes(), requireNonNull(systemMemoryContextSupplier, "systemMemoryContextSupplier is null"), requireNonNull(notificationExecutor, "notificationExecutor is null")); ImmutableList.Builder<ClientBuffer> partitions = ImmutableList.builder(); for (OutputBufferId bufferId : outputBuffers.getBuffers().keySet()) { ClientBuffer partition = new ClientBuffer(taskInstanceId, bufferId); partitions.add(partition); } this.partitions = partitions.build(); state.compareAndSet(OPEN, NO_MORE_BUFFERS); state.compareAndSet(NO_MORE_PAGES, FLUSHING); checkFlushComplete(); }
private synchronized ClientBuffer getBuffer(OutputBufferId id) { ClientBuffer buffer = buffers.get(id); if (buffer != null) { return buffer; } // NOTE: buffers are allowed to be created in the FINISHED state because destroy() can move to the finished state // without a clean "no-more-buffers" message from the scheduler. This happens with limit queries and is ok because // the buffer will be immediately destroyed. checkState(state.get().canAddBuffers() || !outputBuffers.isNoMoreBufferIds(), "No more buffers already set"); // NOTE: buffers are allowed to be created before they are explicitly declared by setOutputBuffers // When no-more-buffers is set, we verify that all created buffers have been declared buffer = new ClientBuffer(taskInstanceId, id); // buffer may have finished immediately before calling this method if (state.get() == FINISHED) { buffer.destroy(); } buffers.put(id, buffer); return buffer; }
if (outputBuffers.isNoMoreBufferIds()) {
private static void assertOutputBuffers(OutputBuffers outputBuffers) { assertNotNull(outputBuffers); assertTrue(outputBuffers.getVersion() > 0); assertTrue(outputBuffers.isNoMoreBufferIds()); Map<OutputBufferId, Integer> buffers = outputBuffers.getBuffers(); assertEquals(buffers.size(), 4); for (int partition = 0; partition < 4; partition++) { assertEquals(buffers.get(new OutputBufferId(partition)), Integer.valueOf(partition)); } } }
checkState(state.canAddBuffers() || !outputBuffers.isNoMoreBufferIds(), "No more buffers already set");
if (outputBuffers.isNoMoreBufferIds()) { this.state.compareAndSet(OPEN, NO_MORE_BUFFERS); this.state.compareAndSet(NO_MORE_PAGES, FLUSHING);
@Override public void setOutputBuffers(OutputBuffers newOutputBuffers) { checkState(!Thread.holdsLock(this), "Can not set output buffers while holding a lock on this"); requireNonNull(newOutputBuffers, "newOutputBuffers is null"); synchronized (this) { // ignore buffers added after query finishes, which can happen when a query is canceled // also ignore old versions, which is normal BufferState state = this.state.get(); if (state.isTerminal() || outputBuffers.getVersion() >= newOutputBuffers.getVersion()) { return; } // verify this is valid state change outputBuffers.checkValidTransition(newOutputBuffers); outputBuffers = newOutputBuffers; // add the new buffers for (OutputBufferId outputBufferId : outputBuffers.getBuffers().keySet()) { getBuffer(outputBufferId); } // update state if no more buffers is set if (outputBuffers.isNoMoreBufferIds()) { this.state.compareAndSet(OPEN, NO_MORE_BUFFERS); this.state.compareAndSet(NO_MORE_PAGES, FLUSHING); } } if (!state.get().canAddBuffers()) { noMoreBuffers(); } checkFlushComplete(); }
@Override public void addOutputBuffers(List<OutputBuffer> newBuffers, boolean noMoreBuffers) { synchronized (this) { if (outputBuffers.isNoMoreBufferIds()) { // a stage can move to a final state (e.g., failed) while scheduling, so ignore // the new buffers return; } OutputBuffers originalOutputBuffers = outputBuffers; // Note: it does not matter which partition id the task is using, in broadcast all tasks read from the same partition for (OutputBuffer newBuffer : newBuffers) { outputBuffers = outputBuffers.withBuffer(newBuffer.getBufferId(), BROADCAST_PARTITION_ID); } if (noMoreBuffers) { outputBuffers = outputBuffers.withNoMoreBufferIds(); } // don't update if nothing changed if (outputBuffers == originalOutputBuffers) { return; } } outputBufferTarget.accept(outputBuffers); } }
if (outputBuffers.isNoMoreBufferIds()) { state.compareAndSet(OPEN, NO_MORE_BUFFERS); state.compareAndSet(NO_MORE_PAGES, FLUSHING);
assertTrue(outputBuffers.isNoMoreBufferIds());