@Override public boolean remove() { try { removeInternal(MAX_RESIZE_INTERNAL_STACK_DEPTH).getSource().release(); totalPoolSize.getAndDecrement(); } catch (PoolResourceException e) { return false; } return true; }
/** * Serializes given object to {@link BufferedItemSource} using given {@link ObjectWriter} * * @param source item to serialize * @param objectWriter writer to be used to serialize given item * @throws IllegalStateException if underlying pool cannot provide {@link BufferedItemSource} * @throws IllegalArgumentException if serialization failed * @return {@link BufferedItemSource} with serialized event */ @Override public ItemSource create(Object source, ObjectWriter objectWriter) { ItemSource<ByteBuf> pooled; try { pooled = bufferedItemSourcePool.getPooled(); } catch (PoolResourceException e) { // FIXME: stop throwing and redirect to failover policy here throw new IllegalStateException(e); } try { objectWriter.writeValue((DataOutput)new ByteBufOutputStream(pooled.getSource()), source); return pooled; } catch (IOException e) { pooled.release(); throw new IllegalArgumentException(e); } }
public void release() { source.release(); }
/** * Clears underlying collection of actions and releases all {@link ItemSource} instances. * <p>MUST be called when request is completed. Otherwise it may lead to excessive resource usage and memory leaks */ public void completed() { for (BulkableAction bulkableAction : actions) { ((BufferedIndex)bulkableAction).release(); } actions.clear(); bulkSource.release(); }
@Override public void shutdown() { objectPool.forEach(pooled -> pooled.getSource().release()); objectPool.clear(); executor.shutdown(); }
/** * Serializes given object to {@link BufferedItemSource} using given {@link ObjectWriter} * * @param source item to serialize * @param objectWriter writer to be used to serialize given item * @throws IllegalStateException if underlying pool cannot provide {@link BufferedItemSource} * @throws IllegalArgumentException if serialization failed * @return {@link BufferedItemSource} with serialized event */ @Override public ItemSource create(Object source, ObjectWriter objectWriter) { ItemSource<ByteBuf> pooled; try { pooled = bufferedItemSourcePool.getPooled(); } catch (PoolResourceException e) { // FIXME: stop throwing and redirect to failover policy here throw new IllegalStateException(e); } try { objectWriter.writeValue((DataOutput)new ByteBufOutputStream(pooled.getSource()), source); return pooled; } catch (IOException e) { pooled.release(); throw new IllegalArgumentException(e); } }
@Test public void releaseResetsTheSource() { // given ReleaseCallback callback = mock(ReleaseCallback.class); CompositeByteBuf byteBuf = spy(createDefaultTestByteBuf()); // then ItemSource source = new BufferedItemSource(byteBuf, callback); source.release(); // then verify(byteBuf).clear(); }
@Override public void shutdown() { objectPool.forEach(pooled -> pooled.getSource().release()); objectPool.clear(); executor.shutdown(); }
@Test public void createExceptionReleasesPooledElement() throws IOException, PoolResourceException { // given ItemSourcePool mockedPool = mock(ItemSourcePool.class); ByteBuf byteBuf = createDefaultTestByteBuf(); ItemSource<ByteBuf> bufferedItemSource = spy(new BufferedItemSource(byteBuf, source -> {})); when(mockedPool.getPooled()).thenReturn(bufferedItemSource); PooledItemSourceFactory pooledItemSourceFactory = new PooledItemSourceFactory(mockedPool); LogEvent logEvent = mock(LogEvent.class); ObjectWriter objectWriter = spy(new ObjectMapper().writerFor(LogEvent.class)); doThrow(new IOException("test exception")).when(objectWriter).writeValue(any(DataOutput.class), eq(logEvent)); ItemSource<ByteBuf> result = null; Exception caught = null; // when try { result = pooledItemSourceFactory.create(logEvent, objectWriter); } catch (Exception e) { caught = e; } // then assertNull(result); verify(bufferedItemSource).release(); assertEquals(IllegalArgumentException.class, caught.getClass()); }
@Override public boolean remove() { try { removeInternal(MAX_RESIZE_INTERNAL_STACK_DEPTH).getSource().release(); totalPoolSize.getAndDecrement(); } catch (PoolResourceException e) { return false; } return true; }
@Test public void releaseDelegatesToGivenCallback() { // given ReleaseCallback callback = mock(ReleaseCallback.class); CompositeByteBuf byteBuf = createDefaultTestByteBuf(); // then ItemSource source = new BufferedItemSource(byteBuf, callback); source.release(); // then ArgumentCaptor<ItemSource> captor = ArgumentCaptor.forClass(ItemSource.class); verify(callback).completed(captor.capture()); assertEquals(source, captor.getValue()); assertTrue(source.getSource() == captor.getValue().getSource()); }
@Test public void defaultReleaseCallbackReturnsPooledElement() throws PoolResourceException { // given BufferedItemSourcePool pool = createDefaultTestBufferedItemSourcePool(false); assertEquals(DEFAULT_TEST_INITIAL_POOL_SIZE, pool.getAvailableSize()); pool.incrementPoolSize(); assertEquals(DEFAULT_TEST_INITIAL_POOL_SIZE + 1, pool.getAvailableSize()); // when ItemSource<ByteBuf> itemSource = pool.getPooled(); assertEquals(DEFAULT_TEST_INITIAL_POOL_SIZE, pool.getAvailableSize()); itemSource.release(); // then assertEquals(DEFAULT_TEST_INITIAL_POOL_SIZE + 1, pool.getAvailableSize()); }
/** * Serializes and writes {@link #actions} into {@link #bulkSource} * * @return underlying buffer filled with serialized actions * @throws IOException if serialization failed */ public ByteBuf serializeRequest() throws IOException { ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(bulkSource.getSource()); for (BulkableAction action : actions) { objectWriter.writeValue((DataOutput) byteBufOutputStream, action); byteBufOutputStream.writeByte(LINE_SEPARATOR); ByteBuf source = ((BufferedIndex)action).getSource().getSource(); bulkSource.getSource().writeBytes(source); byteBufOutputStream.writeByte(LINE_SEPARATOR); } return bulkSource.getSource(); }
@Override public Function<Bulk, Boolean> createFailureHandler(FailoverPolicy failover) { return bulk -> { BufferedBulk bufferedBulk = (BufferedBulk)bulk; LOG.warn(String.format("Batch of %s items failed. Redirecting to %s", bufferedBulk.getActions().size(), failover.getClass().getName())); bufferedBulk.getActions().forEach(failedItem -> { ByteBuf byteBuf = ((BufferedIndex) failedItem).source.getSource(); failover.deliver(byteBuf.toString(0, byteBuf.writerIndex(), Charset.defaultCharset())); }); return true; }; }
/** * Creates ONE pooled {@link BufferedItemSource} */ @Override public final void incrementPoolSize() { CompositeByteBuf buffer = new CompositeByteBuf(byteBufAllocator, false, 2).capacity(estimatedSourceSize); objectPool.add(new BufferedItemSource(buffer, bufferedItemSource -> { bufferedItemSource.getSource().clear(); objectPool.add(bufferedItemSource); })); totalPoolSize.getAndIncrement(); }
/** * Creates ONE pooled {@link BufferedItemSource} */ @Override public final void incrementPoolSize() { CompositeByteBuf buffer = new CompositeByteBuf(byteBufAllocator, false, 2).capacity(estimatedSourceSize); objectPool.add(new BufferedItemSource(buffer, bufferedItemSource -> { bufferedItemSource.getSource().clear(); objectPool.add(bufferedItemSource); })); totalPoolSize.getAndIncrement(); }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new IndexRequest(indexName) .type(ACTION_TYPE) .source((String)source.getSource(), XContentType.JSON); }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new IndexRequest(indexName) .type(ACTION_TYPE) .source((String)source.getSource()); }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new IndexRequest(indexName) .type(ACTION_TYPE) .source((String)source.getSource(), XContentType.JSON); }