@Override public void shutdown() { objectPool.forEach(pooled -> pooled.getSource().release()); objectPool.clear(); executor.shutdown(); }
@Override public boolean remove() { try { removeInternal(MAX_RESIZE_INTERNAL_STACK_DEPTH).getSource().release(); totalPoolSize.getAndDecrement(); } catch (PoolResourceException e) { return false; } return true; }
@Override public void shutdown() { objectPool.forEach(pooled -> pooled.getSource().release()); objectPool.clear(); executor.shutdown(); }
@Override public boolean remove() { try { removeInternal(MAX_RESIZE_INTERNAL_STACK_DEPTH).getSource().release(); totalPoolSize.getAndDecrement(); } catch (PoolResourceException e) { return false; } return true; }
/** * Serializes and writes {@link #actions} into {@link #bulkSource} * * @return underlying buffer filled with serialized actions * @throws IOException if serialization failed */ public ByteBuf serializeRequest() throws IOException { ByteBufOutputStream byteBufOutputStream = new ByteBufOutputStream(bulkSource.getSource()); for (BulkableAction action : actions) { objectWriter.writeValue((DataOutput) byteBufOutputStream, action); byteBufOutputStream.writeByte(LINE_SEPARATOR); ByteBuf source = ((BufferedIndex)action).getSource().getSource(); bulkSource.getSource().writeBytes(source); byteBufOutputStream.writeByte(LINE_SEPARATOR); } return bulkSource.getSource(); }
@Override public Function<Bulk, Boolean> createFailureHandler(FailoverPolicy failover) { return bulk -> { BufferedBulk bufferedBulk = (BufferedBulk)bulk; LOG.warn(String.format("Batch of %s items failed. Redirecting to %s", bufferedBulk.getActions().size(), failover.getClass().getName())); bufferedBulk.getActions().forEach(failedItem -> { ByteBuf byteBuf = ((BufferedIndex) failedItem).source.getSource(); failover.deliver(byteBuf.toString(0, byteBuf.writerIndex(), Charset.defaultCharset())); }); return true; }; }
/** * Creates ONE pooled {@link BufferedItemSource} */ @Override public final void incrementPoolSize() { CompositeByteBuf buffer = new CompositeByteBuf(byteBufAllocator, false, 2).capacity(estimatedSourceSize); objectPool.add(new BufferedItemSource(buffer, bufferedItemSource -> { bufferedItemSource.getSource().clear(); objectPool.add(bufferedItemSource); })); totalPoolSize.getAndIncrement(); }
/** * Creates ONE pooled {@link BufferedItemSource} */ @Override public final void incrementPoolSize() { CompositeByteBuf buffer = new CompositeByteBuf(byteBufAllocator, false, 2).capacity(estimatedSourceSize); objectPool.add(new BufferedItemSource(buffer, bufferedItemSource -> { bufferedItemSource.getSource().clear(); objectPool.add(bufferedItemSource); })); totalPoolSize.getAndIncrement(); }
/** * Serializes given object to {@link BufferedItemSource} using given {@link ObjectWriter} * * @param source item to serialize * @param objectWriter writer to be used to serialize given item * @throws IllegalStateException if underlying pool cannot provide {@link BufferedItemSource} * @throws IllegalArgumentException if serialization failed * @return {@link BufferedItemSource} with serialized event */ @Override public ItemSource create(Object source, ObjectWriter objectWriter) { ItemSource<ByteBuf> pooled; try { pooled = bufferedItemSourcePool.getPooled(); } catch (PoolResourceException e) { // FIXME: stop throwing and redirect to failover policy here throw new IllegalStateException(e); } try { objectWriter.writeValue((DataOutput)new ByteBufOutputStream(pooled.getSource()), source); return pooled; } catch (IOException e) { pooled.release(); throw new IllegalArgumentException(e); } }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new IndexRequest(indexName) .type(ACTION_TYPE) .source((String)source.getSource()); }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new IndexRequest(indexName) .type(ACTION_TYPE) .source((String)source.getSource(), XContentType.JSON); }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new IndexRequest(indexName) .type(ACTION_TYPE) .source((String)source.getSource(), XContentType.JSON); }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new TestBatchItem(source.getSource()); }
@Override public Object createBatchItem(String indexName, ItemSource source) { return new Index.Builder(source.getSource()) .index(indexName) .type(ACTION_TYPE) .build(); }
@Test public void releaseDelegatesToGivenCallback() { // given ReleaseCallback callback = mock(ReleaseCallback.class); CompositeByteBuf byteBuf = createDefaultTestByteBuf(); // then ItemSource source = new BufferedItemSource(byteBuf, callback); source.release(); // then ArgumentCaptor<ItemSource> captor = ArgumentCaptor.forClass(ItemSource.class); verify(callback).completed(captor.capture()); assertEquals(source, captor.getValue()); assertTrue(source.getSource() == captor.getValue().getSource()); }
@Test public void doesntChangeTheSource() { // given String expected = "expectedSource"; // when ItemSource<String> itemSource = new StringItemSource(expected); String actualSource = itemSource.getSource(); // then Assert.assertEquals(expected, actualSource); }
@Test public void messageOnlyItemSourceLayout() { // given ItemAppenderFactory factory = new ItemAppenderFactory(); String formattedIndexName = UUID.randomUUID().toString(); BatchDelivery batchDelivery = mock(BatchDelivery.class); ItemSourceLayout itemSourceLayout = spy(new TestItemSourceaLayout(mock(Configuration.class))); ItemSourceAppender itemAppender = (ItemSourceAppender) factory.createInstance(true, (AbstractLayout)itemSourceLayout, batchDelivery); String expectedMessage = UUID.randomUUID().toString(); ItemSource itemSource = new StringItemSource(expectedMessage); when(itemSourceLayout.serialize(any(Message.class))).thenReturn(itemSource); LogEvent logEvent = mock(LogEvent.class); Message message = mock(Message.class); when(logEvent.getMessage()).thenReturn(message); // when itemAppender.append(formattedIndexName, logEvent); // then assertEquals(ItemSourceAppender.class, itemAppender.getClass()); verify(itemSourceLayout).serialize(messageCaptor.capture()); assertEquals(message, messageCaptor.getValue()); verify(batchDelivery).add(indexNameCaptor.capture(), itemSourceCaptor.capture()); assertEquals(formattedIndexName, indexNameCaptor.getValue()); assertEquals(expectedMessage, itemSourceCaptor.getValue().getSource()); }
@Test public void nonMessageOnlyItemSourceLayout() { // given ItemAppenderFactory factory = new ItemAppenderFactory(); String formattedIndexName = UUID.randomUUID().toString(); BatchDelivery batchDelivery = mock(BatchDelivery.class); ItemSourceLayout itemSourceLayout = spy(new TestItemSourceaLayout(mock(Configuration.class))); ItemSourceAppender itemAppender = (ItemSourceAppender) factory.createInstance(false, (AbstractLayout)itemSourceLayout, batchDelivery); String expectedMessage = UUID.randomUUID().toString(); ItemSource itemSource = new StringItemSource(expectedMessage); when(itemSourceLayout.serialize(any(LogEvent.class))).thenReturn(itemSource); LogEvent logEvent = createDefaultTestLogEvent(); // when itemAppender.append(formattedIndexName, logEvent); // then assertEquals(ItemSourceAppender.class, itemAppender.getClass()); verify(itemSourceLayout).serialize(logEventCaptor.capture()); assertEquals(logEvent, logEventCaptor.getValue()); verify(batchDelivery).add(indexNameCaptor.capture(), itemSourceCaptor.capture()); assertEquals(formattedIndexName, indexNameCaptor.getValue()); assertEquals(expectedMessage, itemSourceCaptor.getValue().getSource()); }
@Test public void doesNotChangeTheSource() { // given CompositeByteBuf expectedSource = createDefaultTestByteBuf(); ItemSource itemSource = new BufferedItemSource(expectedSource, mock(ReleaseCallback.class)); // when Object actualSource = itemSource.getSource(); // then assertEquals(expectedSource, actualSource); assertTrue(expectedSource == actualSource); }
@Test public void releaseHasNoSideEffect() { // given String expected = "expectedSource"; ItemSource<String> itemSource = new StringItemSource(expected); // when itemSource.release(); // then Assert.assertEquals(expected, itemSource.getSource()); }