protected ChunkProcessor<I> createChunkProcessor() { BatchRetryTemplate batchRetryTemplate = createRetryOperations(); FaultTolerantChunkProcessor<I, O> chunkProcessor = new FaultTolerantChunkProcessor<>(getProcessor(), getWriter(), batchRetryTemplate); chunkProcessor.setBuffering(!isReaderTransactionalQueue()); chunkProcessor.setProcessorTransactional(processorTransactional); SkipPolicy writeSkipPolicy = createSkipPolicy(); writeSkipPolicy = getFatalExceptionAwareProxy(writeSkipPolicy); chunkProcessor.setWriteSkipPolicy(writeSkipPolicy); chunkProcessor.setProcessSkipPolicy(writeSkipPolicy); chunkProcessor.setRollbackClassifier(getRollbackClassifier()); chunkProcessor.setKeyGenerator(keyGenerator); detectStreamInReader(); ArrayList<StepListener> listeners = new ArrayList<>(getItemListeners()); listeners.addAll(skipListeners); chunkProcessor.setListeners(listeners); chunkProcessor.setChunkMonitor(chunkMonitor); return chunkProcessor; }
private void callSkipListeners(final Chunk<I> inputs, final Chunk<O> outputs) { for (SkipWrapper<I> wrapper : inputs.getSkips()) { I item = wrapper.getItem(); if (item == null) { continue; } Throwable e = wrapper.getException(); callProcessSkipListener(item, e); } for (SkipWrapper<O> wrapper : outputs.getSkips()) { Throwable e = wrapper.getException(); try { getListener().onSkipInWrite(wrapper.getItem(), e); } catch (RuntimeException ex) { throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); } } // Clear skips if we are possibly going to process this chunk again outputs.clearSkips(); inputs.clearSkips(); }
BatchRetryTemplate.createState(getInputKeys(inputs), rollbackClassifier)); callSkipListeners(inputs, outputs);
@Test public void testTransformWithExceptionAndNoRollback() throws Exception { processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { throw new DataIntegrityViolationException("Planned"); } return item; } }); processor.setProcessSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor .setRollbackClassifier(new BinaryExceptionClassifier( Collections .<Class<? extends Throwable>> singleton(DataIntegrityViolationException.class), false)); Chunk<String> inputs = new Chunk<>(Arrays.asList("1", "2")); processor.process(contribution, inputs); assertEquals(1, list.size()); }
@Test public void testOnErrorInWrite() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "fail")); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void onWriteError(Exception e, List<? extends String> item) { writeError.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk);// Process foo, fail processor.process(contribution, chunk);// Process foo processAndExpectPlannedRuntimeException(chunk);// Process fail assertEquals("[foo, fail, fail]", writeError.toString()); }
inputIterator.next(); try { writeItems(items); doAfterWrite(items); contribution.incrementWriteCount(1); inputIterator.remove(); doOnWriteError(e, items); if (!shouldSkip(itemWriteSkipPolicy, cause, -1) && !rollbackClassifier.classify(cause)) { inputIterator.remove(); outputIterator.remove(); checkSkipPolicy(inputIterator, outputIterator, cause, contribution, recovery);
processor.setProcessorTransactional(false); processor.setProcessSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { processor.setRollbackClassifier(new BinaryExceptionClassifier(Collections .<Class<? extends Throwable>> singleton(IllegalArgumentException.class), false)); processor.afterPropertiesSet(); Chunk<String> inputs = new Chunk<>(Arrays.asList("1", "2", "skip", "skip", "3", "fail", "fail", "4", "5")); processor.process(contribution, inputs); assertEquals(5, list.size()); assertEquals("[1, 2, 3, 4, 5]", list.toString());
/** * An Error can be retried or skipped but by default it is just propagated * * @throws Exception */ @Test public void testWriteSkipOnError() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { if (items.contains("fail")) { assertFalse("Expected Error!", true); } } }); Chunk<String> inputs = new Chunk<>( Arrays.asList("3", "fail", "2")); try { processor.process(contribution, inputs); fail("Expected Error"); } catch (Error e) { assertEquals("Expected Error!", e.getMessage()); } processor.process(contribution, inputs); }
if (chunkProcessor instanceof FaultTolerantChunkProcessor) { FaultTolerantChunkProcessor faultTolerantChunkProcessor = (FaultTolerantChunkProcessor) chunkProcessor; faultTolerantChunkProcessor.setRollbackClassifier(rollbackClassifier); faultTolerantChunkProcessor.setWriteSkipPolicy(skipPolicy); faultTolerantChunkProcessor.setProcessSkipPolicy(skipPolicy);
@Test public void testAfterWriteAllPassedInRecovery() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "bar")); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { // Fail if there is more than one item if (items.size() > 1) { throw new RuntimeException("Planned failure!"); } list.addAll(items); } }, batchRetryTemplate); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void afterWrite(List<? extends String> item) { after.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk); processor.process(contribution, chunk); processor.process(contribution, chunk); assertEquals("[foo, bar]", list.toString()); assertEquals("[foo, bar]", after.toString()); }
@Test public void testOnErrorInWriteAllItemsFail() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "bar")); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { // Always fail in writer throw new RuntimeException("Planned failure!"); } }, batchRetryTemplate); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void onWriteError(Exception e, List<? extends String> item) { writeError.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk);// Process foo, bar processAndExpectPlannedRuntimeException(chunk);// Process foo processAndExpectPlannedRuntimeException(chunk);// Process bar assertEquals("[foo, bar, foo, bar]", writeError.toString()); }
@Test // BATCH-2663 public void testFilterCountOnSkipInWriteWithoutRetry() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { return null; } return item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("fail", "1", "2")); processAndExpectPlannedRuntimeException(inputs); // (first attempt) Process fail, 1, 2 // item 1 is filtered out so it is removed from the chunk => now inputs = [fail, 2] // using NeverRetryPolicy by default => now scanning processAndExpectPlannedRuntimeException(inputs); // (scanning) Process fail processor.process(contribution, inputs); // (scanning) Process 2 assertEquals(1, list.size()); assertEquals("[2]", list.toString()); assertEquals(1, contribution.getWriteSkipCount()); assertEquals(1, contribution.getFilterCount()); }
@Test public void testFilterCountOnSkip() throws Exception { processor.setProcessSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { throw new RuntimeException("Skippable"); } if (item.equals("3")) { return null; } return item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("3", "1", "2")); try { processor.process(contribution, inputs); fail("Expected Exception"); } catch (Exception e) { assertEquals("Skippable", e.getMessage()); } processor.process(contribution, inputs); assertEquals(1, list.size()); assertEquals(1, contribution.getSkipCount()); assertEquals(1, contribution.getFilterCount()); }
/** * Overrides the buffering settings in the chunk processor if it is fault tolerant. * @param chunkProcessor the chunk processor that is going to be used in the workers */ private void setNonBuffering(ChunkProcessor<T> chunkProcessor) { if (chunkProcessor instanceof FaultTolerantChunkProcessor<?, ?>) { ((FaultTolerantChunkProcessor<?, ?>) chunkProcessor).setBuffering(false); } }
@Before public void setUp() { batchRetryTemplate = new BatchRetryTemplate(); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { if (items.contains("fail")) { throw new RuntimeException("Planned failure!"); } list.addAll(items); } }, batchRetryTemplate); batchRetryTemplate.setRetryPolicy(new NeverRetryPolicy()); }
inputIterator.next(); try { writeItems(items); doAfterWrite(items); contribution.incrementWriteCount(1); inputIterator.remove(); if (!shouldSkip(itemWriteSkipPolicy, e, -1) && !rollbackClassifier.classify(e)) { inputIterator.remove(); outputIterator.remove(); checkSkipPolicy(inputIterator, outputIterator, e, contribution);
processor.setProcessorTransactional(false); processor.setProcessSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { processor.afterPropertiesSet(); Chunk<String> inputs = new Chunk<>(Arrays.asList("1", "2", "skip", "skip", "3", "fail", "fail", "4", "5")); try { processor.process(contribution, inputs); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { processor.process(contribution, inputs); fail("Expected IllegalArgumentException"); } catch (IllegalArgumentException e) { assertEquals("Expected Skippable Exception!", e.getMessage()); processor.process(contribution, inputs); assertEquals(5, list.size()); assertEquals("[1, 2, 3, 4, 5]", list.toString());
@Test public void testWriteSkipOnException() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { if (items.contains("fail")) { throw new RuntimeException("Expected Exception!"); } } }); Chunk<String> inputs = new Chunk<>( Arrays.asList("3", "fail", "2")); try { processor.process(contribution, inputs); fail("Expected RuntimeException"); } catch (RuntimeException e) { assertEquals("Expected Exception!", e.getMessage()); } processor.process(contribution, inputs); try { processor.process(contribution, inputs); fail("Expected RuntimeException"); } catch (RuntimeException e) { assertEquals("Expected Exception!", e.getMessage()); } assertEquals(1, contribution.getSkipCount()); assertEquals(1, contribution.getWriteCount()); assertEquals(0, contribution.getFilterCount()); }
@Test public void testAfterWrite() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "fail", "bar")); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void afterWrite(List<? extends String> item) { after.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk); processor.process(contribution, chunk); assertEquals(2, chunk.getItems().size()); processAndExpectPlannedRuntimeException(chunk); assertEquals(1, chunk.getItems().size()); processor.process(contribution, chunk); assertEquals(0, chunk.getItems().size()); // foo is written once because it the failure is detected before it is // committed the first time assertEquals("[foo, bar]", list.toString()); // the after listener is called once per successful item, which is // important assertEquals("[foo, bar]", after.toString()); }
@Test // BATCH-2663 public void testFilterCountOnSkipInWriteWithRetry() throws Exception { SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy(); retryPolicy.setMaxAttempts(3); batchRetryTemplate.setRetryPolicy(retryPolicy); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { return null; } return item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("fail", "1", "2")); processAndExpectPlannedRuntimeException(inputs); // (first attempt) Process fail, 1, 2 // item 1 is filtered out so it is removed from the chunk => now inputs = [fail, 2] processAndExpectPlannedRuntimeException(inputs); // (first retry) Process fail, 2 processAndExpectPlannedRuntimeException(inputs); // (second retry) Process fail, 2 // retry exhausted (maxAttempts = 3) => now scanning processAndExpectPlannedRuntimeException(inputs); // (scanning) Process fail processor.process(contribution, inputs); // (scanning) Process 2 assertEquals(1, list.size()); assertEquals("[2]", list.toString()); assertEquals(1, contribution.getWriteSkipCount()); assertEquals(3, contribution.getFilterCount()); }