@Test public void testOnErrorInWrite() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "fail")); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void onWriteError(Exception e, List<? extends String> item) { writeError.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk);// Process foo, fail processor.process(contribution, chunk);// Process foo processAndExpectPlannedRuntimeException(chunk);// Process fail assertEquals("[foo, fail, fail]", writeError.toString()); }
protected ChunkProcessor<I> createChunkProcessor() { BatchRetryTemplate batchRetryTemplate = createRetryOperations(); FaultTolerantChunkProcessor<I, O> chunkProcessor = new FaultTolerantChunkProcessor<>(getProcessor(), getWriter(), batchRetryTemplate); chunkProcessor.setBuffering(!isReaderTransactionalQueue()); chunkProcessor.setProcessorTransactional(processorTransactional); SkipPolicy writeSkipPolicy = createSkipPolicy(); writeSkipPolicy = getFatalExceptionAwareProxy(writeSkipPolicy); chunkProcessor.setWriteSkipPolicy(writeSkipPolicy); chunkProcessor.setProcessSkipPolicy(writeSkipPolicy); chunkProcessor.setRollbackClassifier(getRollbackClassifier()); chunkProcessor.setKeyGenerator(keyGenerator); detectStreamInReader(); ArrayList<StepListener> listeners = new ArrayList<>(getItemListeners()); listeners.addAll(skipListeners); chunkProcessor.setListeners(listeners); chunkProcessor.setChunkMonitor(chunkMonitor); return chunkProcessor; }
@Test public void testAfterWrite() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "fail", "bar")); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void afterWrite(List<? extends String> item) { after.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk); processor.process(contribution, chunk); assertEquals(2, chunk.getItems().size()); processAndExpectPlannedRuntimeException(chunk); assertEquals(1, chunk.getItems().size()); processor.process(contribution, chunk); assertEquals(0, chunk.getItems().size()); // foo is written once because it the failure is detected before it is // committed the first time assertEquals("[foo, bar]", list.toString()); // the after listener is called once per successful item, which is // important assertEquals("[foo, bar]", after.toString()); }
/** * An Error can be retried or skipped but by default it is just propagated * * @throws Exception */ @Test public void testWriteSkipOnError() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { if (items.contains("fail")) { assertFalse("Expected Error!", true); } } }); Chunk<String> inputs = new Chunk<>( Arrays.asList("3", "fail", "2")); try { processor.process(contribution, inputs); fail("Expected Error"); } catch (Error e) { assertEquals("Expected Error!", e.getMessage()); } processor.process(contribution, inputs); }
@Test public void testAfterWriteAllPassedInRecovery() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "bar")); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { // Fail if there is more than one item if (items.size() > 1) { throw new RuntimeException("Planned failure!"); } list.addAll(items); } }, batchRetryTemplate); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void afterWrite(List<? extends String> item) { after.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk); processor.process(contribution, chunk); processor.process(contribution, chunk); assertEquals("[foo, bar]", list.toString()); assertEquals("[foo, bar]", after.toString()); }
@Test public void testOnErrorInWriteAllItemsFail() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "bar")); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { // Always fail in writer throw new RuntimeException("Planned failure!"); } }, batchRetryTemplate); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void onWriteError(Exception e, List<? extends String> item) { writeError.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk);// Process foo, bar processAndExpectPlannedRuntimeException(chunk);// Process foo processAndExpectPlannedRuntimeException(chunk);// Process bar assertEquals("[foo, bar, foo, bar]", writeError.toString()); }
@Test // BATCH-2663 public void testFilterCountOnSkipInWriteWithoutRetry() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { return null; } return item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("fail", "1", "2")); processAndExpectPlannedRuntimeException(inputs); // (first attempt) Process fail, 1, 2 // item 1 is filtered out so it is removed from the chunk => now inputs = [fail, 2] // using NeverRetryPolicy by default => now scanning processAndExpectPlannedRuntimeException(inputs); // (scanning) Process fail processor.process(contribution, inputs); // (scanning) Process 2 assertEquals(1, list.size()); assertEquals("[2]", list.toString()); assertEquals(1, contribution.getWriteSkipCount()); assertEquals(1, contribution.getFilterCount()); }
@Test public void testWriteSkipOnExceptionWithTrivialChunk() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override
@Test public void testWriteSkipOnException() throws Exception { processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { if (items.contains("fail")) { throw new RuntimeException("Expected Exception!"); } } }); Chunk<String> inputs = new Chunk<>( Arrays.asList("3", "fail", "2")); try { processor.process(contribution, inputs); fail("Expected RuntimeException"); } catch (RuntimeException e) { assertEquals("Expected Exception!", e.getMessage()); } processor.process(contribution, inputs); try { processor.process(contribution, inputs); fail("Expected RuntimeException"); } catch (RuntimeException e) { assertEquals("Expected Exception!", e.getMessage()); } assertEquals(1, contribution.getSkipCount()); assertEquals(1, contribution.getWriteCount()); assertEquals(0, contribution.getFilterCount()); }
@Test // BATCH-2663 public void testFilterCountOnSkipInWriteWithRetry() throws Exception { SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy(); retryPolicy.setMaxAttempts(3); batchRetryTemplate.setRetryPolicy(retryPolicy); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemProcessor(new ItemProcessor<String, String>() { @Override public String process(String item) throws Exception { if (item.equals("1")) { return null; } return item; } }); Chunk<String> inputs = new Chunk<>(Arrays.asList("fail", "1", "2")); processAndExpectPlannedRuntimeException(inputs); // (first attempt) Process fail, 1, 2 // item 1 is filtered out so it is removed from the chunk => now inputs = [fail, 2] processAndExpectPlannedRuntimeException(inputs); // (first retry) Process fail, 2 processAndExpectPlannedRuntimeException(inputs); // (second retry) Process fail, 2 // retry exhausted (maxAttempts = 3) => now scanning processAndExpectPlannedRuntimeException(inputs); // (scanning) Process fail processor.process(contribution, inputs); // (scanning) Process 2 assertEquals(1, list.size()); assertEquals("[2]", list.toString()); assertEquals(1, contribution.getWriteSkipCount()); assertEquals(3, contribution.getFilterCount()); }
retryPolicy.setMaxAttempts(2); batchRetryTemplate.setRetryPolicy(retryPolicy); processor.setWriteSkipPolicy(new LimitCheckingItemSkipPolicy(1, Collections.<Class<? extends Throwable>, Boolean> singletonMap( IllegalArgumentException.class, true)));
retryPolicy.setMaxAttempts(2); batchRetryTemplate.setRetryPolicy(retryPolicy); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override
retryPolicy.setMaxAttempts(2); batchRetryTemplate.setRetryPolicy(retryPolicy); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processor.setItemWriter(new ItemWriter<String>() { @Override
FaultTolerantChunkProcessor faultTolerantChunkProcessor = (FaultTolerantChunkProcessor) chunkProcessor; faultTolerantChunkProcessor.setRollbackClassifier(rollbackClassifier); faultTolerantChunkProcessor.setWriteSkipPolicy(skipPolicy); faultTolerantChunkProcessor.setProcessSkipPolicy(skipPolicy);
protected ChunkProcessor<I> createChunkProcessor() { BatchRetryTemplate batchRetryTemplate = createRetryOperations(); FaultTolerantChunkProcessor<I, O> chunkProcessor = new FaultTolerantChunkProcessor<>(getProcessor(), getWriter(), batchRetryTemplate); chunkProcessor.setBuffering(!isReaderTransactionalQueue()); chunkProcessor.setProcessorTransactional(processorTransactional); SkipPolicy writeSkipPolicy = createSkipPolicy(); writeSkipPolicy = getFatalExceptionAwareProxy(writeSkipPolicy); chunkProcessor.setWriteSkipPolicy(writeSkipPolicy); chunkProcessor.setProcessSkipPolicy(writeSkipPolicy); chunkProcessor.setRollbackClassifier(getRollbackClassifier()); chunkProcessor.setKeyGenerator(keyGenerator); detectStreamInReader(); ArrayList<StepListener> listeners = new ArrayList<StepListener>(getItemListeners()); listeners.addAll(skipListeners); chunkProcessor.setListeners(listeners); chunkProcessor.setChunkMonitor(chunkMonitor); return chunkProcessor; }
protected ChunkProcessor<I> createChunkProcessor() { BatchRetryTemplate batchRetryTemplate = createRetryOperations(); FaultTolerantChunkProcessor<I, O> chunkProcessor = new FaultTolerantChunkProcessor<I, O>(getProcessor(), getWriter(), batchRetryTemplate); chunkProcessor.setBuffering(!isReaderTransactionalQueue()); chunkProcessor.setProcessorTransactional(processorTransactional); SkipPolicy writeSkipPolicy = createSkipPolicy(); writeSkipPolicy = getFatalExceptionAwareProxy(writeSkipPolicy); chunkProcessor.setWriteSkipPolicy(writeSkipPolicy); chunkProcessor.setProcessSkipPolicy(writeSkipPolicy); chunkProcessor.setRollbackClassifier(getRollbackClassifier()); chunkProcessor.setKeyGenerator(keyGenerator); detectStreamInReader(); ArrayList<StepListener> listeners = new ArrayList<StepListener>(getItemListeners()); listeners.addAll(skipListeners); chunkProcessor.setListeners(listeners); chunkProcessor.setChunkMonitor(chunkMonitor); return chunkProcessor; }
protected ChunkProcessor<I> createChunkProcessor() { BatchRetryTemplate batchRetryTemplate = createRetryOperations(); FaultTolerantChunkProcessor<I, O> chunkProcessor = new FaultTolerantChunkProcessor<>(getProcessor(), getWriter(), batchRetryTemplate); chunkProcessor.setBuffering(!isReaderTransactionalQueue()); chunkProcessor.setProcessorTransactional(processorTransactional); SkipPolicy writeSkipPolicy = createSkipPolicy(); writeSkipPolicy = getFatalExceptionAwareProxy(writeSkipPolicy); chunkProcessor.setWriteSkipPolicy(writeSkipPolicy); chunkProcessor.setProcessSkipPolicy(writeSkipPolicy); chunkProcessor.setRollbackClassifier(getRollbackClassifier()); chunkProcessor.setKeyGenerator(keyGenerator); detectStreamInReader(); ArrayList<StepListener> listeners = new ArrayList<StepListener>(getItemListeners()); listeners.addAll(skipListeners); chunkProcessor.setListeners(listeners); chunkProcessor.setChunkMonitor(chunkMonitor); return chunkProcessor; }
/** * @return {@link ChunkProcessor} configured for fault-tolerance. */ @Override protected SimpleChunkProcessor<T, S> configureChunkProcessor() { BatchRetryTemplate batchRetryTemplate = configureRetry(); FaultTolerantChunkProcessor<T, S> chunkProcessor = new FaultTolerantChunkProcessor<T, S>(getItemProcessor(), getItemWriter(), batchRetryTemplate); chunkProcessor.setBuffering(!isReaderTransactionalQueue()); chunkProcessor.setProcessorTransactional(processorTransactional); SkipPolicy writeSkipPolicy = skipPolicy != null ? skipPolicy : new LimitCheckingItemSkipPolicy(skipLimit, getSkippableExceptionClasses()); writeSkipPolicy = getFatalExceptionAwareProxy(writeSkipPolicy); chunkProcessor.setWriteSkipPolicy(writeSkipPolicy); chunkProcessor.setProcessSkipPolicy(writeSkipPolicy); chunkProcessor.setRollbackClassifier(getRollbackClassifier()); chunkProcessor.setKeyGenerator(keyGenerator); chunkProcessor.setChunkMonitor(chunkMonitor); return chunkProcessor; }