public FaultTolerantStepFactoryBeanTests() throws Exception { reader = new SkipReaderStub<>(); processor = new SkipProcessorStub<>(); writer = new SkipWriterStub<>(); }
@Override public T process(T item) throws Exception { processed.add(item); committed.add(item); try { checkFailure(item); } catch (Exception e) { if (filter) { return null; } else { throw e; } } return item; } }
@Test public void testFilterInProcessor() throws Exception { processor.setFailures("4"); processor.setFilter(true); factory.setCommitInterval(30); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 2, 3, 4, 5]", processor.getProcessed().toString()); assertEquals("[1, 2, 3, 4, 5]", processor.getCommitted().toString()); assertEquals("[1, 2, 3, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 5]", writer.getCommitted().toString()); }
@Test public void testSkipInProcessor() throws Exception { processor.setFailures("4"); factory.setCommitInterval(30); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 2, 3, 4, 1, 2, 3, 5]", processor.getProcessed().toString()); assertEquals("[1, 2, 3, 5]", processor.getCommitted().toString()); assertEquals("[1, 2, 3, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 5]", writer.getCommitted().toString()); }
@Test public void testMultipleSkipsInNonTransactionalProcessor() throws Exception { processor.setFailures("2", "4"); factory.setCommitInterval(30); factory.setProcessorTransactional(false); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 3, 5]", writer.getWritten().toString()); assertEquals("[1, 3, 5]", writer.getCommitted().toString()); // If non-transactional, we should only process each item once assertEquals("[1, 2, 3, 4, 5]", processor.getProcessed().toString()); }
writer.clear(); factory.setItemWriter(writer); processor.clear(); factory.setItemProcessor(processor); factory.setSkippableExceptionClasses(skippable); processor.setFailures("2"); assertEquals("[1, 2, 3, 4, 5]", processor.getCommitted().toString()); List<String> processed = new ArrayList<>(processor.getProcessed()); Collections.sort(processed); assertEquals("[1, 2, 3, 4, 5]", processed.toString());
/** * Scenario: Exception in processor that should cause rollback */ @Test public void testProcessorDefaultRollbackOnRuntimeException() throws Exception { reader.setItems("1", "2", "3", "4"); processor.setFailures("1", "3"); processor.setExceptionType(SkippableRuntimeException.class); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals(2, stepExecution.getSkipCount()); assertEquals(2, stepExecution.getRollbackCount()); }
@Test public void testMultipleSkipsInWriter() throws Exception { writer.setFailures("2", "4"); factory.setCommitInterval(30); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 3, 5]", processor.getCommitted().toString()); assertEquals("[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]", processor.getProcessed().toString()); assertEquals(2, stepExecution.getWriteSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(3, stepExecution.getWriteCount()); assertEquals(0, stepExecution.getFilterCount()); }
@Test public void testProcessFilter() throws Exception { processor.setFailures("4"); processor.setFilter(true); ItemProcessListenerStub<String, String> listenerStub = new ItemProcessListenerStub<>(); factory.setListeners(new StepListener[] { listenerStub }); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(0, stepExecution.getSkipCount()); assertEquals(0, stepExecution.getReadSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(1, stepExecution.getFilterCount()); assertEquals(0, stepExecution.getRollbackCount()); assertTrue(listenerStub.isFilterEncountered()); // writer skips "4" assertTrue(reader.getRead().contains("4")); assertFalse(writer.getWritten().contains("4")); List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,2,3,5")); assertEquals(expectedOutput, writer.getWritten()); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
@SuppressWarnings("unchecked") @Before public void setUp() throws Exception { reader = new SkipReaderStub<>(); processor = new SkipProcessorStub<>(); writer = new SkipWriterStub<>(); factory = new FaultTolerantStepFactoryBean<>(); factory.setBeanName("stepName"); ResourcelessTransactionManager transactionManager = new ResourcelessTransactionManager(); factory.setTransactionManager(transactionManager); factory.setCommitInterval(2); reader.clear(); reader.setItems("1", "2", "3", "4", "5"); factory.setItemReader(reader); processor.clear(); factory.setItemProcessor(processor); writer.clear(); factory.setItemWriter(writer); factory.setSkipLimit(2); factory.setSkippableExceptionClasses(getExceptionMap(Exception.class)); MapJobRepositoryFactoryBean repositoryFactory = new MapJobRepositoryFactoryBean(); repositoryFactory.setTransactionManager(transactionManager); repositoryFactory.afterPropertiesSet(); repository = repositoryFactory.getObject(); factory.setJobRepository(repository); jobExecution = repository.createJobExecution("skipJob", new JobParameters()); stepExecution = jobExecution.createStepExecution(factory.getName()); repository.add(stepExecution); }
@Test public void testSkipInWriterNonTransactionalProcessor() throws Exception { writer.setFailures("4"); factory.setCommitInterval(30); factory.setProcessorTransactional(false); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 2, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 3, 4, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5]", processor.getProcessed().toString()); }
@Test public void testMultithreadedSkipInWriter() throws Exception { writer.setFailures("1", "2", "3", "4", "5"); factory.setCommitInterval(3); factory.setSkipLimit(10); factory.setTaskExecutor(new SimpleAsyncTaskExecutor()); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[]", writer.getCommitted().toString()); assertEquals("[]", processor.getCommitted().toString()); assertEquals(5, stepExecution.getSkipCount()); }
/** * Check items causing errors are skipped as expected. */ @Test public void testProcessSkip() throws Exception { processor.setFailures("4"); writer.setFailures("4"); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(1, stepExecution.getSkipCount()); assertEquals(0, stepExecution.getReadSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(1, stepExecution.getProcessSkipCount()); assertEquals(1, stepExecution.getRollbackCount()); // writer skips "4" assertTrue(reader.getRead().contains("4")); assertFalse(writer.getWritten().contains("4")); List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,2,3,5")); assertEquals(expectedOutput, writer.getWritten()); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
@SuppressWarnings("unchecked") @Before public void setUp() throws Exception { factory = new FaultTolerantStepFactoryBean<>(); factory.setBeanName("stepName"); factory.setTransactionManager(new ResourcelessTransactionManager()); factory.setCommitInterval(2); reader.clear(); reader.setItems("1", "2", "3", "4", "5"); factory.setItemReader(reader); processor.clear(); factory.setItemProcessor(processor); writer.clear(); factory.setItemWriter(writer); factory.setSkipLimit(2); factory .setSkippableExceptionClasses(getExceptionMap(SkippableException.class, SkippableRuntimeException.class)); MapJobRepositoryFactoryBean repositoryFactory = new MapJobRepositoryFactoryBean(); repositoryFactory.afterPropertiesSet(); repository = repositoryFactory.getObject(); factory.setJobRepository(repository); jobExecution = repository.createJobExecution("skipJob", new JobParameters()); stepExecution = jobExecution.createStepExecution(factory.getName()); repository.add(stepExecution); }
@Test public void testMultipleSkipsInProcessor() throws Exception { processor.setFailures("2", "4"); factory.setCommitInterval(30); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 3, 5]", processor.getCommitted().toString()); assertEquals("[1, 3, 5]", writer.getWritten().toString()); assertEquals("[1, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 1, 3, 4, 1, 3, 5]", processor.getProcessed().toString()); }
/** * Scenario: Exception in processor that should cause rollback because of * checked exception */ @Test public void testProcessorDefaultRollbackOnCheckedException() throws Exception { reader.setItems("1", "2", "3", "4"); processor.setFailures("1", "3"); processor.setExceptionType(SkippableException.class); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals(2, stepExecution.getSkipCount()); assertEquals(2, stepExecution.getRollbackCount()); }
@Test public void testSkipInWriter() throws Exception { writer.setFailures("4"); factory.setCommitInterval(30); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 2, 3, 5]", processor.getCommitted().toString()); assertEquals("[1, 2, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 3, 4, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]", processor.getProcessed().toString()); assertEquals(1, stepExecution.getWriteSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(4, stepExecution.getWriteCount()); assertEquals(0, stepExecution.getFilterCount()); }
@Test public void testMultipleSkipsInWriterNonTransactionalProcessor() throws Exception { writer.setFailures("2", "4"); factory.setCommitInterval(30); factory.setProcessorTransactional(false); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5]", processor.getProcessed().toString()); }
@Test public void testProcessSkipWithNoRollbackForCheckedException() throws Exception { processor.setFailures("4"); processor.setExceptionType(SkippableException.class); factory.setNoRollbackExceptionClasses(getExceptionList(SkippableException.class)); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals(1, stepExecution.getSkipCount()); assertEquals(0, stepExecution.getReadSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(1, stepExecution.getProcessSkipCount()); assertEquals(0, stepExecution.getRollbackCount()); // skips "4" assertTrue(reader.getRead().contains("4")); assertFalse(writer.getCommitted().contains("4")); List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,2,3,5")); assertEquals(expectedOutput, writer.getCommitted()); }
@Test public void testSkipInWriterTransactionalReader() throws Exception { writer.setFailures("4"); ItemReader<String> reader = new ListItemReader<>(TransactionAwareProxyFactory.createTransactionalList(Arrays.asList("1", "2", "3", "4", "5"))); factory.setItemReader(reader); factory.setCommitInterval(30); factory.setSkipLimit(10); factory.setIsReaderTransactionalQueue(true); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[]", writer.getCommitted().toString()); assertEquals("[1, 2, 3, 4]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]", processor.getProcessed().toString()); }