@Test public void testSkipInWriterNonTransactionalProcessor() throws Exception { writer.setFailures("4"); factory.setCommitInterval(30); factory.setProcessorTransactional(false); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 2, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 3, 4, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5]", processor.getProcessed().toString()); }
@Test public void testMultipleSkipsInWriterNonTransactionalProcessor() throws Exception { writer.setFailures("2", "4"); factory.setCommitInterval(30); factory.setProcessorTransactional(false); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5]", processor.getProcessed().toString()); }
@Test public void testSkipInWriterTransactionalReader() throws Exception { writer.setFailures("4"); ItemReader<String> reader = new ListItemReader<>(TransactionAwareProxyFactory.createTransactionalList(Arrays.asList("1", "2", "3", "4", "5"))); factory.setItemReader(reader); factory.setCommitInterval(30); factory.setSkipLimit(10); factory.setIsReaderTransactionalQueue(true); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[]", writer.getCommitted().toString()); assertEquals("[1, 2, 3, 4]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]", processor.getProcessed().toString()); }
@Test public void testMultithreadedSkipInWriter() throws Exception { writer.setFailures("1", "2", "3", "4", "5"); factory.setCommitInterval(3); factory.setSkipLimit(10); factory.setTaskExecutor(new SimpleAsyncTaskExecutor()); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[]", writer.getCommitted().toString()); assertEquals("[]", processor.getCommitted().toString()); assertEquals(5, stepExecution.getSkipCount()); }
/** * Check items causing errors are skipped as expected. */ @Test public void testSkipOverLimit() throws Exception { reader.setFailures("2"); writer.setFailures("4"); factory.setSkipLimit(1); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(1, stepExecution.getSkipCount()); // writer did not skip "2" as it never made it to writer, only "4" did assertTrue(reader.getRead().contains("4")); assertFalse(writer.getCommitted().contains("4")); // failure on "4" tripped the skip limit so we never got to "5" List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,3")); assertEquals(expectedOutput, writer.getCommitted()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
/** * Check items causing errors are skipped as expected. */ @Test public void testSkipOverLimitOnReadWithListener() throws Exception { reader.setFailures("1", "3", "5"); writer.setFailures(); final List<Throwable> listenerCalls = new ArrayList<>(); factory.setListeners(new StepListener[] { new SkipListenerSupport<String, String>() { @Override public void onSkipInRead(Throwable t) { listenerCalls.add(t); } } }); factory.setCommitInterval(2); factory.setSkipLimit(2); Step step = factory.getObject(); step.execute(stepExecution); // 1,3 skipped inside a committed chunk. 5 tripped the skip // limit but it was skipped in a chunk that rolled back, so // it will re-appear on a restart and the listener is not called. assertEquals(2, listenerCalls.size()); assertEquals(2, stepExecution.getReadSkipCount()); assertEquals(BatchStatus.FAILED, stepExecution.getStatus()); }
/** * Check items causing errors are skipped as expected. */ @Test public void testSkipOnWriteNotDoubleCounted() throws Exception { reader.setItems(StringUtils.commaDelimitedListToStringArray("1,2,3,4,5,6,7")); reader.setFailures(StringUtils.commaDelimitedListToStringArray("2,3")); writer.setFailures("4", "5"); factory.setSkipLimit(4); factory.setCommitInterval(3); // includes all expected skips Step step = factory.getObject(); step.execute(stepExecution); assertEquals(4, stepExecution.getSkipCount()); assertEquals(2, stepExecution.getReadSkipCount()); assertEquals(2, stepExecution.getWriteSkipCount()); // skipped 2,3,4,5 List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,6,7")); assertEquals(expectedOutput, writer.getCommitted()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
@Test public void testMultipleSkipsInWriter() throws Exception { writer.setFailures("2", "4"); factory.setCommitInterval(30); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 3, 5]", processor.getCommitted().toString()); assertEquals("[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]", processor.getProcessed().toString()); assertEquals(2, stepExecution.getWriteSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(3, stepExecution.getWriteCount()); assertEquals(0, stepExecution.getFilterCount()); }
@Test public void testSkipInWriter() throws Exception { writer.setFailures("4"); factory.setCommitInterval(30); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals("[1, 2, 3, 5]", processor.getCommitted().toString()); assertEquals("[1, 2, 3, 5]", writer.getCommitted().toString()); assertEquals("[1, 2, 3, 4, 1, 2, 3, 4, 5]", writer.getWritten().toString()); assertEquals("[1, 2, 3, 4, 5, 1, 2, 3, 4, 5]", processor.getProcessed().toString()); assertEquals(1, stepExecution.getWriteSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(4, stepExecution.getWriteCount()); assertEquals(0, stepExecution.getFilterCount()); }
/** * Check items causing errors are skipped as expected. */ @Test public void testSkipOnReadNotDoubleCounted() throws Exception { reader.setItems(StringUtils.commaDelimitedListToStringArray("1,2,3,4,5,6")); reader.setFailures(StringUtils.commaDelimitedListToStringArray("2,3,5")); writer.setFailures("4"); factory.setSkipLimit(4); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(4, stepExecution.getSkipCount()); assertEquals(3, stepExecution.getReadSkipCount()); assertEquals(1, stepExecution.getWriteSkipCount()); // skipped 2,3,4,5 List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,6")); assertEquals(expectedOutput, writer.getCommitted()); // reader exceptions should not cause rollback, 1 writer exception // causes 2 rollbacks assertEquals(2, stepExecution.getRollbackCount()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
/** * Scenario: Exception in writer that should not cause rollback and scan */ @Test public void testWriterDefaultRollbackOnRuntimeException() throws Exception { writer.setFailures("2", "3"); writer.setExceptionType(SkippableRuntimeException.class); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals(2, stepExecution.getSkipCount()); assertEquals(4, stepExecution.getRollbackCount()); }
@Test public void testReprocessingAfterWriterRollback() throws Exception { reader.setItems("1", "2", "3", "4"); writer.setFailures("4"); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(1, stepExecution.getSkipCount()); assertEquals(2, stepExecution.getRollbackCount()); // 1,2,3,4,3,4 - one scan until the item is // identified and finally skipped on the second attempt assertEquals("[1, 2, 3, 4, 3, 4]", processor.getProcessed().toString()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
/** * Scenario: Exception in writer that should not cause rollback and scan */ @Test public void testWriterDefaultRollbackOnCheckedException() throws Exception { writer.setFailures("2", "3"); writer.setExceptionType(SkippableException.class); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals(2, stepExecution.getSkipCount()); assertEquals(4, stepExecution.getRollbackCount()); }
/** * Scenario: Exception in writer that should not cause rollback and scan */ @Test public void testWriterDefaultRollbackOnError() throws Exception { writer.setFailures("2", "3"); writer.setExceptionType(AssertionError.class); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.FAILED, stepExecution.getStatus()); assertEquals(0, stepExecution.getSkipCount()); assertEquals(1, stepExecution.getRollbackCount()); }
/** * Check items causing errors are skipped as expected. */ @Test public void testWriteSkip() throws Exception { writer.setFailures("4"); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(1, stepExecution.getSkipCount()); assertEquals(0, stepExecution.getReadSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(1, stepExecution.getWriteSkipCount()); assertEquals(2, stepExecution.getRollbackCount()); // writer skips "4" assertTrue(reader.getRead().contains("4")); assertFalse(writer.getCommitted().contains("4")); List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,2,3,5")); assertEquals(expectedOutput, writer.getCommitted()); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
/** * Check items causing errors are skipped as expected. */ @Test public void testProcessSkip() throws Exception { processor.setFailures("4"); writer.setFailures("4"); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(1, stepExecution.getSkipCount()); assertEquals(0, stepExecution.getReadSkipCount()); assertEquals(5, stepExecution.getReadCount()); assertEquals(1, stepExecution.getProcessSkipCount()); assertEquals(1, stepExecution.getRollbackCount()); // writer skips "4" assertTrue(reader.getRead().contains("4")); assertFalse(writer.getWritten().contains("4")); List<String> expectedOutput = Arrays.asList(StringUtils.commaDelimitedListToStringArray("1,2,3,5")); assertEquals(expectedOutput, writer.getWritten()); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
/** * Scenario: Exception in writer that should not cause rollback and scan */ @Test public void testWriterNoRollbackOnRuntimeException() throws Exception { writer.setFailures("2", "3"); writer.setExceptionType(SkippableRuntimeException.class); factory.setNoRollbackExceptionClasses(getExceptionList(SkippableRuntimeException.class)); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals(2, stepExecution.getSkipCount()); // Two multi-item chunks rolled back. When the item was encountered on // its own it can proceed assertEquals(2, stepExecution.getRollbackCount()); }
@SuppressWarnings("unchecked") @Test public void testNonSkippableException() throws Exception { // nothing is skippable factory.setSkippableExceptionClasses(getExceptionMap(NonExistentException.class)); factory.setCommitInterval(1); // no failures on read reader.setItems("1", "2", "3", "4", "5"); writer.setFailures("1"); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.FAILED, stepExecution.getStatus()); assertEquals(1, reader.getRead().size()); assertEquals(ExitStatus.FAILED.getExitCode(), stepExecution.getExitStatus().getExitCode()); assertTrue(stepExecution.getExitStatus().getExitDescription().contains("Intended Failure")); assertStepExecutionsAreEqual(stepExecution, repository.getLastStepExecution(jobExecution.getJobInstance(), step .getName())); }
/** * Scenario: Exception in writer that should not cause rollback and scan */ @Test public void testWriterNoRollbackOnCheckedException() throws Exception { writer.setFailures("2", "3"); writer.setExceptionType(SkippableException.class); factory.setNoRollbackExceptionClasses(getExceptionList(SkippableException.class)); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.COMPLETED, stepExecution.getStatus()); assertEquals(2, stepExecution.getSkipCount()); // Two multi-item chunks rolled back. When the item was encountered on // its own it can proceed assertEquals(2, stepExecution.getRollbackCount()); }
/** * Check items causing errors are skipped as expected. */ @Test public void testReadSkipWithPolicyExceptionInWriter() throws Exception { // Should be ignored factory.setSkipLimit(0); factory.setSkipPolicy(new SkipPolicy() { @Override public boolean shouldSkip(Throwable t, int skipCount) throws SkipLimitExceededException { throw new RuntimeException("Planned exception in SkipPolicy"); } }); writer.setFailures("2"); Step step = factory.getObject(); step.execute(stepExecution); assertEquals(BatchStatus.FAILED, stepExecution.getStatus()); assertEquals(0, stepExecution.getWriteSkipCount()); assertEquals(2, stepExecution.getReadCount()); }