/** * Responsible for the writing portion of the chunking loop. In this implementation, delegates to the * {{@link #doPersist(StepContribution, Chunk)}. * * @param contribution a {@link StepContribution} * @param chunk a {@link Chunk} * @throws Exception thrown if error occurs during the writing portion of the chunking loop. */ protected void persist(final StepContribution contribution, final Chunk<O> chunk) throws Exception { doPersist(contribution, chunk); contribution.incrementWriteCount(chunk.getItems().size()); }
@Override protected void write(StepContribution contribution, Chunk<T> inputs, Chunk<T> outputs) throws Exception { doWrite(outputs.getItems()); // Do not update the step contribution until the chunks are // actually processed updateStepContribution(contribution, stepContributionSource); } });
/** * Implements writing and all related listener calls * * @param contribution a {@link StepContribution} * @param chunk a {@link Chunk} * @throws Exception thrown if error occurs during the writing portion of the chunking loop. */ protected final void doPersist(final StepContribution contribution, final Chunk<O> chunk) throws Exception { try { List<O> items = chunk.getItems(); listener.beforeWrite(items); itemWriter.write(items); listener.afterWrite(items); } catch (Exception e) { listener.onWriteError(e, chunk.getItems()); throw e; } } }
@Test public void testRetry() throws Exception { logger.debug("Starting simple scenario"); List<String> items = new ArrayList<>(chunk.getItems()); int before = items.size(); items.removeAll(Collections.singleton("fail")); boolean error = true; while (error && count++ < BACKSTOP_LIMIT) { try { statefulRetry(chunk); error = false; } catch (Exception e) { error = true; } } logger.debug("Chunk: " + chunk); assertTrue("Backstop reached. Probably an infinite loop...", count < BACKSTOP_LIMIT); assertFalse(chunk.getItems().contains("fail")); assertEquals(items, chunk.getItems()); assertEquals(before-chunk.getItems().size(), chunk.getSkips().size()); }
/** * @param chunk Chunk to retry */ private void statefulRetry(Chunk<String> chunk) throws Exception { if (retryAttempts <= retryLimit) { try { // N.B. a classic stateful retry goes straight to recovery here logger.debug(String.format("Retry (attempts=%d) chunk: %s", retryAttempts, chunk)); doWrite(chunk.getItems()); retryAttempts = 0; } catch (Exception e) { retryAttempts++; // stateful retry always rethrow throw e; } } else { try { logger.debug(String.format("Recover (attempts=%d) chunk: %s", retryAttempts, chunk)); recover(chunk); } finally { retryAttempts = 0; } } // recovery return; }
/** * Simple implementation delegates to the {@link #doWrite(List)} method and * increments the write count in the contribution. Subclasses can handle * more complicated scenarios, e.g.with fault tolerance. If output items are * skipped they should be removed from the inputs as well. * * @param contribution the current step contribution * @param inputs the inputs that gave rise to the outputs * @param outputs the outputs to write * @throws Exception if there is a problem */ protected void write(StepContribution contribution, Chunk<I> inputs, Chunk<O> outputs) throws Exception { try { doWrite(outputs.getItems()); } catch (Exception e) { /* * For a simple chunk processor (no fault tolerance) we are done * here, so prevent any more processing of these inputs. */ inputs.clear(); throw e; } contribution.incrementWriteCount(outputs.size()); }
@Test public void test() throws Exception { Chunk<String> chunk = provider.provide(null); assertNotNull(chunk); assertEquals(0, chunk.getItems().size()); } }
@Override protected Chunk<O> getAdjustedOutputs(Chunk<I> inputs, Chunk<O> outputs) { @SuppressWarnings("unchecked") UserData<O> data = (UserData<O>) inputs.getUserData(); Chunk<O> previous = data.getOutputs(); Chunk<O> next = new Chunk<>(outputs.getItems(), previous.getSkips()); next.setBusy(previous.isBusy()); // Remember for next time if there are skips accumulating data.setOutputs(next); return next; }
@Override public Object doWithRetry(RetryContext context) throws Exception { contextHolder.set(context); if (!data.scanning()) { chunkMonitor.setChunkSize(inputs.size()); try { doWrite(outputs.getItems()); } catch (Exception e) { if (rollbackClassifier.classify(e)) { throw e; } /* * If the exception is marked as no-rollback, we need to * override that, otherwise there's no way to write the * rest of the chunk or to honour the skip listener * contract. */ throw new ForceRollbackForWriteSkipException( "Force rollback on skippable exception so that skipped item can be located.", e); } contribution.incrementWriteCount(outputs.size()); } else { scan(contribution, inputs, outputs, chunkMonitor, false); } return null; } };
@Test public void testAfterWrite() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "fail", "bar")); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void afterWrite(List<? extends String> item) { after.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk); processor.process(contribution, chunk); assertEquals(2, chunk.getItems().size()); processAndExpectPlannedRuntimeException(chunk); assertEquals(1, chunk.getItems().size()); processor.process(contribution, chunk); assertEquals(0, chunk.getItems().size()); // foo is written once because it the failure is detected before it is // committed the first time assertEquals("[foo, bar]", list.toString()); // the after listener is called once per successful item, which is // important assertEquals("[foo, bar]", after.toString()); }
@Override @SuppressWarnings({ "unchecked", "rawtypes" }) public Object doWithRetry(RetryContext context) throws Exception { chunkMonitor.setChunkSize(chunk.size()); try { doPersist(contribution, chunk); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // Per section 9.2.7 of JSR-352, the SkipListener receives all the items within the chunk ((MulticasterBatchListener) getListener()).onSkipInWrite(chunk.getItems(), e); } else { getListener().onRetryWriteException((List<Object>) chunk.getItems(), e); if (rollbackClassifier.classify(e)) { throw e; } } /* * If the exception is marked as no-rollback, we need to * override that, otherwise there's no way to write the * rest of the chunk or to honour the skip listener * contract. */ throw new ForceRollbackForWriteSkipException( "Force rollback on skippable exception so that skipped item can be located.", e); } contribution.incrementWriteCount(chunk.size()); return null; } };
final UserData<O> data = (UserData<O>) inputs.getUserData(); final Chunk<O> cache = data.getOutputs(); final Iterator<O> cacheIterator = cache.isEmpty() ? null : new ArrayList<>(cache.getItems()).iterator(); final AtomicInteger count = new AtomicInteger(0);
@Test public void testProvide() throws Exception { provider = new FaultTolerantChunkProvider<>(new ListItemReader<>(Arrays.asList("foo", "bar")), new RepeatTemplate()); Chunk<String> chunk = provider.provide(contribution); assertNotNull(chunk); assertEquals(2, chunk.getItems().size()); }
@Test public void testProvide() throws Exception { provider = new SimpleChunkProvider<>(new ListItemReader<>(Arrays.asList("foo", "bar")), new RepeatTemplate()); Chunk<String> chunk = provider.provide(contribution); assertNotNull(chunk); assertEquals(2, chunk.getItems().size()); }
@Test public void testProvideWithOverflow() throws Exception { provider = new SimpleChunkProvider<String>(new ListItemReader<>(Arrays.asList("foo", "bar")), new RepeatTemplate()) { @Override protected String read(StepContribution contribution, Chunk<String> chunk) throws SkipOverflowException, Exception { chunk.skip(new RuntimeException("Planned")); throw new SkipOverflowException("Overflow"); } }; Chunk<String> chunk = null; chunk = provider.provide(contribution); assertNotNull(chunk); assertEquals(0, chunk.getItems().size()); assertEquals(1, chunk.getErrors().size()); }
@Test public void testProvideWithOverflow() throws Exception { provider = new FaultTolerantChunkProvider<>(new ItemReader<String>() { @Override public String read() throws Exception, UnexpectedInputException, ParseException { throw new RuntimeException("Planned"); } }, new RepeatTemplate()); provider.setSkipPolicy(new LimitCheckingItemSkipPolicy(Integer.MAX_VALUE, Collections.<Class<? extends Throwable>,Boolean>singletonMap(Exception.class, Boolean.TRUE))); provider.setMaxSkipsOnRead(10); Chunk<String> chunk = null; chunk = provider.provide(contribution); assertNotNull(chunk); assertEquals(0, chunk.getItems().size()); assertEquals(10, chunk.getErrors().size()); } }
/** * Responsible for the writing portion of the chunking loop. In this implementation, delegates to the * {{@link #doPersist(StepContribution, Chunk)}. * * @param contribution a {@link StepContribution} * @param chunk a {@link Chunk} * @throws Exception thrown if error occurs during the writing portion of the chunking loop. */ protected void persist(final StepContribution contribution, final Chunk<O> chunk) throws Exception { doPersist(contribution, chunk); contribution.incrementWriteCount(chunk.getItems().size()); }