@Override protected Chunk<O> getAdjustedOutputs(Chunk<I> inputs, Chunk<O> outputs) { @SuppressWarnings("unchecked") UserData<O> data = (UserData<O>) inputs.getUserData(); Chunk<O> previous = data.getOutputs(); Chunk<O> next = new Chunk<>(outputs.getItems(), previous.getSkips()); next.setBusy(previous.isBusy()); // Remember for next time if there are skips accumulating data.setOutputs(next); return next; }
protected Chunk<O> transform(StepContribution contribution, Chunk<I> inputs) throws Exception { Chunk<O> outputs = new Chunk<>(); for (Chunk<I>.ChunkIterator iterator = inputs.iterator(); iterator.hasNext();) { final I item = iterator.next(); O output; try { output = doProcess(item); } catch (Exception e) { /* * For a simple chunk processor (no fault tolerance) we are done * here, so prevent any more processing of these inputs. */ inputs.clear(); throw e; } if (output != null) { outputs.add(output); } else { iterator.remove(); } } return outputs; }
@Override public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; }
/** * Extension point for subclasses to allow them to memorise the contents of * the inputs, in case they are needed for accounting purposes later. The * default implementation sets up some user data to remember the original * size of the inputs. If this method is overridden then some or all of * {@link #isComplete(Chunk)}, {@link #getFilterCount(Chunk, Chunk)} and * {@link #getAdjustedOutputs(Chunk, Chunk)} might also need to be, to * ensure that the user data is handled consistently. * * @param inputs the inputs for the process */ protected void initializeUserData(Chunk<I> inputs) { inputs.setUserData(inputs.size()); }
/** * Extension point for subclasses to calculate the filter count. Defaults to * the difference between input size and output size. * * @param inputs the inputs after transformation * @param outputs the outputs after transformation * * @return the difference in sizes * * @see #initializeUserData(Chunk) */ protected int getFilterCount(Chunk<I> inputs, Chunk<O> outputs) { return (Integer) inputs.getUserData() - outputs.size(); }
@Override protected Chunk<O> transform(final StepContribution contribution, Chunk<I> inputs) throws Exception { Chunk<O> outputs = new Chunk<>(); @SuppressWarnings("unchecked") final UserData<O> data = (UserData<O>) inputs.getUserData(); final Chunk<O> cache = data.getOutputs(); final Iterator<O> cacheIterator = cache.isEmpty() ? null : new ArrayList<>(cache.getItems()).iterator(); final AtomicInteger count = new AtomicInteger(0); for (final Chunk<I>.ChunkIterator iterator = inputs.iterator(); iterator.hasNext();) { getInputKey(item), rollbackClassifier)); if (output != null) { outputs.add(output); outputs.add(cacheIterator.next());
final UserData<O> data = (UserData<O>) inputs.getUserData(); if (outputs.isEmpty() || inputs.isEmpty()) { data.scanning(false); inputs.setBusy(false); chunkMonitor.resetOffset(); return; Chunk<I>.ChunkIterator inputIterator = inputs.iterator(); Chunk<O>.ChunkIterator outputIterator = outputs.iterator(); if (!inputs.getSkips().isEmpty()) { if (outputIterator.hasNext()) { outputIterator.remove(); if (outputs.isEmpty()) { data.scanning(false); inputs.setBusy(false); chunkMonitor.resetOffset();
@Override protected boolean isComplete(Chunk<I> inputs) { /* * Need to remember the write skips across transactions, otherwise they * keep coming back. Since we register skips with the inputs they will * not be processed again but the output skips need to be saved for * registration later with the listeners. The inputs are going to be the * same for all transactions processing the same chunk, but the outputs * are not, so we stash them in user data on the inputs. */ @SuppressWarnings("unchecked") UserData<O> data = (UserData<O>) inputs.getUserData(); Chunk<O> previous = data.getOutputs(); return inputs.isEmpty() && previous.getSkips().isEmpty(); }
@Override public Object recover(RetryContext context) throws Exception { Throwable e = context.getLastThrowable(); if (outputs.size() > 1 && !rollbackClassifier.classify(e)) { throw new RetryException("Invalid retry state during write caused by " + "exception that does not classify for rollback: ", e); } Chunk<I>.ChunkIterator inputIterator = inputs.iterator(); for (Chunk<O>.ChunkIterator outputIterator = outputs.iterator(); outputIterator.hasNext();) { inputIterator.next(); outputIterator.next(); checkSkipPolicy(inputIterator, outputIterator, e, contribution, true); if (!rollbackClassifier.classify(e)) { throw new RetryException( "Invalid retry state during recovery caused by exception that does not classify for rollback: ", e); } } return null; }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override protected void initializeUserData(Chunk<I> inputs) { @SuppressWarnings("unchecked") UserData<O> data = (UserData<O>) inputs.getUserData(); if (data == null) { data = new UserData<>(); inputs.setUserData(data); data.setOutputs(new Chunk<>()); } else { // BATCH-2663: re-initialize filter count when scanning the chunk if (data.scanning()) { data.filterCount = 0; } } }
private void callSkipListeners(final Chunk<I> inputs, final Chunk<O> outputs) { for (SkipWrapper<I> wrapper : inputs.getSkips()) { I item = wrapper.getItem(); if (item == null) { continue; } Throwable e = wrapper.getException(); callProcessSkipListener(item, e); } for (SkipWrapper<O> wrapper : outputs.getSkips()) { Throwable e = wrapper.getException(); try { getListener().onSkipInWrite(wrapper.getItem(), e); } catch (RuntimeException ex) { throw new SkipListenerFailedException("Fatal exception in SkipListener.", ex, e); } } // Clear skips if we are possibly going to process this chunk again outputs.clearSkips(); inputs.clearSkips(); }
/** * Simple implementation delegates to the {@link #doWrite(List)} method and * increments the write count in the contribution. Subclasses can handle * more complicated scenarios, e.g.with fault tolerance. If output items are * skipped they should be removed from the inputs as well. * * @param contribution the current step contribution * @param inputs the inputs that gave rise to the outputs * @param outputs the outputs to write * @throws Exception if there is a problem */ protected void write(StepContribution contribution, Chunk<I> inputs, Chunk<O> outputs) throws Exception { try { doWrite(outputs.getItems()); } catch (Exception e) { /* * For a simple chunk processor (no fault tolerance) we are done * here, so prevent any more processing of these inputs. */ inputs.clear(); throw e; } contribution.incrementWriteCount(outputs.size()); }
@Override public Chunk<T> provide(StepContribution contribution) throws Exception { return new Chunk<>(); }
@Override public Object doWithRetry(RetryContext context) throws Exception { contextHolder.set(context); if (!data.scanning()) { chunkMonitor.setChunkSize(inputs.size()); try { doWrite(outputs.getItems()); } catch (Exception e) { if (rollbackClassifier.classify(e)) { throw e; } /* * If the exception is marked as no-rollback, we need to * override that, otherwise there's no way to write the * rest of the chunk or to honour the skip listener * contract. */ throw new ForceRollbackForWriteSkipException( "Force rollback on skippable exception so that skipped item can be located.", e); } contribution.incrementWriteCount(outputs.size()); } else { scan(contribution, inputs, outputs, chunkMonitor, false); } return null; } };
protected Chunk<O> transform(StepContribution contribution, Chunk<I> inputs) throws Exception { Chunk<O> outputs = new Chunk<O>(); for (Chunk<I>.ChunkIterator iterator = inputs.iterator(); iterator.hasNext();) { final I item = iterator.next(); O output = doProcess(item); if (output != null) { outputs.add(output); } else { iterator.remove(); } } return outputs; }
@Test public void testProvideWithOverflow() throws Exception { provider = new SimpleChunkProvider<String>(new ListItemReader<>(Arrays.asList("foo", "bar")), new RepeatTemplate()) { @Override protected String read(StepContribution contribution, Chunk<String> chunk) throws SkipOverflowException, Exception { chunk.skip(new RuntimeException("Planned")); throw new SkipOverflowException("Overflow"); } }; Chunk<String> chunk = null; chunk = provider.provide(contribution); assertNotNull(chunk); assertEquals(0, chunk.getItems().size()); assertEquals(1, chunk.getErrors().size()); }