/** * Replace the chunk processor in the tasklet provided with one that can act as a master in the Remote Chunking * pattern. * * @param tasklet a ChunkOrientedTasklet * @param chunkWriter an ItemWriter that can send the chunks to remote workers * @param stepContributionSource a StepContributionSource used to gather results from the workers */ private void replaceChunkProcessor(ChunkOrientedTasklet<?> tasklet, ItemWriter<T> chunkWriter, final StepContributionSource stepContributionSource) { setField(tasklet, "chunkProcessor", new SimpleChunkProcessor<T, T>(new PassThroughItemProcessor<>(), chunkWriter) { @Override protected void write(StepContribution contribution, Chunk<T> inputs, Chunk<T> outputs) throws Exception { doWrite(outputs.getItems()); // Do not update the step contribution until the chunks are // actually processed updateStepContribution(contribution, stepContributionSource); } }); }
/** * Creates a {@link PassThroughItemProcessor} and uses it to create an * instance of {@link Tasklet}. */ public TestingChunkOrientedTasklet(ItemReader<T> itemReader, ItemWriter<T> itemWriter, RepeatOperations repeatOperations) { this(itemReader, new PassThroughItemProcessor<>(), itemWriter, repeatOperations); }
@Override protected AbstractBeanDefinition parseInternal(Element element, ParserContext parserContext) { String id = element.getAttribute(ID_ATTRIBUTE); Assert.hasText(id, "The id attribute must be specified"); String inputChannel = element.getAttribute(INPUT_CHANNEL_ATTRIBUTE); Assert.hasText(inputChannel, "The input-channel attribute must be specified"); String outputChannel = element.getAttribute(OUTPUT_CHANNEL_ATTRIBUTE); Assert.hasText(outputChannel, "The output-channel attribute must be specified"); String itemProcessor = element.getAttribute(ITEM_PROCESSOR_ATTRIBUTE); String itemWriter = element.getAttribute(ITEM_WRITER_ATTRIBUTE); Assert.hasText(itemWriter, "The item-writer attribute must be specified"); BeanDefinitionRegistry beanDefinitionRegistry = parserContext.getRegistry(); BeanDefinitionBuilder chunkProcessorBuilder = BeanDefinitionBuilder .genericBeanDefinition(SimpleChunkProcessor.class) .addPropertyReference(ITEM_WRITER_PROPERTY_NAME, itemWriter); if(StringUtils.hasText(itemProcessor)) { chunkProcessorBuilder.addPropertyReference(ITEM_PROCESSOR_PROPERTY_NAME, itemProcessor); } else { chunkProcessorBuilder.addPropertyValue(ITEM_PROCESSOR_PROPERTY_NAME, new PassThroughItemProcessor<>()); } BeanDefinition chunkProcessorChunkHandler = BeanDefinitionBuilder .genericBeanDefinition(ChunkProcessorChunkHandler.class) .addPropertyValue(CHUNK_PROCESSOR_PROPERTY_NAME, chunkProcessorBuilder.getBeanDefinition()) .getBeanDefinition(); beanDefinitionRegistry.registerBeanDefinition(CHUNK_PROCESSOR_CHUNK_HANDLER_BEAN_NAME_PREFIX + id, chunkProcessorChunkHandler); new ServiceActivatorParser(id).parse(element, parserContext); return null; }
/** * Create an {@link IntegrationFlow} with a {@link ChunkProcessorChunkHandler} * configured as a service activator listening to the input channel and replying * on the output channel. * * @return the integration flow */ @SuppressWarnings({"unchecked", "rawtypes"}) public IntegrationFlow build() { Assert.notNull(this.itemWriter, "An ItemWriter must be provided"); Assert.notNull(this.inputChannel, "An InputChannel must be provided"); Assert.notNull(this.outputChannel, "An OutputChannel must be provided"); if(this.itemProcessor == null) { this.itemProcessor = new PassThroughItemProcessor(); } SimpleChunkProcessor<I, O> chunkProcessor = new SimpleChunkProcessor<>(this.itemProcessor, this.itemWriter); ChunkProcessorChunkHandler<I> chunkProcessorChunkHandler = new ChunkProcessorChunkHandler<>(); chunkProcessorChunkHandler.setChunkProcessor(chunkProcessor); return IntegrationFlows .from(this.inputChannel) .handle(chunkProcessorChunkHandler, SERVICE_ACTIVATOR_METHOD_NAME) .channel(this.outputChannel) .get(); }
@Before public void setUp() { batchRetryTemplate = new BatchRetryTemplate(); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { if (items.contains("fail")) { throw new RuntimeException("Planned failure!"); } list.addAll(items); } }, batchRetryTemplate); batchRetryTemplate.setRetryPolicy(new NeverRetryPolicy()); }
@Test public void testOnErrorInWriteAllItemsFail() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "bar")); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { // Always fail in writer throw new RuntimeException("Planned failure!"); } }, batchRetryTemplate); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void onWriteError(Exception e, List<? extends String> item) { writeError.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk);// Process foo, bar processAndExpectPlannedRuntimeException(chunk);// Process foo processAndExpectPlannedRuntimeException(chunk);// Process bar assertEquals("[foo, bar, foo, bar]", writeError.toString()); }
@Test public void testAfterWriteAllPassedInRecovery() throws Exception { Chunk<String> chunk = new Chunk<>(Arrays.asList("foo", "bar")); processor = new FaultTolerantChunkProcessor<>( new PassThroughItemProcessor<>(), new ItemWriter<String>() { @Override public void write(List<? extends String> items) throws Exception { // Fail if there is more than one item if (items.size() > 1) { throw new RuntimeException("Planned failure!"); } list.addAll(items); } }, batchRetryTemplate); processor.setListeners(Arrays .asList(new ItemListenerSupport<String, String>() { @Override public void afterWrite(List<? extends String> item) { after.addAll(item); } })); processor.setWriteSkipPolicy(new AlwaysSkipItemSkipPolicy()); processAndExpectPlannedRuntimeException(chunk); processor.process(contribution, chunk); processor.process(contribution, chunk); assertEquals("[foo, bar]", list.toString()); assertEquals("[foo, bar]", after.toString()); }
@Test public void testSimpleStep() throws Exception { StepParserStepFactoryBean<Object, Object> fb = new StepParserStepFactoryBean<>(); fb.setHasChunkElement(true); fb.setBeanName("step1"); fb.setAllowStartIfComplete(true); fb.setJobRepository(new JobRepositorySupport()); fb.setStartLimit(5); fb.setTransactionManager(new ResourcelessTransactionManager()); fb.setListeners(new StepListener[] { new StepExecutionListenerSupport() }); fb.setIsolation(Isolation.DEFAULT); fb.setTransactionTimeout(-1); fb.setPropagation(Propagation.REQUIRED); fb.setChunkCompletionPolicy(new DummyCompletionPolicy()); fb.setTaskExecutor(new SyncTaskExecutor()); fb.setItemReader(new DummyItemReader()); fb.setItemProcessor(new PassThroughItemProcessor<>()); fb.setItemWriter(new DummyItemWriter()); fb.setStreams(new ItemStream[] {new FlatFileItemReader<>() }); Object step = fb.getObject(); assertTrue(step instanceof TaskletStep); Object tasklet = ReflectionTestUtils.getField(step, "tasklet"); assertTrue(tasklet instanceof ChunkOrientedTasklet<?>); }
fb.setTaskExecutor(new SyncTaskExecutor()); fb.setItemReader(new DummyItemReader()); fb.setItemProcessor(new PassThroughItemProcessor<>()); fb.setItemWriter(new DummyItemWriter()); fb.setStreams(new ItemStream[] {new FlatFileItemReader<>() });
.<String, String>chunk(3) .reader(reader) .processor(new PassThroughItemProcessor<>()) .writer(new DummyItemWriter()) .listener(new AnnotationBasedStepExecutionListener());
@Bean @StepScope @Autowired public ItemProcessor<Binding, Binding> processor() { return new PassThroughItemProcessor<Binding>(); }
processor = new PassThroughItemProcessor<Quad>();