@Override public RepeatStatus doInIteration(RepeatContext context) throws Exception { I item = provide(contribution, chunk); if(item != null) { contribution.incrementReadCount(); } else { return RepeatStatus.FINISHED; } O processedItem = transform(contribution, item); if(processedItem == null) { filterCount.incrementAndGet(); } else { output.add(processedItem); } return RepeatStatus.CONTINUABLE; } });
@Override public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; }
@Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { contribution.incrementReadCount(); contribution.incrementWriteCount(1); return RepeatStatus.continueIf(count.incrementAndGet() < maxCount); }
@Override public Chunk<String> provide(StepContribution contribution) throws Exception { contribution.incrementReadCount(); Chunk<String> chunk = new Chunk<>(); chunk.add("foo"); return chunk; } @Override
@Override public Chunk<String> provide(StepContribution contribution) throws Exception { contribution.incrementReadCount(); Chunk<String> chunk = new Chunk<>(); chunk.add("foo"); chunk.setEnd(); return chunk; } @Override
@Test public void testApplyContribution() throws Exception { StepContribution contribution = execution.createStepContribution(); contribution.incrementReadSkipCount(); contribution.incrementWriteSkipCount(); contribution.incrementReadCount(); contribution.incrementWriteCount(7); contribution.incrementFilterCount(1); execution.apply(contribution); assertEquals(1, execution.getReadSkipCount()); assertEquals(1, execution.getWriteSkipCount()); assertEquals(1, execution.getReadCount()); assertEquals(7, execution.getWriteCount()); assertEquals(1, execution.getFilterCount()); }
contribution.incrementReadCount(); return RepeatStatus.FINISHED;
@Override public RepeatStatus doInIteration(RepeatContext context) throws Exception { I item = provide(contribution, chunk); if(item != null) { contribution.incrementReadCount(); } else { return RepeatStatus.FINISHED; } O processedItem = transform(contribution, item); if(processedItem == null) { filterCount.incrementAndGet(); } else { output.add(processedItem); } return RepeatStatus.CONTINUABLE; } });
@Override public RepeatStatus doInIteration(RepeatContext context) throws Exception { I item = provide(contribution, chunk); if(item != null) { contribution.incrementReadCount(); } else { return RepeatStatus.FINISHED; } O processedItem = transform(contribution, item); if(processedItem == null) { filterCount.incrementAndGet(); } else { output.add(processedItem); } return RepeatStatus.CONTINUABLE; } });
@Override public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item instanceof FlushFlag) { return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; }
@Override public RepeatStatus doInIteration(RepeatContext context) throws Exception { I item = provide(contribution, chunk); if(item != null) { contribution.incrementReadCount(); } else { return RepeatStatus.FINISHED; } O processedItem = transform(contribution, item); if(processedItem == null) { filterCount.incrementAndGet(); } else { output.add(processedItem); } return RepeatStatus.CONTINUABLE; } });
@Override public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; }
public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; }
@Override public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; }
@Override public RepeatStatus doInIteration(final RepeatContext context) throws Exception { I item = null; try { item = read(contribution, inputs); } catch (SkipOverflowException e) { // read() tells us about an excess of skips by throwing an // exception return RepeatStatus.FINISHED; } if (item == null) { inputs.setEnd(); return RepeatStatus.FINISHED; } inputs.add(item); contribution.incrementReadCount(); return RepeatStatus.CONTINUABLE; }
private void saveStats(List<ExecJob> execs, StepContribution contribution) throws Exception { if (CollectionUtils.isEmpty(execs) || contribution == null) { return; } for (ExecJob execJob : execs) { PigStats stats = execJob.getStatistics(); // embedded pig contains no stats and further more throws Exceptions // use CDH3 compatible comparison if (stats != null && !stats.getClass().getName().contains("EmbeddedPigStats")) { // compute the input stats manually List<InputStats> inputStats = stats.getInputStats(); for (InputStats is : inputStats) { for (int i = 0; i < safeLongToInt(is.getNumberRecords()); i++) { contribution.incrementReadCount(); } } contribution.incrementWriteCount(safeLongToInt(stats.getRecordWritten())); // Skip information not available yet // workaround: query the internal map/reduce jobs ? //contribution.incrementReadSkipCount(safeLongToInt(count.getValue())); // //for (int i = 0; i < safeLongToInt(count.getValue()); i++) { // contribution.incrementWriteSkipCount(); //} } } }
@Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { RepeatStatus status = RepeatStatus.CONTINUABLE; if (chunkContext.getStepContext().getStepExecution().getReadCount() < resources.length) { // On each iteration we add as an attachment a resource Resource resource = resources[chunkContext.getStepContext().getStepExecution().getReadCount()]; helper.addAttachment(resource.getFilename(), resource.getFile()); // We confirm that we read one resource contribution.incrementReadCount(); } else { // We send the e-mail on the last iteration this.mailSender.send(helper.getMimeMessage()); // We confirm the number of attachments contribution.incrementWriteCount(resources.length); status = RepeatStatus.FINISHED; } return status; }
@SuppressWarnings("deprecation") private void saveCounters(Job job, StepContribution contribution) { Counters counters = null; try { counters = job.getCounters(); } catch (Exception ex) { if (RuntimeException.class.isAssignableFrom(ex.getClass())) { throw (RuntimeException)ex; } else { // ignore - we just can't get stats } } if (counters == null) { return; } // TODO: remove deprecation suppress when we don't want to rely on org.apache.hadoop.mapred Counter count = counters.findCounter(Task.Counter.MAP_INPUT_RECORDS); for (int i = 0; i < safeLongToInt(count.getValue()); i++) { contribution.incrementReadCount(); } count = counters.findCounter(Task.Counter.MAP_SKIPPED_RECORDS); contribution.incrementReadSkipCount(safeLongToInt(count.getValue())); count = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS); contribution.incrementWriteCount(safeLongToInt(count.getValue())); count = counters.findCounter(Task.Counter.REDUCE_SKIPPED_RECORDS); for (int i = 0; i < safeLongToInt(count.getValue()); i++) { contribution.incrementWriteSkipCount(); } }
@Override public RepeatStatus execute(StepContribution contribution, ChunkContext chunkContext) throws Exception { ExecutionContext context = chunkContext.getStepContext().getStepExecution().getExecutionContext(); RepeatStatus result; if (!opened) { reader.open(context); opened = true; } try { T item = reader.read(); reader.update(context); if (item != null) { // We keep trace of non-null read contribution.incrementReadCount(); result = finished(contribution, doRead(contribution, context, item)); } else { result = finished(contribution, ExitStatus.COMPLETED); } } catch (Exception e) { result = finished(contribution, ExitStatus.FAILED.addExitDescription(e)); } reader.update(context); return result; }