@Override public I recover(RetryContext context) throws Exception { Throwable e = context.getLastThrowable(); if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { contribution.incrementReadSkipCount(); logger.debug("Skipping after failed process", e); return null; } else { if (rollbackClassifier.classify(e)) { // Default is to rollback unless the classifier // allows us to continue throw new RetryException("Non-skippable exception in recoverer while reading", e); } throw new BatchRuntimeException(e); } }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override public I doWithRetry(RetryContext arg0) throws Exception { while (true) { try { return doProvide(contribution, chunk); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); getListener().onSkipInRead(e); logger.debug("Skipping failed input", e); } else { getListener().onRetryReadException(e); if(rollbackClassifier.classify(e)) { throw e; } else { throw e; } } } } } };
@Test public void testApplyContribution() throws Exception { StepContribution contribution = execution.createStepContribution(); contribution.incrementReadSkipCount(); contribution.incrementWriteSkipCount(); contribution.incrementReadCount(); contribution.incrementWriteCount(7); contribution.incrementFilterCount(1); execution.apply(contribution); assertEquals(1, execution.getReadSkipCount()); assertEquals(1, execution.getWriteSkipCount()); assertEquals(1, execution.getReadCount()); assertEquals(7, execution.getWriteCount()); assertEquals(1, execution.getFilterCount()); }
@Override public I recover(RetryContext context) throws Exception { Throwable e = context.getLastThrowable(); if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { contribution.incrementReadSkipCount(); logger.debug("Skipping after failed process", e); return null; } else { if (rollbackClassifier.classify(e)) { // Default is to rollback unless the classifier // allows us to continue throw new RetryException("Non-skippable exception in recoverer while reading", e); } throw new BatchRuntimeException(e); } }
@Override public I recover(RetryContext context) throws Exception { Throwable e = context.getLastThrowable(); if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { contribution.incrementReadSkipCount(); logger.debug("Skipping after failed process", e); return null; } else { if (rollbackClassifier.classify(e)) { // Default is to rollback unless the classifier // allows us to continue throw new RetryException("Non-skippable exception in recoverer while reading", e); } throw new BatchRuntimeException(e); } }
@Override public I recover(RetryContext context) throws Exception { Throwable e = context.getLastThrowable(); if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { contribution.incrementReadSkipCount(); logger.debug("Skipping after failed process", e); return null; } else { if (rollbackClassifier.classify(e)) { // Default is to rollback unless the classifier // allows us to continue throw new RetryException("Non-skippable exception in recoverer while reading", e); } throw new BatchRuntimeException(e); } }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override protected I read(StepContribution contribution, Chunk<I> chunk) throws Exception { while (true) { try { return doRead(); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); if (chunk.getErrors().size() >= maxSkipsOnRead) { throw new SkipOverflowException("Too many skips on read"); } logger.debug("Skipping failed input", e); } else { if (rollbackClassifier.classify(e)) { throw new NonSkippableReadException("Non-skippable exception during read", e); } logger.debug("No-rollback for non-skippable exception (ignored)", e); } } } }
@Override public I doWithRetry(RetryContext arg0) throws Exception { while (true) { try { return doProvide(contribution, chunk); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); getListener().onSkipInRead(e); logger.debug("Skipping failed input", e); } else { getListener().onRetryReadException(e); if(rollbackClassifier.classify(e)) { throw e; } else { throw e; } } } } } };
@Override public I doWithRetry(RetryContext arg0) throws Exception { while (true) { try { return doProvide(contribution, chunk); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); getListener().onSkipInRead(e); logger.debug("Skipping failed input", e); } else { getListener().onRetryReadException(e); if(rollbackClassifier.classify(e)) { throw e; } else { throw e; } } } } } };
@Override public I doWithRetry(RetryContext arg0) throws Exception { while (true) { try { return doProvide(contribution, chunk); } catch (Exception e) { if (shouldSkip(skipPolicy, e, contribution.getStepSkipCount())) { // increment skip count and try again contribution.incrementReadSkipCount(); chunk.skip(e); getListener().onSkipInRead(e); logger.debug("Skipping failed input", e); } else { getListener().onRetryReadException(e); if(rollbackClassifier.classify(e)) { throw e; } else { throw e; } } } } } };
@SuppressWarnings("deprecation") private void saveCounters(Job job, StepContribution contribution) { Counters counters = null; try { counters = job.getCounters(); } catch (Exception ex) { if (RuntimeException.class.isAssignableFrom(ex.getClass())) { throw (RuntimeException)ex; } else { // ignore - we just can't get stats } } if (counters == null) { return; } // TODO: remove deprecation suppress when we don't want to rely on org.apache.hadoop.mapred Counter count = counters.findCounter(Task.Counter.MAP_INPUT_RECORDS); for (int i = 0; i < safeLongToInt(count.getValue()); i++) { contribution.incrementReadCount(); } count = counters.findCounter(Task.Counter.MAP_SKIPPED_RECORDS); contribution.incrementReadSkipCount(safeLongToInt(count.getValue())); count = counters.findCounter(Task.Counter.REDUCE_OUTPUT_RECORDS); contribution.incrementWriteCount(safeLongToInt(count.getValue())); count = counters.findCounter(Task.Counter.REDUCE_SKIPPED_RECORDS); for (int i = 0; i < safeLongToInt(count.getValue()); i++) { contribution.incrementWriteSkipCount(); } }
contribution.incrementReadSkipCount(safeLongToInt(stats.getCounterValue(Task.Counter.MAP_SKIPPED_RECORDS))); contribution.incrementWriteCount(safeLongToInt(stats.getCounterValue(Task.Counter.REDUCE_OUTPUT_RECORDS)));