/** * Post-processes rejected lines(Logging, keeping count, etc). * * @param line the line that was rejected by the producer. * @param context the context in which the rejection occured. * @param reason the reason why this line was rejected. */ public void reject(Text line, FijiTableContext context, String reason) { if (mRejectedLineCounter % mLogRate == 0L) { LOG.error("Rejecting line: {} with reason: {}", line.toString(), reason); } mRejectedLineCounter++; //TODO(FIJIMRLIB-9) Abort this bulk importer job early if rejected records exceed a threshold context.incrementCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_REJECTED); //TODO(FIJIMRLIB-4) Allow this to emit to a rejected output so that import can be reattempted. }
/** * Post-processes incomplete lines(Logging, keeping count, etc). * * @param line the line that was marked incomplete incomplete by the producer. * @param context the context in which the incompletion occured. * @param reason the reason why this line was incomplete. */ public void incomplete(Text line, FijiTableContext context, String reason) { if (mIncompleteLineCounter % mLogRate == 0L) { LOG.error("Incomplete line: {} with reason: {}", line.toString(), reason); } mIncompleteLineCounter++; //TODO(FIJIMRLIB-9) Abort this bulk importer job early if incomplete records exceed a threshold context.incrementCounter(JobHistoryCounters.BULKIMPORTER_RECORDS_INCOMPLETE); //TODO(FIJIMRLIB-4) Add a strict mode where we reject incomplete lines }
/** {@inheritDoc} */ @Override protected void map(FijiRowData input, Context context) throws IOException { Preconditions.checkNotNull(mContext); mPivoter.produce(input, mContext); mContext.incrementCounter(JobHistoryCounters.PIVOTER_ROWS_PROCESSED); }
/** {@inheritDoc} */ @Override public void produce(final FijiRowData row, final FijiTableContext context) throws IOException { final Iterable<FijiCell<Object>> cells; if (mColumn.isFullyQualified()) { cells = row.asIterable(mColumn.getFamily(), mColumn.getQualifier()); } else { cells = row.asIterable(mColumn.getFamily()); } for (FijiCell<Object> cell : cells) { context.incrementCounter(Counters.CELLS_PROCESSED); final DecodedCell<Object> original = new DecodedCell<Object>(cell.getWriterSchema(), cell.getData()); final DecodedCell<Object> rewritten = rewriteCell(original); if (rewritten != original) { context.put( row.getEntityId(), mColumn.getFamily(), mColumn.getQualifier(), cell.getTimestamp(), rewritten.getData()); context.incrementCounter(Counters.CELLS_REWRITTEN); } } }