/** * Handles common parameter initialization that a subclass might want to leverage. * @param context * @param conf */ protected void doSetup(Context context, Configuration conf) { // If a custom separator has been used, // decode it back from Base64 encoding. separator = conf.get(ImportTsv.SEPARATOR_CONF_KEY); if (separator == null) { separator = ImportTsv.DEFAULT_SEPARATOR; } else { separator = Bytes.toString(Base64.getDecoder().decode(separator)); } // Should never get 0 as we are setting this to a valid value in job configuration. ts = conf.getLong(ImportTsv.TIMESTAMP_CONF_KEY, 0); skipBadLines = context.getConfiguration().getBoolean(ImportTsv.SKIP_LINES_CONF_KEY, true); badLineCount = context.getCounter("ImportTsv", "Bad Lines"); }
@Override protected void reduce(AvroKey<GenericRecord> key, Iterable<AvroValue<GenericRecord>> values, Context context) throws IOException, InterruptedException { int numVals = 0; AvroValue<GenericRecord> valueToRetain = null; for (AvroValue<GenericRecord> value : values) { if (valueToRetain == null) { valueToRetain = value; } else if (this.deltaComparatorOptional.isPresent()) { valueToRetain = this.deltaComparatorOptional.get().compare(valueToRetain, value) >= 0 ? valueToRetain : value; } numVals++; } this.outKey.datum(valueToRetain.datum()); if (numVals > 1) { context.getCounter(EVENT_COUNTER.MORE_THAN_1).increment(1); context.getCounter(EVENT_COUNTER.DEDUPED).increment(numVals - 1); } context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1); context.write(this.outKey, NullWritable.get()); }
/** {@inheritDoc} */ @Override public void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException, InterruptedException { ctx.getCounter(TestCounter.COUNTER1).increment(1); ctx.getCounter(TestCounter.COUNTER2).increment(1); int sum = 0; for (IntWritable value : values) sum += value.get(); ctx.write(key, new IntWritable(sum)); } }
public static void incrementCounter(TaskID taskId, String groupName, String counterName, long incr) { @SuppressWarnings("rawtypes") Reducer.Context context = contextMap.get(taskId); if(context != null) { context.getCounter(groupName, counterName).increment(incr); } }
@Override protected void reduce(Text key, Iterable<BytesWritable> values, Context context){ Counter cheeseCounter = context.getCounter(CounterEnum.CHEESE); Counter fleesCounter = context.getCounter(CounterEnum.FLEES); System.out.println(cheeseCounter.getName() + ": " + cheeseCounter.getValue()); System.out.println(fleesCounter.getName() + ": " + fleesCounter.getValue()); } }
public static void incrementCounter(TaskID taskId, String groupName, String counterName, long incr) { Reducer<?,?,?,?>.Context context = contextMap.get(taskId); if (context != null) { context.getCounter(groupName, counterName).increment(incr); } }
@Override public void reduce(WritableComparable<?> key, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { Vector merged = VectorWritable.merge(vectors.iterator()).get(); result.set(new SequentialAccessSparseVector(merged)); ctx.write(key, result); ctx.getCounter(Stats.NUM_USERS).increment(1); } }
@Override public void setup(Context context) throws IOException { refsChecked = context.getCounter(Counters.REFERENCES_CHECKED); rowsWritten = context.getCounter(Counters.ROWS_WRITTEN); }
public static void incrementCounter(TaskID taskId, Enum counterName, long incr) { Reducer<?,?,?,?>.Context context = contextMap.get(taskId); if (context != null) { context.getCounter(counterName).increment(incr); } }
protected void collectStats(final LongWritable key, final double sum, final Context context) { final long level = (key.get() % numLevels) + minLevel; Double max = maxPerLevel.get(level); if ((max == null) || (sum > max)) { max = sum; maxPerLevel.put(level, max); } context.getCounter("Entries per level", "level " + Long.toString(level)).increment(1); }
public static void incrementCounter(TaskID taskId, Enum<?> counterName, long incr) { Reducer<?,?,?,?>.Context context = contextMap.get(taskId); if (context != null) { context.getCounter(counterName).increment(incr); } }
@Override protected void reduce(Text key, Iterable<BytesWritable> values, Context context){ Counter cheeseCounter = context.getCounter(CounterEnum.CHEESE); Counter fleesCounter = context.getCounter(CounterEnum.FLEES); System.out.println(cheeseCounter.getName() + ": " + cheeseCounter.getValue()); System.out.println(fleesCounter.getName() + ": " + fleesCounter.getValue()); } }
protected void collectStats(final long key, final Context context) { final long level = (key % numLevels) + minLevel; context.getCounter("Entries per level", "level " + Long.toString(level)).increment(1); } }
private void incrementCounter(Context context,Enum <?> counterId,long amount) { Counter counter=context.getCounter(counterId); if(counter!=null) { counter.increment(amount); }; };
@Override public void reduce(WritableComparable<?> key, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { Vector merged = VectorWritable.merge(vectors.iterator()).get(); result.set(new SequentialAccessSparseVector(merged)); ctx.write(key, result); ctx.getCounter(Stats.NUM_USERS).increment(1); } }
public static void incrementCounter(TaskID taskId, String groupName, String counterName, long incr) { Reducer<?,?,?,?>.Context context = contextMap.get(taskId); if (context != null) { context.getCounter(groupName, counterName).increment(incr); } }
@Override protected void reduce(AvroKey<GenericRecord> key, Iterable<AvroValue<GenericRecord>> values, Context context) throws IOException, InterruptedException { int numVals = 0; for (AvroValue<GenericRecord> av : values) { outKey.datum(av.datum()); numVals++; } if (numVals > 1) { context.getCounter(EVENT_COUNTER.MORE_THAN_1).increment(1); context.getCounter(EVENT_COUNTER.DEDUPED).increment(numVals - 1); } context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1); context.write(outKey, NullWritable.get()); } }
String keyString = Bytes.toStringBinary(key.getBytes(), 0, key.getLength()); LOG.error("LinkedListError: key=" + keyString + ", lost big or tiny families"); context.getCounter(Counts.LOST_FAMILIES).increment(1); context.write(key, LOSTFAM); ordinal = Counts.UNDEFINED.ordinal(); context.write(key, new BytesWritable(addPrefixFlag(ordinal, bs))); context.getCounter(Counts.UNDEFINED).increment(1); } else { ordinal = Counts.EXTRA_UNDEF_REFERENCES.ordinal(); context.getCounter("undef", keyString).increment(1); context.getCounter(Counts.UNREFERENCED).increment(1); if (rows.addAndGet(1) < MISSING_ROWS_TO_LOG) { String keyString = Bytes.toStringBinary(key.getBytes(), 0, key.getLength()); context.getCounter("unref", keyString).increment(1); new BytesWritable(addPrefixFlag(Counts.EXTRAREFERENCES.ordinal(), refs.get(i)))); context.getCounter(Counts.EXTRAREFERENCES).increment(refs.size() - 1); context.getCounter(Counts.REFERENCED).increment(1);
/** {@inheritDoc} */ @Override public void reduce(Text key, Iterable<IntWritable> values, Context ctx) throws IOException, InterruptedException { ctx.getCounter(TestCounter.COUNTER1).increment(1); ctx.getCounter(TestCounter.COUNTER3).increment(1); } }
@Override public void setup(Context context) throws IOException { refsChecked = context.getCounter(Counters.REFERENCES_CHECKED); rowsWritten = context.getCounter(Counters.ROWS_WRITTEN); }