public void map(ImmutableBytesWritable row, Result values, OutputCollector<ImmutableBytesWritable, Result> output, Reporter reporter) throws IOException { // Count every row containing data, whether it's in qualifiers or values reporter.incrCounter(Counters.ROWS, 1); }
private void createDpDir(final Path dpPath) throws IOException { if (!fs.exists(dpPath)) { fs.mkdirs(dpPath); if (reporter != null) { reporter.incrCounter(counterGroup, Operator.HIVE_COUNTER_CREATED_DYNAMIC_PARTITIONS, 1); } } }
private void createDpDirCheckSrc(final Path dpStagingPath, final Path dpFinalPath) throws IOException { if (!fs.exists(dpStagingPath) && !fs.exists(dpFinalPath)) { fs.mkdirs(dpStagingPath); // move task will create dp final path if (reporter != null) { reporter.incrCounter(counterGroup, Operator.HIVE_COUNTER_CREATED_DYNAMIC_PARTITIONS, 1); } } }
private void incrCounter(String line) { String trimmedLine = line.substring(counterPrefix.length()).trim(); String[] columns = trimmedLine.split(","); if (columns.length == 3) { try { reporter.incrCounter(columns[0], columns[1], Long.parseLong(columns[2])); } catch (NumberFormatException e) { LOG.warn("Cannot parse counter increment '" + columns[2] + "' from line " + line); } } else { LOG.warn("Cannot parse counter line: " + line); } }
@Override public void func(Operator op) { Map<String, Long> opStats = op.getStats(); for (Map.Entry<String, Long> e : opStats.entrySet()) { if (rp != null) { rp.incrCounter(groupName, e.getKey(), e.getValue()); } } } }
private void incrCounter(String line) { String trimmedLine = line.substring(counterPrefix.length()).trim(); String[] columns = trimmedLine.split(","); if (columns.length == 3) { try { reporter.incrCounter(columns[0], columns[1], Long.parseLong(columns[2])); } catch (NumberFormatException e) { LOG.warn("Cannot parse counter increment '" + columns[2] + "' from line " + line); } } else { LOG.warn("Cannot parse counter line: " + line); } }
@SuppressWarnings({"unchecked"}) public void write(Object key, Object value) throws IOException { reporter.incrCounter(COUNTERS_GROUP, counterName, 1); writer.write(key, value); }
@Override public void func(Operator op) { Map<String, Long> opStats = op.getStats(); for (Map.Entry<String, Long> e : opStats.entrySet()) { if (rp != null) { rp.incrCounter(groupName, e.getKey(), e.getValue()); } } } }
@Test @SuppressWarnings({ "deprecation", "unchecked" }) public void shouldRegInReportEveryIncomingRow() throws IOException { int iterationNumber = 999; RowCounter.RowCounterMapper mapper = new RowCounter.RowCounterMapper(); Reporter reporter = mock(Reporter.class); for (int i = 0; i < iterationNumber; i++) mapper.map(mock(ImmutableBytesWritable.class), mock(Result.class), mock(OutputCollector.class), reporter); Mockito.verify(reporter, times(iterationNumber)).incrCounter( any(), anyLong()); }
} else if(numTuples > 1) { reporter.incrCounter(CollisionCounter.NUM_COLLISIONS, 1); reporter.incrCounter(CollisionCounter.MAX_COLLISIONS, numTuples - numCollisions);
reporter.incrCounter(counterGroup, Operator.HIVE_COUNTER_CREATED_FILES, 1);
reporter.incrCounter(HiveConf.getVar(hconf, HiveConf.ConfVars.HIVECOUNTERGROUP), Operator.HIVECOUNTERCREATEDFILES, 1);
@Override public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException { int slotId = Integer.parseInt(value.toString().trim()); long[] range = HtmlCore.getPageRange(slotId, pages, slotpages); for (long i=range[0]; i<range[1]; i++) { key.set(i); Text v = new Text(Long.toString(i)); output.collect(key, v); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+v.getLength()); } } }
public void reduce (final IntWritable key, final Iterator<Text> values, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { int i; double next_rank = 0; double previous_rank = 0; while (values.hasNext()) { String cur_value_str = values.next().toString(); if( cur_value_str.charAt(0) == 's' ) previous_rank = Double.parseDouble( cur_value_str.substring(1) ); else next_rank += Double.parseDouble( cur_value_str.substring(1) ) ; } next_rank = next_rank * mixing_c + random_coeff; output.collect( key, new Text("v" + next_rank ) ); if( change_reported == 0 ) { double diff = Math.abs(previous_rank-next_rank); if( diff > converge_threshold ) { reporter.incrCounter(PrCounters.CONVERGE_CHECK, 1); change_reported = 1; } } } }
@Override public void map(LongWritable key, Text value, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { int slotId = Integer.parseInt(value.toString().trim()); long[] range = HtmlCore.getPageRange(slotId, pages, slotpages); generator.fireRandom(slotId); rand = new Random(slotId * 1000 + 101); Text k = new Text(); for (long i=range[0]; i<range[1]; i++) { String classname = "/class" + rand.nextInt(groups); k.set(classname); value.set(generator.genBayesWords()); output.collect(k, value); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, k.getLength()+value.getLength()); if (0==(i % 10000)) { log.info("still running: " + (i - range[0]) + " of " + slotpages); } } } }
@Override public void reduce(LongWritable key, Iterator<JoinBytesInt> values, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException { v.clear(); while (values.hasNext()) { v.add(values.next()); } if (0!=v.ulen) { if (v.refs > 0) { Text value = new Text( new String(v.url) + delim + v.refs + delim + (rand.nextInt(99) + 1) ); output.collect( key, value); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+value.getLength()); } else { missed++; } } else { errors++; } } }
reporter.incrCounter(PrCounters.CONVERGE_CHECK, 1); break;
public void map(LongWritable key, Text value, OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException { int slotId = Integer.parseInt(value.toString().trim()); html.fireRandom(slotId); long[] range = HtmlCore.getPageRange(slotId, pages, slotpages); /** * For output collect */ for (long i=range[0]; i<range[1]; i++) { key.set(i); long[] linkids = html.genPureLinkIds(); for (int j=0; j<linkids.length; j++) { String to = Long.toString(linkids[j]); Text v = new Text(to); output.collect(key, v); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+v.getLength()); } if (0==(i % 10000)) { log.info("still running: " + (i - range[0]) + " of " + slotpages); } } } }
Text value = new Text(visit.nextAccess(new String(vitem.url))); output.collect(key, value); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+value.getLength());
output.collect(new LongWritable(count), new VectorWritable(p)); reporter.setStatus(Long.toString(count + 1) + " samples generated"); reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8 + p.getNumNondefaultElements() * 8);