congrats Icon
New! Tabnine Pro 14-day free trial
Start a free trial
Tabnine Logo
OutputCollector
Code IndexAdd Tabnine to your IDE (free)

How to use
OutputCollector
in
org.apache.hadoop.mapred

Best Java code snippets using org.apache.hadoop.mapred.OutputCollector (Showing top 20 results out of 918)

Refine searchRefine arrow

  • Text
  • IntWritable
  • Reporter
  • LongWritable
  • NullWritable
  • StringTokenizer
origin: apache/ignite

/** {@inheritDoc} */
@Override public void map(LongWritable key, Text val, OutputCollector<Text, IntWritable> output, Reporter reporter)
    throws IOException {
  assert wasConfigured : "Mapper should be configured";
  String line = val.toString();
  StringTokenizer tokenizer = new StringTokenizer(line);
  while (tokenizer.hasMoreTokens()) {
    word.set(tokenizer.nextToken());
    output.collect(word, one);
  }
  HadoopErrorSimulator.instance().onMap();
}
origin: io.hops/hadoop-mapreduce-client-core

public void map(K key, Text value,
        OutputCollector<Text, LongWritable> output,
        Reporter reporter)
 throws IOException {
 // get input text
 String text = value.toString();       // value is line of text
 // tokenize the value
 StringTokenizer st = new StringTokenizer(text);
 while (st.hasMoreTokens()) {
  // output <token,1> pairs
  output.collect(new Text(st.nextToken()), new LongWritable(1));
 }  
}

origin: intel-hadoop/HiBench

  public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException
  {
    final String[] line = value.toString().split("\t");
    output.collect(new IntWritable(Integer.parseInt(line[0])), new Text(line[1]) );
  }
}
origin: apache/flink

@Override
public void map(LongWritable k, Text v, OutputCollector<Text, LongWritable> out, Reporter rep)
    throws IOException {
  // normalize and split the line
  String line = v.toString();
  String[] tokens = line.toLowerCase().split("\\W+");
  // emit the pairs
  for (String token : tokens) {
    if (token.length() > 0) {
      out.collect(new Text(token), new LongWritable(1L));
    }
  }
}
origin: apache/flink

@Override
public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r)
    throws IOException {
  int commentCnt = 0;
  while (vs.hasNext()) {
    String v = vs.next().toString();
    if (v.startsWith(this.countPrefix)) {
      commentCnt++;
    }
  }
  out.collect(k, new IntWritable(commentCnt));
}
origin: intel-hadoop/HiBench

  @Override
  public void map(LongWritable key, Text value,
      OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException {

    int slotId = Integer.parseInt(value.toString().trim());
    long[] range = HtmlCore.getPageRange(slotId, pages, slotpages);
    
    for (long i=range[0]; i<range[1]; i++) {
      key.set(i);
      Text v = new Text(Long.toString(i));
      output.collect(key, v);
      reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+v.getLength());
    }
  }
}
origin: intel-hadoop/HiBench

  public void reduce (final IntWritable key, final Iterator<Text> values, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException
  {
    int i;
    double next_rank = 0;
    double previous_rank = 0;
    while (values.hasNext()) {
      String cur_value_str = values.next().toString();
      if( cur_value_str.charAt(0) == 's' )
        previous_rank = Double.parseDouble( cur_value_str.substring(1) );
      else
        next_rank += Double.parseDouble( cur_value_str.substring(1) ) ;
    }
    next_rank = next_rank * mixing_c + random_coeff;
    output.collect( key, new Text("v" + next_rank ) );
    if( change_reported == 0 ) {
      double diff = Math.abs(previous_rank-next_rank);
      if( diff > converge_threshold ) {
        reporter.incrCounter(PrCounters.CONVERGE_CHECK, 1);
        change_reported = 1;
      }
    }
  }
}
origin: intel-hadoop/HiBench

  @Override
  public void map(LongWritable key, Text value,
      OutputCollector<Text, Text> output, Reporter reporter)
      throws IOException {
    int slotId = Integer.parseInt(value.toString().trim());
    long[] range = HtmlCore.getPageRange(slotId, pages, slotpages);
    generator.fireRandom(slotId);
    rand = new Random(slotId * 1000 + 101);
    
    Text k = new Text();
    for (long i=range[0]; i<range[1]; i++) {
      String classname = "/class" + rand.nextInt(groups);
      k.set(classname);
      value.set(generator.genBayesWords());
      output.collect(k, value);
      reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED,
        k.getLength()+value.getLength());
      if (0==(i % 10000)) {
        log.info("still running: " + (i - range[0]) + " of " + slotpages);
      }
    }
  }
}
origin: intel-hadoop/HiBench

  @Override
  public void map(LongWritable key, Text value,
      OutputCollector<LongWritable, JoinBytesInt> output, Reporter reporter) throws IOException {
    
    String[] items = value.toString().split("[,\t]");
    key.set(Long.parseLong(items[0]));
    uitem.url= items[1].getBytes();
    uitem.ulen = (byte) uitem.url.length;
    
    output.collect(key, uitem);
  }
}
origin: apache/flink

@Override
public void map(final IntWritable k, final Text v,
    final OutputCollector<IntWritable, Text> out, final Reporter r) throws IOException {
  out.collect(k, v);
  out.collect(k, new Text(v.toString().toUpperCase()));
}
origin: intel-hadoop/HiBench

  @Override
  public void reduce(LongWritable key, Iterator<JoinBytesInt> values,
      OutputCollector<LongWritable, Text> output, Reporter reporter) throws IOException {
    v.clear();
    while (values.hasNext()) {
      v.add(values.next());
    }
    
    if (0!=v.ulen) {
      if (v.refs > 0) {
        Text value = new Text(
            new String(v.url) +
            delim +
            v.refs +
            delim +
            (rand.nextInt(99) + 1)
            );
        output.collect(
            key, value);
            
        reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED, 8+value.getLength());
      } else {
        missed++;
      }
    } else {
      errors++;                    
    }
  }
}
origin: intel-hadoop/HiBench

         Reporter reporter
         ) throws IOException {
String field = key.toString();
reporter.setStatus("starting " + field + " ::host = " + hostName);
 String sSum = "";
 while (values.hasNext())
  sSum += values.next().toString() + ";";
 output.collect(key, new Text(sSum));
 reporter.setStatus("finished " + field + " ::host = " + hostName);
 return;
 while (values.hasNext())
  fSum += Float.parseFloat(values.next().toString());
 output.collect(key, new Text(String.valueOf(fSum)));
 reporter.setStatus("finished " + field + " ::host = " + hostName);
 return;
  lSum += Long.parseLong(values.next().toString());
 output.collect(key, new Text(String.valueOf(lSum)));
reporter.setStatus("finished " + field + " ::host = " + hostName);
origin: apache/avro

public void output(ByteBuffer datum) {
 try {
  collector.collect(new TetherData(datum), NullWritable.get());
 } catch (Throwable e) {
  LOG.warn("Error: "+e, e);
  synchronized (this) {
   error = e.toString();
  }
 }
}
origin: apache/avro

 public void map(LongWritable key, Text value,
        OutputCollector<AvroKey<Long>,AvroValue<Utf8>> out,
        Reporter reporter) throws IOException {
  out.collect(new AvroKey<>(key.get()),
        new AvroValue<>(new Utf8(value.toString())));
 }
}
origin: apache/avro

 public void map(LongWritable key, Text value,
         OutputCollector<AvroWrapper<Pair<Long,Utf8>>,NullWritable> out,
         Reporter reporter) throws IOException {
  out.collect(new AvroWrapper<>(new Pair<>(key.get(), new Utf8(value.toString()))),
        NullWritable.get());
 }
}
origin: intel-hadoop/HiBench

  @Override
  public void map(LongWritable key, Text value,
      OutputCollector<LongWritable, NullWritable> output, Reporter reporter) throws IOException {

    String delimiter = "[ \t]";
    String[] pair = value.toString().split(delimiter);
    
    output.collect(
        new LongWritable(Long.parseLong(pair[0])),
        NullWritable.get()
        );
  }
}
origin: apache/avro

 public void reduce(AvroKey<Long> key, Iterator<AvroValue<Utf8>> values,
           OutputCollector<LongWritable, Text> out,
           Reporter reporter) throws IOException {
  while (values.hasNext()) {
   AvroValue<Utf8> value = values.next();
   out.collect(new LongWritable(key.datum()),
         new Text(value.datum().toString()));
  }
 }
}
origin: intel-hadoop/HiBench

String[] numbers = value.toString().split("\t");
int i = 0;
long numSamples = Long.parseLong(numbers[i++]);
  Vector p = new RandomAccessSparseVector(dimension);
  p.assign(vec);
  output.collect(new LongWritable(count), new VectorWritable(p));
  reporter.setStatus(Long.toString(count + 1) + " samples generated");
  reporter.incrCounter(HiBench.Counters.BYTES_DATA_GENERATED,
      8 + p.getNumNondefaultElements() * 8);
origin: intel-hadoop/HiBench

  @Override
  public void reduce(LongWritable key, Iterator<NullWritable> values,
      OutputCollector<NullWritable, Text> output, Reporter reporter)
          throws IOException {

    output.collect(NullWritable.get(), new Text(key.toString()));
  }
}
origin: apache/flink

@Override
public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r)
    throws IOException {
  while (v.hasNext()) {
    out.collect(new IntWritable(k.get() % 4), v.next());
  }
}
org.apache.hadoop.mapredOutputCollector

Javadoc

Collects the <key, value> pairs output by Mappers and Reducers.

OutputCollector is the generalization of the facility provided by the Map-Reduce framework to collect data output by either the Mapper or the Reducer i.e. intermediate outputs or the output of the job.

Most used methods

  • collect
    Adds a key/value pair to the output.

Popular in Java

  • Updating database using SQL prepared statement
  • scheduleAtFixedRate (ScheduledExecutorService)
  • notifyDataSetChanged (ArrayAdapter)
  • compareTo (BigDecimal)
  • BufferedImage (java.awt.image)
    The BufferedImage subclass describes an java.awt.Image with an accessible buffer of image data. All
  • RandomAccessFile (java.io)
    Allows reading from and writing to a file in a random-access manner. This is different from the uni-
  • LinkedHashMap (java.util)
    LinkedHashMap is an implementation of Map that guarantees iteration order. All optional operations a
  • Pattern (java.util.regex)
    Patterns are compiled regular expressions. In many cases, convenience methods such as String#matches
  • DataSource (javax.sql)
    An interface for the creation of Connection objects which represent a connection to a database. This
  • JFrame (javax.swing)
  • Top 12 Jupyter Notebook Extensions
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimAtomGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyStudentsTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now