org.apache.hadoop.mapreduce.Mapper$Context.getNumReduceTasks java code examples

Common ways to obtain Mapper$Context

private void myMethod () {
 Mapper$Context m =WrappedMapper wrappedMapper;new Context(wrappedMapper)
Mockito mockito;mockito.mock(Mapper.Context.class)
Smart code suggestions by Tabnine
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}

@Override
protected void setup(Context context) throws IOException, InterruptedException {
 // For class path debugging
 LOG.info("classpath: " + System.getProperty("java.class.path"));
 ClassLoader loader = AvroKeyMapper.class.getClassLoader();
 LOG.info("com.linkedin.events.fixed_16: " + loader.getResource("com/linkedin/events/fixed_16.class"));
 LOG.info("org.apache.avro.Schema: " + loader.getResource("org/apache/avro/Schema.class"));
 // We need to use RelaxedAvroSerialization to skip the illegal field names
 // during schema parsing.
 //keySchema = AvroJob.getMapOutputKeySchema(context.getConfiguration());
 keySchema = RelaxedAvroSerialization.getKeyWriterSchema(context.getConfiguration());
 outValue = new AvroValue<GenericRecord>();
 outKey = new AvroKey<GenericRecord>();
 outKey.datum(new GenericData.Record(keySchema));
 if (context.getNumReduceTasks() == 0)
  mapOnly = true;
}

@Override
protected void setup(
  Mapper<Rectangle, Iterable<S>, IntWritable, OGCJTSShape>.Context context)
    throws IOException, InterruptedException {
 super.setup(context);
 columnBoundaries = SpatialSite.getReduceSpace(context.getConfiguration());
 if (columnBoundaries == null)
  key.set(new Random().nextInt(context.getNumReduceTasks()));
}

@Override
protected void setup(Context context)
  throws IOException, InterruptedException {
 // TODO Auto-generated method stub
 super.setup(context);
 Configuration conf = context.getConfiguration();
 this.imageWidth = conf.getInt("width", 1000);
 this.imageHeight = conf.getInt("height", 1000);
 this.inputMBR = (Rectangle) OperationsParams.getShape(conf, InputMBR);
 this.outputValue = new IntWritable(0);
 this.plotter = Plotter.getPlotter(conf);
 this.smooth = plotter.isSmooth();
 this.numReducers = Math.max(1, context.getNumReduceTasks());
 this.random = new Random();
}

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}

@Override
protected void setup(Context context) throws IOException {
 // check if the map task attempt progress is 0
 assertEquals("Invalid progress in map setup", 
        0.0f, context.getProgress(), 0f);
 
 // define the progress boundaries
 if (context.getNumReduceTasks() == 0) {
  progressRange = 1f;
 } else {
  progressRange = 0.667f;
 }
}

@Override
public void cleanup(Context context)
throws IOException, InterruptedException {
 final int nReds = context.getNumReduceTasks();
 if (nReds > 0) {
  final SleepSplit split = (SleepSplit) context.getInputSplit();
  int id = split.getId();
  final int nMaps = split.getNumMaps();
  //This is a hack to pass the sleep duration via Gridmix key
  //TODO: We need to come up with better solution for this.
  
  final GridmixKey key = new GridmixKey(GridmixKey.REDUCE_SPEC, 0, 0L);
  for (int i = id, idx = 0; i < nReds; i += nMaps) {
   key.setPartition(i);
   key.setReduceOutputBytes(split.getReduceDurations(idx++));
   id += nReds;
   context.write(key, NullWritable.get());
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}

  public void generateKey() {
    int numReduceTasks = context.getNumReduceTasks();
    set( RandomTools.getInt(numReduceTasks) );
  }
}

if (context.getNumReduceTasks() > 0 
  && context.getCounter(TaskCounter.SPILLED_RECORDS).getValue() == 0) {
 LOG.info("Boosting the map phase progress.");

final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}

Popular methods of Mapper$Context

Popular in Java

Making http post requests using okhttp
putExtra (Intent)
scheduleAtFixedRate (ScheduledExecutorService)
orElseThrow (Optional)
Return the contained value, if present, otherwise throw an exception to be created by the provided s
Enumeration (java.util)
A legacy iteration interface.New code should use Iterator instead. Iterator replaces the enumeration
CountDownLatch (java.util.concurrent)
A synchronization aid that allows one or more threads to wait until a set of operations being perfor
JButton (javax.swing)
BasicDataSource (org.apache.commons.dbcp)
Basic implementation of javax.sql.DataSource that is configured via JavaBeans properties. This is no
StringUtils (org.apache.commons.lang)
Operations on java.lang.String that arenull safe. * IsEmpty/IsBlank - checks if a String contains
SAXParseException (org.xml.sax)
Encapsulate an XML parse error or warning.> This module, both source code and documentation, is in t
PhpStorm for WordPress

How to use getNumReduceTasksmethodin org.apache.hadoop.mapreduce.Mapper$Context

Best Java code snippets using org.apache.hadoop.mapreduce.Mapper$Context.getNumReduceTasks (Showing top 16 results out of 315)

How to use
getNumReduceTasks
method
in
org.apache.hadoop.mapreduce.Mapper$Context