Tabnine Logo
Mapper$Context.getNumReduceTasks
Code IndexAdd Tabnine to your IDE (free)

How to use
getNumReduceTasks
method
in
org.apache.hadoop.mapreduce.Mapper$Context

Best Java code snippets using org.apache.hadoop.mapreduce.Mapper$Context.getNumReduceTasks (Showing top 16 results out of 315)

origin: apache/hbase

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: apache/incubator-gobblin

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}
origin: confluentinc/camus

@Override
protected void setup(Context context) throws IOException, InterruptedException {
 // For class path debugging
 LOG.info("classpath: " + System.getProperty("java.class.path"));
 ClassLoader loader = AvroKeyMapper.class.getClassLoader();
 LOG.info("com.linkedin.events.fixed_16: " + loader.getResource("com/linkedin/events/fixed_16.class"));
 LOG.info("org.apache.avro.Schema: " + loader.getResource("org/apache/avro/Schema.class"));
 // We need to use RelaxedAvroSerialization to skip the illegal field names
 // during schema parsing.
 //keySchema = AvroJob.getMapOutputKeySchema(context.getConfiguration());
 keySchema = RelaxedAvroSerialization.getKeyWriterSchema(context.getConfiguration());
 outValue = new AvroValue<GenericRecord>();
 outKey = new AvroKey<GenericRecord>();
 outKey.datum(new GenericData.Record(keySchema));
 if (context.getNumReduceTasks() == 0)
  mapOnly = true;
}
origin: ch.cern.hadoop/hadoop-gridmix

final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {
origin: ch.cern.hadoop/hadoop-gridmix

if (context.getNumReduceTasks() > 0 
  && context.getCounter(TaskCounter.SPILLED_RECORDS).getValue() == 0) {
 LOG.info("Boosting the map phase progress.");
origin: org.apache.hbase/hbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: com.aliyun.hbase/alihbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   CellWritableComparable[] startKeyWraps =
     new CellWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: org.apache.hbase/hbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: com.aliyun.hbase/alihbase-mapreduce

 @Override
 public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
    RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
   byte[][] startKeys = regionLocator.getStartKeys();
   if (startKeys.length != reduceNum) {
    throw new IOException("Region split after job initialization");
   }
   KeyValueWritableComparable[] startKeyWraps =
     new KeyValueWritableComparable[startKeys.length - 1];
   for (int i = 1; i < startKeys.length; ++i) {
    startKeyWraps[i - 1] =
      new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
   }
   KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
 }
}
origin: ch.cern.hadoop/hadoop-gridmix

@Override
public void cleanup(Context context)
throws IOException, InterruptedException {
 final int nReds = context.getNumReduceTasks();
 if (nReds > 0) {
  final SleepSplit split = (SleepSplit) context.getInputSplit();
  int id = split.getId();
  final int nMaps = split.getNumMaps();
  //This is a hack to pass the sleep duration via Gridmix key
  //TODO: We need to come up with better solution for this.
  
  final GridmixKey key = new GridmixKey(GridmixKey.REDUCE_SPEC, 0, 0L);
  for (int i = id, idx = 0; i < nReds; i += nMaps) {
   key.setPartition(i);
   key.setReduceOutputBytes(split.getReduceDurations(idx++));
   id += nReds;
   context.write(key, NullWritable.get());
  }
 }
}
origin: org.apache.gobblin/gobblin-compaction

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}
origin: com.linkedin.gobblin/gobblin-compaction

@Override
protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context)
  throws IOException, InterruptedException {
 if (context.getNumReduceTasks() == 0) {
  context.write(key, NullWritable.get());
 } else {
  populateComparableKeyRecord(key.datum(), this.outKey.datum());
  this.outValue.datum(key.datum());
  try {
   context.write(this.outKey, this.outValue);
  } catch (AvroRuntimeException e) {
   final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths();
   throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e);
  }
 }
 context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1);
}
origin: aseldawy/spatialhadoop2

@Override
protected void setup(Context context)
  throws IOException, InterruptedException {
 // TODO Auto-generated method stub
 super.setup(context);
 Configuration conf = context.getConfiguration();
 this.imageWidth = conf.getInt("width", 1000);
 this.imageHeight = conf.getInt("height", 1000);
 this.inputMBR = (Rectangle) OperationsParams.getShape(conf, InputMBR);
 this.outputValue = new IntWritable(0);
 this.plotter = Plotter.getPlotter(conf);
 this.smooth = plotter.isSmooth();
 this.numReducers = Math.max(1, context.getNumReduceTasks());
 this.random = new Random();
}
origin: aseldawy/spatialhadoop2

@Override
protected void setup(
  Mapper<Rectangle, Iterable<S>, IntWritable, OGCJTSShape>.Context context)
    throws IOException, InterruptedException {
 super.setup(context);
 columnBoundaries = SpatialSite.getReduceSpace(context.getConfiguration());
 if (columnBoundaries == null)
  key.set(new Random().nextInt(context.getNumReduceTasks()));
}

origin: ch.cern.hadoop/hadoop-mapreduce-client-jobclient

@Override
protected void setup(Context context) throws IOException {
 // check if the map task attempt progress is 0
 assertEquals("Invalid progress in map setup", 
        0.0f, context.getProgress(), 0f);
 
 // define the progress boundaries
 if (context.getNumReduceTasks() == 0) {
  progressRange = 1f;
 } else {
  progressRange = 0.667f;
 }
}

origin: io.github.repir/RepIRTools

  public void generateKey() {
    int numReduceTasks = context.getNumReduceTasks();
    set( RandomTools.getInt(numReduceTasks) );
  }
}
org.apache.hadoop.mapreduceMapper$ContextgetNumReduceTasks

Popular methods of Mapper$Context

  • write
  • getConfiguration
  • getCounter
  • getInputSplit
  • progress
  • setStatus
  • getTaskAttemptID
  • nextKeyValue
  • getCurrentValue
  • getCurrentKey
  • getJobID
  • getInputFormatClass
  • getJobID,
  • getInputFormatClass,
  • getLocalCacheFiles,
  • getOutputCommitter,
  • getCredentials,
  • getLocalCacheArchives,
  • getStatus,
  • getCacheArchives,
  • getCacheFiles

Popular in Java

  • Start an intent from android
  • notifyDataSetChanged (ArrayAdapter)
  • startActivity (Activity)
  • setContentView (Activity)
  • Container (java.awt)
    A generic Abstract Window Toolkit(AWT) container object is a component that can contain other AWT co
  • Thread (java.lang)
    A thread is a thread of execution in a program. The Java Virtual Machine allows an application to ha
  • Format (java.text)
    The base class for all formats. This is an abstract base class which specifies the protocol for clas
  • TreeSet (java.util)
    TreeSet is an implementation of SortedSet. All optional operations (adding and removing) are support
  • IOUtils (org.apache.commons.io)
    General IO stream manipulation utilities. This class provides static utility methods for input/outpu
  • LoggerFactory (org.slf4j)
    The LoggerFactory is a utility class producing Loggers for various logging APIs, most notably for lo
  • Top PhpStorm plugins
Tabnine Logo
  • Products

    Search for Java codeSearch for JavaScript code
  • IDE Plugins

    IntelliJ IDEAWebStormVisual StudioAndroid StudioEclipseVisual Studio CodePyCharmSublime TextPhpStormVimGoLandRubyMineEmacsJupyter NotebookJupyter LabRiderDataGripAppCode
  • Company

    About UsContact UsCareers
  • Resources

    FAQBlogTabnine AcademyTerms of usePrivacy policyJava Code IndexJavascript Code Index
Get Tabnine for your IDE now