@Override public void setup(Context context) throws IOException { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); int reduceNum = context.getNumReduceTasks(); Configuration conf = context.getConfiguration(); TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME)); try (Connection conn = ConnectionFactory.createConnection(conf); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { byte[][] startKeys = regionLocator.getStartKeys(); if (startKeys.length != reduceNum) { throw new IOException("Region split after job initialization"); } CellWritableComparable[] startKeyWraps = new CellWritableComparable[startKeys.length - 1]; for (int i = 1; i < startKeys.length; ++i) { startKeyWraps[i - 1] = new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i])); } CellWritableComparablePartitioner.START_KEYS = startKeyWraps; } } }
@Override protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context) throws IOException, InterruptedException { if (context.getNumReduceTasks() == 0) { context.write(key, NullWritable.get()); } else { populateComparableKeyRecord(key.datum(), this.outKey.datum()); this.outValue.datum(key.datum()); try { context.write(this.outKey, this.outValue); } catch (AvroRuntimeException e) { final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths(); throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e); } } context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1); }
@Override protected void setup(Context context) throws IOException, InterruptedException { // For class path debugging LOG.info("classpath: " + System.getProperty("java.class.path")); ClassLoader loader = AvroKeyMapper.class.getClassLoader(); LOG.info("com.linkedin.events.fixed_16: " + loader.getResource("com/linkedin/events/fixed_16.class")); LOG.info("org.apache.avro.Schema: " + loader.getResource("org/apache/avro/Schema.class")); // We need to use RelaxedAvroSerialization to skip the illegal field names // during schema parsing. //keySchema = AvroJob.getMapOutputKeySchema(context.getConfiguration()); keySchema = RelaxedAvroSerialization.getKeyWriterSchema(context.getConfiguration()); outValue = new AvroValue<GenericRecord>(); outKey = new AvroKey<GenericRecord>(); outKey.datum(new GenericData.Record(keySchema)); if (context.getNumReduceTasks() == 0) mapOnly = true; }
final int nReduces = ctxt.getNumReduceTasks(); if (nReduces > 0) {
if (context.getNumReduceTasks() > 0 && context.getCounter(TaskCounter.SPILLED_RECORDS).getValue() == 0) { LOG.info("Boosting the map phase progress.");
@Override public void setup(Context context) throws IOException { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); int reduceNum = context.getNumReduceTasks(); Configuration conf = context.getConfiguration(); TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME)); try (Connection conn = ConnectionFactory.createConnection(conf); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { byte[][] startKeys = regionLocator.getStartKeys(); if (startKeys.length != reduceNum) { throw new IOException("Region split after job initialization"); } CellWritableComparable[] startKeyWraps = new CellWritableComparable[startKeys.length - 1]; for (int i = 1; i < startKeys.length; ++i) { startKeyWraps[i - 1] = new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i])); } CellWritableComparablePartitioner.START_KEYS = startKeyWraps; } } }
@Override public void setup(Context context) throws IOException { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); int reduceNum = context.getNumReduceTasks(); Configuration conf = context.getConfiguration(); TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME)); try (Connection conn = ConnectionFactory.createConnection(conf); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { byte[][] startKeys = regionLocator.getStartKeys(); if (startKeys.length != reduceNum) { throw new IOException("Region split after job initialization"); } CellWritableComparable[] startKeyWraps = new CellWritableComparable[startKeys.length - 1]; for (int i = 1; i < startKeys.length; ++i) { startKeyWraps[i - 1] = new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i])); } CellWritableComparablePartitioner.START_KEYS = startKeyWraps; } } }
@Override public void setup(Context context) throws IOException { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); int reduceNum = context.getNumReduceTasks(); Configuration conf = context.getConfiguration(); TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME)); try (Connection conn = ConnectionFactory.createConnection(conf); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { byte[][] startKeys = regionLocator.getStartKeys(); if (startKeys.length != reduceNum) { throw new IOException("Region split after job initialization"); } KeyValueWritableComparable[] startKeyWraps = new KeyValueWritableComparable[startKeys.length - 1]; for (int i = 1; i < startKeys.length; ++i) { startKeyWraps[i - 1] = new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i])); } KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps; } } }
@Override public void setup(Context context) throws IOException { cfRenameMap = createCfRenameMap(context.getConfiguration()); filter = instantiateFilter(context.getConfiguration()); int reduceNum = context.getNumReduceTasks(); Configuration conf = context.getConfiguration(); TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME)); try (Connection conn = ConnectionFactory.createConnection(conf); RegionLocator regionLocator = conn.getRegionLocator(tableName)) { byte[][] startKeys = regionLocator.getStartKeys(); if (startKeys.length != reduceNum) { throw new IOException("Region split after job initialization"); } KeyValueWritableComparable[] startKeyWraps = new KeyValueWritableComparable[startKeys.length - 1]; for (int i = 1; i < startKeys.length; ++i) { startKeyWraps[i - 1] = new KeyValueWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i])); } KeyValueWritableComparablePartitioner.START_KEYS = startKeyWraps; } } }
@Override public void cleanup(Context context) throws IOException, InterruptedException { final int nReds = context.getNumReduceTasks(); if (nReds > 0) { final SleepSplit split = (SleepSplit) context.getInputSplit(); int id = split.getId(); final int nMaps = split.getNumMaps(); //This is a hack to pass the sleep duration via Gridmix key //TODO: We need to come up with better solution for this. final GridmixKey key = new GridmixKey(GridmixKey.REDUCE_SPEC, 0, 0L); for (int i = id, idx = 0; i < nReds; i += nMaps) { key.setPartition(i); key.setReduceOutputBytes(split.getReduceDurations(idx++)); id += nReds; context.write(key, NullWritable.get()); } } }
@Override protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context) throws IOException, InterruptedException { if (context.getNumReduceTasks() == 0) { context.write(key, NullWritable.get()); } else { populateComparableKeyRecord(key.datum(), this.outKey.datum()); this.outValue.datum(key.datum()); try { context.write(this.outKey, this.outValue); } catch (AvroRuntimeException e) { final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths(); throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e); } } context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1); }
@Override protected void map(AvroKey<GenericRecord> key, NullWritable value, Context context) throws IOException, InterruptedException { if (context.getNumReduceTasks() == 0) { context.write(key, NullWritable.get()); } else { populateComparableKeyRecord(key.datum(), this.outKey.datum()); this.outValue.datum(key.datum()); try { context.write(this.outKey, this.outValue); } catch (AvroRuntimeException e) { final Path[] paths = ((CombineFileSplit) context.getInputSplit()).getPaths(); throw new IOException("Unable to process paths " + StringUtils.join(paths, ','), e); } } context.getCounter(EVENT_COUNTER.RECORD_COUNT).increment(1); }
@Override protected void setup(Context context) throws IOException, InterruptedException { // TODO Auto-generated method stub super.setup(context); Configuration conf = context.getConfiguration(); this.imageWidth = conf.getInt("width", 1000); this.imageHeight = conf.getInt("height", 1000); this.inputMBR = (Rectangle) OperationsParams.getShape(conf, InputMBR); this.outputValue = new IntWritable(0); this.plotter = Plotter.getPlotter(conf); this.smooth = plotter.isSmooth(); this.numReducers = Math.max(1, context.getNumReduceTasks()); this.random = new Random(); }
@Override protected void setup( Mapper<Rectangle, Iterable<S>, IntWritable, OGCJTSShape>.Context context) throws IOException, InterruptedException { super.setup(context); columnBoundaries = SpatialSite.getReduceSpace(context.getConfiguration()); if (columnBoundaries == null) key.set(new Random().nextInt(context.getNumReduceTasks())); }
@Override protected void setup(Context context) throws IOException { // check if the map task attempt progress is 0 assertEquals("Invalid progress in map setup", 0.0f, context.getProgress(), 0f); // define the progress boundaries if (context.getNumReduceTasks() == 0) { progressRange = 1f; } else { progressRange = 0.667f; } }
public void generateKey() { int numReduceTasks = context.getNumReduceTasks(); set( RandomTools.getInt(numReduceTasks) ); } }