private String oomErrorMessage(Reporter reporter) { return "Got OOM while mapping record #: " + recordCounter + " from input: " + ((FileSplit) reporter.getInputSplit()).getPath().getName(); }
@Override public void map(AvroWrapper<IN> wrapper, NullWritable value, OutputCollector<KO, VO> collector, Reporter reporter) throws IOException { if (mapper == null) { TaggedInputSplit is = (TaggedInputSplit) reporter.getInputSplit(); Class<? extends AvroMapper> mapperClass = is.getMapperClass(); mapper = (AvroMapper<IN,OUT>) ReflectionUtils.newInstance(mapperClass, conf); } if (out == null) out = new MapCollector<OUT,K,V,KO,VO>(collector, isMapOnly); mapper.map(wrapper.datum(), out, reporter); } }
@Override public InputSplit getInputSplit() throws UnsupportedOperationException { return wrappedReporter.getInputSplit(); }
@Override public InputSplit getInputSplit() { return NULL.getInputSplit(); }
@Override public InputSplit getInputSplit() throws UnsupportedOperationException { return wrappedReporter.getInputSplit(); }
@Override public InputSplit getInputSplit() throws UnsupportedOperationException { return wrappedReporter.getInputSplit(); }
@Override public InputSplit getInputSplit() throws UnsupportedOperationException { return wrappedReporter.getInputSplit(); }
@Override public InputSplit getInputSplit() throws UnsupportedOperationException { return wrappedReporter.getInputSplit(); }
@Override public InputSplit getInputSplit() throws UnsupportedOperationException { return wrappedReporter.getInputSplit(); }
/** * test Reporter.NULL * */ @Test (timeout=5000) public void testReporter(){ Reporter nullReporter=Reporter.NULL; assertNull(nullReporter.getCounter(null)); assertNull(nullReporter.getCounter("group", "name")); // getInputSplit method removed try{ assertNull(nullReporter.getInputSplit()); }catch(UnsupportedOperationException e){ assertEquals( "NULL reporter has no input",e.getMessage()); } assertEquals(0,nullReporter.getProgress(),0.01); } }
public void map(WritableComparable key, Writable value, OutputCollector<WritableComparable, Writable> out, Reporter reporter) throws IOException { System.out.println("map: " + key + ", " + value); out.collect((WritableComparable) value, key); InputSplit split = reporter.getInputSplit(); if (split.getClass() != MyInputFormat.MySplit.class) { throw new IOException("Got wrong split in MyMapper! " + split.getClass().getName()); } } }
public void map(LongWritable key, Text val, OutputCollector<Text, Text> output, Reporter reporter) throws IOException { FileSplit fileSplit = (FileSplit) reporter.getInputSplit(); String fileName = fileSplit.getPath().getName(); location.set(fileName); String line = val.toString(); StringTokenizer itr = new StringTokenizer(line.toLowerCase()); while (itr.hasMoreTokens()) { word.set(itr.nextToken()); output.collect(word, location); } } }
@SuppressWarnings("unchecked") public void map(K1 key, V1 value, OutputCollector<K2, V2> outputCollector, Reporter reporter) throws IOException { if (mapper == null) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit) reporter.getInputSplit(); mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(inputSplit .getMapperClass(), conf); } mapper.map(key, value, outputCollector, reporter); }
@SuppressWarnings("unchecked") public void map(K1 key, V1 value, OutputCollector<K2, V2> outputCollector, Reporter reporter) throws IOException { if (mapper == null) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit) reporter.getInputSplit(); mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(inputSplit .getMapperClass(), conf); } mapper.map(key, value, outputCollector, reporter); }
@SuppressWarnings("unchecked") public void map(K1 key, V1 value, OutputCollector<K2, V2> outputCollector, Reporter reporter) throws IOException { if (mapper == null) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit) reporter.getInputSplit(); mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(inputSplit .getMapperClass(), conf); } mapper.map(key, value, outputCollector, reporter); }
@SuppressWarnings("unchecked") public void map(K1 key, V1 value, OutputCollector<K2, V2> outputCollector, Reporter reporter) throws IOException { if (mapper == null) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit) reporter.getInputSplit(); mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(inputSplit .getMapperClass(), conf); } mapper.map(key, value, outputCollector, reporter); }
@SuppressWarnings("unchecked") public void map(K1 key, V1 value, OutputCollector<K2, V2> outputCollector, Reporter reporter) throws IOException { if (mapper == null) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit) reporter.getInputSplit(); mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(inputSplit .getMapperClass(), conf); } mapper.map(key, value, outputCollector, reporter); }
@SuppressWarnings("unchecked") public void map(K1 key, V1 value, OutputCollector<K2, V2> outputCollector, Reporter reporter) throws IOException { if (mapper == null) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit) reporter.getInputSplit(); mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(inputSplit .getMapperClass(), conf); } mapper.map(key, value, outputCollector, reporter); }
@SuppressWarnings("unchecked") public void map(K1 key, V1 value, OutputCollector<K2, V2> outputCollector, Reporter reporter) throws IOException { if (mapper == null) { // Find the Mapper from the TaggedInputSplit. TaggedInputSplit inputSplit = (TaggedInputSplit) reporter.getInputSplit(); mapper = (Mapper<K1, V1, K2, V2>) ReflectionUtils.newInstance(inputSplit .getMapperClass(), conf); } mapper.map(key, value, outputCollector, reporter); }
@Override public void map(AvroWrapper<IN> wrapper, NullWritable value, OutputCollector<KO, VO> collector, Reporter reporter) throws IOException { if (mapper == null) { TaggedInputSplit is = (TaggedInputSplit) reporter.getInputSplit(); Class<? extends AvroMapper> mapperClass = is.getMapperClass(); mapper = (AvroMapper<IN,OUT>) ReflectionUtils.newInstance(mapperClass, conf); } if (out == null) out = new MapCollector<OUT,K,V,KO,VO>(collector, isMapOnly); mapper.map(wrapper.datum(), out, reporter); } }