public LazyInteger(LazyInteger copy) { super(copy); data = new IntWritable(copy.data.get()); }
public Tuple2<Text, IntWritable> call(Tuple2<String, Integer> record) { return new Tuple2(new Text(record._1), new IntWritable(record._2)); } }
@Override public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { int commentCnt = 0; while (vs.hasNext()) { String v = vs.next().toString(); if (v.startsWith("Comment")) { commentCnt++; } } out.collect(new IntWritable(42), new IntWritable(commentCnt)); }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { final String[] line = value.toString().split("\t"); output.collect(new IntWritable(Integer.parseInt(line[0])), new Text(line[1]) ); } }
@SuppressWarnings("unchecked") @Test public void sequenceFile() { String outputDir = new File(tempDir, "output").getAbsolutePath(); List<Tuple2<Integer, String>> pairs = Arrays.asList( new Tuple2<>(1, "a"), new Tuple2<>(2, "aa"), new Tuple2<>(3, "aaa") ); JavaPairRDD<Integer, String> rdd = sc.parallelizePairs(pairs); rdd.mapToPair(pair -> new Tuple2<>(new IntWritable(pair._1()), new Text(pair._2()))) .saveAsHadoopFile(outputDir, IntWritable.class, Text.class, SequenceFileOutputFormat.class); // Try reading the output back as an object file JavaPairRDD<Integer, String> readRDD = sc.sequenceFile(outputDir, IntWritable.class, Text.class).mapToPair(pair -> new Tuple2<>(pair._1().get(), pair._2().toString())); assertEquals(pairs, readRDD.collect()); }
private void runAndVerify(String str, int months, String expResult, GenericUDF udf) throws HiveException { DeferredObject valueObj0 = new DeferredJavaObject(new Text(str)); DeferredObject valueObj1 = new DeferredJavaObject(new IntWritable(months)); DeferredObject[] args = { valueObj0, valueObj1 }; Text output = (Text) udf.evaluate(args); assertEquals("add_months() test ", expResult, output != null ? output.toString() : null); }
@Override public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { while (v.hasNext()) { out.collect(new IntWritable(k.get() % 4), v.next()); } }
@Override public void map(Object key, Text value, Context context) throws IOException, InterruptedException { String items[] = value.toString().split("\\s+"); context.write(new IntWritable(items.length), value); } }
public void reduce (final IntWritable key, final Iterator<V1> values, final OutputCollector<IntWritable, IntWritable> output, final Reporter reporter) throws IOException { int count = 0; while (values.hasNext()) { values.next(); count++; } output.collect( key, new IntWritable( count ) ); } }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { String line_text = value.toString(); int tabpos = line_text.indexOf("\t"); if( tabpos > 0 ) { int out_key = Integer.parseInt(line_text.substring(0, tabpos)); output.collect( new IntWritable(out_key) , new Text(line_text.substring(tabpos+1)) ); } else { output.collect( new IntWritable(Integer.parseInt(line_text)) , new Text("") ); } } }
@SuppressWarnings("unchecked") @Test public void sequenceFile() { String outputDir = new File(tempDir, "output").getAbsolutePath(); List<Tuple2<Integer, String>> pairs = Arrays.asList( new Tuple2<>(1, "a"), new Tuple2<>(2, "aa"), new Tuple2<>(3, "aaa") ); JavaPairRDD<Integer, String> rdd = sc.parallelizePairs(pairs); rdd.mapToPair(pair -> new Tuple2<>(new IntWritable(pair._1()), new Text(pair._2()))) .saveAsHadoopFile(outputDir, IntWritable.class, Text.class, SequenceFileOutputFormat.class); // Try reading the output back as an object file JavaPairRDD<Integer, String> readRDD = sc.sequenceFile(outputDir, IntWritable.class, Text.class).mapToPair(pair -> new Tuple2<>(pair._1().get(), pair._2().toString())); assertEquals(pairs, readRDD.collect()); }
@Override public Tuple2<Text, IntWritable> map(Tuple2<String, Integer> value) throws Exception { return new Tuple2<Text, IntWritable>(new Text(value.f0), new IntWritable(value.f1)); }
@Override public void reduce(IntWritable k, Iterator<Text> vs, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { int commentCnt = 0; while (vs.hasNext()) { String v = vs.next().toString(); if (v.startsWith(this.countPrefix)) { commentCnt++; } } out.collect(k, new IntWritable(commentCnt)); }
private void runAndVerify(String str, String delim, Integer count, String expResult, GenericUDF udf) throws HiveException { DeferredObject valueObj0 = new DeferredJavaObject(str != null ? new Text(str) : null); DeferredObject valueObj1 = new DeferredJavaObject(delim != null ? new Text(delim) : delim); DeferredObject valueObj2 = new DeferredJavaObject(count != null ? new IntWritable(count) : null); DeferredObject[] args = { valueObj0, valueObj1, valueObj2 }; Text output = (Text) udf.evaluate(args); assertEquals("substring_index() test ", expResult, output != null ? output.toString() : null); }
@Override public Object copyObject(Object o) { return o == null ? null : new IntWritable(((IntWritable) o).get()); }
@Override public void reduce(IntWritable k, Iterator<IntWritable> v, OutputCollector<IntWritable, IntWritable> out, Reporter r) throws IOException { int sum = 0; while (v.hasNext()) { sum += v.next().get(); } out.collect(k, new IntWritable(sum)); }
private void runAndVerify(Timestamp ts, int months, Text dateFormat, String expResult, GenericUDF udf) throws HiveException { DeferredObject valueObj0 = new DeferredJavaObject(new TimestampWritableV2(ts)); DeferredObject valueObj1 = new DeferredJavaObject(new IntWritable(months)); DeferredObject valueObj2 = new DeferredJavaObject(dateFormat); DeferredObject[] args = {valueObj0, valueObj1, valueObj2}; Text output = (Text) udf.evaluate(args); assertEquals("add_months() test for timestamp", expResult, output != null ? output.toString() : null); }
public void map (final LongWritable key, final Text value, final OutputCollector<IntWritable, Text> output, final Reporter reporter) throws IOException { final String[] line = value.toString().split("\t"); output.collect(new IntWritable(Integer.parseInt(line[0])), new Text(line[1]) ); } }
@SuppressWarnings("unchecked") @Test public void sequenceFile() { String outputDir = new File(tempDir, "output").getAbsolutePath(); List<Tuple2<Integer, String>> pairs = Arrays.asList( new Tuple2<>(1, "a"), new Tuple2<>(2, "aa"), new Tuple2<>(3, "aaa") ); JavaPairRDD<Integer, String> rdd = sc.parallelizePairs(pairs); rdd.mapToPair(pair -> new Tuple2<>(new IntWritable(pair._1()), new Text(pair._2()))) .saveAsHadoopFile(outputDir, IntWritable.class, Text.class, SequenceFileOutputFormat.class); // Try reading the output back as an object file JavaPairRDD<Integer, String> readRDD = sc.sequenceFile(outputDir, IntWritable.class, Text.class).mapToPair(pair -> new Tuple2<>(pair._1().get(), pair._2().toString())); assertEquals(pairs, readRDD.collect()); }
@Override protected void setup(Context context) { mCount = new IntWritable(0); mText = new Text(""); }