public void testStringToDate() throws HiveException { GenericUDFDateDiff udf = new GenericUDFDateDiff(); ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.javaStringObjectInspector; ObjectInspector[] arguments = {valueOI1, valueOI2}; udf.initialize(arguments); DeferredObject valueObj1 = new DeferredJavaObject(new Text("2009-07-20")); DeferredObject valueObj2 = new DeferredJavaObject(new Text("2009-07-22")); DeferredObject[] args = {valueObj1, valueObj2}; IntWritable output = (IntWritable) udf.evaluate(args); assertEquals("date_iff() test for STRING failed ", "-2", output.toString()); // Test with null args args = new DeferredObject[] { new DeferredJavaObject(null), valueObj2 }; assertNull("date_add() 1st arg null", udf.evaluate(args)); args = new DeferredObject[] { valueObj1, new DeferredJavaObject(null) }; assertNull("date_add() 2nd arg null", udf.evaluate(args)); args = new DeferredObject[] { new DeferredJavaObject(null), new DeferredJavaObject(null) }; assertNull("date_add() both args null", udf.evaluate(args)); }
public void testTimestampToDate() throws HiveException { GenericUDFDateDiff udf = new GenericUDFDateDiff(); ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.writableTimestampObjectInspector; ObjectInspector[] arguments = {valueOI1, valueOI2}; udf.initialize(arguments); DeferredObject valueObj1 = new DeferredJavaObject(new TimestampWritableV2( Timestamp.valueOf(LocalDateTime.of(109, 06, 20, 0, 0, 0, 0).toString()))); DeferredObject valueObj2 = new DeferredJavaObject(new TimestampWritableV2( Timestamp.valueOf(LocalDateTime.of(109, 06, 17, 0, 0, 0, 0).toString()))); DeferredObject[] args = {valueObj1, valueObj2}; IntWritable output = (IntWritable) udf.evaluate(args); assertEquals("datediff() test for TIMESTAMP failed ", "3", output.toString()); // Test with null args args = new DeferredObject[] { new DeferredJavaObject(null), valueObj2 }; assertNull("date_add() 1st arg null", udf.evaluate(args)); args = new DeferredObject[] { valueObj1, new DeferredJavaObject(null) }; assertNull("date_add() 2nd arg null", udf.evaluate(args)); args = new DeferredObject[] { new DeferredJavaObject(null), new DeferredJavaObject(null) }; assertNull("date_add() both args null", udf.evaluate(args)); }
public void testDateWritablepToDate() throws HiveException { GenericUDFDateDiff udf = new GenericUDFDateDiff(); ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableDateObjectInspector; ObjectInspector valueOI2 = PrimitiveObjectInspectorFactory.writableDateObjectInspector; ObjectInspector[] arguments = {valueOI1, valueOI2}; udf.initialize(arguments); DeferredObject valueObj1 = new DeferredJavaObject(new DateWritableV2(Date.of(109, 06, 20))); DeferredObject valueObj2 = new DeferredJavaObject(new DateWritableV2(Date.of(109, 06, 10))); DeferredObject[] args = {valueObj1, valueObj2}; IntWritable output = (IntWritable) udf.evaluate(args); assertEquals("datediff() test for DATEWRITABLE failed ", "10", output.toString()); // Test with null args args = new DeferredObject[] { new DeferredJavaObject(null), valueObj2 }; assertNull("date_add() 1st arg null", udf.evaluate(args)); args = new DeferredObject[] { valueObj1, new DeferredJavaObject(null) }; assertNull("date_add() 2nd arg null", udf.evaluate(args)); args = new DeferredObject[] { new DeferredJavaObject(null), new DeferredJavaObject(null) }; assertNull("date_add() both args null", udf.evaluate(args)); }
public void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException { Iterator<IntWritable> iterator = values.iterator(); StringBuilder builder = new StringBuilder(); while (iterator.hasNext()) { builder.append("," + iterator.next().toString()); } v.set(builder.toString().replaceFirst(",", "")); context.write(key, v); } }
public void map(Text key, IntWritable value, Context context) throws Exception { String line = value.toString(); //this will be your key in the final output /* Perform operations on the line */ /* standard values = <return value from sybase query.>; */ /*Perform benchmark calculations and obtain benchmark values */ context.write(line,benchmarkValue); }
@Override public String toString() { return "<key: " + (key == null ? "null" : key.toString()) + ", partition: " + partition.toString() + ">"; }
public static Map<String, Integer> importParameter(SequenceFile.Reader sequenceFileReader) throws IOException { Map<String, Integer> hashMap = new HashMap<String, Integer>(); IntWritable intWritable = new IntWritable(); Text text = new Text(); while (sequenceFileReader.next(intWritable, text)) { if (intWritable.get() % 100000 == 0) { sLogger.info("Imported term " + text.toString() + " with index " + intWritable.toString()); } hashMap.put(text.toString(), intWritable.get()); } return hashMap; }
SequenceFile.Reader read = new SequenceFile.Reader(fs, new Path("<path do dictionary>"), conf); IntWritable dicKey = new IntWritable(); Text text = new Text(); HashMap<Integer, String> dictionaryMap = new HashMap(); while (read.next(text, dicKey)) { dictionaryMap.put(Integer.parseInt(dicKey.toString()), text.toString()); } read.close();
public static MD5Hash getMD5Hash(IntWritable key) throws IOException { return MD5Hash.digest(key.toString()); }
public List<Text> evaluate(List<String> values, String prefix, boolean useIndexAsPrefix) { if (values == null) { return null; } if (prefix == null) { prefix = ""; } List<IntWritable> hashValues = ArrayHashValuesUDF.hashValues(values, null, MurmurHash3.DEFAULT_NUM_FEATURES, useIndexAsPrefix); final int len = hashValues.size(); final Text[] stringValues = new Text[len]; for (int i = 0; i < len; i++) { IntWritable v = hashValues.get(i); stringValues[i] = new Text(prefix + v.toString()); } return Arrays.asList(stringValues); } }
public List<Text> evaluate(List<String> values, String prefix, boolean useIndexAsPrefix) { if (values == null) { return null; } if (prefix == null) { prefix = ""; } List<IntWritable> hashValues = ArrayHashValuesUDF.hashValues(values, null, MurmurHash3.DEFAULT_NUM_FEATURES, useIndexAsPrefix); final int len = hashValues.size(); final Text[] stringValues = new Text[len]; for (int i = 0; i < len; i++) { IntWritable v = hashValues.get(i); stringValues[i] = new Text(prefix + v.toString()); } return Arrays.asList(stringValues); } }
public List<Text> evaluate(List<String> values, String prefix, boolean useIndexAsPrefix) { if(values == null) { return null; } if(prefix == null) { prefix = ""; } List<IntWritable> hashValues = ArrayHashValuesUDF.hashValues(values, null, MurmurHash3.DEFAULT_NUM_FEATURES, useIndexAsPrefix); final int len = hashValues.size(); final Text[] stringValues = new Text[len]; for(int i = 0; i < len; i++) { IntWritable v = hashValues.get(i); stringValues[i] = val(prefix + v.toString()); } return Arrays.asList(stringValues); } }
@Override protected Text convertVertexToLine( Vertex<LongWritable, KMeansVertexValue, NullWritable> vertex) throws IOException { StringBuffer sb = new StringBuffer(vertex.getId().toString()); sb.append(delimiter); sb.append(vertex.getValue().getClusterId().toString()); return new Text(sb.toString()); }
value.toString()+"\"");
this.writeStyle("value", value.toString());
import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import java.io.IOException; class DumpDict { public static void main(String[] args) { try { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); SequenceFile.Reader read = new SequenceFile.Reader(fs, new Path(args[0]), conf); IntWritable dicKey = new IntWritable(); Text text = new Text(); // HashMap dictionaryMap = new HashMap(); while (read.next(text, dicKey)) { // dictionaryMap.put(Integer.parseInt(dicKey.toString()), text.toString()); System.out.println(dicKey.toString()+" "+text.toString()); } read.close(); } catch (IOException e) { System.out.println(e.toString()); } } }
@Override public void writeVertex( Vertex<LongWritable, IntWritable, FloatWritable> vertex) throws IOException, InterruptedException { getRecordWriter().write( new Text(vertex.getId().toString()), new Text(vertex.getValue().toString())); } }
@Override public void writeVertex(Vertex<LongWritable, IntWritable, FloatWritable> vertex) throws IOException, InterruptedException { getRecordWriter().write( new Text(vertex.getId().toString()), new Text(vertex.getValue().toString())); } }
LOG.info(key.toString() + " " + sb.toString()); this.outputValue.set(sb.toString()); context.write(key, this.outputValue);
while (reader.next(key, value)) { System.out.println(value.toString() + " belongs to cluster " + key.toString());