sb.append(((ByteObjectInspector) poi).get(o)); break; sb.append(((LongObjectInspector) poi).get(o)); break; sb.append(((FloatObjectInspector) poi).get(o)); break; SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)); appendWithQuotes(sb, s); break; break; case DATE: Date d = ((DateObjectInspector) poi).getPrimitiveJavaObject(o); appendWithQuotes(sb, d.toString()); break; case TIMESTAMP: { Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o); appendWithQuotes(sb, t.toString()); break; throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
boolean[] needsEscape) throws IOException { PrimitiveObjectInspector.PrimitiveCategory category = oi.getPrimitiveCategory(); switch (category) { case BOOLEAN: { boolean b = ((BooleanObjectInspector) oi).get(o); if (b) { out.write(trueBytes, 0, trueBytes.length); LazyInteger.writeUTF8(out, ((ByteObjectInspector) oi).get(o)); break; LazyLong.writeUTF8(out, ((LongObjectInspector) oi).get(o)); break; float f = ((FloatObjectInspector) oi).get(o); ByteBuffer b = Text.encode(String.valueOf(f)); out.write(b.array(), 0, b.limit()); Text t = ((StringObjectInspector) oi).getPrimitiveWritableObject(o); writeEscaped(out, t.getBytes(), 0, t.getLength(), escaped, escapeChar, needsEscape); ((DateObjectInspector) oi).getPrimitiveWritableObject(o)); break; ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o)); break;
sb.append(((ByteObjectInspector) poi).get(o)); break; sb.append(((LongObjectInspector) poi).get(o)); break; sb.append(((FloatObjectInspector) poi).get(o)); break; sb.append('"'); sb.append(escapeString(((StringObjectInspector) poi) .getPrimitiveJavaObject(o))); sb.append('"'); break; sb.append('"'); sb.append(((DateObjectInspector) poi) .getPrimitiveWritableObject(o)); sb.append('"'); break; sb.append('"'); sb.append(((TimestampObjectInspector) poi) .getPrimitiveWritableObject(o)); sb.append('"'); break; + poi.getPrimitiveCategory());
switch (inspector.getPrimitiveCategory()) { case VOID: return; break; case LONG: recordConsumer.addLong(((LongObjectInspector) inspector).get(value)); break; case SHORT: break; case TIMESTAMP: Timestamp ts = ((TimestampObjectInspector) inspector).getPrimitiveJavaObject(value); recordConsumer.addBinary(NanoTimeUtils.getNanoTime(ts, false).toBinary()); break; case DECIMAL: HiveDecimal vDecimal = ((HiveDecimal) inspector.getPrimitiveJavaObject(value)); DecimalTypeInfo decTypeInfo = (DecimalTypeInfo) inspector.getTypeInfo(); recordConsumer.addBinary(decimalToBinary(vDecimal, decTypeInfo)); break; case DATE: Date vDate = ((DateObjectInspector) inspector).getPrimitiveJavaObject(value); recordConsumer.addInteger(DateWritable.dateToDays(vDate)); break; default:
private Object serializePrimitive(TypeInfo typeInfo, PrimitiveObjectInspector fieldOI, Object structFieldData, Schema schema) throws AvroSerdeException { switch(fieldOI.getPrimitiveCategory()) { case BINARY: if (schema.getType() == Type.BYTES){ return AvroSerdeUtils.getBufferFromBytes((byte[])fieldOI.getPrimitiveJavaObject(structFieldData)); } else if (schema.getType() == Type.FIXED){ Fixed fixed = new GenericData.Fixed(schema, (byte[])fieldOI.getPrimitiveJavaObject(structFieldData)); return fixed; } else { return vc.getValue(); case DATE: Date date = ((DateObjectInspector)fieldOI).getPrimitiveJavaObject(structFieldData); return DateWritableV2.dateToDays(date); case TIMESTAMP: Timestamp timestamp = ((TimestampObjectInspector) fieldOI).getPrimitiveJavaObject(structFieldData); return timestamp.toEpochMilli(); case UNKNOWN:
PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1); PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2); if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) { return poi1.getPrimitiveCategory().compareTo( poi2.getPrimitiveCategory()); int v1 = ((ByteObjectInspector) poi1).get(o1); int v2 = ((ByteObjectInspector) poi2).get(o2); return v1 - v2; long v1 = ((LongObjectInspector) poi1).get(o1); long v2 = ((LongObjectInspector) poi2).get(o2); return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0); float v1 = ((FloatObjectInspector) poi1).get(o1); float v2 = ((FloatObjectInspector) poi2).get(o2); .getPrimitiveWritableObject(o1); DateWritableV2 d2 = ((DateObjectInspector) poi2) .getPrimitiveWritableObject(o2); return d1.compareTo(d2); .getPrimitiveWritableObject(o1); TimestampWritableV2 t2 = ((TimestampObjectInspector) poi2) .getPrimitiveWritableObject(o2); return t1.compareTo(t2);
if (((StringObjectInspector) inspector).preferWritable()) { return ((StringObjectInspector) inspector).getPrimitiveWritableObject(hiveObj) .toString(); } else { return ((StringObjectInspector) inspector).getPrimitiveJavaObject(hiveObj); return ((BooleanObjectInspector) inspector).get(hiveObj); } else if (inspector instanceof FloatObjectInspector) { return ((FloatObjectInspector) inspector).get(hiveObj); } else if (inspector instanceof DoubleObjectInspector) { return ((DoubleObjectInspector) inspector).get(hiveObj); } else if (inspector instanceof LongObjectInspector) { return ((LongObjectInspector) inspector).get(hiveObj); } else if (inspector instanceof ShortObjectInspector) { return ((ShortObjectInspector) inspector).get(hiveObj); } else if (inspector instanceof ByteObjectInspector) { return ((ByteObjectInspector) inspector).get(hiveObj); } else if (inspector instanceof HiveDecimalObjectInspector) { if (((HiveDecimalObjectInspector) inspector).preferWritable()) { return ((DateObjectInspector) inspector).getPrimitiveJavaObject(hiveObj); } else if (inspector instanceof PrimitiveObjectInspector) { return ((PrimitiveObjectInspector) inspector).getPrimitiveJavaObject(hiveObj); } else if (inspector instanceof StructObjectInspector) {
switch (oi.getPrimitiveCategory()) { case VOID: result = null; case STRING: StringObjectInspector soi = (StringObjectInspector) oi; String s = soi.getPrimitiveJavaObject(o).trim(); try { result = Date.valueOf(s); result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get(); break; case TIMESTAMP: result = DateWritable.timeToDate( ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getSeconds()); break; default: throw new RuntimeException("Cannot convert to Date from: " + oi.getTypeName());
switch (oi.getPrimitiveCategory()) { case VOID: break; case STRING: StringObjectInspector soi = (StringObjectInspector) oi; String s = soi.getPrimitiveJavaObject(o).trim(); try { if (s.length() == DATE_LENGTH) { result = ((DateObjectInspector) oi).getPrimitiveWritableObject(o).get(); break; case TIMESTAMP: result = DateWritableV2.timeToDate( ((TimestampObjectInspector) oi).getPrimitiveWritableObject(o).getSeconds()); break; case TIMESTAMPLOCALTZ: String tstz = oi.getPrimitiveWritableObject(o).toString(); int divSpace = tstz.indexOf(' '); if (divSpace == -1) { default: throw new RuntimeException("Cannot convert to Date from: " + oi.getTypeName());
switch(primitiveObjectInspector.getPrimitiveCategory()) { case DECIMAL: DecimalTypeInfo decimalTypeInfo = (DecimalTypeInfo) primitiveObjectInspector.getTypeInfo(); HiveDecimalObjectInspector decimalOI = (HiveDecimalObjectInspector) primitiveObjectInspector; BigDecimal bigDecimal = decimalOI.getPrimitiveJavaObject(field).bigDecimalValue(); return new DecimalConverter(decimalType).toInternal(bigDecimal); case TIMESTAMP: Timestamp ts = ((TimestampObjectInspector) primitiveObjectInspector).getPrimitiveJavaObject(field); return DataStructureConverters.getConverterForType(DataTypes.TIMESTAMP).toInternal(ts); case DATE: int days = ((DateObjectInspector) primitiveObjectInspector).getPrimitiveWritableObject(field).getDays(); return days; case CHAR: return vc.getValue(); default: return primitiveObjectInspector.getPrimitiveJavaObject(field);
case PRIMITIVE: PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; switch (poi.getPrimitiveCategory()) { byte b = 0; if (objectForField != null) { b = boi.get(objectForField); long l = 0; if (objectForField != null) { l = loi.get(objectForField); TimestampWritableV2 ts = tsoi.getPrimitiveWritableObject(objectForField); out.writeTimestamp(ts, getTimeStampByteNum(timestampPrecision)); return; DateWritableV2 dw = dtoi.getPrimitiveWritableObject(objectForField); out.writeDate(dw); return; return; default: throw new SerDeException("Unrecognized type: " + poi.getPrimitiveCategory());
private void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDateStats().setBitVectors(v);; } }
@Override protected void updateMax(Object maxValue, DateObjectInspector maxFieldOI) { if ((maxValue != null) && (max == null || max.compareTo(maxFieldOI.getPrimitiveWritableObject(maxValue)) < 0)) { // DateWritable is mutable, DateStatsAgg needs its own copy max = new DateWritable(maxFieldOI.getPrimitiveWritableObject(maxValue)); } } };
private void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { DateWritable v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } }
private static long formatDateAsLong(Object object, DateObjectInspector inspector) { if (object instanceof LazyDate) { return ((LazyDate) object).getWritableObject().getDays(); } if (object instanceof DateWritable) { return ((DateWritable) object).getDays(); } // Hive will return java.sql.Date at midnight in JVM time zone long millisLocal = inspector.getPrimitiveJavaObject(object).getTime(); // Convert it to midnight in UTC long millisUtc = DateTimeZone.getDefault().getMillisKeepLocal(DateTimeZone.UTC, millisLocal); // Convert midnight UTC to days return TimeUnit.MILLISECONDS.toDays(millisUtc); }
@Override void write(Object obj) throws IOException { super.write(obj); if (obj != null) { // Using the Writable here as it's used directly for writing as well as for stats. DateWritable val = ((DateObjectInspector) inspector).getPrimitiveWritableObject(obj); indexStatistics.updateDate(val); writer.write(val.getDays()); if (createBloomFilter) { bloomFilter.addLong(val.getDays()); } } }
private static void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDateStats().setBitVectors(buf); ; } }
retValue.set(inputDateOI.getPrimitiveWritableObject(arguments[0].get()) .getTimeInSeconds()); return retValue; Timestamp timestamp = inputTimestampOI.getPrimitiveJavaObject(arguments[0].get()); setValueFromTs(retValue, timestamp); return retValue;
@SuppressWarnings("deprecation") @Override public void set(ValueOption<?> target, Object value) { if (value == null) { target.setNull(); } else if (primitive) { java.sql.Date entity = inspector.getPrimitiveJavaObject(value); setDate(target, entity); } else { DateWritable writable = inspector.getPrimitiveWritableObject(value); setDate(target, writable.get()); } }
retValue.set(inputDateOI.getPrimitiveWritableObject(arguments[0].get()) .getTimeInSeconds()); return retValue; return retValue; Timestamp timestamp = inputTimestampOI.getPrimitiveJavaObject(arguments[0].get()); setValueFromTs(retValue, timestamp); return retValue;