private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = MapJoinBytesTableContainer.getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
@Override public Object getStructFieldData(Object data, StructField fieldRef) { if (data == null) { return null; } LazyBinaryStruct struct = (LazyBinaryStruct) data; MyField f = (MyField) fieldRef; int fieldID = f.getFieldID(); assert (fieldID >= 0 && fieldID < fields.size()); return struct.getField(fieldID); }
/** * Create a hierarchical LazyBinaryObject based on the given typeInfo. */ public static LazyBinaryObject createLazyBinaryObject(ObjectInspector oi) { ObjectInspector.Category c = oi.getCategory(); switch (c) { case PRIMITIVE: return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi); case MAP: return new LazyBinaryMap((LazyBinaryMapObjectInspector) oi); case LIST: return new LazyBinaryArray((LazyBinaryListObjectInspector) oi); case STRUCT: return new LazyBinaryStruct((LazyBinaryStructObjectInspector) oi); case UNION: return new LazyBinaryUnion((LazyBinaryUnionObjectInspector) oi); } throw new RuntimeException("Hive LazyBinarySerDe Internal error."); }
/** * Get one field out of the struct. * * If the field is a primitive field, return the actual object. Otherwise * return the LazyObject. This is because PrimitiveObjectInspector does not * have control over the object used by the user - the user simply directly * use the Object instead of going through Object * PrimitiveObjectInspector.get(Object). * * @param fieldID * The field ID * @return The field as a LazyObject */ public Object getField(int fieldID) { if (!parsed) { parse(); } return uncheckedGetField(fieldID); }
@Override public List<Object> getStructFieldsDataAsList(Object data) { if (data == null) { return null; } LazyBinaryStruct struct = (LazyBinaryStruct) data; return struct.getFieldsAsList(); }
/** * Deserialize a table record to a lazybinary struct. */ @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } BinaryComparable b = (BinaryComparable) field; if (b.getLength() == 0) { return null; } byteArrayRef.setData(b.getBytes()); cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getLength()); lastOperationSerialize = false; lastOperationDeserialize = true; return cachedLazyBinaryStruct; }
/** * Returns the statistics after (de)serialization) */ @Override public SerDeStats getSerDeStats() { // must be different assert (lastOperationSerialize != lastOperationDeserialize); if (lastOperationSerialize) { stats.setRawDataSize(serializedSize); } else { stats.setRawDataSize(cachedLazyBinaryStruct.getRawDataSerializedSize()); } return stats; } }
/** * Get the values of the fields as an ArrayList. * * @return The values of the fields as an ArrayList. */ public ArrayList<Object> getFieldsAsList() { if (!parsed) { parse(); } if (cachedList == null) { cachedList = new ArrayList<Object>(fields.length); for (int i = 0; i < fields.length; i++) { cachedList.add(uncheckedGetField(i)); } } else { assert fields.length == cachedList.size(); for (int i = 0; i < fields.length; i++) { cachedList.set(i, uncheckedGetField(i)); } } return cachedList; }
ArrayList<Object> fields; if (lazyObject instanceof LazyBinaryStruct) { fields = ((LazyBinaryStruct) lazyObject).getFieldsAsList(); } else { fields = (ArrayList<Object>) lazyObject;
/** * Deserialize a table record to a lazybinary struct. */ @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } if (field instanceof BytesWritable) { BytesWritable b = (BytesWritable) field; if (b.getSize() == 0) { return null; } // For backward-compatibility with hadoop 0.17 byteArrayRef.setData(b.get()); cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getSize()); } else if (field instanceof Text) { Text t = (Text) field; if (t.getLength() == 0) { return null; } byteArrayRef.setData(t.getBytes()); cachedLazyBinaryStruct.init(byteArrayRef, 0, t.getLength()); } else { throw new SerDeException(getClass().toString() + ": expects either BytesWritable or Text object!"); } return cachedLazyBinaryStruct; }
/** * Returns the statistics after (de)serialization) */ @Override public SerDeStats getSerDeStats() { // must be different assert (lastOperationSerialize != lastOperationDeserialize); if (lastOperationSerialize) { stats.setRawDataSize(serializedSize); } else { stats.setRawDataSize(cachedLazyBinaryStruct.getRawDataSerializedSize()); } return stats; } }
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
/** * Get one field out of the struct. * * If the field is a primitive field, return the actual object. Otherwise * return the LazyObject. This is because PrimitiveObjectInspector does not * have control over the object used by the user - the user simply directly * use the Object instead of going through Object * PrimitiveObjectInspector.get(Object). * * @param fieldID * The field ID * @return The field as a LazyObject */ public Object getField(int fieldID) { if (!parsed) { parse(); } return uncheckedGetField(fieldID); }
@Override public List<Object> getStructFieldsDataAsList(Object data) { if (data == null) { return null; } LazyBinaryStruct struct = (LazyBinaryStruct) data; return struct.getFieldsAsList(); }
public static List<Object> getComplexFieldsAsList(LazyBinaryStruct lazyBinaryStruct, ArrayList<Object> objectArrayBuffer, LazyBinaryStructObjectInspector lazyBinaryStructObjectInspector) { List<? extends StructField> fields = lazyBinaryStructObjectInspector.getAllStructFieldRefs(); for (int i = 0; i < fields.size(); i++) { StructField field = fields.get(i); ObjectInspector objectInspector = field.getFieldObjectInspector(); Category category = objectInspector.getCategory(); Object object = lazyBinaryStruct.getField(i); if (category == Category.PRIMITIVE) { objectArrayBuffer.set(i, object); } else { objectArrayBuffer.set(i, ObjectInspectorUtils.copyToStandardObject( object, objectInspector, ObjectInspectorCopyOption.WRITABLE)); } } return objectArrayBuffer; }
/** * Deserialize a table record to a lazybinary struct. */ @Override public Object deserialize(Writable field) throws SerDeException { if (byteArrayRef == null) { byteArrayRef = new ByteArrayRef(); } BinaryComparable b = (BinaryComparable) field; if (b.getLength() == 0) { return null; } byteArrayRef.setData(b.getBytes()); cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getLength()); lastOperationSerialize = false; lastOperationDeserialize = true; return cachedLazyBinaryStruct; }
/** * Create a hierarchical LazyBinaryObject based on the given typeInfo. */ public static LazyBinaryObject createLazyBinaryObject(ObjectInspector oi) { ObjectInspector.Category c = oi.getCategory(); switch (c) { case PRIMITIVE: return createLazyBinaryPrimitiveClass((PrimitiveObjectInspector) oi); case MAP: return new LazyBinaryMap((LazyBinaryMapObjectInspector) oi); case LIST: return new LazyBinaryArray((LazyBinaryListObjectInspector) oi); case STRUCT: return new LazyBinaryStruct((LazyBinaryStructObjectInspector) oi); } throw new RuntimeException("Hive LazyBinarySerDe Internal error."); }
/** * Returns the statistics after (de)serialization) */ @Override public SerDeStats getSerDeStats() { // must be different assert (lastOperationSerialize != lastOperationDeserialize); if (lastOperationSerialize) { stats.setRawDataSize(serializedSize); } else { stats.setRawDataSize(cachedLazyBinaryStruct.getRawDataSerializedSize()); } return stats; } }
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int)ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = getComplexFieldsAsList( valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
/** * Get one field out of the struct. * * If the field is a primitive field, return the actual object. Otherwise * return the LazyObject. This is because PrimitiveObjectInspector does not * have control over the object used by the user - the user simply directly * use the Object instead of going through Object * PrimitiveObjectInspector.get(Object). * * @param fieldID * The field ID * @return The field as a LazyObject */ public Object getField(int fieldID) { if (!parsed) { parse(); } return uncheckedGetField(fieldID); }