@Override protected LazyObjectBase createLazyObjectBase(ObjectInspector objectInspector) { return LazyFactory.createLazyObject(objectInspector); } }
@Override public LazyObjectBase createKey(ObjectInspector inspector) throws SerDeException { return LazyFactory.createLazyObject(inspector, keyMapping.binaryStorage.get(0)); }
protected LazyObjectBase createLazyField(int fieldID, StructField fieldRef) throws SerDeException { return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector()); }
/** * Creates a LazyObject based on the LazyObjectInspector. Will create binary variants for * primitive objects when the switch <code>typeBinary</code> is specified as true. * * @param oi ObjectInspector * @param typeBinary Boolean value used as switch to return variants of LazyPrimitive * objects which are initialized from a binary format for the data. * @return LazyObject<? extends ObjectInspector> */ public static LazyObject<? extends ObjectInspector> createLazyObject(ObjectInspector oi, boolean typeBinary) { if (oi.getCategory() == Category.PRIMITIVE) { return createLazyPrimitiveClass((PrimitiveObjectInspector) oi, typeBinary); } else { return createLazyObject(oi); } }
/** * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} * @param objectInspector ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ private Object toLazyListObject(Object obj, ObjectInspector objectInspector) { if (obj == null) { return null; } List<?> listObj = (List<?>) obj; LazyArray retList = (LazyArray) LazyFactory.createLazyObject(objectInspector); List<Object> lazyList = retList.getList(); ObjectInspector listElementOI = ((ListObjectInspector) objectInspector).getListElementObjectInspector(); for (int i = 0; i < listObj.size(); i++) { lazyList.add(toLazyObject(listObj.get(i), listElementOI)); } return retList; }
/** * Create the {@link LazyObjectBase lazy field} * */ public static LazyObjectBase createLazyField(ColumnMapping[] columnMappings, int fieldID, ObjectInspector inspector) { ColumnMapping colMap = columnMappings[fieldID]; if (colMap.getQualifierName() == null && !colMap.isHbaseRowKey()) { // a column family return new LazyHBaseCellMap((LazyMapObjectInspector) inspector); } return LazyFactory.createLazyObject(inspector, colMap.getBinaryStorage().get(0)); }
/** * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} * @param oi ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ private LazyObject<? extends ObjectInspector> toLazyPrimitiveObject(Object obj, ObjectInspector oi) { if (obj == null) { return null; } LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(oi); ByteArrayRef ref = new ByteArrayRef(); String objAsString = obj.toString().trim(); ref.setData(objAsString.getBytes()); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; }
public void parseMultiDelimit(byte[] rawRow, byte[] fieldDelimit) { if (rawRow == null || fieldDelimit == null) { return; } if (fields == null) { List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs(); fields = new LazyObject[fieldRefs.size()]; for (int i = 0; i < fields.length; i++) { fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector()); } fieldInited = new boolean[fields.length]; startPosition = new int[fields.length + 1]; } // the indexes of the delimiters int[] delimitIndexes = findIndexes(rawRow, fieldDelimit); int diff = fieldDelimit.length - 1; // first field always starts from 0, even when missing startPosition[0] = 0; for (int i = 1; i < fields.length; i++) { if (delimitIndexes[i - 1] != -1) { int start = delimitIndexes[i - 1] + fieldDelimit.length; startPosition[i] = start - i * diff; } else { startPosition[i] = length + 1; } } startPosition[fields.length] = length + 1; Arrays.fill(fieldInited, false); parsed = true; }
/** * Convert the given object to a lazy object using the given {@link ObjectInspector} * * @param obj Object to be converted to a {@link LazyObject} * @param objectInspector ObjectInspector used for the conversion * @return the created {@link LazyObject lazy object} * */ @SuppressWarnings({ "rawtypes", "unchecked" }) private Object toLazyMapObject(Object obj, ObjectInspector objectInspector) { if (obj == null) { return null; } // avro guarantees that the key will be of type string. So we just need to worry about // deserializing the value here LazyMap lazyMap = (LazyMap) LazyFactory.createLazyObject(objectInspector); Map map = lazyMap.getMap(); Map<Object, Object> origMap = (Map) obj; ObjectInspector keyObjectInspector = ((MapObjectInspector) objectInspector).getMapKeyObjectInspector(); ObjectInspector valueObjectInspector = ((MapObjectInspector) objectInspector).getMapValueObjectInspector(); for (Entry entry : origMap.entrySet()) { Object value = entry.getValue(); map.put(toLazyPrimitiveObject(entry.getKey(), keyObjectInspector), toLazyObject(value, valueObjectInspector)); } return lazyMap; }
public static void assertMapDecode(String encodedMap, Map<? extends Object, ? extends Object> expectedMap) { LazyMap lazyMap = (LazyMap) createLazyObject(getLazySimpleMapObjectInspector( LAZY_STRING_OBJECT_INSPECTOR, getLazyStringObjectInspector(false, (byte) 0), (byte) 2, (byte) 3, new Text("\\N"), false, (byte) 0)); lazyMap.init(newByteArrayRef(encodedMap), 0, encodedMap.length()); Map<Object, Object> map = lazyMap.getMap(); assertEquals(map, expectedMap); }
/** * Create an initialize a {@link LazyObject} with the given bytes for the given fieldID. * * @param fieldID * field for which the object is to be created * @param bytes * value with which the object is to be initialized with * @return initialized {@link LazyObject} * */ public LazyObject<? extends ObjectInspector> toLazyObject(int fieldID, byte[] bytes) { ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector(); LazyObject<? extends ObjectInspector> lazyObject = LazyFactory .createLazyObject(fieldOI); ByteArrayRef ref = new ByteArrayRef(); ref.setData(bytes); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; }
/** * Create an initialize a {@link LazyObject} with the given bytes for the given fieldID. * * @param fieldID field for which the object is to be created * @param bytes value with which the object is to be initialized with * @return initialized {@link LazyObject} * */ public LazyObject<? extends ObjectInspector> toLazyObject(int fieldID, byte[] bytes) { ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector(); LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(fieldOI); ByteArrayRef ref = new ByteArrayRef(); ref.setData(bytes); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; } }
/** * Initialize the SerDe given the parameters. serialization.format: separator * char or byte code (only supports byte-value up to 127) columns: * ","-separated column names columns.types: ",", ":", or ";"-separated column * types * * @see org.apache.hadoop.hive.serde2.AbstractSerDe#initialize(Configuration, Properties) */ @Override public void initialize(Configuration job, Properties tbl) throws SerDeException { super.initialize(job, tbl); serdeParams = new LazySerDeParameters(job, tbl, getClass().getName()); // Create the ObjectInspectors for the fields cachedObjectInspector = LazyFactory.createLazyStructInspector(serdeParams .getColumnNames(), serdeParams.getColumnTypes(), new LazyObjectInspectorParametersImpl(serdeParams)); cachedLazyStruct = (LazyStruct) LazyFactory .createLazyObject(cachedObjectInspector); serializedSize = 0; stats = new SerDeStats(); lastOperationSerialize = false; lastOperationDeserialize = false; }
@Override protected LazyObjectBase createLazyField(final int fieldID, final StructField fieldRef) throws SerDeException { if (columnsMapping[fieldID].hbaseRowKey) { return keyFactory.createKey(fieldRef.getFieldObjectInspector()); } if (columnsMapping[fieldID].hbaseTimestamp) { return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector()); } if (valueFactories != null) { return valueFactories.get(fieldID).createValueObject(fieldRef.getFieldObjectInspector()); } // fallback to default return HBaseSerDeHelper.createLazyField(columnsMapping, fieldID, fieldRef.getFieldObjectInspector()); }
@Override public void initialize(Configuration conf, Properties tbl) throws SerDeException { // get the SerDe parameters super.initialize(conf, tbl); serdeParams = new LazySerDeParameters(conf, tbl, getClass().getName()); fieldDelimited = tbl.getProperty(serdeConstants.FIELD_DELIM); if (fieldDelimited == null || fieldDelimited.isEmpty()) { throw new SerDeException("This table does not have serde property \"field.delim\"!"); } // get the collection separator and map key separator // TODO: use serdeConstants.COLLECTION_DELIM when the typo is fixed collSep = LazyUtils.getByte(tbl.getProperty(COLLECTION_DELIM), DEFAULT_SEPARATORS[1]); keySep = LazyUtils.getByte(tbl.getProperty(serdeConstants.MAPKEY_DELIM), DEFAULT_SEPARATORS[2]); serdeParams.setSeparator(1, collSep); serdeParams.setSeparator(2, keySep); // Create the ObjectInspectors for the fields cachedObjectInspector = LazyFactory.createLazyStructInspector(serdeParams .getColumnNames(), serdeParams.getColumnTypes(), serdeParams .getSeparators(), serdeParams.getNullSequence(), serdeParams .isLastColumnTakesRest(), serdeParams.isEscaped(), serdeParams .getEscapeChar()); cachedLazyStruct = (LazyStruct) LazyFactory.createLazyObject(cachedObjectInspector); assert serdeParams.getColumnNames().size() == serdeParams.getColumnTypes().size(); numColumns = serdeParams.getColumnNames().size(); }
LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(), binaryStorage.get(1));
field = LazyFactory.createLazyObject(oi.getObjectInspectors().get(tag)); fieldInited = false; parsed = true;
/** * Get the element without checking out-of-bound index. */ private Object uncheckedGetElement(int index) { if (elementInited[index]) { return arrayElements[index].getObject(); } elementInited[index] = true; int elementStart = startPosition[index]; int elementLength = startPosition[index + 1] - elementStart - 1; if (arrayElements[index] == null) { arrayElements[index] = LazyFactory.createLazyObject(oi.getListElementObjectInspector()); } if (isNull(oi.getNullSequence(), bytes, elementStart, elementLength)) { arrayElements[index].setNull(); } else { arrayElements[index].init(bytes, elementStart, elementLength); } return arrayElements[index].getObject(); }
/** * Get the value object with the index without checking parsed. * * @param index * The index into the array starting from 0 */ private Object uncheckedGetValue(int index) { if (valueInited[index]) { return valueObjects[index].getObject(); } valueInited[index] = true; Text nullSequence = oi.getNullSequence(); int valueIBegin = keyEnd[index] + 1; int valueILength = valueLength[index]; if (valueObjects[index] == null) { valueObjects[index] = LazyFactory.createLazyObject(oi.getMapValueObjectInspector()); } if (isNull(oi.getNullSequence(), bytes, valueIBegin, valueILength)) { valueObjects[index].setNull(); } else { valueObjects[index].init(bytes, valueIBegin, valueILength); } return valueObjects[index].getObject(); }
/** * Create an initialize a {@link LazyObject} with the given bytes for the given fieldID. * * @param fieldID field for which the object is to be created * @param bytes value with which the object is to be initialized with * * @return initialized {@link LazyObject} * */ @Override public LazyObject<? extends ObjectInspector> toLazyObject(int fieldID, byte[] bytes) { ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector(); LazyObject<? extends ObjectInspector> lazyObject = LazyFactory.createLazyObject(fieldOI); ByteArrayRef ref = new ByteArrayRef(); ref.setData(bytes); // initialize the lazy object lazyObject.init(ref, 0, ref.getData().length); return lazyObject; } }