public static void configureJobConf(TableDesc tableDesc, JobConf jobConf) throws Exception { HBaseSerDeParameters serdeParams = new HBaseSerDeParameters(jobConf, tableDesc.getProperties(), HBaseSerDe.class.getName()); serdeParams.getKeyFactory().configureJobConf(tableDesc, jobConf); } }
public static ObjectInspector createLazyHBaseStructInspector( LazySerDeParameters serdeParams, int index, HBaseKeyFactory keyFactory, List<HBaseValueFactory> valueFactories) throws SerDeException { List<TypeInfo> columnTypes = serdeParams.getColumnTypes(); ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>( columnTypes.size()); for (int i = 0; i < columnTypes.size(); i++) { if (i == index) { columnObjectInspectors.add(keyFactory.createKeyObjectInspector(columnTypes.get(i))); } else { columnObjectInspectors.add(valueFactories.get(i).createValueObjectInspector( columnTypes.get(i))); } } return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( serdeParams.getColumnNames(), columnObjectInspectors, null, serdeParams.getSeparators()[0], serdeParams, ObjectInspectorOptions.JAVA); }
@Override public DecomposedPredicate decomposePredicate( JobConf jobConf, Deserializer deserializer, ExprNodeDesc predicate) { HBaseKeyFactory keyFactory = ((HBaseSerDe) deserializer).getKeyFactory(); return keyFactory.decomposePredicate(jobConf, deserializer, predicate); }
private HBaseKeyFactory initKeyFactory(Configuration conf, Properties tbl) throws SerDeException { try { HBaseKeyFactory keyFactory = createKeyFactory(conf, tbl); if (keyFactory != null) { keyFactory.init(this, tbl); } return keyFactory; } catch (Exception e) { throw new SerDeException(e); } }
@Override protected LazyObjectBase createLazyField(final int fieldID, final StructField fieldRef) throws SerDeException { if (columnsMapping[fieldID].hbaseRowKey) { return keyFactory.createKey(fieldRef.getFieldObjectInspector()); } if (columnsMapping[fieldID].hbaseTimestamp) { return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector()); } if (valueFactories != null) { return valueFactories.get(fieldID).createValueObject(fieldRef.getFieldObjectInspector()); } // fallback to default return HBaseSerDeHelper.createLazyField(columnsMapping, fieldID, fieldRef.getFieldObjectInspector()); }
Object value = values.get(keyIndex); byte[] key = keyFactory.serializeKey(value, field); if (key == null) { throw new SerDeException("HBase row key cannot be NULL");
private HBaseKeyFactory initKeyFactory(Configuration conf, Properties tbl) throws SerDeException { try { HBaseKeyFactory keyFactory = createKeyFactory(conf, tbl); if (keyFactory != null) { keyFactory.init(this, tbl); } return keyFactory; } catch (Exception e) { throw new SerDeException(e); } }
@Override protected LazyObjectBase createLazyField(final int fieldID, final StructField fieldRef) throws SerDeException { if (columnsMapping[fieldID].hbaseRowKey) { return keyFactory.createKey(fieldRef.getFieldObjectInspector()); } if (columnsMapping[fieldID].hbaseTimestamp) { return LazyFactory.createLazyObject(fieldRef.getFieldObjectInspector()); } if (valueFactories != null) { return valueFactories.get(fieldID).createValueObject(fieldRef.getFieldObjectInspector()); } // fallback to default return HBaseSerDeHelper.createLazyField(columnsMapping, fieldID, fieldRef.getFieldObjectInspector()); }
Object value = values.get(keyIndex); byte[] key = keyFactory.serializeKey(value, field); if (key == null) { throw new SerDeException("HBase row key cannot be NULL");
public static ObjectInspector createLazyHBaseStructInspector(HBaseSerDeParameters hSerdeParams, Properties tbl) throws SerDeException { List<TypeInfo> columnTypes = hSerdeParams.getColumnTypes(); ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>( columnTypes.size()); for (int i = 0; i < columnTypes.size(); i++) { if (i == hSerdeParams.getKeyIndex()) { columnObjectInspectors.add(hSerdeParams.getKeyFactory() .createKeyObjectInspector(columnTypes.get(i))); } else { columnObjectInspectors.add(hSerdeParams.getValueFactories().get(i) .createValueObjectInspector(columnTypes.get(i))); } } List<String> structFieldComments = StringUtils.isEmpty(tbl.getProperty("columns.comments")) ? new ArrayList<>(Collections.nCopies(columnTypes.size(), "")) : Arrays.asList(tbl.getProperty("columns.comments").split("\0", columnTypes.size())); return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( hSerdeParams.getColumnNames(), columnObjectInspectors, structFieldComments, hSerdeParams.getSerdeParams().getSeparators()[0], hSerdeParams.getSerdeParams(), ObjectInspectorOptions.JAVA); } }
public static void configureJobConf(TableDesc tableDesc, JobConf jobConf) throws Exception { HBaseSerDeParameters serdeParams = new HBaseSerDeParameters(jobConf, tableDesc.getProperties(), HBaseSerDe.class.getName()); serdeParams.getKeyFactory().configureJobConf(tableDesc, jobConf); } }
@Override public DecomposedPredicate decomposePredicate( JobConf jobConf, Deserializer deserializer, ExprNodeDesc predicate) { HBaseKeyFactory keyFactory = ((HBaseSerDe) deserializer).getKeyFactory(); return keyFactory.decomposePredicate(jobConf, deserializer, predicate); }
public static ObjectInspector createLazyHBaseStructInspector( LazySerDeParameters serdeParams, int index, HBaseKeyFactory keyFactory, List<HBaseValueFactory> valueFactories) throws SerDeException { List<TypeInfo> columnTypes = serdeParams.getColumnTypes(); ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>( columnTypes.size()); for (int i = 0; i < columnTypes.size(); i++) { if (i == index) { columnObjectInspectors.add(keyFactory.createKeyObjectInspector(columnTypes.get(i))); } else { columnObjectInspectors.add(valueFactories.get(i).createValueObjectInspector( columnTypes.get(i))); } } return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector( serdeParams.getColumnNames(), columnObjectInspectors, null, serdeParams.getSeparators()[0], serdeParams, ObjectInspectorOptions.JAVA); } }