@Override public void write(Object value) { byte[] vBinary = inspector.getPrimitiveJavaObject(value); recordConsumer.addBinary(Binary.fromByteArray(vBinary)); } }
@Override public void write(Object value) { byte[] vBinary = inspector.getPrimitiveJavaObject(value); recordConsumer.addBinary(Binary.fromByteArray(vBinary)); } }
@Override public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) { final byte[] value = ((BinaryObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue); ((NullableVarBinaryVector) outputVV).getMutator().setSafe(outputIndex, value, 0, value.length); } }
private void initValue() { try { Object val = bloomFilterDynamicValue.getValue(); if (val != null) { BinaryObjectInspector boi = (BinaryObjectInspector) bloomFilterDynamicValue.getObjectInspector(); byte[] bytes = boi.getPrimitiveJavaObject(val); bloomFilter = BloomFilter.deserialize(new ByteArrayInputStream(bytes)); } else { bloomFilter = null; } initialized = true; } catch (Exception err) { throw new RuntimeException(err); } }
private void initValue() { InputStream in = null; try { Object val = bloomFilterDynamicValue.getValue(); if (val != null) { BinaryObjectInspector boi = (BinaryObjectInspector) bloomFilterDynamicValue.getObjectInspector(); byte[] bytes = boi.getPrimitiveJavaObject(val); in = new NonSyncByteArrayInputStream(bytes); bloomFilter = BloomKFilter.deserialize(in); } else { bloomFilter = null; } initialized = true; } catch (Exception err) { throw new RuntimeException(err); } finally { IOUtils.closeStream(in); } }
private static void unpackDoubleStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) throws UnsupportedDoubleException { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDoubleStats().setNumDVs(v); } else if (fName.equals("max")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setHighValue(d); } else if (fName.equals("min")) { double d = ((DoubleObjectInspector) oi).get(o); if (Double.isInfinite(d) || Double.isNaN(d)) { throw new UnsupportedDoubleException(); } statsObj.getStatsData().getDoubleStats().setLowValue(d); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDoubleStats().setBitVectors(buf); ; } }
byte[] b = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o); Text txt = new Text(); txt.set(b, 0, b.length);
private static void unpackLongStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setNumDVs(v); } else if (fName.equals("max")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setHighValue(v); } else if (fName.equals("min")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getLongStats().setLowValue(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getLongStats().setBitVectors(buf); ; } }
private static void unpackStringStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumDVs(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setMaxColLen(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getStringStats().setBitVectors(buf); ; } }
private static void unpackDecimalStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDecimalStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDecimalStats().setNumDVs(v); } else if (fName.equals("max")) { HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDecimalStats().setHighValue(convertToThriftDecimal(d)); } else if (fName.equals("min")) { HiveDecimal d = ((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDecimalStats().setLowValue(convertToThriftDecimal(d)); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDecimalStats().setBitVectors(buf); ; } }
private static void unpackDateStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getDateStats().setNumDVs(v); } else if (fName.equals("max")) { DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setHighValue(new Date(v.getDays())); } else if (fName.equals("min")) { DateWritableV2 v = ((DateObjectInspector) oi).getPrimitiveWritableObject(o); statsObj.getStatsData().getDateStats().setLowValue(new Date(v.getDays())); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getDateStats().setBitVectors(buf); ; } }
t.set(bytes.getBytes(), 0, bytes.getLength()); } else { t.set(binaryOI.getPrimitiveJavaObject(input));
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial != null) { NumericStatsAgg myagg = (NumericStatsAgg) agg; // Update min if min is lesser than the smallest value seen so far Object minValue = soi.getStructFieldData(partial, minField); myagg.updateMin(minValue, minFieldOI); // Update max if max is greater than the largest value seen so far Object maxValue = soi.getStructFieldData(partial, maxField); myagg.updateMax(maxValue, maxFieldOI); // Update the null counter Object countNull = soi.getStructFieldData(partial, countNullsField); myagg.countNulls += countNullsFieldOI.get(countNull); // Merge numDistinctValue Estimators Object numDistinct = soi.getStructFieldData(partial, ndvField); byte[] buf = ndvFieldOI.getPrimitiveJavaObject(numDistinct); if (buf != null && buf.length != 0) { if (myagg.numDV == null) { myagg.numDV = NumDistinctValueEstimatorFactory.getNumDistinctValueEstimator(buf); } else { myagg.numDV.mergeEstimators(NumDistinctValueEstimatorFactory .getNumDistinctValueEstimator(buf)); } } } } }
byte[] buf = ndvFieldOI.getPrimitiveJavaObject(partialValue);
return; case BINARY: VARBINARY.writeSlice(builder, Slices.wrappedBuffer(((BinaryObjectInspector) inspector).getPrimitiveJavaObject(object))); return; case DECIMAL:
@Override public void generateJson(JsonGenerator gen, Object obj) throws JsonGenerationException, IOException { if (obj == null) { gen.writeNull(); } else { byte[] bytes = binaryInspector.getPrimitiveJavaObject(obj); gen.writeBinary(bytes); } } }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial == null) { return; } try { HLLBuffer myagg = (HLLBuffer) agg; byte[] partialBuffer = this.inputAndPartialBinaryOI.getPrimitiveJavaObject(partial); myagg.merge(partialBuffer); } catch (Exception e) { LOG.error("Error", e); throw new HiveException(e); } }
/** * @param obj * @param objInsp * @return */ private String getByteString(Object obj, PrimitiveObjectInspector objInsp) { switch (objInsp.getPrimitiveCategory()) { case STRING: StringObjectInspector strInspector = (StringObjectInspector) objInsp; return strInspector.getPrimitiveJavaObject(obj); case BINARY: BinaryObjectInspector binInspector = (BinaryObjectInspector) objInsp; return new String(binInspector.getPrimitiveJavaObject(obj)); /// XXX TODO interpret other types, like ints or doubled default: return null; } }
@Override public void merge(AggregationBuffer agg, Object partial) throws HiveException { if (partial == null) { return; } try { HLLBuffer myagg = (HLLBuffer) agg; byte[] partialBuffer = this.partialBufferOI .getPrimitiveJavaObject(partial); myagg.merge(partialBuffer); } catch (Exception e) { throw new HiveException(e); } }
break; case BINARY: byte[] vBinary = ((BinaryObjectInspector) inspector).getPrimitiveJavaObject(value); recordConsumer.addBinary(Binary.fromByteArray(vBinary)); break;