private static void unpackBinaryStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setNumNulls(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setMaxColLen(v); } }
private void unpackBinaryStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setNumNulls(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setMaxColLen(v); } }
@Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats(); BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats(); aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); } }
@Override void compare(ColumnStatisticsData colstats, int offset) { Assert.assertEquals("binary max length", maxLens.get(offset), (Long) colstats.getBinaryStats().getMaxColLen()); Assert.assertEquals("binary min length", avgLens.get(offset), colstats.getBinaryStats().getAvgColLen(), 0.01); Assert.assertEquals("binary num nulls", numNulls.get(offset), (Long) colstats.getBinaryStats().getNumNulls()); }
@Override void compareAggr(ColumnStatisticsObj obj) { compareCommon(obj); Assert.assertEquals("aggr binary max length", getMaxLen(), obj.getStatsData().getBinaryStats().getMaxColLen()); Assert.assertEquals("aggr binary min length", getAvgLen(), obj.getStatsData().getBinaryStats().getAvgColLen(), 0.01); Assert.assertEquals("aggr binary num nulls", getNumNulls(), obj.getStatsData().getBinaryStats().getNumNulls()); } }
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integeral : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); default: return null; } }
BinaryColumnStatsData bcsd = csd.getBinaryStats(); ret.addAll(Lists.newArrayList( "", "", "" + bcsd.getNumNulls(), "",
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integer : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); case Date: return statData.getDateStats().getNumNulls(); default: return null; } }
@Override public ColumnStatisticsObj aggregate(List<ColStatsObjWithSourceInfo> colStatsWithSourceInfo, List<String> partNames, boolean areAllPartsFound) throws MetaException { ColumnStatisticsObj statsObj = null; String colType = null; String colName = null; BinaryColumnStatsData aggregateData = null; for (ColStatsObjWithSourceInfo csp : colStatsWithSourceInfo) { ColumnStatisticsObj cso = csp.getColStatsObj(); if (statsObj == null) { colName = cso.getColName(); colType = cso.getColType(); statsObj = ColumnStatsAggregatorFactory.newColumnStaticsObj(colName, colType, cso.getStatsData().getSetField()); } BinaryColumnStatsData newData = cso.getStatsData().getBinaryStats(); if (aggregateData == null) { aggregateData = newData.deepCopy(); } else { aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); } } ColumnStatisticsData columnStatisticsData = new ColumnStatisticsData(); columnStatisticsData.setBinaryStats(aggregateData); statsObj.setStatsData(columnStatisticsData); return statsObj; } }
ColumnStatisticsData csd = cso.getStatsData(); if (csd.isSetBinaryStats()) { BinaryColumnStatsData bcsd = csd.getBinaryStats(); appendColumnStats(tableInfo, "", "", bcsd.getNumNulls(), "", bcsd.getAvgColLen(), bcsd.getMaxColLen(), "", "");
BinaryColumnStatsData binaryStatsData = columnStatistics.getStatsData().getBinaryStats(); OptionalLong maxColumnLength = binaryStatsData.isSetMaxColLen() ? OptionalLong.of(binaryStatsData.getMaxColLen()) : OptionalLong.empty(); OptionalDouble averageColumnLength = binaryStatsData.isSetAvgColLen() ? OptionalDouble.of(binaryStatsData.getAvgColLen()) : OptionalDouble.empty();
cs.setAvgColLen(JavaDataModel.get().primitive1()); } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) { cs.setAvgColLen(csd.getBinaryStats().getAvgColLen()); cs.setNumNulls(csd.getBinaryStats().getNumNulls()); } else if (colTypeLowerCase.equals(serdeConstants.TIMESTAMP_TYPE_NAME) || colTypeLowerCase.equals(serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME)) {
ColumnStatisticsData csd = cso.getStatsData(); if (csd.isSetBinaryStats()) { BinaryColumnStatsData bcsd = csd.getBinaryStats(); appendColumnStatsNoFormatting(colBuffer, "", "", bcsd.getNumNulls(), "", bcsd.getAvgColLen(), bcsd.getMaxColLen(), "", "");
cs.setAvgColLen(JavaDataModel.get().primitive1()); } else if (colTypeLowerCase.equals(serdeConstants.BINARY_TYPE_NAME)) { cs.setAvgColLen(csd.getBinaryStats().getAvgColLen()); cs.setNumNulls(csd.getBinaryStats().getNumNulls()); } else if (colTypeLowerCase.equals(serdeConstants.TIMESTAMP_TYPE_NAME)) { cs.setAvgColLen(JavaDataModel.get().lengthOfTimestamp());
BinaryColumnStatsData oldBinaryStatsData = oldStatObj.getStatsData().getBinaryStats(); BinaryColumnStatsData newBinaryStatsData = newStatObj.getStatsData().getBinaryStats(); if (newBinaryStatsData.isSetMaxColLen()) { oldBinaryStatsData.setMaxColLen(newBinaryStatsData.getMaxColLen());
stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null); } else if (statsObj.getStatsData().isSetBinaryStats()) { BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats(); mColStats.setBinaryStats( binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null,
stringStats.isSetAvgColLen() ? stringStats.getAvgColLen() : null); } else if (statsObj.getStatsData().isSetBinaryStats()) { BinaryColumnStatsData binaryStats = statsObj.getStatsData().getBinaryStats(); mColStats.setBinaryStats( binaryStats.isSetNumNulls() ? binaryStats.getNumNulls() : null,
private void unpackBinaryStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setNumNulls(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getBinaryStats().setMaxColLen(v); } }
@Override public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats(); BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats(); aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); } }
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integeral : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); default: return null; } }