/** * Convertes to StringColumnStatsDataInspector * if it's a StringColumnStatsData. * @param cso ColumnStatisticsObj * @return StringColumnStatsDataInspector */ public static StringColumnStatsDataInspector stringInspectorFromStats(ColumnStatisticsObj cso) { StringColumnStatsDataInspector columnStats; if (cso.getStatsData().getStringStats() instanceof StringColumnStatsDataInspector) { columnStats = (StringColumnStatsDataInspector)(cso.getStatsData().getStringStats()); } else { columnStats = new StringColumnStatsDataInspector(cso.getStatsData().getStringStats()); } return columnStats; }
private static void unpackStringStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumDVs(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setMaxColLen(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; byte[] buf = ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getStringStats().setBitVectors(buf); ; } }
private void unpackStringStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumDVs(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setMaxColLen(v); } else if (fName.equals("ndvbitvector")) { PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi; String v = ((StringObjectInspector) poi).getPrimitiveJavaObject(o); statsObj.getStatsData().getStringStats().setBitVectors(v);; } }
Map<String, StringColumnStatsData> extractedAdjustedStatsMap = new HashMap<>(); for (Map.Entry<String, ColumnStatisticsData> entry : adjustedStatsMap.entrySet()) { extractedAdjustedStatsMap.put(entry.getKey(), entry.getValue().getStringStats());
@Override void compare(ColumnStatisticsData colstats, int offset) { Assert.assertEquals("str num nulls", numNulls.get(offset), (Long) colstats.getStringStats().getNumNulls()); Assert.assertEquals("str num dvs", numDvs.get(offset), (Long) colstats.getStringStats().getNumDVs()); Assert.assertEquals("str low val", (long)maxLens.get(offset), colstats.getStringStats().getMaxColLen()); Assert.assertEquals("str high val", avgLens.get(offset), colstats.getStringStats().getAvgColLen(), 0.01); }
@Override void compareAggr(ColumnStatisticsObj obj) { compareCommon(obj); Assert.assertEquals("aggr str num nulls", getNumNulls(), obj.getStatsData().getStringStats().getNumNulls()); Assert.assertEquals("aggr str num dvs", getNumDvs(), obj.getStatsData().getStringStats().getNumDVs()); Assert.assertEquals("aggr str low val", getMaxLen(), obj.getStatsData().getStringStats().getMaxColLen()); Assert.assertEquals("aggr str high val", getAvgLen(), obj.getStatsData().getStringStats().getAvgColLen(), 0.01); } }
colName, columnStatisticsData.getStringStats().getNumDVs(), partNames.size(), colStatsWithSourceInfo.size()); statsObj.setStatsData(columnStatisticsData);
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integeral : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); default: return null; } }
convertToString(bcsd.getBitVectors()))); } else if (csd.isSetStringStats()) { StringColumnStatsData scsd = csd.getStringStats(); ret.addAll(Lists.newArrayList( "", "", "" + scsd.getNumNulls(), "" + scsd.getNumDVs(),
private Long getNullcountFor(StatType type, ColumnStatisticsData statData) { switch(type) { case Integer : return statData.getLongStats().getNumNulls(); case Double: return statData.getDoubleStats().getNumNulls(); case String: return statData.getStringStats().getNumNulls(); case Boolean: return statData.getBooleanStats().getNumNulls(); case Binary: return statData.getBinaryStats().getNumNulls(); case Date: return statData.getDateStats().getNumNulls(); default: return null; } }
bcsd.getMaxColLen(), "", ""); } else if (csd.isSetStringStats()) { StringColumnStatsData scsd = csd.getStringStats(); appendColumnStats(tableInfo, "", "", scsd.getNumNulls(), scsd.getNumDVs(), scsd.getAvgColLen(), scsd.getMaxColLen(), "", "");
StringColumnStatsData stringStatsData = columnStatistics.getStatsData().getStringStats(); OptionalLong maxColumnLength = stringStatsData.isSetMaxColLen() ? OptionalLong.of(stringStatsData.getMaxColLen()) : OptionalLong.empty(); OptionalDouble averageColumnLength = stringStatsData.isSetAvgColLen() ? OptionalDouble.of(stringStatsData.getAvgColLen()) : OptionalDouble.empty();
|| colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME)) { cs.setCountDistint(csd.getStringStats().getNumDVs()); cs.setNumNulls(csd.getStringStats().getNumNulls()); cs.setAvgColLen(csd.getStringStats().getAvgColLen()); } else if (colTypeLowerCase.equals(serdeConstants.BOOLEAN_TYPE_NAME)) { if (csd.getBooleanStats().getNumFalses() > 0 && csd.getBooleanStats().getNumTrues() > 0) {
bcsd.getAvgColLen(), bcsd.getMaxColLen(), "", ""); } else if (csd.isSetStringStats()) { StringColumnStatsData scsd = csd.getStringStats(); appendColumnStatsNoFormatting(colBuffer, "", "", scsd.getNumNulls(), scsd.getNumDVs(), scsd.getAvgColLen(), scsd.getMaxColLen(), "", "");
StringColumnStatsData stringStats = new StringColumnStatsData(); statsData.setStringStats(stringStats); statsData.getStringStats().setAvgColLen(avgColLen); statsData.getStringStats().setMaxColLen(maxColLen); statsData.getStringStats().setNumDVs(numDVs); statsData.getStringStats().setNumNulls(numNulls); assertEquals(colStats.getStatsDesc().getPartName(), partName); assertEquals(colStats2.getColName(), colName[1]); assertEquals(colStats2.getStatsData().getStringStats().getMaxColLen(), maxColLen); assertEquals(colStats2.getStatsData().getStringStats().getAvgColLen(), avgColLen, 0.01); assertEquals(colStats2.getStatsData().getStringStats().getNumNulls(), numNulls); assertEquals(colStats2.getStatsData().getStringStats().getNumDVs(), numDVs);
|| colTypeLowerCase.startsWith(serdeConstants.CHAR_TYPE_NAME) || colTypeLowerCase.startsWith(serdeConstants.VARCHAR_TYPE_NAME)) { cs.setCountDistint(csd.getStringStats().getNumDVs()); cs.setNumNulls(csd.getStringStats().getNumNulls()); cs.setAvgColLen(csd.getStringStats().getAvgColLen()); } else if (colTypeLowerCase.equals(serdeConstants.BOOLEAN_TYPE_NAME)) { if (csd.getBooleanStats().getNumFalses() > 0 && csd.getBooleanStats().getNumTrues() > 0) {
StringColumnStatsData oldStringStatsData = oldStatObj.getStatsData().getStringStats(); StringColumnStatsData newStringStatsData = newStatObj.getStatsData().getStringStats(); if (newStringStatsData.isSetMaxColLen()) { oldStringStatsData.setMaxColLen(newStringStatsData.getMaxColLen());
low, high); } else if (statsObj.getStatsData().isSetStringStats()) { StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats(); mColStats.setStringStats( stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null,
low, high); } else if (statsObj.getStatsData().isSetStringStats()) { StringColumnStatsData stringStats = statsObj.getStatsData().getStringStats(); mColStats.setStringStats( stringStats.isSetNumNulls() ? stringStats.getNumNulls() : null,
private void unpackStringStats(ObjectInspector oi, Object o, String fName, ColumnStatisticsObj statsObj) { if (fName.equals("countnulls")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumNulls(v); } else if (fName.equals("numdistinctvalues")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setNumDVs(v); } else if (fName.equals("avglength")) { double d = ((DoubleObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setAvgColLen(d); } else if (fName.equals("maxlength")) { long v = ((LongObjectInspector) oi).get(o); statsObj.getStatsData().getStringStats().setMaxColLen(v); } }