@Override public String[] getRow() { int[] rowIndex = rowIndices.get(i); String[] row = new String[rowIndex.length]; for (int x = 0; x < row.length; x++) { row[x] = dict.getValueFromId(rowIndex[x]); } return row; }
@Override public String decode(byte[] bytes, int offset, int len) { int id = BytesUtil.readUnsigned(bytes, offset, len); try { String value = dict.getValueFromId(id); return value; } catch (IllegalArgumentException e) { logger.error("Can't get dictionary value from " + dict + " (id = " + id + ")"); return ""; } }
void writeData(DataOutput out) throws IOException { out.writeInt(rowIndices.size()); if (rowIndices.size() > 0) { int n = rowIndices.get(0).length; out.writeInt(n); if (this.useDictionary == true) { dict.write(out); for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { out.writeInt(row[j]); } } } else { for (int i = 0; i < rowIndices.size(); i++) { int[] row = rowIndices.get(i); for (int j = 0; j < n; j++) { // NULL_STR is tricky, but we don't want to break the current snapshots out.writeUTF(dict.getValueFromId(row[j]) == null ? NULL_STR : dict.getValueFromId(row[j])); } } } } }
public List<T> enumeratorValues() { List<T> ret = Lists.newArrayListWithExpectedSize(getSize()); for (int i = getMinId(); i <= getMaxId(); i++) { ret.add(getValueFromId(i)); } return ret; }
@Override public Object deserialize(ByteBuffer in) { int id = BytesUtil.readUnsigned(in, dict.getSizeOfId()); return dict.getValueFromId(id); }
@Override public boolean moveNext() throws IOException { String minValue = null; int curDictIndex = 0; // multi-merge dictionary forest for (int i = 0; i < dictionaryList.size(); i++) { Dictionary<String> dict = dictionaryList.get(i); if (dict == null) continue; int curKey = curKeys.get(i); if (curKey > dict.getMaxId()) continue; String curValue = dict.getValueFromId(curKey); if (minValue == null || dataType.compare(minValue, curValue) > 0) { minValue = curValue; curDictIndex = i; } } if (minValue == null) { curValue = null; return false; } curValue = minValue; curKeys.set(curDictIndex, curKeys.get(curDictIndex) + 1); return true; }
private long runQueryValue(Dictionary<String> dict, int cardnality, int testTimes) { long startTime = System.currentTimeMillis(); int step = 1; for (int i = 0; i < testTimes; i++) { for (int j = 0; j < cardnality; j++) { step |= dict.getValueFromId(j).length(); } } return System.currentTimeMillis() - startTime; }
private long runQueryValueBytes(Dictionary<String> dict, int cardnality, int testTimes) { long startTime = System.currentTimeMillis(); int step = 1; for (int i = 0; i < testTimes; i++) { for (int j = 0; j < cardnality; j++) { //step |= dict.getValueBytesFromId(j).length; step |= dict.getValueFromId(j).length(); } } return System.currentTimeMillis() - startTime; }
private long runQueryValueBytes2(Dictionary<String> dict, int cardnality, int testTimes) { long startTime = System.currentTimeMillis(); int step = 1; byte[] returnValue = new byte[2048]; for (int i = 0; i < testTimes; i++) { for (int j = 0; j < cardnality; j++) { step |= dict.getValueFromId(j).length(); } } return System.currentTimeMillis() - startTime; }
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); ByteArray raw = rawIterator.next(); int key = BytesUtil.readUnsigned(raw.array(), raw.offset(), raw.length()); String colValue = rawColDict.getValueFromId(key); tuple.setDimensionValue(literalTupleIdx, colValue); } };
@Override public boolean contains(Dictionary other) { if (other.getSize() > this.getSize()) { return false; } for (int i = other.getMinId(); i <= other.getMaxId(); ++i) { T v = (T) other.getValueFromId(i); if (!this.containsValue(v)) { return false; } } return true; }
@Override public boolean contains(Dictionary other) { if (other.getSize() > this.getSize()) { return false; } for (int i = other.getMinId(); i <= other.getMaxId(); ++i) { T v = (T) other.getValueFromId(i); if (!this.containsValue(v)) { return false; } } return true; }
Collection<Object> inValues = Lists.newArrayList(); for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { Object dictVal = dict.getValueFromId(i); Object computedVal = builtInFunctionTupleFilter.invokeFunction(dictVal); Class clazz = Primitives.wrap(computedVal.getClass());
@Override public List<ByteArray> reEncodeDictionary(List<ByteArray> value, MeasureDesc measureDesc, Map<TblColRef, Dictionary<String>> oldDicts, Map<TblColRef, Dictionary<String>> newDicts) { TblColRef colRef = getRawColumn(measureDesc.getFunction()); Dictionary<String> sourceDict = oldDicts.get(colRef); Dictionary<String> mergedDict = newDicts.get(colRef); int valueSize = value.size(); byte[] newIdBuf = new byte[valueSize * mergedDict.getSizeOfId()]; int bufOffset = 0; for (ByteArray c : value) { int oldId = BytesUtil.readUnsigned(c.array(), c.offset(), c.length()); int newId; String v = sourceDict.getValueFromId(oldId); if (v == null) { newId = mergedDict.nullId(); } else { newId = mergedDict.getIdFromValue(v); } BytesUtil.writeUnsigned(newId, newIdBuf, bufOffset, mergedDict.getSizeOfId()); c.reset(newIdBuf, bufOffset, mergedDict.getSizeOfId()); bufOffset += mergedDict.getSizeOfId(); } return value; } };
private TupleFilter translateFunctionTupleFilter(BuiltInFunctionTupleFilter builtInFunctionTupleFilter) { if (!builtInFunctionTupleFilter.isValid()) return null; TblColRef columnRef = builtInFunctionTupleFilter.getColumn(); Dictionary<?> dict = dimEncMap.getDictionary(columnRef); if (dict == null) return null; CompareTupleFilter translated = new CompareTupleFilter(builtInFunctionTupleFilter.isReversed() ? FilterOperatorEnum.NOTIN : FilterOperatorEnum.IN); translated.addChild(new ColumnTupleFilter(columnRef)); try { int translatedInClauseMaxSize = KylinConfig.getInstanceFromEnv().getTranslatedInClauseMaxSize(); for (int i = dict.getMinId(); i <= dict.getMaxId(); i++) { Object dictVal = dict.getValueFromId(i); if ((Boolean) builtInFunctionTupleFilter.invokeFunction(dictVal)) { translated.addChild(new ConstantTupleFilter(dictVal)); if (translated.getChildren().size() > translatedInClauseMaxSize) { return null; } } } logger.debug("getting a in clause with {} children", translated.getChildren().size()); } catch (Exception e) { logger.debug(e.getMessage()); return null; } return translated; }
String v = sourceDict.getValueFromId(idInSourceDict); if (v == null) { idInMergedDict = mergedDict.nullId();
@Override public String decode(byte[] bytes, int offset, int len) { int id = BytesUtil.readUnsigned(bytes, offset, len); try { String value = dict.getValueFromId(id); return value; } catch (IllegalArgumentException e) { logger.error("Can't get dictionary value from " + dict + " (id = " + id + ")"); return ""; } }
public List<T> enumeratorValues() { List<T> ret = Lists.newArrayListWithExpectedSize(getSize()); for (int i = getMinId(); i <= getMaxId(); i++) { ret.add(getValueFromId(i)); } return ret; }
@Override public Object deserialize(ByteBuffer in) { int id = BytesUtil.readUnsigned(in, dict.getSizeOfId()); return dict.getValueFromId(id); }
@Override public void fillTuple(Tuple tuple, int row) { if (expectRow++ != row) throw new IllegalStateException(); ByteArray raw = rawIterator.next(); int key = BytesUtil.readUnsigned(raw.array(), raw.offset(), raw.length()); String colValue = rawColDict.getValueFromId(key); tuple.setDimensionValue(literalTupleIdx, colValue); } };