/** * Comparing the 2 byte buffers. This is used in case of data load sorting step. * * @param byteBuffer1 * @param byteBuffer2 * @return */ public int compareTo(ByteBuffer byteBuffer1, ByteBuffer byteBuffer2) { // Short circuit equal case if (byteBuffer1 == byteBuffer2) { return 0; } int len1 = byteBuffer1.remaining(); int len2 = byteBuffer2.remaining(); byte[] buffer1 = new byte[len1]; byte[] buffer2 = new byte[len2]; byteBuffer1.get(buffer1); byteBuffer2.get(buffer2); return compareTo(buffer1, buffer2); }
/** * This method will do min/max comparison of values and update if required * * @param minMaxValueCompare1 * @param minMaxValueCompare2 * @param isMinValueComparison */ private byte[][] compareAndUpdateMinMax(byte[][] minMaxValueCompare1, byte[][] minMaxValueCompare2, boolean isMinValueComparison) { // Compare and update min max values byte[][] updatedMinMaxValues = new byte[minMaxValueCompare1.length][]; System.arraycopy(minMaxValueCompare1, 0, updatedMinMaxValues, 0, minMaxValueCompare1.length); for (int i = 0; i < minMaxValueCompare1.length; i++) { int compare = ByteUtil.UnsafeComparer.INSTANCE .compareTo(minMaxValueCompare2[i], minMaxValueCompare1[i]); if (isMinValueComparison) { if (compare < 0) { updatedMinMaxValues[i] = minMaxValueCompare2[i]; } } else if (compare > 0) { updatedMinMaxValues[i] = minMaxValueCompare2[i]; } } return updatedMinMaxValues; }
private int compareFilterValue(int surrogateKey, List<Integer> sortedSurrogates, byte[] byteValueOfFilterMember, String filterKey) { byte[] dictionaryValue = getDictionaryBytesFromSurrogate(surrogateKey); int cmp = -1; //fortify fix if (null == dictionaryValue) { cmp = -1; } else if (this.getDataType() != DataTypes.STRING) { cmp = compareFilterKeyWithDictionaryKey( new String(dictionaryValue, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)), filterKey, this.getDataType()); } else { cmp = ByteUtil.UnsafeComparer.INSTANCE.compareTo(dictionaryValue, byteValueOfFilterMember); } return cmp; }
private boolean isScanRequired(byte[] blkMaxVal, byte[] blkMinVal, byte[][] filterValues, boolean isMinMaxSet) { if (!isMinMaxSet) { // scan complete data if min max is not written for a given column return true; } boolean isScanRequired = false; for (int k = 0; k < filterValues.length; k++) { // filter value should be in range of max and min value i.e // max>filtervalue>min // so filter-max should be negative int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMaxVal); // and filter-min should be positive int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blkMinVal); // if any filter value is in range than this block needs to be // scanned if (maxCompare <= 0 && minCompare >= 0) { isScanRequired = true; break; } } return isScanRequired; }
@Override public int compareTo(int rowId, byte[] compareValue) { // now to get the row from memory block we need to do following thing // 1. first get the current offset // 2. if it's not a last row- get the next row offset // Subtract the current row offset + 2 bytes(to skip the data length) with next row offset // else subtract the current row offset with complete data // length // get the offset of set of data int currentDataOffset = dataOffsets[rowId]; int length = 0; // calculating the length of data if (rowId < numberOfRows - 1) { length = dataOffsets[rowId + 1] - (currentDataOffset + getLengthSize()); } else { // for last record length = this.dataLength - currentDataOffset; } return ByteUtil.UnsafeComparer.INSTANCE .compareTo(data, currentDataOffset, length, compareValue, 0, compareValue.length); }
private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues, DataType dataType) { boolean isScanRequired = false; Object minValue = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockMinValue, dataType); for (int k = 0; k < filterValues.length; k++) { if (ByteUtil.UnsafeComparer.INSTANCE .compareTo(filterValues[k], CarbonCommonConstants.EMPTY_BYTE_ARRAY) == 0) { return true; } // so filter-min should be positive Object data = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValues[k], dataType); SerializableComparator comparator = Comparator.getComparator(dataType); int minCompare = comparator.compare(data, minValue); // if any filter value is in range than this block needs to be // scanned less than min range. if (minCompare > 0) { isScanRequired = true; break; } } return isScanRequired; }
/** * Compare method for bytes * * @param buffer1 * @param buffer2 * @return */ public static int compare(byte[] buffer1, byte[] buffer2) { // Short circuit equal case if (buffer1 == buffer2) { return 0; } int len1 = buffer1.length; int len2 = buffer2.length; int offset1 = 0; int offset2 = 0; // Call UnsafeComparer compareTo for comparision. return ByteUtil.UnsafeComparer.INSTANCE .compareTo(buffer1, offset1, len1, buffer2, offset2, len2); }
private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues, boolean isMinMaxSet) { if (!isMinMaxSet) { // scan complete data if min max is not written for a given column return true; } boolean isScanRequired = false; for (int k = 0; k < filterValues.length; k++) { // filter value should be in range of max and min value i.e // max>filtervalue>min // so filter-max should be negative int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue); // if any filter value is in range than this block needs to be // scanned less than equal to max range. if (maxCompare <= 0) { isScanRequired = true; break; } } return isScanRequired; }
private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues, DataType dataType) { boolean isScanRequired = false; Object maxValue = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(blockMaxValue, dataType); for (int k = 0; k < filterValues.length; k++) { if (ByteUtil.UnsafeComparer.INSTANCE .compareTo(filterValues[k], CarbonCommonConstants.EMPTY_BYTE_ARRAY) == 0) { return true; } // filter value should be in range of max and min value i.e // max>filtervalue>min // so filter-max should be negative Object data = DataTypeUtil.getDataBasedOnDataTypeForNoDictionaryColumn(filterValues[k], dataType); SerializableComparator comparator = Comparator.getComparator(dataType); int maxCompare = comparator.compare(data, maxValue); // if any filter value is in range than this block needs to be // scanned less than equal to max range. if (maxCompare <= 0) { isScanRequired = true; break; } } return isScanRequired; }
private boolean isScanRequired(byte[] blockMaxValue, byte[][] filterValues, boolean isMinMaxSet) { if (!isMinMaxSet) { // scan complete data if min max is not written for a given column return true; } boolean isScanRequired = false; for (int k = 0; k < filterValues.length; k++) { // filter value should be in range of max and min value i.e // max>filtervalue>min // so filter-max should be negative int maxCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMaxValue); // if any filter value is in range than this block needs to be // scanned less than equal to max range. if (maxCompare < 0) { isScanRequired = true; break; } } return isScanRequired; }
private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues, boolean isMinMaxSet) { if (!isMinMaxSet) { // scan complete data if min max is not written for a given column return true; } boolean isScanRequired = false; for (int k = 0; k < filterValues.length; k++) { // and filter-min should be positive int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue); // if any filter applied is not in range of min and max of block // then since its a less than equal to fiter validate whether the block // min range is less than equal to applied filter member if (minCompare > 0) { isScanRequired = true; break; } } return isScanRequired; }
@Override public void update(byte[] value) { // return if min/max need not be written if (isIgnoreMinMaxFlagSet(value)) { return; } // input value is LV encoded byte[] newValue = getActualValue(value); if (min == null) { min = newValue; } if (null == max) { max = newValue; } if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(min, newValue) > 0) { min = newValue; } if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(max, newValue) < 0) { max = newValue; } }
private boolean isScanRequired(byte[] blockMinValue, byte[][] filterValues, boolean isMinMaxSet) { if (!isMinMaxSet) { // scan complete data if min max is not written for a given column return true; } boolean isScanRequired = false; for (int k = 0; k < filterValues.length; k++) { // and filter-min should be positive int minCompare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterValues[k], blockMinValue); // if any filter applied is not in range of min and max of block // then since its a less than equal to fiter validate whether the block // min range is less than equal to applied filter member if (minCompare >= 0) { isScanRequired = true; break; } } return isScanRequired; }
@Override public int compareTo(int rowId, byte[] compareValue) { // rowId is the inverted index, but the null bitset is based on actual data int nullBitSetRowId = rowId; if (isExplicitSorted()) { nullBitSetRowId = getInvertedIndex(rowId); } byte[] nullBitSet = getNullBitSet(nullBitSetRowId, columnPage.getColumnSpec().getColumnType()); if (nullBitSet != null && ByteUtil.UnsafeComparer.INSTANCE.compareTo(nullBitSet, compareValue) == 0) { // check if the compare value is a null value // if the compare value is null and the data is also null we can directly return 0 return 0; } else { byte[] chunkData = this.getChunkDataInBytes(rowId); return ByteUtil.UnsafeComparer.INSTANCE.compareTo(chunkData, compareValue); } }
/** * Return negative value if {@code buffer1} less than {@code buffer2}, * return 0 if they are equal, otherwise return positive value. * @param buffer1 value to compare * @param buffer2 value to compare * @return compare result */ public int compareTo(byte[] buffer1, byte[] buffer2) { // Short circuit equal case if (buffer1 == buffer2) { return 0; } int len1 = buffer1.length; int len2 = buffer2.length; int minLength = (len1 <= len2) ? len1 : len2; return compareTo(buffer1, buffer2, len1, len2, minLength); }
@Override public int compareTo(ColumnWithRowId o) { return ByteUtil.UnsafeComparer.INSTANCE.compareTo(column, o.column); }
@Override public int compare(byte[] filterMember1, byte[] filterMember2) { return ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterMember1, filterMember2); }
/** * to compare the two byte array * * @param rowId index of first byte array * @param compareValue value of to be compared * @return compare result */ @Override public int compareTo(int rowId, byte[] compareValue) { return ByteUtil.UnsafeComparer.INSTANCE .compareTo(data, rowId * columnValueSize, columnValueSize, compareValue, 0, columnValueSize); }
@Override public int compareTo(ColumnWithRowId o) { return UnsafeComparer.INSTANCE .compareTo(column, 2, column.length - 2, o.column, 2, o.column.length - 2); }