private void writeMap(ListVector arrowVector, MapColumnVector hiveVector, MapTypeInfo typeInfo, int size, VectorizedRowBatch vectorizedRowBatch, boolean isNative) { final ListTypeInfo structListTypeInfo = toStructListTypeInfo(typeInfo); final ListColumnVector structListVector = toStructListVector(hiveVector); write(arrowVector, structListVector, structListTypeInfo, size, vectorizedRowBatch, isNative); final ArrowBuf validityBuffer = arrowVector.getValidityBuffer(); for (int rowIndex = 0; rowIndex < size; rowIndex++) { if (hiveVector.isNull[rowIndex]) { BitVectorHelper.setValidityBit(validityBuffer, rowIndex, 0); } else { BitVectorHelper.setValidityBitToOne(validityBuffer, rowIndex); } } }
/** * Set the element at the given index to null. * * @param index position of element */ public void setNull(int index) { handleSafe(index); // not really needed to set the bit to 0 as long as // the buffer always starts from 0. BitVectorHelper.setValidityBit(validityBuffer, index, 0); }
/** * Set the element at the given index to the value set in data holder. * * @param index position of element * @param holder data holder for value of element */ public void set(int index, TimeStampMilliTZHolder holder) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); }
private void splitAndTransferValidityBuffer(int startIndex, int length, ListVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + byteSizeTarget, offset); byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1);
/** * Set the bit at a given index to provided value (1 or 0). Internally * takes care of allocating the buffer if the caller didn't do so. * * @param validityBuffer validity buffer of the vector * @param allocator allocator for the buffer * @param valueCount number of values to allocate/set * @param index index to be set * @param value value to set * @return ArrowBuf */ public static ArrowBuf setValidityBit(ArrowBuf validityBuffer, BufferAllocator allocator, int valueCount, int index, int value) { if (validityBuffer == null) { validityBuffer = allocator.buffer(getValidityBufferSize(valueCount)); } setValidityBit(validityBuffer, index, value); if (index == (valueCount - 1)) { validityBuffer.writerIndex(getValidityBufferSize(valueCount)); } return validityBuffer; }
private void splitAndTransferValidityBuffer(int startIndex, int length, StructVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); int byteSizeTarget = BitVectorHelper.getValidityBufferSize(length); int offset = startIndex % 8; byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + i, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + i + 1, offset); byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(validityBuffer, firstByteSource + byteSizeTarget, offset); byte b1 = BitVectorHelper.getBitsFromCurrentByte(validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1);
int startByteIndex = BitVectorHelper.byteIndex(firstBitIndex); final int lastBitIndex = firstBitIndex + count; final int endByteIndex = BitVectorHelper.byteIndex(lastBitIndex); final int startByteBitIndex = BitVectorHelper.bitIndex(firstBitIndex); final int endBytebitIndex = BitVectorHelper.bitIndex(lastBitIndex); if (count < 8 && startByteIndex == endByteIndex) { bitMask |= (byte) (1L << i); BitVectorHelper.setBitMaskedByte(validityBuffer, startByteIndex, bitMask); BitVectorHelper.setBitMaskedByte(valueBuffer, startByteIndex, bitMask); } else { BitVectorHelper.setBitMaskedByte(validityBuffer, startByteIndex, bitMask); BitVectorHelper.setBitMaskedByte(valueBuffer, startByteIndex, bitMask); ++startByteIndex; final int byteIndex = BitVectorHelper.byteIndex(lastBitIndex - endBytebitIndex); final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7)); BitVectorHelper.setBitMaskedByte(validityBuffer, byteIndex, bitMask); BitVectorHelper.setBitMaskedByte(valueBuffer, byteIndex, bitMask);
@Override public void setInitialCapacity(int numRecords, double density) { validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); super.setInitialCapacity(numRecords, density); }
/** * Set the bit at provided index to 1. * * @param validityBuffer validity buffer of the vector * @param index index to be set */ public static void setValidityBitToOne(ArrowBuf validityBuffer, int index) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); final byte bitMask = (byte) (1L << bitIndex); currentByte |= bitMask; validityBuffer.setByte(byteIndex, currentByte); }
/** * Get the number of elements that are null in the vector. * * @return the number of null elements. */ public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); }
break; case BIT: generator.writeNumber(BitVectorHelper.get(buffer, index)); break; case VARBINARY: {
/** * Set the element at the given index to null. * * @param index position of element */ public void setNull(int index) { handleSafe(index); // not really needed to set the bit to 0 as long as // the buffer always starts from 0. BitVectorHelper.setValidityBit(validityBuffer, index, 0); }
/** * Set the element at the given index to the value set in data holder. * * @param index position of element * @param holder data holder for value of element */ public void set(int index, Float8Holder holder) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); }
private void splitAndTransferValidityBuffer(int startIndex, int length, BaseVariableWidthVector target) { assert startIndex + length <= valueCount; int firstByteSource = BitVectorHelper.byteIndex(startIndex); int lastByteSource = BitVectorHelper.byteIndex(valueCount - 1); int byteSizeTarget = getValidityBufferSizeFromCount(length); int offset = startIndex % 8; byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + i, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + i + 1, offset); byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); byte b2 = BitVectorHelper.getBitsFromNextByte(this.validityBuffer, firstByteSource + byteSizeTarget, offset); byte b1 = BitVectorHelper.getBitsFromCurrentByte(this.validityBuffer, firstByteSource + byteSizeTarget - 1, offset); target.validityBuffer.setByte(byteSizeTarget - 1, b1);
@Override protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException { final int bufferSize = BitVectorHelper.getValidityBufferSize(count); ArrowBuf buf = allocator.buffer(bufferSize); // C++ integration test fails without this. buf.setZero(0, bufferSize); for (int i = 0; i < count; i++) { parser.nextToken(); BitVectorHelper.setValidityBit(buf, i, parser.readValueAs(Boolean.class) ? 1 : 0); } buf.writerIndex(bufferSize); return buf; } };
@Override public void setInitialCapacity(int numRecords) { validityAllocationSizeInBytes = BitVectorHelper.getValidityBufferSize(numRecords); super.setInitialCapacity(numRecords); }
/** * Set the bit at a given index to provided value (1 or 0). * * @param validityBuffer validity buffer of the vector * @param index index to be set * @param value value to set */ public static void setValidityBit(ArrowBuf validityBuffer, int index, int value) { final int byteIndex = byteIndex(index); final int bitIndex = bitIndex(index); byte currentByte = validityBuffer.getByte(byteIndex); final byte bitMask = (byte) (1L << bitIndex); if (value != 0) { currentByte |= bitMask; } else { currentByte -= (bitMask & currentByte); } validityBuffer.setByte(byteIndex, currentByte); }
/** * Get the number of elements that are null in the vector. * * @return the number of null elements. */ @Override public int getNullCount() { return BitVectorHelper.getNullCount(validityBuffer, valueCount); }
/** * Set the element at the given index to null. * * @param index position of element */ public void setNull(int index) { handleSafe(index); // not really needed to set the bit to 0 as long as // the buffer always starts from 0. BitVectorHelper.setValidityBit(validityBuffer, index, 0); }
/** * Set the element at the given index to the value set in data holder. * * @param index position of element * @param holder data holder for value of element */ public void set(int index, TimeStampMicroHolder holder) { BitVectorHelper.setValidityBitToOne(validityBuffer, index); setValue(index, holder.value); }