@Override final boolean isNullAt(int rowId) { // TODO: Workaround if vector has all non-null values, see ARROW-1948 if (accessor.getValueCount() > 0 && accessor.getValidityBuffer().capacity() == 0) { return false; } else { return super.isNullAt(rowId); } }
private void readList(FieldVector arrowVector, ListColumnVector hiveVector, ListTypeInfo typeInfo) { final int size = arrowVector.getValueCount(); final ArrowBuf offsets = arrowVector.getOffsetBuffer(); final int OFFSET_WIDTH = 4; read(arrowVector.getChildrenFromFields().get(0), hiveVector.child, typeInfo.getListElementTypeInfo()); for (int i = 0; i < size; i++) { if (arrowVector.isNull(i)) { VectorizedBatchUtil.setNullColIsNullValue(hiveVector, i); } else { hiveVector.isNull[i] = false; final int offset = offsets.getInt(i * OFFSET_WIDTH); hiveVector.offsets[i] = offset; hiveVector.lengths[i] = offsets.getInt((i + 1) * OFFSET_WIDTH) - offset; } } }
@Override final UTF8String getUTF8String(int rowId) { accessor.get(rowId, stringResult); if (stringResult.isSet == 0) { return null; } else { return UTF8String.fromAddress(null, stringResult.buffer.memoryAddress() + stringResult.start, stringResult.end - stringResult.start); } } }
public void loadDataBuffer(SerializedField metadata, ArrowBuf buffer) { final int actualLength = metadata.getBufferLength(); final int valueCount = metadata.getValueCount(); final int expectedLength = valueCount * 1; assert actualLength == expectedLength : String.format("Expected to load %d bytes but actually loaded %d bytes in data buffer", expectedLength, actualLength); vector.valueBuffer = buffer.slice(0, actualLength); vector.valueBuffer.retain(1); vector.valueBuffer.writerIndex(actualLength); } }
private void resizeBuffer(int sizeInBytes) { int targetSize = Numbers.nextPowerOfTwo(sizeInBytes); final ArrowBuf oldBuf = buf; buf = allocator.buffer(targetSize); PlatformDependent.copyMemory(oldBuf.memoryAddress(), buf.memoryAddress(), oldBuf.capacity()); buf.writerIndex(oldBuf.writerIndex()); oldBuf.release(); }
public static void setNotNullAndZero(FieldVector vector){ List<ArrowBuf> buffers = vector.getFieldBuffers(); ArrowBuf bits = buffers.get(0); writeWordwise(bits.memoryAddress(), bits.capacity(), ON); ArrowBuf values = buffers.get(1); writeWordwise(values.memoryAddress(), values.capacity(), OFF); }
/** * End the current value. * * @param index index of the value to end * @param size number of elements in the list that was written */ public void endValue(int index, int size) { final int currentOffset = offsetBuffer.getInt((index + 1) * OFFSET_WIDTH); offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, currentOffset + size); }
@Override public BytesWritable getPrimitiveWritableObject(Object o) { final VarBinaryHolder h = (VarBinaryHolder)o; final byte[] buf = new byte[h.end-h.start]; h.buffer.getBytes(h.start, buf, 0, h.end-h.start); return new BytesWritable(buf); }
public void reset() { buf.readerIndex(0); buf.writerIndex(0); buf.setZero(0, buf.capacity()); }
@Override public void output() { byte[] ba = ((com.dremio.exec.expr.fn.hll.UnionAccum) work.obj).getOutputBytes(); buffer = buffer.reallocIfNeeded(ba.length); out.buffer = buffer; out.start = 0; out.end = ba.length; out.buffer.setBytes(0, ba); out.isSet = 1; }
private void resizeBuffer(int newCapacity) { final int blockWidth = this.blockWidth; final int sizeInBytes = Numbers.nextPowerOfTwo(newCapacity * blockWidth); final ArrowBuf oldBuf = buf; buf = allocator.buffer(sizeInBytes); final int oldCapacity = this.capacity; fillZeros(buf.memoryAddress() + oldCapacity * blockWidth, (newCapacity - oldCapacity) * blockWidth); PlatformDependent.copyMemory(oldBuf.memoryAddress(), buf.memoryAddress(), oldCapacity * blockWidth); buf.writerIndex(oldBuf.writerIndex()); oldBuf.release(); this.capacity = newCapacity; }
@Override public synchronized void close() { if(buf != null){ buf.release(); buf = null; } }
@Override public void loadFieldBuffers(ArrowFieldNode fieldNode, List<ArrowBuf> ownBuffers) { if (ownBuffers.size() != 1) { throw new IllegalArgumentException("Illegal buffer count, expected " + 1 + ", got: " + ownBuffers.size()); } ArrowBuf bitBuffer = ownBuffers.get(0); validityBuffer.release(); validityBuffer = BitVectorHelper.loadValidityBuffer(fieldNode, bitBuffer, allocator); valueCount = fieldNode.getLength(); validityAllocationSizeInBytes = validityBuffer.capacity(); }
@Nullable @Override public ArrowBuf apply(@Nullable ArrowBuf buf) { int writerIndex = buf.writerIndex(); ArrowBuf newBuf = buf.transferOwnership(context.getAllocator()).buffer; newBuf.writerIndex(writerIndex); buf.release(); return newBuf; } }).toList();
public void setup() { buffer = buffer.reallocIfNeeded(100); byte[] buf = new byte[right.end - right.start]; right.buffer.getBytes(right.start, buf, 0, right.end - right.start); String inputFormat = new String(buf); outputFormat = new java.text.DecimalFormat(inputFormat); }
private void cleanPartition() { partition = null; resetValues(); for (VectorWrapper<?> vw : internal) { ValueVector vv = vw.getValueVector(); if ((vv instanceof BaseValueVector)) { ArrowBuf validityBuffer = vv.getValidityBuffer(); validityBuffer.setZero(0, validityBuffer.capacity()); } } lagCopiedToInternal = false; }
protected final void setBytes(int index, byte[] value, int start, int length) { /* end offset of current last element in the vector. this will * be the start offset of new element we are trying to store. */ final int startOffset = getstartOffset(index); /* set new end offset */ offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, startOffset + length); /* store the var length data in value buffer */ valueBuffer.setBytes(startOffset, value, start, length); }