public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(FieldType fieldType) { AddOrGetResult<T> result = super.addOrGetVector(fieldType); reader = new UnionListReader(this); return result; }
@Override public boolean allocateNewSafe() { boolean dataAlloc = false; try { allocateOffsetBuffer(offsetAllocationSizeInBytes); dataAlloc = vector.allocateNewSafe(); } catch (Exception e) { e.printStackTrace(); clear(); return false; } finally { if (!dataAlloc) { clear(); } } return dataAlloc; }
@Override public Iterator<ValueVector> iterator() { return Collections.<ValueVector>singleton(getDataVector()).iterator(); }
public int startNewValue(int index) { while (index >= getOffsetBufferValueCapacity()) { reallocOffsetBuffer(); } int offset = offsetBuffer.getInt(index * OFFSET_WIDTH); offsetBuffer.setInt((index + 1) * OFFSET_WIDTH, offset); setValueCount(index + 1); return offset; }
public void setValueCount(int valueCount) { this.valueCount = valueCount; while (valueCount > getOffsetBufferValueCapacity()) { reallocOffsetBuffer(); } final int childValueCount = valueCount == 0 ? 0 : offsetBuffer.getInt(valueCount * OFFSET_WIDTH); vector.setValueCount(childValueCount); } }
public void load(SerializedField metadata, ArrowBuf buffer) { /* release the current buffers (if any) */ vector.clear(); /* get the metadata for all children */ final SerializedField offsetMetadata = metadata.getChild(0); final SerializedField vectorMetadata = metadata.getChild(1); final int offsetLength = offsetMetadata.getBufferLength(); final int vectorLength = vectorMetadata.getBufferLength(); /* load inner offset buffer */ loadOffsetBuffer(offsetMetadata, buffer); /* load inner data vector */ if (vector.getDataVector() == BaseRepeatedValueVector.DEFAULT_DATA_VECTOR) { vector.addOrGetVector(FieldType.nullable(getArrowMinorType(metadata.getMajorType().getMinorType()).getType())); } TypeHelper.load(vector.vector, vectorMetadata, buffer.slice(offsetLength, vectorLength)); }
@Override public ArrowBuf[] getBuffers(boolean clear) { final ArrowBuf[] buffers; if (getBufferSize() == 0) { buffers = new ArrowBuf[0]; } else { List<ArrowBuf> list = new ArrayList<>(); list.add(offsetBuffer); list.addAll(Arrays.asList(vector.getBuffers(false))); buffers = list.toArray(new ArrowBuf[list.size()]); } if (clear) { for (ArrowBuf buffer : buffers) { buffer.retain(); } clear(); } return buffers; }
@Override public void clear() { super.clear(); validityBuffer = releaseBuffer(validityBuffer); lastSet = 0; }
private int handleRemainder() { final int remainingRecordCount = ((BaseRepeatedValueVector)flattener.getFlattenField()).getInnerValueCount() - remainderIndex; doAlloc(); int projRecords = flattener.flattenRecords(remainingRecordCount, 0, monitor); if (projRecords < remainingRecordCount) { remainderIndex += projRecords; } else { hasRemainder = false; remainderIndex = 0; flattener.resetGroupIndex(); state = State.CAN_CONSUME; } setValueCount(projRecords); return projRecords; }
@Override public int getValueCapacity() { final int offsetValueCapacity = Math.max(getOffsetBufferValueCapacity() - 1, 0); if (vector == DEFAULT_DATA_VECTOR) { return offsetValueCapacity; } return Math.min(vector.getValueCapacity(), offsetValueCapacity); }
@Override public int getBufferSize() { if (getValueCount() == 0) { return 0; } return ((valueCount + 1) * OFFSET_WIDTH) + vector.getBufferSize(); }
/** * Allocate memory for the vector. We internally use a default value count * of 4096 to allocate memory for at least these many elements in the * vector. * * @return false if memory allocation fails, true otherwise. */ public boolean allocateNewSafe() { boolean success = false; try { /* we are doing a new allocation -- release the current buffers */ clear(); /* allocate validity buffer */ allocateValidityBuffer(validityAllocationSizeInBytes); /* allocate offset and data buffer */ success = super.allocateNewSafe(); } finally { if (!success) { clear(); return false; } } return true; }
final int valueCount = fieldToFlatten.getValueCount(); for ( ; valueIndexLocal < valueCount; valueIndexLocal++) { final int innerValueCount = ((BaseRepeatedValueVector)fieldToFlatten).getInnerValueCountAt(valueIndexLocal); for ( ; innerValueIndexLocal < innerValueCount; innerValueIndexLocal++) {
private int handleInitial(int incomingRecordCount) { doAlloc(); int childCount = incomingRecordCount == 0 ? 0 : ((BaseRepeatedValueVector)flattener.getFlattenField()).getInnerValueCount(); int outputRecords = flattener.flattenRecords(incomingRecordCount, 0, monitor); // TODO - change this to be based on the repeated vector length if (outputRecords < childCount) { setValueCount(outputRecords); hasRemainder = true; remainderIndex = outputRecords; return remainderIndex; } else { setValueCount(outputRecords); flattener.resetGroupIndex(); state = State.CAN_CONSUME; return outputRecords; } }